diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index d5b187a..6517412 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -31,3 +31,9 @@ jobs: - name: Run pre-commit hooks run: | pre-commit run --all-files + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Verify Hugging Face publishing bundle + run: uv run scripts/export_huggingface_viewer.py --check diff --git a/README.md b/README.md index 119406a..583f056 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ Each task runs in an isolated, stateful tool environment and is evaluated with executable checks over the final backend state and side effects. Some tasks also check required properties of the final response. +Browse the 507 tasks, shared scenarios, and agent configuration in the +[ThinkingBox-Bench dataset on Hugging Face](https://huggingface.co/datasets/microsoft/ThinkingBox-Bench). +The Hugging Face dataset is a viewer-friendly representation; this GitHub +repository remains the executable source. + If your goal is to run the published benchmark, follow the complete [ThinkingBox-Bench v1.0 installation and run instructions](releases/thinkingbox_bench_v1/README.md#run-the-benchmark) directly. The remaining setup and examples in this README are intended for diff --git a/releases/thinkingbox_bench_v1/README.md b/releases/thinkingbox_bench_v1/README.md index f7bfb93..a97e954 100644 --- a/releases/thinkingbox_bench_v1/README.md +++ b/releases/thinkingbox_bench_v1/README.md @@ -39,6 +39,52 @@ so these alternatives do not increase the task count. Other datasets and tests in this repository are not part of ThinkingBox-Bench v1.0. +## Hugging Face Dataset Viewer + +Browse this release in the public +[ThinkingBox-Bench dataset on Hugging Face](https://huggingface.co/datasets/microsoft/ThinkingBox-Bench). +Its Viewer exposes three linked subsets: + +| Subset | Rows | Contents | +| - | -: | - | +| `tasks` | 507 | User goal, initial-state patch, expected tool interactions, and rubrics | +| `scenarios` | 5 | Shared world state and available tools, linked by `scenario_id` | +| `agents` | 1 | Agent instructions and built-in tools | + +### Maintainer workflow + +The generated [`huggingface/`](huggingface/) directory is a self-contained +publishing bundle for a separate Hugging Face dataset repository. It provides +the Viewer data without copying the executable benchmark. + +Regenerate the Parquet files from the canonical release sources: + +```bash +uv run scripts/export_huggingface_viewer.py +``` + +Verify that the committed exports are current: + +```bash +uv run scripts/export_huggingface_viewer.py --check +``` + +Publishing updates under `microsoft/` requires membership in the Microsoft +Hugging Face organization with write access. Confirm the authenticated account +before uploading: + +```bash +hf auth login +hf auth whoami +hf upload microsoft/ThinkingBox-Bench \ + releases/thinkingbox_bench_v1/huggingface . \ + --repo-type dataset +``` + +The Hugging Face repository contains the dataset card, license, and derived +Parquet tables. Users run the benchmark from the tagged GitHub sources +documented below. + ## Evaluation method Each task compares the final backend state with its golden expected state using diff --git a/releases/thinkingbox_bench_v1/huggingface/LICENSE.txt b/releases/thinkingbox_bench_v1/huggingface/LICENSE.txt new file mode 100644 index 0000000..aad2f5f --- /dev/null +++ b/releases/thinkingbox_bench_v1/huggingface/LICENSE.txt @@ -0,0 +1,104 @@ +This repository is dual-licensed by content type: + + - Software (code under servers/ and any first-party Python in this repo) + is licensed under the MIT License (SPDX: MIT) — see the MIT section + below. + + - Data (content under dataset/, support/, and releases/) is licensed under + the Community Data License Agreement – Permissive – Version 2.0 (SPDX: + CDLA-Permissive-2.0) — see the CDLA section below. The canonical text + is published at https://cdla.dev/permissive-2-0/. + +When redistributing data from this repository, include this LICENSE.txt +alongside the redistributed data, as required by section 2.1 of the CDLA. + + +================================================================================ +MIT License (applies to software / code) +================================================================================ + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================================ +CDLA-Permissive-2.0 (applies to data) +================================================================================ + +Copyright (c) 2026 Microsoft Corporation + + +Community Data License Agreement – Permissive – Version 2.0 + +This is the Community Data License Agreement – Permissive, Version 2.0 (the +"agreement"). Data Provider(s) and Data Recipient(s) agree as follows: + +1. Provision of the Data + +1.1. A Data Recipient may use, modify, and share the Data made available by +Data Provider(s) under this agreement if that Data Recipient follows the terms +of this agreement. + +1.2. This agreement does not impose any restriction on a Data Recipient's use, +modification, or sharing of any portions of the Data that are in the public +domain or that may be used, modified, or shared under any other legal +exception or limitation. + +2. Conditions for Sharing Data + +2.1. A Data Recipient may share Data, with or without modifications, so long +as the Data Recipient makes available the text of this agreement with the +shared Data. + +3. No Restrictions on Results + +3.1. This agreement does not impose any restriction or obligations with +respect to the use, modification, or sharing of Results. + +4. No Warranty; Limitation of Liability + +4.1. All Data Recipients receive the Data subject to the following terms: + +THE DATA IS PROVIDED ON AN "AS IS" BASIS, WITHOUT REPRESENTATIONS, WARRANTIES +OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT +LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, +MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +NO DATA PROVIDER SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT +LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE DATA OR RESULTS, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGES. + +5. Definitions + +5.1. "Data" means the material received by a Data Recipient under this +agreement. + +5.2. "Data Provider" means any person who is the source of Data provided +under this agreement and in reliance on a Data Recipient's agreement to its +terms. + +5.3. "Data Recipient" means any person who receives Data directly or +indirectly from a Data Provider and agrees to the terms of this agreement. + +5.4. "Results" means any outcome obtained by computational analysis of Data, +including for example machine learning models and models' insights. diff --git a/releases/thinkingbox_bench_v1/huggingface/README.md b/releases/thinkingbox_bench_v1/huggingface/README.md new file mode 100644 index 0000000..f899daf --- /dev/null +++ b/releases/thinkingbox_bench_v1/huggingface/README.md @@ -0,0 +1,99 @@ +--- +pretty_name: ThinkingBox-Bench +license: cdla-permissive-2.0 +language: + - en +task_categories: + - reinforcement-learning +tags: + - agent + - tool-use + - benchmark + - evaluation +configs: + - config_name: tasks + default: true + data_files: + - split: test + path: data/tasks.parquet + - config_name: scenarios + data_files: + - split: test + path: data/scenarios.parquet + - config_name: agents + data_files: + - split: test + path: data/agents.parquet +--- + +# ThinkingBox-Bench + +ThinkingBox-Bench is an executable benchmark for evaluating whether tool-using +LLM agents can reliably complete stateful business workflows. Version 1.0 +contains 507 tool-agent-user tasks across retail and e-commerce, travel and +hospitality, auto insurance, neobank support, and consulting IT/HR support. + +This dataset repository provides a browsable representation of the benchmark. +The executable benchmark, tool servers, and supporting fixtures are maintained +in the +[`microsoft/thinkingbox-data`](https://github.com/microsoft/thinkingbox-data) +GitHub repository. + +## Dataset structure + +| Subset | Rows | Contents | +| - | -: | - | +| `tasks` | 507 | User goal, initial-state patch, expected tool interactions, and rubrics | +| `scenarios` | 5 | Shared world state and available tools, linked by `scenario_id` | +| `agents` | 1 | Agent instructions and built-in tools | + +Select a subset using the Dataset Viewer dropdown. Each task references its +shared scenario through `scenario_id`. Nested task state and expected +interactions are serialized as JSON strings so they remain readable and +portable in the Viewer. + +### Task fields + +| Field | Description | +| - | - | +| `task_ref` | Canonical `file.py:function_name` identifier from the release test list | +| `domain` | Human-readable benchmark domain | +| `scenario_id` | Key linking the task to its shared row in the `scenarios` subset | +| `query` | Initial request sent by the simulated user | +| `user_context` | Instructions and facts available to the simulated user | +| `initial_state_patch_json` | JSON object applied to the scenario's base world state before the task starts | +| `expected_tool_interactions_json` | Ordered golden tool calls that define the expected state changes | +| `rubrics_json` | Additional response requirements evaluated for applicable tasks | +| `source_url` | Tagged GitHub source containing the executable test definition | +| `release_tag` | Immutable `thinkingbox-data` release used to generate the row | + +The dataset does not store a precomputed expected end state. During evaluation, +the scenario's MCP server creates a fresh database, applies +`initial_state_patch_json`, and replays `expected_tool_interactions_json` to +materialize the `golden_db_state`. It then compares the stable hash of that +state with the hash of the database modified by the evaluated agent. This +runtime process ensures the expected state uses the same tool implementation +and database semantics as the agent's attempt. + +## Intended use + +ThinkingBox-Bench v1.0 is intended exclusively for evaluation. Do not use its +task content, expected outcomes, golden state, or tool trajectories for prompt +optimization, fine-tuning, reinforcement learning, reward-model training, or +other model optimization. + +## Run the benchmark + +The Parquet tables are for browsing and analysis; they are not the executable +runtime. Follow the +[ThinkingBox-Bench v1.0 instructions](https://github.com/microsoft/thinkingbox-data/blob/thinkingbox-bench-v1.0/releases/thinkingbox_bench_v1/README.md#run-the-benchmark) +to install ThinkingBox, start the required services, and run all 507 tasks. + +For reproducibility, use the +[`thinkingbox-bench-v1.0`](https://github.com/microsoft/thinkingbox-data/releases/tag/thinkingbox-bench-v1.0) +release. + +## License + +The dataset is licensed under the Community Data License Agreement - +Permissive - Version 2.0 (`CDLA-Permissive-2.0`). See `LICENSE.txt`. diff --git a/releases/thinkingbox_bench_v1/huggingface/data/agents.parquet b/releases/thinkingbox_bench_v1/huggingface/data/agents.parquet new file mode 100644 index 0000000..c0a01b2 Binary files /dev/null and b/releases/thinkingbox_bench_v1/huggingface/data/agents.parquet differ diff --git a/releases/thinkingbox_bench_v1/huggingface/data/scenarios.parquet b/releases/thinkingbox_bench_v1/huggingface/data/scenarios.parquet new file mode 100644 index 0000000..6fd3d13 Binary files /dev/null and b/releases/thinkingbox_bench_v1/huggingface/data/scenarios.parquet differ diff --git a/releases/thinkingbox_bench_v1/huggingface/data/tasks.parquet b/releases/thinkingbox_bench_v1/huggingface/data/tasks.parquet new file mode 100644 index 0000000..75bdd51 Binary files /dev/null and b/releases/thinkingbox_bench_v1/huggingface/data/tasks.parquet differ diff --git a/scripts/export_huggingface_viewer.py b/scripts/export_huggingface_viewer.py new file mode 100644 index 0000000..a7cd03e --- /dev/null +++ b/scripts/export_huggingface_viewer.py @@ -0,0 +1,355 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# /// script +# dependencies = [ +# "pyarrow>=18.0.0", +# "PyYAML>=6.0.2", +# ] +# /// + +"""Generate Hugging Face Dataset Viewer tables for ThinkingBox-Bench v1.0.""" + +from __future__ import annotations + +import argparse +import ast +import json +import shutil +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq +import yaml + +RELEASE_TAG = "thinkingbox-bench-v1.0" +REPOSITORY_URL = "https://github.com/microsoft/thinkingbox-data" +EXPECTED_ROW_COUNTS = {"tasks": 507, "scenarios": 5, "agents": 1} +ROOT = Path(__file__).resolve().parents[1] +RELEASE_DIR = ROOT / "releases" / "thinkingbox_bench_v1" +TEST_LIST = RELEASE_DIR / "testlist_thinkingbox_bench_v1.yaml" +BUNDLE_DIR = RELEASE_DIR / "huggingface" +DATA_DIR = BUNDLE_DIR / "data" +LICENSE_SOURCE = ROOT / "LICENSE.txt" +LICENSE_DESTINATION = BUNDLE_DIR / "LICENSE.txt" +DATASET_CARD = BUNDLE_DIR / "README.md" +EXPECTED_BUNDLE_FILES = { + Path("LICENSE.txt"), + Path("README.md"), + Path("data/agents.parquet"), + Path("data/scenarios.parquet"), + Path("data/tasks.parquet"), +} +EXPECTED_CONFIGS = [ + { + "config_name": "tasks", + "default": True, + "data_files": [{"split": "test", "path": "data/tasks.parquet"}], + }, + { + "config_name": "scenarios", + "data_files": [{"split": "test", "path": "data/scenarios.parquet"}], + }, + { + "config_name": "agents", + "data_files": [{"split": "test", "path": "data/agents.parquet"}], + }, +] + +DOMAIN_BY_SCENARIO = { + "sandbox_external_retail": "retail_and_ecommerce", + "external_booking_v1": "travel_and_hospitality", + "sandbox_auto_insurance": "auto_insurance", + "sandbox_neobank_support_v1": "neobank_support", + "sandbox_consulting": "consulting_it_and_hr", +} + +TASK_SCHEMA = pa.schema( + [ + ("task_ref", pa.string()), + ("domain", pa.string()), + ("scenario_id", pa.string()), + ("query", pa.string()), + ("user_context", pa.string()), + ("initial_state_patch_json", pa.string()), + ("expected_tool_interactions_json", pa.string()), + ("rubrics_json", pa.string()), + ("source_url", pa.string()), + ("release_tag", pa.string()), + ] +) +SCENARIO_SCHEMA = pa.schema( + [ + ("scenario_id", pa.string()), + ("domain", pa.string()), + ("world_state_json", pa.string()), + ("available_tools", pa.list_(pa.string())), + ("source_url", pa.string()), + ("release_tag", pa.string()), + ] +) +AGENT_SCHEMA = pa.schema( + [ + ("agent_id", pa.string()), + ("system_instructions", pa.string()), + ("parallel_tool_calls", pa.bool_()), + ("builtin_tools_json", pa.string()), + ("source_url", pa.string()), + ("release_tag", pa.string()), + ] +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="Verify that committed exports and license match the current sources.", + ) + return parser.parse_args() + + +def load_yaml(path: Path) -> Any: + with path.open(encoding="utf-8") as stream: + return yaml.safe_load(stream) + + +def to_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def source_url(path: Path) -> str: + relative_path = path.relative_to(ROOT).as_posix() + return f"{REPOSITORY_URL}/blob/{RELEASE_TAG}/{relative_path}" + + +def find_test_files() -> dict[str, Path]: + files_by_name: dict[str, Path] = {} + for path in sorted((ROOT / "dataset" / "test_case").rglob("*.py")): + if path.name in files_by_name: + raise ValueError(f"Test filename is ambiguous: {path.name}") + files_by_name[path.name] = path + return files_by_name + + +def load_task_document( + task_ref: str, files_by_name: dict[str, Path] +) -> tuple[dict[str, Any], Path]: + try: + filename, function_name = task_ref.split(":", 1) + path = files_by_name[filename] + except (KeyError, ValueError) as error: + raise ValueError(f"Invalid task reference: {task_ref}") from error + + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + function = next( + ( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == function_name + ), + None, + ) + if function is None: + raise ValueError(f"Function not found for task reference: {task_ref}") + + docstring = ast.get_docstring(function, clean=False) + if not docstring or not docstring.startswith("!"): + raise ValueError(f"Task does not contain a YAML docstring: {task_ref}") + + document = yaml.safe_load(docstring[1:]) + if not isinstance(document, dict): + raise ValueError(f"Task YAML is not a mapping: {task_ref}") + return document, path + + +def build_task_row( + task_ref: str, files_by_name: dict[str, Path] +) -> dict[str, Any]: + document, path = load_task_document(task_ref, files_by_name) + initial_state = document.get("init") + if not isinstance(initial_state, dict) or len(initial_state) != 1: + raise ValueError(f"Task must initialize exactly one scenario: {task_ref}") + + scenario_id, scenario_state = next(iter(initial_state.items())) + if scenario_id not in DOMAIN_BY_SCENARIO or not isinstance(scenario_state, dict): + raise ValueError(f"Unsupported scenario in task: {task_ref}") + + golden_test_case = scenario_state.get("golden_test_case") + if not isinstance(golden_test_case, dict) or not isinstance( + golden_test_case.get("tool_interactions"), list + ): + raise ValueError(f"Task has no golden tool interactions: {task_ref}") + + return { + "task_ref": task_ref, + "domain": DOMAIN_BY_SCENARIO[scenario_id], + "scenario_id": scenario_id, + "query": document.get("query", ""), + "user_context": document.get("user_context", ""), + "initial_state_patch_json": to_json(scenario_state.get("data_patch") or {}), + "expected_tool_interactions_json": to_json( + golden_test_case["tool_interactions"] + ), + "rubrics_json": to_json( + scenario_state.get("rubrics_yesno") + or scenario_state.get("rubrics") + or [] + ), + "source_url": source_url(path), + "release_tag": RELEASE_TAG, + } + + +def build_tasks() -> pa.Table: + task_refs = load_yaml(TEST_LIST) + if ( + not isinstance(task_refs, list) + or len(task_refs) != EXPECTED_ROW_COUNTS["tasks"] + or len(set(task_refs)) != EXPECTED_ROW_COUNTS["tasks"] + ): + raise ValueError( + f"Expected {EXPECTED_ROW_COUNTS['tasks']} unique canonical task references" + ) + + files_by_name = find_test_files() + rows = [build_task_row(task_ref, files_by_name) for task_ref in task_refs] + return pa.Table.from_pylist(rows, schema=TASK_SCHEMA) + + +def build_scenarios() -> pa.Table: + rows = [] + for scenario_id, domain in DOMAIN_BY_SCENARIO.items(): + path = ROOT / "dataset" / "scenario" / f"{scenario_id}.yaml" + document = load_yaml(path) + world_state = document.get("world_state") + tools = document.get("tools") + if not isinstance(world_state, dict) or not isinstance(tools, list): + raise ValueError(f"Invalid scenario document: {path}") + + tool_names = [tool.get("name") for tool in tools] + if not all(isinstance(name, str) for name in tool_names): + raise ValueError(f"Scenario contains an unnamed tool: {path}") + + rows.append( + { + "scenario_id": scenario_id, + "domain": domain, + "world_state_json": to_json(world_state), + "available_tools": tool_names, + "source_url": source_url(path), + "release_tag": RELEASE_TAG, + } + ) + + return pa.Table.from_pylist(rows, schema=SCENARIO_SCHEMA) + + +def build_agents() -> pa.Table: + path = ROOT / "dataset" / "agent" / "think.yaml" + document = load_yaml(path) + row = { + "agent_id": "think", + "system_instructions": document.get("system_instructions", ""), + "parallel_tool_calls": document.get("parallel_tool_calls", False), + "builtin_tools_json": to_json(document.get("builtin_tools") or []), + "source_url": source_url(path), + "release_tag": RELEASE_TAG, + } + return pa.Table.from_pylist([row], schema=AGENT_SCHEMA) + + +def build_tables() -> dict[str, pa.Table]: + tables = { + "tasks": build_tasks(), + "scenarios": build_scenarios(), + "agents": build_agents(), + } + for name, expected_count in EXPECTED_ROW_COUNTS.items(): + actual_count = tables[name].num_rows + if actual_count != expected_count: + raise ValueError( + f"Expected {expected_count} exported {name}, found {actual_count}" + ) + + scenario_rows = { + row["scenario_id"]: row["domain"] for row in tables["scenarios"].to_pylist() + } + for task in tables["tasks"].to_pylist(): + if scenario_rows.get(task["scenario_id"]) != task["domain"]: + raise ValueError( + f"Task {task['task_ref']} has an invalid scenario or domain" + ) + return tables + + +def load_dataset_card_metadata() -> dict[str, Any]: + content = DATASET_CARD.read_text(encoding="utf-8") + sections = content.split("---", 2) + if len(sections) != 3 or sections[0].strip(): + raise ValueError("Dataset card must start with YAML front matter") + metadata = yaml.safe_load(sections[1]) + if not isinstance(metadata, dict): + raise ValueError("Dataset card metadata must be a mapping") + return metadata + + +def check_tables(tables: dict[str, pa.Table]) -> None: + errors = [] + for name, expected in tables.items(): + path = DATA_DIR / f"{name}.parquet" + if not path.exists(): + errors.append(f"Missing {path.relative_to(ROOT)}") + continue + actual = pq.read_table(path) + if not actual.equals(expected): + errors.append(f"Outdated {path.relative_to(ROOT)}") + if ( + not LICENSE_DESTINATION.exists() + or LICENSE_DESTINATION.read_bytes() != LICENSE_SOURCE.read_bytes() + ): + errors.append(f"Outdated {LICENSE_DESTINATION.relative_to(ROOT)}") + actual_files = { + path.relative_to(BUNDLE_DIR) + for path in BUNDLE_DIR.rglob("*") + if path.is_file() + } + if actual_files != EXPECTED_BUNDLE_FILES: + errors.append( + "Unexpected Hugging Face bundle files: " + f"{sorted(str(path) for path in actual_files ^ EXPECTED_BUNDLE_FILES)}" + ) + try: + metadata = load_dataset_card_metadata() + if metadata.get("configs") != EXPECTED_CONFIGS: + errors.append("Dataset card configs do not match the exported tables") + except (OSError, ValueError, yaml.YAMLError) as error: + errors.append(f"Invalid dataset card: {error}") + if errors: + raise SystemExit("\n".join(errors)) + print(f"Verified the Hugging Face publishing bundle ({len(tables)} tables).") + + +def write_tables(tables: dict[str, pa.Table]) -> None: + DATA_DIR.mkdir(parents=True, exist_ok=True) + for name, table in tables.items(): + path = DATA_DIR / f"{name}.parquet" + pq.write_table(table, path, compression="zstd") + print(f"Wrote {path.relative_to(ROOT)} ({table.num_rows} rows).") + shutil.copyfile(LICENSE_SOURCE, LICENSE_DESTINATION) + print(f"Copied {LICENSE_DESTINATION.relative_to(ROOT)}.") + + +def main() -> None: + args = parse_args() + tables = build_tables() + if args.check: + check_tables(tables) + else: + write_tables(tables) + + +if __name__ == "__main__": + main()