Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/audits.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ jobs:
# The JS case-folding table against Python for every code point both runtimes assign; the output
# says how many it could not compare (the runtimes ship different Unicode versions) (#946).
- name: Check the key-normalisation case-folding table
run: python scripts/generate_key_fold.py --check
run: python scripts/generate_key_fold.py --check --require

- name: Smoke-test the CLI
run: |
Expand Down
10 changes: 8 additions & 2 deletions scripts/generate_key_fold.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

python3 scripts/generate_key_fold.py # rewrite the table in place
python3 scripts/generate_key_fold.py --check # exit 1 if the table folds a character differently from Python
python3 scripts/generate_key_fold.py --check --require # fail when node is missing rather than skipping (CI)

`make key-fold` rewrites the table and `make key-fold-check` (also part of `make check` and CI) verifies it:
it compares the engine's normalizeKeyText() with Python's normalize_key_text() for every code point both
Expand All @@ -26,6 +27,7 @@
import argparse
import json
import re
import shutil
import subprocess
import sys
import unicodedata
Expand Down Expand Up @@ -141,17 +143,21 @@ def check_table() -> dict:
"node": node, "built_with": stamp.groups() if stamp else None}


def main() -> int:
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument("--check", action="store_true", help="exit 1 if the table is out of date; write nothing")
args = parser.parse_args()
parser.add_argument("--require", action="store_true", help="fail when node is missing rather than skipping the check")
args = parser.parse_args(argv)
source = ENGINE.read_text(encoding="utf-8")
start, stop = source.find(BEGIN), source.find(END)
if start == -1 or stop == -1:
print(f"error: KEY_FOLD markers not found in {ENGINE}", file=sys.stderr)
return 2
stop += len(END)
if args.check:
if shutil.which("node") is None:
print("node is needed for this check (install Node 18+); skipping")
return 2 if args.require else 0
r = check_table()
print(f"Python {sys.version.split()[0]} (Unicode {unicodedata.unidata_version}), node {r['node']['node']} "
f"(Unicode {r['node']['unicode']})")
Expand Down
35 changes: 33 additions & 2 deletions tests/test_generate_key_fold.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import importlib
import re
import shutil
import subprocess

import pytest

if shutil.which("node") is None: # pragma: no cover - CI has node
pytest.skip("node is needed to compare with toLowerCase()", allow_module_level=True)
HAS_NODE = shutil.which("node") is not None
needs_node = pytest.mark.skipif(not HAS_NODE, reason="node is needed to compare with toLowerCase()")


@pytest.fixture
Expand All @@ -19,6 +20,7 @@ def script(tmp_path, monkeypatch):
return mod


@needs_node
def test_generated_block_carries_the_known_folds_and_the_unicode_versions(script):
block = script.build_block()
assert block.startswith(script.BEGIN) and block.endswith(script.END)
Expand All @@ -28,6 +30,7 @@ def test_generated_block_carries_the_known_folds_and_the_unicode_versions(script
assert block.isascii()


@needs_node
def test_check_agrees_after_a_rewrite_reports_what_it_skipped_and_detects_a_stale_table(script, capsys, monkeypatch):
import sys
monkeypatch.setattr(sys, "argv", ["generate_key_fold.py"])
Expand All @@ -49,6 +52,7 @@ def test_check_agrees_after_a_rewrite_reports_what_it_skipped_and_detects_a_stal
assert "STALE" in stale and "U+00DF" in stale and "make key-fold" in stale


@needs_node
def test_check_notes_when_the_runtimes_know_a_newer_unicode_than_the_table(script, capsys, monkeypatch):
import sys
text = script.ENGINE.read_text(encoding="utf-8")
Expand All @@ -60,7 +64,34 @@ def test_check_notes_when_the_runtimes_know_a_newer_unicode_than_the_table(scrip
assert "knows a newer Unicode than the table was built with" in capsys.readouterr().out


def test_check_skips_when_node_is_missing(script, capsys, monkeypatch):
import sys
monkeypatch.setenv("PATH", "")
monkeypatch.setattr(sys, "argv", ["generate_key_fold.py", "--check"])
assert script.main() == 0
out = capsys.readouterr().out
assert "node is needed for this check (install Node 18+); skipping" in out


def test_check_fails_when_node_is_missing_and_require_is_passed(script, capsys, monkeypatch):
import sys
monkeypatch.setenv("PATH", "")
monkeypatch.setattr(sys, "argv", ["generate_key_fold.py", "--check", "--require"])
assert script.main() == 2
out = capsys.readouterr().out
assert "node is needed for this check (install Node 18+); skipping" in out


def test_rewrite_fails_when_node_is_missing(script, monkeypatch):
import sys
monkeypatch.setenv("PATH", "")
monkeypatch.setattr(sys, "argv", ["generate_key_fold.py"])
with pytest.raises((FileNotFoundError, subprocess.CalledProcessError)):
script.main()


def test_the_repo_copy_still_has_its_markers():
mod = importlib.import_module("scripts.generate_key_fold")
text = mod.ENGINE.read_text(encoding="utf-8")
assert mod.BEGIN in text and mod.END in text and "var KEY_FOLD = {" in text

Loading