From f3b2dd1808b333bfc5486dc287b6168464b77200 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Sat, 19 Sep 2026 01:07:22 +0000
Subject: [PATCH] test: word_tokenize keeps a fraction and a signed number
 whole; CI runs pytest

test_chunking::test_word has failed since 2023 and no workflow ran it.
Two assertions from 2021 predate two merged tokenizer changes: #1 made
a digit run joined by "/" one token, the same as "1.5" and "13:00", so
"3/4" and "11/11/2023" reach a number parser whole; #3 made a leading
"-" the sign of a negative number, so "-2" and "-2.5" are one token.
ovos-number-parser reads a fraction as one "3/4" token and splits it on
"/" itself. The test now states that contract, with a date, a spaced
"/" and a spaced "-" as the counter-cases.

build_tests.yml calls the gh-automations shared workflow with test_path
set to the two unit files, on 3.10 to 3.14, so the suite runs on every
push and pull request to dev. license_tests.py stays with its own
workflow, which installs lichecker.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .github/workflows/build_tests.yml | 26 ++++++++------------------
 test/test_chunking.py             | 27 +++++++++++++++++++++++----
 2 files changed, 31 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml
index 04e1305..a17ab40 100644
--- a/.github/workflows/build_tests.yml
+++ b/.github/workflows/build_tests.yml
@@ -1,25 +1,15 @@
 name: Run Build Tests
 on:
   push:
+    branches: [dev]
+  pull_request:
+    branches: [dev]
   workflow_dispatch:
 
 jobs:
   build_tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          ref: ${{ github.head_ref }}
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: 3.14
-      - name: Install Build Tools
-        run: |
-          python -m pip install build wheel
-      - name: Build Distribution Packages
-        run: |
-          python -m build --wheel
-      - name: Install core repo
-        run: |
-          pip install .
+    uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev
+    secrets: inherit
+    with:
+      python_versions: '["3.10","3.11","3.12","3.13","3.14"]'
+      test_path: 'test/test_chunking.py test/test_tokenization.py'
diff --git a/test/test_chunking.py b/test/test_chunking.py
index 397db5a..f8a9883 100644
--- a/test/test_chunking.py
+++ b/test/test_chunking.py
@@ -13,15 +13,26 @@ def test_word(self):
         self.assertEqual(
             word_tokenize("100%"),
             ['100', '%'])
+        # A digit run joined by "/" is one token, the same as "1.5" and
+        # "13:00": the numeric alternative of _WORD_REGEX takes [,.:/] as
+        # in-number separators since #1, so a fraction or a date reaches a
+        # number parser whole ("3/4", "11/11/2023"). A "/" between spaces
+        # or between letters is still its own token.
         self.assertEqual(
             word_tokenize("2 and 3/4"),
-            ['2', 'and', '3', "/", '4'])
+            ['2', 'and', '3/4'])
         self.assertEqual(
             word_tokenize("2 and (3/4)"),
-            ['2', 'and', '(', '3', '/', '4', ')'])
+            ['2', 'and', '(', '3/4', ')'])
         self.assertEqual(
             word_tokenize("2 and [3/4]"),
-            ['2', 'and', '[', '3', '/', '4', ']'])
+            ['2', 'and', '[', '3/4', ']'])
+        self.assertEqual(
+            word_tokenize("on 11/11/2023 at 13:00"),
+            ['on', '11/11/2023', 'at', '13:00'])
+        self.assertEqual(
+            word_tokenize("3 / 4 and a/b"),
+            ['3', '/', '4', 'and', 'a', '/', 'b'])
         self.assertEqual(
             word_tokenize("2 and 3\\4"),
             ['2', 'and', '3', "\\", '4'])
@@ -41,12 +52,20 @@ def test_word(self):
              (37, 38, ','), (39, 46, 'mycroft'), (47, 49, 'is'),
              (50, 54, 'FOSS'), (54, 55, '!')])
 
+        # A leading "-" on a digit run is the sign of a negative number
+        # since #3 ("-2", "-2.5"); a "-" set apart by a space is an operator
         self.assertEqual(
             word_tokenize("-2"),
-            ['-', '2'])
+            ['-2'])
+        self.assertEqual(
+            word_tokenize("-2.5"),
+            ['-2.5'])
         self.assertEqual(
             word_tokenize("- 2"),
             ['-', '2'])
+        self.assertEqual(
+            word_tokenize("2 - 2"),
+            ['2', '-', '2'])
 
     def test_empty(self):
         test_sent = "this is a   string with  many \t spaces and \n\r\t\r stuff"