test: word_tokenize keeps a fraction and a signed number whole; CI runs pytest

.github/workflows/build_tests.yml

@@ -1,25 +1,15 @@
 name: Run Build Tests
 on:
   push:
+    branches: [dev]
+  pull_request:
+    branches: [dev]
   workflow_dispatch:
 
 jobs:
   build_tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          ref: ${{ github.head_ref }}
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: 3.14
-      - name: Install Build Tools
-        run: |
-          python -m pip install build wheel
-      - name: Build Distribution Packages
-        run: |
-          python -m build --wheel
-      - name: Install core repo
-        run: |
-          pip install .
+    uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev
+    secrets: inherit
+    with:
+      python_versions: '["3.10","3.11","3.12","3.13","3.14"]'
+      test_path: 'test/test_chunking.py test/test_tokenization.py'

test/test_chunking.py

@@ -13,15 +13,26 @@ def test_word(self):
         self.assertEqual(
             word_tokenize("100%"),
             ['100', '%'])
+        # A digit run joined by "/" is one token, the same as "1.5" and
+        # "13:00": the numeric alternative of _WORD_REGEX takes [,.:/] as
+        # in-number separators since #1, so a fraction or a date reaches a
+        # number parser whole ("3/4", "11/11/2023"). A "/" between spaces
+        # or between letters is still its own token.
         self.assertEqual(
             word_tokenize("2 and 3/4"),
-            ['2', 'and', '3', "/", '4'])
+            ['2', 'and', '3/4'])
         self.assertEqual(
             word_tokenize("2 and (3/4)"),
-            ['2', 'and', '(', '3', '/', '4', ')'])
+            ['2', 'and', '(', '3/4', ')'])
         self.assertEqual(
             word_tokenize("2 and [3/4]"),
-            ['2', 'and', '[', '3', '/', '4', ']'])
+            ['2', 'and', '[', '3/4', ']'])
+        self.assertEqual(
+            word_tokenize("on 11/11/2023 at 13:00"),
+            ['on', '11/11/2023', 'at', '13:00'])
+        self.assertEqual(
+            word_tokenize("3 / 4 and a/b"),
+            ['3', '/', '4', 'and', 'a', '/', 'b'])
         self.assertEqual(
             word_tokenize("2 and 3\\4"),
             ['2', 'and', '3', "\\", '4'])
@@ -41,12 +52,20 @@ def test_word(self):
              (37, 38, ','), (39, 46, 'mycroft'), (47, 49, 'is'),
              (50, 54, 'FOSS'), (54, 55, '!')])
 
+        # A leading "-" on a digit run is the sign of a negative number
+        # since #3 ("-2", "-2.5"); a "-" set apart by a space is an operator
         self.assertEqual(
             word_tokenize("-2"),
-            ['-', '2'])
+            ['-2'])
+        self.assertEqual(
+            word_tokenize("-2.5"),
+            ['-2.5'])
         self.assertEqual(
             word_tokenize("- 2"),
             ['-', '2'])
+        self.assertEqual(
+            word_tokenize("2 - 2"),
+            ['2', '-', '2'])
 
     def test_empty(self):
         test_sent = "this is a   string with  many \t spaces and \n\r\t\r stuff"