From a08a84f6878105a19f912d6e1ae4dd60ed2ee959 Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Fri, 7 Aug 2020 13:50:33 +0530 Subject: [PATCH 1/8] adding ordinal number support english --- number_parser/__init__.py | 2 +- number_parser/parser.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/number_parser/__init__.py b/number_parser/__init__.py index 0e270f5..8efbc8a 100644 --- a/number_parser/__init__.py +++ b/number_parser/__init__.py @@ -1 +1 @@ -from number_parser.parser import parse, parse_number +from number_parser.parser import parse, parse_number, parse_ordinal diff --git a/number_parser/parser.py b/number_parser/parser.py index d2de0bc..97c8eb6 100644 --- a/number_parser/parser.py +++ b/number_parser/parser.py @@ -158,6 +158,38 @@ def _normalize_dict(lang_dict): return {_strip_accents(word): number for word, number in lang_dict.items()} +def _is_cardinal_token(token, lang_dict): + if token in lang_dict.all_numbers: + return token + return None + + +def _is_ordinal_token(token, lang_dict): + token = _apply_cardinal_conversion(token, lang_dict) + return _is_cardinal_token(token, lang_dict) + + +def _is_number_token(token, lang_dict): + return _is_cardinal_token(token, lang_dict) or _is_ordinal_token(token, lang_dict) + + +def _apply_cardinal_conversion(input_string, language): + # this will be coming from the LanguageData + CARDINAL_DIRECT_NUMBERS = {'first': 'one', 'second': 'two', 'third': 'three', 'fifth': 'five', 'eighth': 'eight', + 'ninth': 'nine', 'twelfth': 'twelve'} + input_string = input_string.lower() + for word, number in CARDINAL_DIRECT_NUMBERS.items(): + input_string = input_string.replace(word, number) + input_string = re.sub(r'ieth$', 'y', input_string) + input_string = re.sub(r'th$', '', input_string) + return input_string + + +def parse_ordinal(input_string, language='en'): + input_string = _apply_cardinal_conversion(input_string, language) + return parse_number(input_string, language) + + def parse_number(input_string, language='en'): """Converts a single number written in natural language to a numeric type""" lang_data = LanguageData(language) @@ -199,7 +231,7 @@ def parse(input_string, language='en'): current_sentence.append(token) continue - elif compare_token in SENTENCE_SEPARATORS: + if compare_token in SENTENCE_SEPARATORS: if tokens_taken: myvalue = _build_number(tokens_taken, lang_data) for each_number in myvalue: From a350f46f1d8eac99b73bafb3d4f865d7044b8db0 Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Fri, 7 Aug 2020 13:50:49 +0530 Subject: [PATCH 2/8] added tests for ordinal numbers --- tests/test_language_en.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_language_en.py b/tests/test_language_en.py index 44c9318..a5a544f 100644 --- a/tests/test_language_en.py +++ b/tests/test_language_en.py @@ -1,5 +1,5 @@ import pytest -from number_parser import parse, parse_number +from number_parser import parse, parse_number, parse_ordinal from tests import HUNDREDS_DIRECTORY, PERMUTATION_DIRECTORY from tests import _test_files LANG = 'en' @@ -95,3 +95,22 @@ def test_parse_number_till_hundred(self): def test_parse_number_permutations(self): _test_files(PERMUTATION_DIRECTORY, LANG) + + @pytest.mark.parametrize( + "test_input,expected", + [ + ('eleventh', 11), + ("nineteenth", 19), + ('hundredth', 100), + ('one hundred and forty second', 142), + ('thousandth', 1_000), + ("two thousand and fifth", 2_005), + ('millionth', 1_000_000), + ("two million three thousand and nineteenth", 2_003_019), + ('two million twenty three thousand and forty ninth', 2_023_049), + ("two million three thousand nine hundred and eighty fourth", 2_003_984), + ('billionth', 1_000_000_000) + ] + ) + def test_parse_ordinal(self, expected, test_input): + assert parse_ordinal(test_input, LANG) == expected From 465a8687fb9333c0a8151e998cbfbdf1b4ece78f Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Thu, 13 Aug 2020 00:32:50 +0530 Subject: [PATCH 3/8] added test for sentences with ordinals --- tests/test_language_en.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_language_en.py b/tests/test_language_en.py index a5a544f..1612881 100644 --- a/tests/test_language_en.py +++ b/tests/test_language_en.py @@ -114,3 +114,18 @@ def test_parse_number_permutations(self): ) def test_parse_ordinal(self, expected, test_input): assert parse_ordinal(test_input, LANG) == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ + ('eleventh day of summer', "11 day of summer"), + ("nineteenth may two thousand", "19 may 2000"), + ('hundredth and one', "101"), + ('one hundred and forty second', "142"), + ('five thousandth and one', "5001"), + ("thirty seven and fifth", "37 and 5"), + ('eighth month of year two thousand and twentieth', "8 month of year 2020"), + ] + ) + def test_parse_sentences_ordinal(self, expected, test_input): + assert parse(test_input, LANG) == expected From b779c7d36dd0b70c7180e6e82cb6e10ea5330ebe Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Thu, 13 Aug 2020 02:02:51 +0530 Subject: [PATCH 4/8] updating parser codes for better structure --- number_parser/parser.py | 71 +++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/number_parser/parser.py b/number_parser/parser.py index 97c8eb6..92fbd08 100644 --- a/number_parser/parser.py +++ b/number_parser/parser.py @@ -153,41 +153,61 @@ def _normalize_tokens(token_list): return [_strip_accents(token.lower()) for token in token_list] -def _normalize_dict(lang_dict): +def _normalize_dict(lang_data): """Removes the accent from each key of input dictionary""" - return {_strip_accents(word): number for word, number in lang_dict.items()} + return {_strip_accents(word): number for word, number in lang_data.items()} -def _is_cardinal_token(token, lang_dict): - if token in lang_dict.all_numbers: +def _is_cardinal_token(token, lang_data): + """Checks if the given token is a cardinal number and returns token""" + if token in lang_data.all_numbers: return token return None -def _is_ordinal_token(token, lang_dict): - token = _apply_cardinal_conversion(token, lang_dict) - return _is_cardinal_token(token, lang_dict) +def _is_ordinal_token(token, lang_data): + """Checks if the given token is a ordinal number and returns token""" + if _is_cardinal_token(token, lang_data) is None: + return _is_number_token(token, lang_data) + return None -def _is_number_token(token, lang_dict): - return _is_cardinal_token(token, lang_dict) or _is_ordinal_token(token, lang_dict) +def _is_number_token(token, lang_data): + """ + Checks if the given token belongs to either cardinal or ordinal numbers + and returns the cardinal form. + """ + token = _apply_cardinal_conversion(token, lang_data) + return _is_cardinal_token(token, lang_data) -def _apply_cardinal_conversion(input_string, language): - # this will be coming from the LanguageData +def _apply_cardinal_conversion(token, lang_data): + """Converts ordinal tokens to cardinal while leaving other tokens unchanged.""" CARDINAL_DIRECT_NUMBERS = {'first': 'one', 'second': 'two', 'third': 'three', 'fifth': 'five', 'eighth': 'eight', 'ninth': 'nine', 'twelfth': 'twelve'} - input_string = input_string.lower() + for word, number in CARDINAL_DIRECT_NUMBERS.items(): - input_string = input_string.replace(word, number) - input_string = re.sub(r'ieth$', 'y', input_string) - input_string = re.sub(r'th$', '', input_string) - return input_string + token = token.replace(word, number) + + token_cardinal_form_1 = re.sub(r'ieth$', 'y', token) + if _is_cardinal_token(token_cardinal_form_1, lang_data) is not None: + return token_cardinal_form_1 + + token_cardinal_form_2 = re.sub(r'th$', '', token) + if _is_cardinal_token(token_cardinal_form_2, lang_data) is not None: + return token_cardinal_form_2 + + return token def parse_ordinal(input_string, language='en'): - input_string = _apply_cardinal_conversion(input_string, language) - return parse_number(input_string, language) + """Converts a single number in ordinal or cardinal form to it's numeric equivalent""" + lang_data = LanguageData(language) + tokens = _tokenize(input_string, language) + normalized_tokens = _normalize_tokens(tokens) + processed_tokens = [_apply_cardinal_conversion(token, lang_data) for token in normalized_tokens] + output_string = ' '.join(processed_tokens) + return parse_number(output_string, language) def parse_number(input_string, language='en'): @@ -226,6 +246,8 @@ def parse(input_string, language='en'): for token in tokens: compare_token = _strip_accents(token.lower()) + ordinal_number = _is_ordinal_token(compare_token, lang_data) + if compare_token.isspace() or compare_token == "": if not tokens_taken: current_sentence.append(token) @@ -244,18 +266,25 @@ def parse(input_string, language='en'): current_sentence = [] continue - elif (compare_token in lang_data.all_numbers or - (compare_token in lang_data.skip_tokens and len(tokens_taken) != 0)): + elif ((compare_token in lang_data.all_numbers or + (compare_token in lang_data.skip_tokens and len(tokens_taken) != 0)) and ordinal_number is None): tokens_taken.append(compare_token) else: + if ordinal_number is not None: + tokens_taken.append(ordinal_number) + if tokens_taken: myvalue = _build_number(tokens_taken, lang_data) for each_number in myvalue: current_sentence.append(each_number) current_sentence.append(" ") tokens_taken = [] - current_sentence.append(token) + + if ordinal_number is None: + current_sentence.append(token) + else: + current_sentence.pop() # Handling extra space when breaking on ordinal numbers. if tokens_taken: myvalue = _build_number(tokens_taken, lang_data) From 66f688b9be1c9caa90d348118081631846207dac Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Thu, 13 Aug 2020 02:03:35 +0530 Subject: [PATCH 5/8] added tests to check edge cases in ordinal parsing --- tests/test_language_en.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_language_en.py b/tests/test_language_en.py index 1612881..5b777fe 100644 --- a/tests/test_language_en.py +++ b/tests/test_language_en.py @@ -109,7 +109,11 @@ def test_parse_number_permutations(self): ("two million three thousand and nineteenth", 2_003_019), ('two million twenty three thousand and forty ninth', 2_023_049), ("two million three thousand nine hundred and eighty fourth", 2_003_984), - ('billionth', 1_000_000_000) + ('billionth', 1_000_000_000), + ('fiftieth fifth', 55), # Is being processed as fifty five for the time-being. + ('with goldsmith', None), + ('th th', None), + ('fifth fiftieth', None) ] ) def test_parse_ordinal(self, expected, test_input): @@ -120,11 +124,13 @@ def test_parse_ordinal(self, expected, test_input): [ ('eleventh day of summer', "11 day of summer"), ("nineteenth may two thousand", "19 may 2000"), - ('hundredth and one', "101"), + ('hundredth and one', "100 and 1"), ('one hundred and forty second', "142"), - ('five thousandth and one', "5001"), + ('five thousandth and one', "5000 and 1"), ("thirty seven and fifth", "37 and 5"), ('eighth month of year two thousand and twentieth', "8 month of year 2020"), + ('He crieth, a path with fifty fifth steps', "He crieth, a path with 55 steps"), + ('twentieth seventh fiftieth third', "20 7 50 3") ] ) def test_parse_sentences_ordinal(self, expected, test_input): From 08bee2f691dad37d9f475042cbc73ca0cedca1b1 Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Thu, 13 Aug 2020 02:22:11 +0530 Subject: [PATCH 6/8] added edge tests parse_ordinal --- tests/test_language_en.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_language_en.py b/tests/test_language_en.py index 5b777fe..88b1668 100644 --- a/tests/test_language_en.py +++ b/tests/test_language_en.py @@ -110,10 +110,13 @@ def test_parse_number_permutations(self): ('two million twenty three thousand and forty ninth', 2_023_049), ("two million three thousand nine hundred and eighty fourth", 2_003_984), ('billionth', 1_000_000_000), - ('fiftieth fifth', 55), # Is being processed as fifty five for the time-being. ('with goldsmith', None), ('th th', None), - ('fifth fiftieth', None) + ('fifth fiftieth', None), + # Some ambiguos cases + ('fiftieth fifth', 55), + ('fiftieth five', 55), + ('fifty five', 55) ] ) def test_parse_ordinal(self, expected, test_input): From 3392bdf70ef94cd4861eb483b326a669bc6b2a1e Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Fri, 14 Aug 2020 13:03:32 +0530 Subject: [PATCH 7/8] updating readme for ordinal support --- README.rst | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 2d25148..7a4d0c3 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ number-parser ``number-parser`` is a simple library that allows you to convert numbers written in the natural language to it's equivalent numeric forms. It currently supports cardinal numbers in the following -languages - English, Hindi, Spanish and Russian. +languages - English, Hindi, Spanish and Russian and ordinal numbers in English. Installation ============ @@ -20,18 +20,21 @@ number-parser requires Python 3.6+. Usage ===== -The library provides two major APIs which corresponds to the following two common usages. +The library provides three major APIs which corresponds to the following common usages. Interface #1: Multiple numbers ------------------------------ Identifying the numbers in a text string, converting them to corresponding numeric values while ignoring non-numeric words. +This also supports ordinal number conversion (for English only). >>> from number_parser import parse >>> parse("I have two hats and thirty seven coats") 'I have 2 hats and 37 coats' >>> parse("One, Two, Three go") '1, 2, 3 go' +>>> parse("First day of year two thousand") +'1 day of year 2000' Interface #2: Single number @@ -41,21 +44,33 @@ Converting a single number written in words to it's corresponding integer. >>> from number_parser import parse_number >>> parse_number("two thousand and twenty") 2020 ->>> output = parse_number("not_a_number") ->>> output -None +>>> parse_number("not_a_number") + + +Interface #3: Single number Ordinal +------------------------------------- + +Converting a single ordinal number written in words to it's corresponding integer. (Support for only English) + +>>> from number_parser import parse_ordinal +>>> parse_ordinal("twenty third") +23 +>>> parse_ordinal("seventy fifth") +75 Language Support ---------------- The default language is English, you can pass the language parameter with corresponding locale for other languages. +It currently supports cardinal numbers in the following +languages - English, Hindi, Spanish and Russian and ordinal numbers in English. >>> from number_parser import parse, parse_number >>> parse("Hay tres gallinas y veintitrés patos", language='es') 'Hay 3 gallinas y 23 patos' >>> parse_number("चौदह लाख बत्तीस हज़ार पाँच सौ चौबीस", language='hi') -1432524 +1432524 Supported cases --------------- @@ -72,7 +87,7 @@ Accurately handling usage of conjunction while forming the number. Handling ambiguous cases without proper separators. >>> parse("two thousand thousand") -2000 1000 +'2000 1000' >>> parse_number("two thousand two million") 2002000000 From 5fab19a1cba79b817c9ee6f96d067896e4a02341 Mon Sep 17 00:00:00 2001 From: arnavkapoor Date: Fri, 14 Aug 2020 13:03:49 +0530 Subject: [PATCH 8/8] adding comment --- number_parser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/number_parser/parser.py b/number_parser/parser.py index 92fbd08..92cb690 100644 --- a/number_parser/parser.py +++ b/number_parser/parser.py @@ -181,7 +181,7 @@ def _is_number_token(token, lang_data): return _is_cardinal_token(token, lang_data) -def _apply_cardinal_conversion(token, lang_data): +def _apply_cardinal_conversion(token, lang_data): # Currently only for English language. """Converts ordinal tokens to cardinal while leaving other tokens unchanged.""" CARDINAL_DIRECT_NUMBERS = {'first': 'one', 'second': 'two', 'third': 'three', 'fifth': 'five', 'eighth': 'eight', 'ninth': 'nine', 'twelfth': 'twelve'}