diff --git a/Jenkinsfile b/Jenkinsfile index fd58716ec..d60e5b861 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-27-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv b/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv new file mode 100644 index 000000000..66d556c36 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/quantities.tsv @@ -0,0 +1,10 @@ +हजार +हज़ार +लाख +करोड़ +करोड +अरब +खरब +मिलियन +बिलियन +ट्रिलियन \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/money.py b/nemo_text_processing/text_normalization/hi/taggers/money.py index 16f389ae7..4d600010c 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/money.py +++ b/nemo_text_processing/text_normalization/hi/taggers/money.py @@ -92,6 +92,40 @@ def __init__(self, cardinal: GraphFst): ) guarded_decimal_digits = has_3plus_sig_digits @ decimal_digits + # Load scale words dynamically from TSV + quantity_graph = ( + pynutil.insert('quantity: "') + + pynini.string_file(get_abs_path("data/money/quantities.tsv")).optimize() + + pynutil.insert('"') + ) + + # Allow an optional space between the number and the quantity word + delete_space_opt = pynini.closure(pynutil.delete(" "), 0, 1) + + # Path for scaled money with decimals (e.g. ₹२.५ करोड़) + graph_scaled_decimal = ( + optional_graph_negative + + currency_major + + insert_space + + integer + + pynini.cross(".", " ") + + decimal_digits + + delete_space_opt + + insert_space + + quantity_graph + ).optimize() + + # Path for scaled money without decimals (e.g. ₹५ करोड़) + graph_scaled_major = ( + optional_graph_negative + + currency_major + + insert_space + + integer + + delete_space_opt + + insert_space + + quantity_graph + ).optimize() + graph_decimal_path = ( optional_graph_negative + currency_major @@ -150,6 +184,8 @@ def _build_major_and_minor(sym_maj_labels, int_graph): graph_currencies = ( pynutil.add_weight(graph_major_only_singular | graph_major_and_minor_singular, -0.001) | pynutil.add_weight(graph_decimal_path, -0.0005) + | pynutil.add_weight(graph_scaled_decimal, -0.0002) + | pynutil.add_weight(graph_scaled_major, -0.0002) | graph_major_only | graph_major_and_minor ) diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/money.py b/nemo_text_processing/text_normalization/hi/verbalizers/money.py index 1e5da99e4..aa6d3106a 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/money.py @@ -46,6 +46,12 @@ def __init__(self): currency_minor = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + quantity = pynutil.delete('quantity: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + drop_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + sp + pynutil.delete("true") + sp, 0, 1 + ) + graph_major_only = integer_part + sp + currency_major all_major_names = [maj for maj, _ in load_labels(get_abs_path("data/money/major_minor_currencies.tsv"))] @@ -97,6 +103,47 @@ def __init__(self): ) graph_decimal_money = pynini.union(*decimal_graphs) - graph = graph_major_only | graph_major_minor | pynutil.add_weight(graph_minor_only, -0.1) | graph_decimal_money + scaled_decimal_graphs = [] + scaled_major_graphs = [] + + for major in all_major_names: + graph_major_slot = pynutil.delete('currency_maj: "') + pynutil.delete(major) + pynutil.delete('"') + + scaled_decimal_graphs.append( + drop_preserve_order + + graph_major_slot + + sp + + integer_part + + sp + + pynutil.insert(" दशमलव ") + + fractional_part + + sp + + quantity + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + ) + + scaled_major_graphs.append( + drop_preserve_order + + graph_major_slot + + sp + + integer_part + + sp + + quantity + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + ) + + graph_scaled_decimal = pynini.union(*scaled_decimal_graphs) + graph_scaled_major = pynini.union(*scaled_major_graphs) + + graph = ( + graph_major_only + | graph_major_minor + | pynutil.add_weight(graph_minor_only, -0.1) + | graph_decimal_money + | graph_scaled_decimal + | graph_scaled_major + ) self.fst = self.delete_tokens(graph).optimize() diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt index e5f157872..0bec8c449 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt @@ -141,4 +141,9 @@ $1.2000~एक डॉलर बीस सेंट ₹५,५६,३२०~पाँच लाख छप्पन हज़ार तीन सौ बीस रुपए ₹१,००,०००~एक लाख रुपए ₹२,१४८~दो हज़ार एक सौ अड़तालीस रुपए -₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए \ No newline at end of file +₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए +₹२.५ करोड़~दो दशमलव पाँच करोड़ रुपए +₹५ करोड़~पाँच करोड़ रुपए +$१.५ मिलियन~एक दशमलव पाँच मिलियन डॉलर +£१२ लाख~बारह लाख पाउंड +₹१.५ हजार~एक दशमलव पाँच हजार रुपए \ No newline at end of file