From 7e17588a33638159daa8b7884a082d5f0dfb18ab Mon Sep 17 00:00:00 2001 From: Sgt_Sinky <1676751+Sinkleberg@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:47:55 -0400 Subject: [PATCH] Allow disabling type inference for selected columns --- CHANGELOG.rst | 2 + csvkit/cli.py | 40 +++ csvkit/utilities/csvjoin.py | 5 +- csvkit/utilities/csvjson.py | 5 +- csvkit/utilities/csvlook.py | 5 +- csvkit/utilities/csvpy.py | 7 +- csvkit/utilities/csvsort.py | 5 +- csvkit/utilities/csvsql.py | 5 +- csvkit/utilities/csvstat.py | 5 +- csvkit/utilities/in2csv.py | 5 +- docs/common_arguments.rst | 10 + docs/contributing.rst | 2 + docs/scripts/csvjoin.rst | 7 +- docs/scripts/csvjson.rst | 5 +- docs/scripts/csvlook.rst | 5 +- docs/scripts/csvpy.rst | 7 +- docs/scripts/csvsort.rst | 5 +- docs/scripts/csvsql.rst | 5 +- docs/scripts/csvstat.rst | 5 +- docs/scripts/in2csv.rst | 7 +- docs/tricks.rst | 2 + tests/test_cli.py | 147 +++++++++++- .../test_selective_inference.py | 227 ++++++++++++++++++ 23 files changed, 476 insertions(+), 42 deletions(-) create mode 100644 tests/test_utilities/test_selective_inference.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5fdf4a1a3..03081928c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,8 @@ Unreleased ---------- +- feat: Tools that support type inference add :code:`--no-inference-columns` to preserve selected columns as text while inferring the remaining columns. + - feat: :doc:`/scripts/csvcut` adds an :code:`--ignore-unknown-columns` option to skip identifiers in :code:`-c/--columns` that do not match a column in the input. - feat: :doc:`/scripts/csvclean` adds a :code:`--remove-empty-columns` option to remove empty columns from standard output. - feat: :doc:`/scripts/in2csv` guesses the ``ndjson`` format for files with :code:`.ndjson`, :code:`.jsonl` and :code:`.jl` extensions. diff --git a/csvkit/cli.py b/csvkit/cli.py index e99acad34..03c4651e2 100644 --- a/csvkit/cli.py +++ b/csvkit/cli.py @@ -29,6 +29,31 @@ QUOTING_CHOICES = sorted(getattr(csv, name) for name in dir(csv) if name.startswith('QUOTE_')) +def _non_empty_column_identifiers(value): + if not value.strip(): + raise argparse.ArgumentTypeError('column selection must not be empty') + return value + + +class _SelectiveTypeTester(agate.TypeTester): + """Resolve text-only columns against each table's actual headers.""" + + def __init__(self, types, text_type, columns, column_offset): + super().__init__(types=types) + self._types = types + self._text_type = text_type + self._columns = columns + self._column_offset = column_offset + + def run(self, rows, column_names): + indices = parse_column_identifiers(self._columns, column_names, self._column_offset) + if not indices: + raise ColumnIdentifierError('No columns match the selection %r.' % self._columns) + force = {column_names[index]: self._text_type for index in indices} + # A fresh tester prevents forced names leaking between inputs with different headers. + return agate.TypeTester(types=self._types, force=force).run(rows, column_names) + + class LazyFile: """ A proxy for a File object that delays opening it until @@ -127,6 +152,18 @@ def add_arguments(self): """ raise NotImplementedError('add_arguments must be provided by each subclass of CSVKitUtility.') + def add_type_inference_arguments(self): + """Register global and column-specific inference options consistently.""" + group = self.argparser.add_mutually_exclusive_group() + group.add_argument( + '-I', '--no-inference', dest='no_inference', action='store_true', + help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' + 'when parsing the input.') + group.add_argument( + '--no-inference-columns', dest='no_inference_columns', metavar='COLUMNS', + type=_non_empty_column_identifiers, + help='Disable type inference only for these column indices, names or ranges, e.g. "1,id,3-5".') + def run(self): """ A wrapper around the main loop of the utility which handles opening and @@ -386,6 +423,9 @@ def get_column_types(self): else: types.insert(1, number_type) + columns = getattr(self.args, 'no_inference_columns', None) + if columns is not None: + return _SelectiveTypeTester(types, text_type, columns, self.get_column_offset()) return agate.TypeTester(types=types) def get_column_offset(self): diff --git a/csvkit/utilities/csvjoin.py b/csvkit/utilities/csvjoin.py index 07d4ee500..78eeebaf3 100644 --- a/csvkit/utilities/csvjoin.py +++ b/csvkit/utilities/csvjoin.py @@ -37,10 +37,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): if isatty(sys.stdin) and self.args.input_paths == ['-']: diff --git a/csvkit/utilities/csvjson.py b/csvkit/utilities/csvjson.py index 2cd52b985..2eb9de638 100644 --- a/csvkit/utilities/csvjson.py +++ b/csvkit/utilities/csvjson.py @@ -49,10 +49,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): """ diff --git a/csvkit/utilities/csvlook.py b/csvkit/utilities/csvlook.py index 13ec786d0..a34d52d8d 100644 --- a/csvkit/utilities/csvlook.py +++ b/csvkit/utilities/csvlook.py @@ -29,10 +29,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): if self.additional_input_expected(): diff --git a/csvkit/utilities/csvpy.py b/csvkit/utilities/csvpy.py index 4689553e8..1c3603bb1 100644 --- a/csvkit/utilities/csvpy.py +++ b/csvkit/utilities/csvpy.py @@ -10,7 +10,7 @@ class CSVPy(CSVKitUtility): description = 'Load a CSV file into a CSV reader and then drop into a Python shell.' - override_flags = ['l', 'zero', 'add-bom'] + override_flags = ['l', 'add-bom'] def add_arguments(self): self.argparser.add_argument( @@ -26,10 +26,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): if self.input_file == sys.stdin: diff --git a/csvkit/utilities/csvsort.py b/csvkit/utilities/csvsort.py index fdada773e..d3bf052eb 100644 --- a/csvkit/utilities/csvsort.py +++ b/csvkit/utilities/csvsort.py @@ -37,10 +37,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): if self.args.names_only: diff --git a/csvkit/utilities/csvsql.py b/csvkit/utilities/csvsql.py index 8e2a81936..452c6376c 100644 --- a/csvkit/utilities/csvsql.py +++ b/csvkit/utilities/csvsql.py @@ -83,10 +83,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() self.argparser.add_argument( '--chunk-size', dest='chunk_size', type=int, help='Chunk size for batch insert into the table. Requires --insert.') diff --git a/csvkit/utilities/csvstat.py b/csvkit/utilities/csvstat.py index 052a75d88..09342ed0e 100644 --- a/csvkit/utilities/csvstat.py +++ b/csvkit/utilities/csvstat.py @@ -143,10 +143,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing the input.') + self.add_type_inference_arguments() def main(self): if self.args.names_only: diff --git a/csvkit/utilities/in2csv.py b/csvkit/utilities/in2csv.py index 85dcd2d8c..f43ee964f 100644 --- a/csvkit/utilities/in2csv.py +++ b/csvkit/utilities/in2csv.py @@ -60,10 +60,7 @@ def add_arguments(self): '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024, help='Limit CSV dialect sniffing to the specified number of bytes. ' 'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.') - self.argparser.add_argument( - '-I', '--no-inference', dest='no_inference', action='store_true', - help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) ' - 'when parsing CSV input.') + self.add_type_inference_arguments() # This is called only from open_excel_input_file(), but is a separate method to use caching. @functools.lru_cache diff --git a/docs/common_arguments.rst b/docs/common_arguments.rst index e279b6785..26edf7c9f 100644 --- a/docs/common_arguments.rst +++ b/docs/common_arguments.rst @@ -64,6 +64,16 @@ For example, to disable CSV sniffing, set :code:`--snifflimit 0` and then, if ne To disable type inference, add the :code:`--no-inference` flag. To prevent text values from being converted to dates or datetimes, set the :code:`--date-format` and/or :code:`--datetime-format` options to a non-occurring value, like ``-``. +To preserve only certain columns as text, use :code:`--no-inference-columns COLUMNS` with :doc:`/scripts/csvjoin`, :doc:`/scripts/csvjson`, :doc:`/scripts/csvlook`, :doc:`/scripts/csvpy` (with :code:`--agate`), :doc:`/scripts/csvsort`, :doc:`/scripts/csvsql`, :doc:`/scripts/csvstat` or :doc:`/scripts/in2csv`. Unselected columns retain normal type inference. For example, keep leading zeroes in an identifier and preserve date-looking codes while still parsing numeric amounts: + +.. code-block:: bash + + csvjson --no-inference-columns id,date_code input.csv + +The selector accepts comma-separated column names, numeric column IDs and inclusive ranges, such as :code:`1,id,3-5`. IDs start at 1, or at 0 with :code:`--zero`. For headerless input, use the generated names (a, b, c, ...) or numeric IDs. Unknown or out-of-range columns produce an error. The same selector is resolved against each input file's headers. You cannot combine this option with :code:`-I/--no-inference`. + +Selected text columns still use the configured NULL conversion rules. Use :code:`--blanks` to preserve the default NULL spellings as text, or :code:`--null-value` to specify additional NULL values. Selective inference still buffers input to infer the unselected columns; it does not enable the all-text streaming shortcuts. + The output of csvkit's tools is always formatted with "default" formatting options. This means that when executing multiple csvkit commands (either with a pipe or through intermediary files) it is only ever necessary to specify these arguments the first time (and doing so for subsequent commands will likely cause them to fail). See the documentation of :doc:`/scripts/csvclean` for a description of the default formatting options. diff --git a/docs/contributing.rst b/docs/contributing.rst index e1fd6d743..1152f754d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -86,6 +86,8 @@ Currently, the following tools buffer: * :doc:`/scripts/csvstat` * :doc:`/scripts/in2csv` unless :code:`--format ndjson --no-inference` is set, or unless :code:`--format csv --no-inference --snifflimit 0` is set and :code:`--no-header-row` and :code:`--skip-lines` aren't set +The :code:`--no-inference-columns` option keeps inference enabled for unselected columns, so it does not enable the all-text streaming shortcuts in :doc:`/scripts/csvjson` or :doc:`/scripts/in2csv`. Those shortcuts require global :code:`--no-inference`. + Legalese ======== diff --git a/docs/scripts/csvjoin.rst b/docs/scripts/csvjoin.rst index e7ea6eff7..6459b1501 100644 --- a/docs/scripts/csvjoin.rst +++ b/docs/scripts/csvjoin.rst @@ -14,7 +14,7 @@ Merges two or more CSV tables together using a method analogous to SQL JOIN oper [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]] [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT] [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-c COLUMNS] - [--outer] [--left] [--right] [-y SNIFF_LIMIT] [-I] + [--outer] [--left] [--right] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE [FILE ...]] Execute a SQL-like join to merge CSV files on a specified column or columns. @@ -48,12 +48,17 @@ Merges two or more CSV tables together using a method analogous to SQL JOIN oper -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". Note that the join operation requires reading all files into memory. Don't try this on very large files. See also: :doc:`../common_arguments`. +The :code:`--no-inference-columns` selector is resolved separately against each input file. Use column names when column positions differ between files. + Examples ======== diff --git a/docs/scripts/csvjson.rst b/docs/scripts/csvjson.rst index 4f995f0f7..11441e72a 100644 --- a/docs/scripts/csvjson.rst +++ b/docs/scripts/csvjson.rst @@ -16,7 +16,7 @@ Converts a CSV file into JSON or GeoJSON (depending on flags): [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-i INDENT] [-k KEY] [--lat LAT] [--lon LON] [--type TYPE] [--geometry GEOMETRY] [--crs CRS] [--no-bbox] [--stream] - [-y SNIFF_LIMIT] [-I] + [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Convert a CSV file into JSON (or GeoJSON). @@ -56,6 +56,9 @@ Converts a CSV file into JSON or GeoJSON (depending on flags): -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". See also: :doc:`../common_arguments`. diff --git a/docs/scripts/csvlook.rst b/docs/scripts/csvlook.rst index 4b04d7360..30931205c 100644 --- a/docs/scripts/csvlook.rst +++ b/docs/scripts/csvlook.rst @@ -17,7 +17,7 @@ Renders a CSV to the command line in a Markdown-compatible, fixed-width format: [--max-rows MAX_ROWS] [--max-columns MAX_COLUMNS] [--max-column-width MAX_COLUMN_WIDTH] [--max-precision MAX_PRECISION] [--no-number-ellipsis] - [-y SNIFF_LIMIT] [-I] + [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Render a CSV file in the console as a Markdown-compatible, fixed-width table. @@ -47,6 +47,9 @@ Renders a CSV to the command line in a Markdown-compatible, fixed-width format: -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". If a table is too wide to display properly try piping the output to ``less -S`` or truncating it using :doc:`csvcut`. diff --git a/docs/scripts/csvpy.rst b/docs/scripts/csvpy.rst index ff1177661..ee90226c1 100644 --- a/docs/scripts/csvpy.rst +++ b/docs/scripts/csvpy.rst @@ -14,7 +14,7 @@ Loads a CSV file into a :class:`agate.csv.Reader` object and then drops into a P [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]] [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT] [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [--dict] [--agate] - [--no-number-ellipsis] [-y SNIFF_LIMIT] [-I] + [--no-number-ellipsis] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Load a CSV file into a CSV reader and then drop into a Python shell. @@ -35,6 +35,11 @@ Loads a CSV file into a :class:`agate.csv.Reader` object and then drops into a P -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". + +Type inference options apply when :code:`--agate` is set. The default CSV reader and :code:`--dict` reader already return text. This tool will automatically use the IPython shell if it is installed, otherwise it will use the running Python shell. diff --git a/docs/scripts/csvsort.rst b/docs/scripts/csvsort.rst index 04997c021..bba92487b 100644 --- a/docs/scripts/csvsort.rst +++ b/docs/scripts/csvsort.rst @@ -14,7 +14,7 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data: [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]] [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT] [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-n] [-c COLUMNS] - [-r] [-i] [-y SNIFF_LIMIT] [-I] + [-r] [-i] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Sort CSV files. Like the Unix "sort" command, but for tabular data. @@ -40,6 +40,9 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data: -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". See also: :doc:`../common_arguments`. diff --git a/docs/scripts/csvsql.rst b/docs/scripts/csvsql.rst index ac7c11b70..1ff92ba6e 100644 --- a/docs/scripts/csvsql.rst +++ b/docs/scripts/csvsql.rst @@ -20,7 +20,7 @@ Generate SQL statements for a CSV file or execute those statements directly on a [--after-insert AFTER_INSERT] [--tables TABLE_NAMES] [--no-constraints] [--unique-constraint UNIQUE_CONSTRAINT] [--no-create] [--create-if-not-exists] [--overwrite] - [--db-schema DB_SCHEMA] [-y SNIFF_LIMIT] [-I] + [--db-schema DB_SCHEMA] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [--chunk-size CHUNK_SIZE] [FILE [FILE ...]] @@ -84,6 +84,9 @@ Generate SQL statements for a CSV file or execute those statements directly on a -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". --chunk-size CHUNK_SIZE Chunk size for batch insert into the table. Requires --insert. diff --git a/docs/scripts/csvstat.rst b/docs/scripts/csvstat.rst index d43ca6d93..9ea51cadf 100644 --- a/docs/scripts/csvstat.rst +++ b/docs/scripts/csvstat.rst @@ -18,7 +18,7 @@ Prints descriptive statistics for all columns in a CSV file. Will intelligently [--unique] [--min] [--max] [--sum] [--mean] [--median] [--stdev] [--len] [--max-precision] [--freq] [--freq-count FREQ_COUNT] [--count] - [--decimal-format DECIMAL_FORMAT] [-G] [-y SNIFF_LIMIT] [-I] + [--decimal-format DECIMAL_FORMAT] [-G] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Print descriptive statistics for each column in a CSV file. @@ -68,6 +68,9 @@ Prints descriptive statistics for all columns in a CSV file. Will intelligently -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing the input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". See also: :doc:`../common_arguments`. diff --git a/docs/scripts/in2csv.rst b/docs/scripts/in2csv.rst index de89522be..b2eb630bb 100644 --- a/docs/scripts/in2csv.rst +++ b/docs/scripts/in2csv.rst @@ -28,7 +28,7 @@ The header line is required though the columns may be in any order: [-f {csv,dbf,fixed,geojson,json,ndjson,xls,xlsx}] [-s SCHEMA] [-k KEY] [-n] [--sheet SHEET] [--write-sheets WRITE_SHEETS] [--use-sheet-names] [--reset-dimensions] - [--encoding-xls ENCODING_XLS] [-y SNIFF_LIMIT] [-I] + [--encoding-xls ENCODING_XLS] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS] [FILE] Convert common, but less awesome, tabular data formats to CSV. @@ -64,11 +64,16 @@ The header line is required though the columns may be in any order: -I, --no-inference Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) when parsing CSV input. + --no-inference-columns COLUMNS + Disable type inference only for these column indices, + names or ranges, e.g. "1,id,3-5". Some command-line flags only pertain to specific input formats. See also: :doc:`../common_arguments`. +Type inference options do not apply to DBF input, which uses its embedded field types. For Excel input, selecting a column preserves text cells as text; it cannot recover leading zeroes that were not stored in a numeric cell. + .. note:: The "ndjson" format refers to "newline delimited JSON", as used by many streaming APIs. diff --git a/docs/tricks.rst b/docs/tricks.rst index a3c9f4419..67691af76 100644 --- a/docs/tricks.rst +++ b/docs/tricks.rst @@ -123,6 +123,8 @@ These may be symptoms of csvkit's type inference being too aggressive for your d For some data, type inference can be error prone. If necessary you can disable it with the :code:`--no-inference` option. This will force all columns to be treated as regular text. +To keep identifiers or date-looking codes as text while inferring other columns, use :code:`--no-inference-columns`, for example :code:`csvsort --no-inference-columns id -c amount input.csv`. This preserves leading zeroes in :code:`id` and sorts :code:`amount` numerically. See :doc:`common_arguments` for column selectors and NULL handling. + To prevent values from being converted to dates or datetimes, set the :code:`--date-format` and/or :code:`--datetime-format` options to a non-occurring value, like ``-``. Slow performance diff --git a/tests/test_cli.py b/tests/test_cli.py index c4fe55d2f..6487776bb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,13 @@ +import datetime +import decimal +import io import unittest +from contextlib import redirect_stderr -from csvkit.cli import ColumnIdentifierError, match_column_identifier, parse_column_identifiers +import agate + +from csvkit.cli import ColumnIdentifierError, CSVKitUtility, match_column_identifier, parse_column_identifiers +from csvkit.utilities.csvsort import CSVSort class TestCli(unittest.TestCase): @@ -90,3 +97,141 @@ def test_range_notation_open_ended(self): self.assertEqual(target, parse_column_identifiers('1,4:', self.headers)) self.assertEqual(list(range(0, len(self.headers))), parse_column_identifiers('1:', self.headers)) + + +class InferenceUtility(CSVKitUtility): + def add_arguments(self): + self.add_type_inference_arguments() + + +class TestInferenceControls(unittest.TestCase): + def test_default_inference(self): + utility = CSVSort([]) + table = agate.Table( + [['001', '10.5', '2026-10-08', 'true']], + ['identifier', 'amount', 'visit', 'active'], column_types=utility.get_column_types(), + ) + self.assertEqual( + tuple(table.rows[0]), + (decimal.Decimal(1), decimal.Decimal('10.5'), datetime.date(2026, 10, 8), True), + ) + + def test_global_inference_preserves_positional_filename(self): + for flag in ('-I', '--no-inference'): + for args in ([flag, 'input.csv'], ['input.csv', flag]): + with self.subTest(args=args): + utility = CSVSort(args) + self.assertEqual(utility.args.input_path, 'input.csv') + self.assertIs(utility.args.no_inference, True) + table = agate.Table([['001', '2']], ['id', 'amount'], + column_types=utility.get_column_types()) + self.assertEqual(tuple(table.rows[0]), ('001', '2')) + + +class TestSelectiveInference(unittest.TestCase): + def get_tester(self, columns, options=()): + # Supply the new parser destination to the existing shared construction API. + # This also exercises base policy without depending on new utility registration. + utility = CSVSort(list(options)) + utility.args.no_inference_columns = columns + return utility.get_column_types() + + def test_selects_names_indices_and_ranges(self): + headers = ['identifier', 'amount', 'visit', 'active'] + rows = [['001', '10.5', '2026-10-08', 'true']] + for columns, options, expected in ( + ('identifier', (), ('001', decimal.Decimal('10.5'), datetime.date(2026, 10, 8), True)), + ('1,visit', (), ('001', decimal.Decimal('10.5'), '2026-10-08', True)), + ('1-2', (), ('001', '10.5', datetime.date(2026, 10, 8), True)), + ('2:3', (), (decimal.Decimal(1), '10.5', '2026-10-08', True)), + ('0,2', ('--zero',), ('001', decimal.Decimal('10.5'), '2026-10-08', True)), + ('0-1', ('--zero',), ('001', '10.5', datetime.date(2026, 10, 8), True)), + ): + with self.subTest(columns=columns, options=options): + table = agate.Table(rows, headers, column_types=self.get_tester(columns, options)) + self.assertEqual(tuple(table.rows[0]), expected) + + def test_unicode_header_and_literal_values(self): + table = agate.Table([['001', 'café 雨', '2']], ['識別子', 'label', 'amount'], + column_types=self.get_tester('識別子')) + self.assertEqual(tuple(table.rows[0]), ('001', 'café 雨', decimal.Decimal(2))) + + def test_default_and_custom_nulls_apply_to_selected_text(self): + table = agate.Table([['', '2'], ['na', '3'], ['MISSING', '4']], ['id', 'amount'], + column_types=self.get_tester('id', ['--null-value', 'MISSING'])) + self.assertEqual(list(table.columns['id']), [None, None, None]) + self.assertIsInstance(table.column_types[1], agate.Number) + + def test_blanks_and_custom_nulls_apply_to_selected_text(self): + table = agate.Table([['', '2'], ['na', '3'], ['MISSING', '4']], ['id', 'amount'], + column_types=self.get_tester('id', ['--blanks', '--null-value', 'MISSING'])) + self.assertEqual(list(table.columns['id']), ['', 'na', None]) + self.assertIsInstance(table.column_types[1], agate.Number) + + def test_locale_and_date_formats_remain_active_for_other_columns(self): + table = agate.Table( + [['001', '1.234,50', '08/10/2026', '2026|10|08 15:30']], + ['id', 'amount', 'day', 'moment'], + column_types=self.get_tester( + 'id', ['--locale', 'de_DE', '--date-format', '%d/%m/%Y', '--datetime-format', '%Y|%m|%d %H:%M'], + ), + ) + self.assertEqual( + tuple(table.rows[0]), + ('001', decimal.Decimal('1234.50'), datetime.date(2026, 10, 8), datetime.datetime(2026, 10, 8, 15, 30)), + ) + + def test_no_leading_zeroes_remains_active_for_unselected_column(self): + table = agate.Table([['001', '10.5']], ['id', 'amount'], + column_types=self.get_tester('amount', ['--no-leading-zeroes'])) + self.assertEqual(tuple(table.rows[0]), ('001', '10.5')) + + def test_reused_tester_resolves_each_schema_independently(self): + for selector in ('1', 'id'): + with self.subTest(selector=selector): + tester = self.get_tester(selector) + first = agate.Table([['001', '2']], ['id', 'amount'], column_types=tester) + second = agate.Table([['2', '001']], ['amount', 'id'], column_types=tester) + self.assertEqual(tuple(first.rows[0]), ('001', decimal.Decimal(2))) + expected = ('2', decimal.Decimal(1)) if selector == '1' else (decimal.Decimal(2), '001') + self.assertEqual(tuple(second.rows[0]), expected) + + def test_unknown_and_invalid_selectors_raise(self): + for selector in ('missing', '0', '4', '1-4', '1-nope', '3-1'): + with self.subTest(selector=selector): + with self.assertRaises(ColumnIdentifierError): + agate.Table([['001', '2']], ['id', 'amount'], column_types=self.get_tester(selector)) + + def test_reused_named_selector_does_not_ignore_missing_column(self): + tester = self.get_tester('id') + agate.Table([['001', '2']], ['id', 'amount'], column_types=tester) + with self.assertRaises(ColumnIdentifierError): + agate.Table([['2', '001']], ['amount', 'other'], column_types=tester) + + def test_empty_header_reports_no_matching_selection(self): + with self.assertRaises(ColumnIdentifierError): + self.get_tester('id').run([], []) + + def test_parser_registers_separate_option(self): + utility = InferenceUtility(['--no-inference-columns', 'id', 'input.csv']) + self.assertEqual(utility.args.no_inference_columns, 'id') + self.assertEqual(utility.args.input_path, 'input.csv') + self.assertIs(utility.args.no_inference, False) + + def test_parser_rejects_empty_selector(self): + for selector in ('', ' '): + with self.subTest(selector=selector): + error = io.StringIO() + with redirect_stderr(error), self.assertRaises(SystemExit) as raised: + InferenceUtility(['--no-inference-columns', selector]) + self.assertEqual(raised.exception.code, 2) + self.assertIn('must not be empty', error.getvalue()) + + def test_parser_rejects_global_and_selected_inference(self): + for args in (['-I', '--no-inference-columns', 'id'], ['--no-inference-columns', 'id', '--no-inference']): + with self.subTest(args=args): + error = io.StringIO() + with redirect_stderr(error), self.assertRaises(SystemExit) as raised: + InferenceUtility(args) + self.assertEqual(raised.exception.code, 2) + self.assertIn('not allowed with argument', error.getvalue()) diff --git a/tests/test_utilities/test_selective_inference.py b/tests/test_utilities/test_selective_inference.py new file mode 100644 index 000000000..2522be38c --- /dev/null +++ b/tests/test_utilities/test_selective_inference.py @@ -0,0 +1,227 @@ +import io +import json +import sys +from contextlib import redirect_stderr +from decimal import Decimal +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import agate +from openpyxl import Workbook + +from csvkit.exceptions import ColumnIdentifierError +from csvkit.utilities.csvjoin import CSVJoin +from csvkit.utilities.csvjson import CSVJSON +from csvkit.utilities.csvlook import CSVLook +from csvkit.utilities.csvpy import CSVPy +from csvkit.utilities.csvsort import CSVSort +from csvkit.utilities.csvsql import CSVSQL +from csvkit.utilities.csvstat import CSVStat +from csvkit.utilities.in2csv import In2CSV +from tests.utils import CSVKitTestCase, stdin_as_string + +DATA = 'id,amount,date,active\n2,10,01/02/2024,true\n10,2,03/04/2024,false\n001,3,05/06/2024,true\n' +SELECTIVE = ['--no-inference-columns', 'id,date', '--date-format', '%m/%d/%Y'] + + +class InputMixin: + def output(self, args, data=DATA): + with stdin_as_string(io.BytesIO(data.encode('utf-8'))): + return self.get_output(['-y', '0'] + args) + + def rows(self, args, data=DATA): + return list(agate.csv.reader(io.StringIO(self.output(args, data)))) + + +class TestSelectiveCSVJSON(InputMixin, CSVKitTestCase): + Utility = CSVJSON + + def test_selective_names_and_types(self): + rows = json.loads(self.output(SELECTIVE)) + self.assertEqual(rows[2], {'id': '001', 'amount': 3, 'date': '05/06/2024', 'active': True}) + + def test_selective_ids_ranges_and_zero(self): + for args in ( + ['--no-inference-columns', '1-2'], + ['--zero', '--no-inference-columns', '0-1'], + ): + with self.subTest(args=args): + row = json.loads(self.output(args + ['--date-format', '%m/%d/%Y']))[2] + self.assertEqual(row, {'id': '001', 'amount': '3', 'date': '2024-05-06', 'active': True}) + + def test_selective_headerless(self): + rows = json.loads(self.output(['-H', '--zero', '--no-inference-columns', '0'], '001,2\n002,3\n')) + self.assertEqual(rows, [{'a': '001', 'b': 2}, {'a': '002', 'b': 3}]) + + def test_selective_nulls(self): + for options, expected in ( + ([], None), + (['--blanks'], 'NA'), + (['--null-value', 'MISSING'], None), + (['--blanks', '--null-value', 'MISSING'], 'NA'), + ): + with self.subTest(options=options): + row = json.loads(self.output(['--no-inference-columns', 'id'] + options, + 'id,amount\nNA,2\n001,3\n'))[0] + self.assertEqual(row, {'id': expected, 'amount': 2}) + row = json.loads(self.output(['--no-inference-columns', 'id', '--null-value', 'MISSING'], + 'id,amount\nMISSING,2\n001,3\n'))[0] + self.assertEqual(row, {'id': None, 'amount': 2}) + + def test_selective_stream_keeps_unselected_inference(self): + rows = [json.loads(line) for line in self.output(['--stream'] + SELECTIVE).splitlines()] + self.assertEqual(rows[2], {'id': '001', 'amount': 3, 'date': '05/06/2024', 'active': True}) + + def test_selective_unknown_column(self): + with self.assertRaises(ColumnIdentifierError): + self.output(['--no-inference-columns', 'missing']) + + def test_selective_conflicts_with_global(self): + for utility in (CSVJoin, CSVJSON, CSVLook, CSVPy, CSVSort, CSVSQL, CSVStat, In2CSV): + with self.subTest(utility=utility.__name__), redirect_stderr(io.StringIO()) as stderr: + with self.assertRaises(SystemExit) as error: + utility(['-I', '--no-inference-columns', 'id']) + self.assertEqual(error.exception.code, 2) + self.assertIn('not allowed with argument', stderr.getvalue()) + + def test_legacy_default_and_global(self): + row = json.loads(self.output(['--date-format', '%m/%d/%Y']))[2] + self.assertEqual(row, {'id': 1, 'amount': 3, 'date': '2024-05-06', 'active': True}) + row = json.loads(self.output(['--no-inference']))[2] + self.assertEqual(row, {'id': '001', 'amount': '3', 'date': '05/06/2024', 'active': 'true'}) + + def test_legacy_global_stream_shortcut(self): + with patch.object(CSVJSON, 'read_csv_to_table', side_effect=AssertionError('buffered input')): + rows = [json.loads(line) for line in self.output(['--stream', '-I']).splitlines()] + self.assertEqual(rows[2]['id'], '001') + self.assertEqual(rows[2]['amount'], '3') + + def test_legacy_global_does_not_consume_filename(self): + with TemporaryDirectory() as directory: + filename = Path(directory) / 'input.csv' + filename.write_text(DATA, encoding='utf-8') + row = json.loads(self.get_output(['--no-inference', str(filename)]))[2] + self.assertEqual(row['id'], '001') + + +class TestSelectiveCSVSort(InputMixin, CSVKitTestCase): + Utility = CSVSort + + def test_selective_text_sort(self): + rows = self.rows(['-c', 'id'] + SELECTIVE) + self.assertEqual([row[0] for row in rows[1:]], ['001', '10', '2']) + + def test_selective_other_columns_sort_numerically(self): + rows = self.rows(['-c', 'amount'] + SELECTIVE) + self.assertEqual([row[0] for row in rows[1:]], ['10', '001', '2']) + + def test_legacy_default_sort(self): + rows = self.rows(['-c', 'id']) + self.assertEqual([row[0] for row in rows[1:]], ['1', '2', '10']) + + +class TestSelectiveCSVStat(InputMixin, CSVKitTestCase): + Utility = CSVStat + + def test_selective_types(self): + rows = json.loads(self.output(['--json'] + SELECTIVE)) + self.assertEqual({row['column_name']: row['type'] for row in rows}, + {'id': 'Text', 'amount': 'Number', 'date': 'Text', 'active': 'Boolean'}) + + +class TestSelectiveCSVLook(InputMixin, CSVKitTestCase): + Utility = CSVLook + + def test_selective_display(self): + output = self.output(SELECTIVE) + self.assertIn('001', output) + self.assertIn('05/06/2024', output) + self.assertIn('True', output) + + +class TestSelectiveCSVSQL(InputMixin, CSVKitTestCase): + Utility = CSVSQL + + def test_selective_schema(self): + output = self.output(['--dialect', 'sqlite', '--tables', 'data'] + SELECTIVE) + self.assertIn('id VARCHAR', output) + self.assertIn('amount FLOAT', output) + self.assertIn('date VARCHAR', output) + self.assertIn('active BOOLEAN', output) + + def test_selective_sqlite_query(self): + rows = self.rows(['--tables', 'data', '--query', + 'SELECT id, amount FROM data ORDER BY amount'] + SELECTIVE) + self.assertEqual(rows, [['id', 'amount'], ['10', '2.0'], ['001', '3.0'], ['2', '10.0']]) + + +class TestSelectiveCSVJoin(CSVKitTestCase): + Utility = CSVJoin + + def join(self, args, left, right): + with TemporaryDirectory() as directory: + paths = [Path(directory) / f'{name}.csv' for name in ('left', 'right')] + for path, content in zip(paths, (left, right)): + path.write_text(content, encoding='utf-8') + return list(self.get_output_as_reader(['-y', '0'] + args + [str(path) for path in paths])) + + def test_selective_names_resolve_per_input(self): + rows = self.join(['-c', 'id', '--no-inference-columns', 'id'], + 'id,amount\n001,2\n002,3\n', 'amount,id\n10,001\n20,002\n') + self.assertEqual(rows[1:], [['001', '2', '10'], ['002', '3', '20']]) + + def test_selective_ids_do_not_leak_forced_names(self): + rows = self.join(['--no-inference-columns', '1'], + 'id,amount\n001,2\n002,3\n', 'amount,id\n010,009\n020,008\n') + self.assertEqual(rows[1:], [['001', '2', '010', '9'], ['002', '3', '020', '8']]) + + +class TestSelectiveIn2CSV(InputMixin, CSVKitTestCase): + Utility = In2CSV + + def test_selective_csv_keeps_unselected_inference(self): + rows = self.rows(['-f', 'csv'] + SELECTIVE) + self.assertEqual(rows[3], ['001', '3', '05/06/2024', 'True']) + + def test_selective_ndjson(self): + data = '\n'.join(json.dumps(dict(zip(('id', 'amount', 'date', 'active'), row))) for row in ( + ('001', '2', '01/02/2024', 'true'), ('002', '3', '03/04/2024', 'false'), + )) + rows = self.rows(['-f', 'ndjson'] + SELECTIVE, data) + self.assertEqual(rows[1:], [['001', '2', '01/02/2024', 'True'], ['002', '3', '03/04/2024', 'False']]) + + def test_selective_xlsx(self): + with TemporaryDirectory() as directory: + filename = Path(directory) / 'input.xlsx' + workbook = Workbook() + worksheet = workbook.active + worksheet.append(['id', 'amount', 'date', 'active']) + worksheet.append(['001', 2, '01/02/2024', 'true']) + worksheet.append(['002', 3, '03/04/2024', 'false']) + workbook.save(filename) + workbook.close() + rows = list(self.get_output_as_reader(SELECTIVE + [str(filename)])) + self.assertEqual(rows[1:], [['001', '2', '01/02/2024', 'True'], ['002', '3', '03/04/2024', 'False']]) + + def test_legacy_csv_stream_shortcut(self): + with patch.object(agate.Table, 'from_csv', side_effect=AssertionError('buffered input')): + rows = self.rows(['-f', 'csv', '-I']) + self.assertEqual(rows[3], ['001', '3', '05/06/2024', 'true']) + + +class TestSelectiveCSVPy(CSVKitTestCase): + Utility = CSVPy + + def test_selective_agate_table(self): + with TemporaryDirectory() as directory: + filename = Path(directory) / 'input.csv' + filename.write_text(DATA, encoding='utf-8') + with patch.dict(sys.modules, {'IPython.frontend.terminal.embed': None}), \ + patch('code.interact') as interact: + self.get_output(['--agate', '--zero', '--no-inference-columns', '0,2', + '--date-format', '%m/%d/%Y', str(filename)]) + table = interact.call_args.kwargs['local']['table'] + self.assertEqual(tuple(table.rows[2]), ('001', Decimal('3'), '05/06/2024', True)) + self.assertEqual(tuple(type(column_type) for column_type in table.column_types), + (agate.Text, agate.Number, agate.Text, agate.Boolean))