Allow disabling type inference for selected columns

CHANGELOG.rst

@@ -1,6 +1,8 @@
 Unreleased
 ----------
 
+-  feat: Tools that support type inference add :code:`--no-inference-columns` to preserve selected columns as text while inferring the remaining columns.
+
 -  feat: :doc:`/scripts/csvcut` adds an :code:`--ignore-unknown-columns` option to skip identifiers in :code:`-c/--columns` that do not match a column in the input.
 -  feat: :doc:`/scripts/csvclean` adds a :code:`--remove-empty-columns` option to remove empty columns from standard output.
 -  feat: :doc:`/scripts/in2csv` guesses the ``ndjson`` format for files with :code:`.ndjson`, :code:`.jsonl` and :code:`.jl` extensions.

csvkit/cli.py

@@ -29,6 +29,31 @@
 QUOTING_CHOICES = sorted(getattr(csv, name) for name in dir(csv) if name.startswith('QUOTE_'))
 
 
+def _non_empty_column_identifiers(value):
+    if not value.strip():
+        raise argparse.ArgumentTypeError('column selection must not be empty')
+    return value
+
+
+class _SelectiveTypeTester(agate.TypeTester):
+    """Resolve text-only columns against each table's actual headers."""
+
+    def __init__(self, types, text_type, columns, column_offset):
+        super().__init__(types=types)
+        self._types = types
+        self._text_type = text_type
+        self._columns = columns
+        self._column_offset = column_offset
+
+    def run(self, rows, column_names):
+        indices = parse_column_identifiers(self._columns, column_names, self._column_offset)
+        if not indices:
+            raise ColumnIdentifierError('No columns match the selection %r.' % self._columns)
+        force = {column_names[index]: self._text_type for index in indices}
+        # A fresh tester prevents forced names leaking between inputs with different headers.
+        return agate.TypeTester(types=self._types, force=force).run(rows, column_names)
+
+
 class LazyFile:
     """
     A proxy for a File object that delays opening it until
@@ -127,6 +152,18 @@ def add_arguments(self):
         """
         raise NotImplementedError('add_arguments must be provided by each subclass of CSVKitUtility.')
 
+    def add_type_inference_arguments(self):
+        """Register global and column-specific inference options consistently."""
+        group = self.argparser.add_mutually_exclusive_group()
+        group.add_argument(
+            '-I', '--no-inference', dest='no_inference', action='store_true',
+            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
+                 'when parsing the input.')
+        group.add_argument(
+            '--no-inference-columns', dest='no_inference_columns', metavar='COLUMNS',
+            type=_non_empty_column_identifiers,
+            help='Disable type inference only for these column indices, names or ranges, e.g. "1,id,3-5".')
+
     def run(self):
         """
         A wrapper around the main loop of the utility which handles opening and
@@ -386,6 +423,9 @@ def get_column_types(self):
                 else:
                     types.insert(1, number_type)
 
+        columns = getattr(self.args, 'no_inference_columns', None)
+        if columns is not None:
+            return _SelectiveTypeTester(types, text_type, columns, self.get_column_offset())
         return agate.TypeTester(types=types)
 
     def get_column_offset(self):

csvkit/utilities/csvjoin.py

@@ -37,10 +37,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         if isatty(sys.stdin) and self.args.input_paths == ['-']:

csvkit/utilities/csvjson.py

@@ -49,10 +49,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         """

csvkit/utilities/csvlook.py

@@ -29,10 +29,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         if self.additional_input_expected():

csvkit/utilities/csvpy.py

@@ -10,7 +10,7 @@
 
 class CSVPy(CSVKitUtility):
     description = 'Load a CSV file into a CSV reader and then drop into a Python shell.'
-    override_flags = ['l', 'zero', 'add-bom']
+    override_flags = ['l', 'add-bom']
 
     def add_arguments(self):
         self.argparser.add_argument(
@@ -26,10 +26,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         if self.input_file == sys.stdin:

csvkit/utilities/csvsort.py

@@ -37,10 +37,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         if self.args.names_only:

csvkit/utilities/csvsql.py

@@ -83,10 +83,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
         self.argparser.add_argument(
             '--chunk-size', dest='chunk_size', type=int,
             help='Chunk size for batch insert into the table. Requires --insert.')

csvkit/utilities/csvstat.py

@@ -143,10 +143,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing the input.')
+        self.add_type_inference_arguments()
 
     def main(self):
         if self.args.names_only:

csvkit/utilities/in2csv.py

@@ -60,10 +60,7 @@ def add_arguments(self):
             '-y', '--snifflimit', dest='sniff_limit', type=int, default=1024,
             help='Limit CSV dialect sniffing to the specified number of bytes. '
                  'Specify "0" to disable sniffing entirely, or "-1" to sniff the entire file.')
-        self.argparser.add_argument(
-            '-I', '--no-inference', dest='no_inference', action='store_true',
-            help='Disable type inference (and --locale, --date-format, --datetime-format, --no-leading-zeroes) '
-                 'when parsing CSV input.')
+        self.add_type_inference_arguments()
 
     # This is called only from open_excel_input_file(), but is a separate method to use caching.
     @functools.lru_cache

docs/common_arguments.rst

@@ -64,6 +64,16 @@ For example, to disable CSV sniffing, set :code:`--snifflimit 0` and then, if ne
 
 To disable type inference, add the :code:`--no-inference` flag. To prevent text values from being converted to dates or datetimes, set the :code:`--date-format` and/or :code:`--datetime-format` options to a non-occurring value, like ``-``.
 
+To preserve only certain columns as text, use :code:`--no-inference-columns COLUMNS` with :doc:`/scripts/csvjoin`, :doc:`/scripts/csvjson`, :doc:`/scripts/csvlook`, :doc:`/scripts/csvpy` (with :code:`--agate`), :doc:`/scripts/csvsort`, :doc:`/scripts/csvsql`, :doc:`/scripts/csvstat` or :doc:`/scripts/in2csv`. Unselected columns retain normal type inference. For example, keep leading zeroes in an identifier and preserve date-looking codes while still parsing numeric amounts:
+
+.. code-block:: bash
+
+   csvjson --no-inference-columns id,date_code input.csv
+
+The selector accepts comma-separated column names, numeric column IDs and inclusive ranges, such as :code:`1,id,3-5`. IDs start at 1, or at 0 with :code:`--zero`. For headerless input, use the generated names (a, b, c, ...) or numeric IDs. Unknown or out-of-range columns produce an error. The same selector is resolved against each input file's headers. You cannot combine this option with :code:`-I/--no-inference`.
+
+Selected text columns still use the configured NULL conversion rules. Use :code:`--blanks` to preserve the default NULL spellings as text, or :code:`--null-value` to specify additional NULL values. Selective inference still buffers input to infer the unselected columns; it does not enable the all-text streaming shortcuts.
+
 The output of csvkit's tools is always formatted with "default" formatting options. This means that when executing multiple csvkit commands (either with a pipe or through intermediary files) it is only ever necessary to specify these arguments the first time (and doing so for subsequent commands will likely cause them to fail).
 
 See the documentation of :doc:`/scripts/csvclean` for a description of the default formatting options.

docs/contributing.rst

@@ -86,6 +86,8 @@ Currently, the following tools buffer:
 * :doc:`/scripts/csvstat`
 * :doc:`/scripts/in2csv` unless :code:`--format ndjson --no-inference` is set, or unless :code:`--format csv --no-inference --snifflimit 0` is set and :code:`--no-header-row` and :code:`--skip-lines` aren't set
 
+The :code:`--no-inference-columns` option keeps inference enabled for unselected columns, so it does not enable the all-text streaming shortcuts in :doc:`/scripts/csvjson` or :doc:`/scripts/in2csv`. Those shortcuts require global :code:`--no-inference`.
+
 Legalese
 ========
 

docs/scripts/csvjoin.rst

@@ -14,7 +14,7 @@ Merges two or more CSV tables together using a method analogous to SQL JOIN oper
                   [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]]
                   [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT]
                   [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-c COLUMNS]
-                  [--outer] [--left] [--right] [-y SNIFF_LIMIT] [-I]
+                  [--outer] [--left] [--right] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                   [FILE [FILE ...]]
 
    Execute a SQL-like join to merge CSV files on a specified column or columns.
@@ -48,12 +48,17 @@ Merges two or more CSV tables together using a method analogous to SQL JOIN oper
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
 
    Note that the join operation requires reading all files into memory. Don't try
    this on very large files.
 
 See also: :doc:`../common_arguments`.
 
+The :code:`--no-inference-columns` selector is resolved separately against each input file. Use column names when column positions differ between files.
+
 Examples
 ========
 

docs/scripts/csvjson.rst

@@ -16,7 +16,7 @@ Converts a CSV file into JSON or GeoJSON (depending on flags):
                   [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-i INDENT]
                   [-k KEY] [--lat LAT] [--lon LON] [--type TYPE]
                   [--geometry GEOMETRY] [--crs CRS] [--no-bbox] [--stream]
-                  [-y SNIFF_LIMIT] [-I]
+                  [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                   [FILE]
 
    Convert a CSV file into JSON (or GeoJSON).
@@ -56,6 +56,9 @@ Converts a CSV file into JSON or GeoJSON (depending on flags):
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
 
 See also: :doc:`../common_arguments`.
 

docs/scripts/csvlook.rst

@@ -17,7 +17,7 @@ Renders a CSV to the command line in a Markdown-compatible, fixed-width format:
                   [--max-rows MAX_ROWS] [--max-columns MAX_COLUMNS]
                   [--max-column-width MAX_COLUMN_WIDTH]
                   [--max-precision MAX_PRECISION] [--no-number-ellipsis]
-                  [-y SNIFF_LIMIT] [-I]
+                  [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                   [FILE]
 
    Render a CSV file in the console as a Markdown-compatible, fixed-width table.
@@ -47,6 +47,9 @@ Renders a CSV to the command line in a Markdown-compatible, fixed-width format:
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
 
 If a table is too wide to display properly try piping the output to ``less -S`` or truncating it using :doc:`csvcut`.
 

docs/scripts/csvpy.rst

@@ -14,7 +14,7 @@ Loads a CSV file into a :class:`agate.csv.Reader` object and then drops into a P
                 [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]]
                 [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT]
                 [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [--dict] [--agate]
-                [--no-number-ellipsis] [-y SNIFF_LIMIT] [-I]
+                [--no-number-ellipsis] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                 [FILE]
 
    Load a CSV file into a CSV reader and then drop into a Python shell.
@@ -35,6 +35,11 @@ Loads a CSV file into a :class:`agate.csv.Reader` object and then drops into a P
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
+
+Type inference options apply when :code:`--agate` is set. The default CSV reader and :code:`--dict` reader already return text.
 
 This tool will automatically use the IPython shell if it is installed, otherwise it will use the running Python shell.
 

docs/scripts/csvsort.rst

@@ -14,7 +14,7 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data:
                   [-S] [--blanks] [--null-value NULL_VALUES [NULL_VALUES ...]]
                   [--date-format DATE_FORMAT] [--datetime-format DATETIME_FORMAT]
                   [-H] [-K SKIP_LINES] [-v] [-l] [--zero] [-V] [-n] [-c COLUMNS]
-                  [-r] [-i] [-y SNIFF_LIMIT] [-I]
+                  [-r] [-i] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                   [FILE]
 
    Sort CSV files. Like the Unix "sort" command, but for tabular data.
@@ -40,6 +40,9 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data:
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
 
 See also: :doc:`../common_arguments`.
 

docs/scripts/csvsql.rst

@@ -20,7 +20,7 @@ Generate SQL statements for a CSV file or execute those statements directly on a
                  [--after-insert AFTER_INSERT] [--tables TABLE_NAMES]
                  [--no-constraints] [--unique-constraint UNIQUE_CONSTRAINT]
                  [--no-create] [--create-if-not-exists] [--overwrite]
-                 [--db-schema DB_SCHEMA] [-y SNIFF_LIMIT] [-I]
+                 [--db-schema DB_SCHEMA] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                  [--chunk-size CHUNK_SIZE]
                  [FILE [FILE ...]]
 
@@ -84,6 +84,9 @@ Generate SQL statements for a CSV file or execute those statements directly on a
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
      --chunk-size CHUNK_SIZE
                            Chunk size for batch insert into the table. Requires
                            --insert.

docs/scripts/csvstat.rst

@@ -18,7 +18,7 @@ Prints descriptive statistics for all columns in a CSV file. Will intelligently
                   [--unique] [--min] [--max] [--sum] [--mean] [--median]
                   [--stdev] [--len] [--max-precision] [--freq]
                   [--freq-count FREQ_COUNT] [--count]
-                  [--decimal-format DECIMAL_FORMAT] [-G] [-y SNIFF_LIMIT] [-I]
+                  [--decimal-format DECIMAL_FORMAT] [-G] [-y SNIFF_LIMIT] [-I | --no-inference-columns COLUMNS]
                   [FILE]
 
    Print descriptive statistics for each column in a CSV file.
@@ -68,6 +68,9 @@ Prints descriptive statistics for all columns in a CSV file. Will intelligently
      -I, --no-inference    Disable type inference (and --locale, --date-format,
                            --datetime-format, --no-leading-zeroes) when parsing
                            the input.
+     --no-inference-columns COLUMNS
+                           Disable type inference only for these column indices,
+                           names or ranges, e.g. "1,id,3-5".
 
 See also: :doc:`../common_arguments`.