csvsort: support mixed ascending and descending columns

CHANGELOG.rst

@@ -1,6 +1,7 @@
 Unreleased
 ----------
 
+-  feat: :doc:`/scripts/csvsort` supports per-column descending order using :code:`~` in :code:`--columns`.
 -  feat: :doc:`/scripts/csvcut` adds an :code:`--ignore-unknown-columns` option to skip identifiers in :code:`-c/--columns` that do not match a column in the input.
 -  feat: :doc:`/scripts/csvclean` adds a :code:`--remove-empty-columns` option to remove empty columns from standard output.
 -  feat: :doc:`/scripts/in2csv` guesses the ``ndjson`` format for files with :code:`.ndjson`, :code:`.jsonl` and :code:`.jl` extensions.

csvkit/cli.py

@@ -73,6 +73,7 @@ class CSVKitUtility:
     description = ''
     epilog = ''
     override_flags = ''
+    literal_options = ()
 
     def __init__(self, args=None, output_file=None, error_file=None):
         """
@@ -81,7 +82,7 @@ def __init__(self, args=None, output_file=None, error_file=None):
         if args is None:
             args = sys.argv[1:]
         if os.name == 'nt':
-            args = _expand_args(args)
+            args = _expand_args(args, literal_options=self.literal_options)
 
         self._init_common_parser()
         self.add_arguments()
@@ -610,10 +611,31 @@ def parse_list(pairs):
 
 
 # Adapted from https://github.com/pallets/click/blame/main/src/click/utils.py
-def _expand_args(args):
+def _expand_args(args, literal_options=()):
     out = []
+    literal_next = False
 
     for arg in args:
+        if literal_next:
+            out.append(arg)
+            literal_next = False
+            continue
+
+        if arg in literal_options:
+            out.append(arg)
+            literal_next = True
+            continue
+
+        # Some option values are column identifiers rather than paths.
+        if any(
+            arg.startswith(f'{option}=') or (
+                option.startswith('-') and not option.startswith('--') and arg.startswith(option)
+            )
+            for option in literal_options
+        ):
+            out.append(arg)
+            continue
+
         arg = os.path.expanduser(arg)
         arg = os.path.expandvars(arg)
 

csvkit/utilities/csvsort.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
+from functools import cmp_to_key
+
 import agate
 
-from csvkit.cli import CSVKitUtility, parse_column_identifiers
+from csvkit.cli import CSVKitUtility, match_column_identifier, parse_column_identifiers
 
 
 def ignore_case_sort(key):
@@ -16,8 +18,36 @@ def inner(row):
     return inner
 
 
+def mixed_direction_sort(columns, descending, ignore_case):
+
+    def compare_rows(left, right):
+        for column, is_descending in zip(columns, descending):
+            left_value = left[column]
+            right_value = right[column]
+
+            if ignore_case:
+                if isinstance(left_value, str):
+                    left_value = left_value.upper()
+                if isinstance(right_value, str):
+                    right_value = right_value.upper()
+
+            if left_value is None:
+                left_value = agate.NullOrder()
+            if right_value is None:
+                right_value = agate.NullOrder()
+
+            comparison = (left_value > right_value) - (left_value < right_value)
+            if comparison:
+                return -comparison if is_descending else comparison
+
+        return 0
+
+    return cmp_to_key(compare_rows)
+
+
 class CSVSort(CSVKitUtility):
     description = 'Sort CSV files. Like the Unix "sort" command, but for tabular data.'
+    literal_options = ('-c', '--columns')
 
     def add_arguments(self):
         self.argparser.add_argument(
@@ -26,10 +56,10 @@ def add_arguments(self):
         self.argparser.add_argument(
             '-c', '--columns', dest='columns',
             help='A comma-separated list of column indices, names or ranges to sort by, e.g. "1,id,3-5". '
-                 'Defaults to all columns.')
+                 'Prefix a column with "~" to sort it in descending order. Defaults to all columns.')
         self.argparser.add_argument(
             '-r', '--reverse', dest='reverse', action='store_true',
-            help='Sort in descending order.')
+            help='Reverse the sort direction of every column.')
         self.argparser.add_argument(
             '-i', '--ignore-case', dest='ignore_case', action='store_true',
             help='Perform case-independent sorting.')
@@ -59,14 +89,36 @@ def main(self):
             **self.reader_kwargs,
         )
 
-        key = parse_column_identifiers(
-            self.args.columns,
-            table.column_names,
-            self.get_column_offset(),
-        )
-
-        if self.args.ignore_case:
-            key = ignore_case_sort(key)
+        identifiers = self.args.columns.split(',') if self.args.columns else []
+        reverse_columns = [
+            identifier.startswith('~') and len(identifier) > 1 and identifier not in table.column_names
+            for identifier in identifiers
+        ]
+
+        if any(reverse_columns):
+            columns = []
+            descending = []
+            for identifier, is_descending in zip(identifiers, reverse_columns):
+                if not identifier:
+                    selected = [match_column_identifier(table.column_names, identifier, self.get_column_offset())]
+                else:
+                    selected = parse_column_identifiers(
+                        identifier[1:] if is_descending else identifier,
+                        table.column_names,
+                        self.get_column_offset(),
+                    )
+                columns.extend(selected)
+                descending.extend([is_descending] * len(selected))
+        else:
+            columns = parse_column_identifiers(self.args.columns, table.column_names, self.get_column_offset())
+            descending = []
+
+        if any(descending):
+            key = mixed_direction_sort(columns, descending, self.args.ignore_case)
+        elif self.args.ignore_case:
+            key = ignore_case_sort(columns)
+        else:
+            key = columns
 
         table = table.order_by(key, reverse=self.args.reverse)
         table.to_csv(self.output_file, **self.writer_kwargs)

docs/scripts/csvsort.rst

@@ -29,9 +29,10 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data:
                            and exit.
      -c COLUMNS, --columns COLUMNS
                            A comma-separated list of column indices, names or
-                           ranges to sort by, e.g. "1,id,3-5". Defaults to all
-                           columns.
-     -r, --reverse         Sort in descending order.
+                           ranges to sort by, e.g. "1,id,3-5". Prefix a column
+                           with "~" to sort it in descending order. Defaults to
+                           all columns.
+     -r, --reverse         Reverse the sort direction of every column.
      -i, --ignore-case     Perform case-independent sorting.
      -y SNIFF_LIMIT, --snifflimit SNIFF_LIMIT
                            Limit CSV dialect sniffing to the specified number of
@@ -61,3 +62,12 @@ View the five states with the most individuals claiming veteran's education bene
 .. code-block:: bash
 
    csvcut -c 1,9 examples/realdata/FY09_EDU_Recipients_by_State.csv | csvsort -r -c 2 | head -n 5
+
+Sort by state ascending, then by total descending within each state:
+
+.. code-block:: bash
+
+   csvsort -c 'state,~total' data.csv
+
+Prefix a range, such as :code:`~2-4`, to sort each column in that range in descending order.
+Use :code:`--reverse` to flip every specified direction.

man/csvsort.1

@@ -58,9 +58,10 @@ optional arguments:
                         and exit.
   \-c COLUMNS, \-\-columns COLUMNS
                         A comma\-separated list of column indices, names or
-                        ranges to sort by, e.g. \(dq1,id,3\-5\(dq. Defaults to all
-                        columns.
-  \-r, \-\-reverse         Sort in descending order.
+                        ranges to sort by, e.g. \(dq1,id,3\-5\(dq. Prefix a column
+                        with \(dq~\(dq to sort it in descending order. Defaults to
+                        all columns.
+  \-r, \-\-reverse         Reverse the sort direction of every column.
   \-i, \-\-ignore\-case     Perform case\-independent sorting.
   \-y SNIFF_LIMIT, \-\-snifflimit SNIFF_LIMIT
                         Limit CSV dialect sniffing to the specified number of
@@ -107,6 +108,21 @@ csvcut \-c 1,9 examples/realdata/FY09_EDU_Recipients_by_State.csv | csvsort \-r
 .fi
 .UNINDENT
 .UNINDENT
+.sp
+Sort by state ascending, then by total descending within each state:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+csvsort \-c \(aqstate,~total\(aq data.csv
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+Prefix a range, such as \fB~2\-4\fP, to sort each column in that range in descending order.
+Use \fB\-\-reverse\fP to flip every specified direction.
 .SH AUTHOR
 Christopher Groskopf and contributors
 .SH COPYRIGHT

tests/test_cli.py

@@ -1,13 +1,17 @@
 import unittest
 
-from csvkit.cli import ColumnIdentifierError, match_column_identifier, parse_column_identifiers
+from csvkit.cli import ColumnIdentifierError, _expand_args, match_column_identifier, parse_column_identifiers
 
 
 class TestCli(unittest.TestCase):
 
     def setUp(self):
         self.headers = ['id', 'name', 'i_work_here', '1', 'more-header-values', 'stuff', 'blueberry']
 
+    def test_expand_args_literal_options(self):
+        args = ['-c', '~priority', '--columns=~value', '-c~name', 'examples/dummy.csv']
+        self.assertEqual(args, _expand_args(args, literal_options=('-c', '--columns')))
+
     def test_match_column_identifier_string(self):
         self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here'))
         self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here', column_offset=0))

tests/test_utilities/test_csvsort.py

@@ -41,6 +41,74 @@ def test_sort_date(self):
         new_order = [str(r[0]) for r in reader]
         self.assertEqual(test_order, new_order)
 
+    def test_mixed_sort_directions(self):
+        data = b'group,value,label\nB,1,first\nA,2,second\nA,10,third\nB,3,fourth\nA,10,fifth\n'
+
+        with stdin_as_string(io.BytesIO(data)):
+            self.assertRows(['-c', 'group,~value'], [
+                ['group', 'value', 'label'],
+                ['A', '10', 'third'],
+                ['A', '10', 'fifth'],
+                ['A', '2', 'second'],
+                ['B', '3', 'fourth'],
+                ['B', '1', 'first'],
+            ])
+
+        with stdin_as_string(io.BytesIO(data)):
+            self.assertRows(['-c', '0,~1', '--zero', '-r'], [
+                ['group', 'value', 'label'],
+                ['B', '1', 'first'],
+                ['B', '3', 'fourth'],
+                ['A', '2', 'second'],
+                ['A', '10', 'third'],
+                ['A', '10', 'fifth'],
+            ])
+
+        for args in (['-c', '~2,1'], ['--columns=~2,1'], ['-c~2,1']):
+            with self.subTest(args=args), stdin_as_string(io.BytesIO(data)):
+                self.assertRows(args, [
+                    ['group', 'value', 'label'],
+                    ['A', '10', 'third'],
+                    ['A', '10', 'fifth'],
+                    ['B', '3', 'fourth'],
+                    ['A', '2', 'second'],
+                    ['B', '1', 'first'],
+                ])
+
+    def test_mixed_sort_directions_with_nulls_and_ignore_case(self):
+        data = b'group,value\nb,apple\nA,banana\na,Apple\nA,\n'
+
+        with stdin_as_string(io.BytesIO(data)):
+            self.assertRows(['-i', '-c', '1,~2'], [
+                ['group', 'value'],
+                ['A', ''],
+                ['A', 'banana'],
+                ['a', 'Apple'],
+                ['b', 'apple'],
+            ])
+
+    def test_mixed_sort_directions_with_range(self):
+        data = b'group,value,label\nA,2,b\nA,2,a\nA,10,c\nB,1,z\n'
+
+        with stdin_as_string(io.BytesIO(data)):
+            self.assertRows(['-c', '1,~2-3'], [
+                ['group', 'value', 'label'],
+                ['A', '10', 'c'],
+                ['A', '2', 'b'],
+                ['A', '2', 'a'],
+                ['B', '1', 'z'],
+            ])
+
+    def test_column_name_starting_with_tilde(self):
+        data = b'~priority,value\n2,second\n1,first\n'
+
+        with stdin_as_string(io.BytesIO(data)):
+            self.assertRows(['-c', '~priority'], [
+                ['~priority', 'value'],
+                ['1', 'first'],
+                ['2', 'second'],
+            ])
+
     def test_ignore_case(self):
         self.assertRows(['-i', 'examples/test_ignore_case.csv'], [
             ['a', 'b', 'c'],