diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5fdf4a1a3..ec8122e5c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,7 @@ Unreleased ---------- +- feat: :doc:`/scripts/csvsort` supports per-column descending order using :code:`~` in :code:`--columns`. - feat: :doc:`/scripts/csvcut` adds an :code:`--ignore-unknown-columns` option to skip identifiers in :code:`-c/--columns` that do not match a column in the input. - feat: :doc:`/scripts/csvclean` adds a :code:`--remove-empty-columns` option to remove empty columns from standard output. - feat: :doc:`/scripts/in2csv` guesses the ``ndjson`` format for files with :code:`.ndjson`, :code:`.jsonl` and :code:`.jl` extensions. diff --git a/csvkit/cli.py b/csvkit/cli.py index e99acad34..cf17859b6 100644 --- a/csvkit/cli.py +++ b/csvkit/cli.py @@ -73,6 +73,7 @@ class CSVKitUtility: description = '' epilog = '' override_flags = '' + literal_options = () def __init__(self, args=None, output_file=None, error_file=None): """ @@ -81,7 +82,7 @@ def __init__(self, args=None, output_file=None, error_file=None): if args is None: args = sys.argv[1:] if os.name == 'nt': - args = _expand_args(args) + args = _expand_args(args, literal_options=self.literal_options) self._init_common_parser() self.add_arguments() @@ -610,10 +611,31 @@ def parse_list(pairs): # Adapted from https://github.com/pallets/click/blame/main/src/click/utils.py -def _expand_args(args): +def _expand_args(args, literal_options=()): out = [] + literal_next = False for arg in args: + if literal_next: + out.append(arg) + literal_next = False + continue + + if arg in literal_options: + out.append(arg) + literal_next = True + continue + + # Some option values are column identifiers rather than paths. + if any( + arg.startswith(f'{option}=') or ( + option.startswith('-') and not option.startswith('--') and arg.startswith(option) + ) + for option in literal_options + ): + out.append(arg) + continue + arg = os.path.expanduser(arg) arg = os.path.expandvars(arg) diff --git a/csvkit/utilities/csvsort.py b/csvkit/utilities/csvsort.py index fdada773e..1dad33f15 100644 --- a/csvkit/utilities/csvsort.py +++ b/csvkit/utilities/csvsort.py @@ -1,8 +1,10 @@ #!/usr/bin/env python +from functools import cmp_to_key + import agate -from csvkit.cli import CSVKitUtility, parse_column_identifiers +from csvkit.cli import CSVKitUtility, match_column_identifier, parse_column_identifiers def ignore_case_sort(key): @@ -16,8 +18,36 @@ def inner(row): return inner +def mixed_direction_sort(columns, descending, ignore_case): + + def compare_rows(left, right): + for column, is_descending in zip(columns, descending): + left_value = left[column] + right_value = right[column] + + if ignore_case: + if isinstance(left_value, str): + left_value = left_value.upper() + if isinstance(right_value, str): + right_value = right_value.upper() + + if left_value is None: + left_value = agate.NullOrder() + if right_value is None: + right_value = agate.NullOrder() + + comparison = (left_value > right_value) - (left_value < right_value) + if comparison: + return -comparison if is_descending else comparison + + return 0 + + return cmp_to_key(compare_rows) + + class CSVSort(CSVKitUtility): description = 'Sort CSV files. Like the Unix "sort" command, but for tabular data.' + literal_options = ('-c', '--columns') def add_arguments(self): self.argparser.add_argument( @@ -26,10 +56,10 @@ def add_arguments(self): self.argparser.add_argument( '-c', '--columns', dest='columns', help='A comma-separated list of column indices, names or ranges to sort by, e.g. "1,id,3-5". ' - 'Defaults to all columns.') + 'Prefix a column with "~" to sort it in descending order. Defaults to all columns.') self.argparser.add_argument( '-r', '--reverse', dest='reverse', action='store_true', - help='Sort in descending order.') + help='Reverse the sort direction of every column.') self.argparser.add_argument( '-i', '--ignore-case', dest='ignore_case', action='store_true', help='Perform case-independent sorting.') @@ -59,14 +89,36 @@ def main(self): **self.reader_kwargs, ) - key = parse_column_identifiers( - self.args.columns, - table.column_names, - self.get_column_offset(), - ) - - if self.args.ignore_case: - key = ignore_case_sort(key) + identifiers = self.args.columns.split(',') if self.args.columns else [] + reverse_columns = [ + identifier.startswith('~') and len(identifier) > 1 and identifier not in table.column_names + for identifier in identifiers + ] + + if any(reverse_columns): + columns = [] + descending = [] + for identifier, is_descending in zip(identifiers, reverse_columns): + if not identifier: + selected = [match_column_identifier(table.column_names, identifier, self.get_column_offset())] + else: + selected = parse_column_identifiers( + identifier[1:] if is_descending else identifier, + table.column_names, + self.get_column_offset(), + ) + columns.extend(selected) + descending.extend([is_descending] * len(selected)) + else: + columns = parse_column_identifiers(self.args.columns, table.column_names, self.get_column_offset()) + descending = [] + + if any(descending): + key = mixed_direction_sort(columns, descending, self.args.ignore_case) + elif self.args.ignore_case: + key = ignore_case_sort(columns) + else: + key = columns table = table.order_by(key, reverse=self.args.reverse) table.to_csv(self.output_file, **self.writer_kwargs) diff --git a/docs/scripts/csvsort.rst b/docs/scripts/csvsort.rst index 04997c021..e6f9c5cd8 100644 --- a/docs/scripts/csvsort.rst +++ b/docs/scripts/csvsort.rst @@ -29,9 +29,10 @@ Sort CSV files. Like the Unix "sort" command, but for tabular data: and exit. -c COLUMNS, --columns COLUMNS A comma-separated list of column indices, names or - ranges to sort by, e.g. "1,id,3-5". Defaults to all - columns. - -r, --reverse Sort in descending order. + ranges to sort by, e.g. "1,id,3-5". Prefix a column + with "~" to sort it in descending order. Defaults to + all columns. + -r, --reverse Reverse the sort direction of every column. -i, --ignore-case Perform case-independent sorting. -y SNIFF_LIMIT, --snifflimit SNIFF_LIMIT Limit CSV dialect sniffing to the specified number of @@ -61,3 +62,12 @@ View the five states with the most individuals claiming veteran's education bene .. code-block:: bash csvcut -c 1,9 examples/realdata/FY09_EDU_Recipients_by_State.csv | csvsort -r -c 2 | head -n 5 + +Sort by state ascending, then by total descending within each state: + +.. code-block:: bash + + csvsort -c 'state,~total' data.csv + +Prefix a range, such as :code:`~2-4`, to sort each column in that range in descending order. +Use :code:`--reverse` to flip every specified direction. diff --git a/man/csvsort.1 b/man/csvsort.1 index 578b20ade..9e9196f5e 100644 --- a/man/csvsort.1 +++ b/man/csvsort.1 @@ -58,9 +58,10 @@ optional arguments: and exit. \-c COLUMNS, \-\-columns COLUMNS A comma\-separated list of column indices, names or - ranges to sort by, e.g. \(dq1,id,3\-5\(dq. Defaults to all - columns. - \-r, \-\-reverse Sort in descending order. + ranges to sort by, e.g. \(dq1,id,3\-5\(dq. Prefix a column + with \(dq~\(dq to sort it in descending order. Defaults to + all columns. + \-r, \-\-reverse Reverse the sort direction of every column. \-i, \-\-ignore\-case Perform case\-independent sorting. \-y SNIFF_LIMIT, \-\-snifflimit SNIFF_LIMIT Limit CSV dialect sniffing to the specified number of @@ -107,6 +108,21 @@ csvcut \-c 1,9 examples/realdata/FY09_EDU_Recipients_by_State.csv | csvsort \-r .fi .UNINDENT .UNINDENT +.sp +Sort by state ascending, then by total descending within each state: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +csvsort \-c \(aqstate,~total\(aq data.csv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Prefix a range, such as \fB~2\-4\fP, to sort each column in that range in descending order. +Use \fB\-\-reverse\fP to flip every specified direction. .SH AUTHOR Christopher Groskopf and contributors .SH COPYRIGHT diff --git a/tests/test_cli.py b/tests/test_cli.py index c4fe55d2f..14e6d4e08 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ import unittest -from csvkit.cli import ColumnIdentifierError, match_column_identifier, parse_column_identifiers +from csvkit.cli import ColumnIdentifierError, _expand_args, match_column_identifier, parse_column_identifiers class TestCli(unittest.TestCase): @@ -8,6 +8,10 @@ class TestCli(unittest.TestCase): def setUp(self): self.headers = ['id', 'name', 'i_work_here', '1', 'more-header-values', 'stuff', 'blueberry'] + def test_expand_args_literal_options(self): + args = ['-c', '~priority', '--columns=~value', '-c~name', 'examples/dummy.csv'] + self.assertEqual(args, _expand_args(args, literal_options=('-c', '--columns'))) + def test_match_column_identifier_string(self): self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here')) self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here', column_offset=0)) diff --git a/tests/test_utilities/test_csvsort.py b/tests/test_utilities/test_csvsort.py index 1d82d4fca..6682ee25b 100644 --- a/tests/test_utilities/test_csvsort.py +++ b/tests/test_utilities/test_csvsort.py @@ -41,6 +41,74 @@ def test_sort_date(self): new_order = [str(r[0]) for r in reader] self.assertEqual(test_order, new_order) + def test_mixed_sort_directions(self): + data = b'group,value,label\nB,1,first\nA,2,second\nA,10,third\nB,3,fourth\nA,10,fifth\n' + + with stdin_as_string(io.BytesIO(data)): + self.assertRows(['-c', 'group,~value'], [ + ['group', 'value', 'label'], + ['A', '10', 'third'], + ['A', '10', 'fifth'], + ['A', '2', 'second'], + ['B', '3', 'fourth'], + ['B', '1', 'first'], + ]) + + with stdin_as_string(io.BytesIO(data)): + self.assertRows(['-c', '0,~1', '--zero', '-r'], [ + ['group', 'value', 'label'], + ['B', '1', 'first'], + ['B', '3', 'fourth'], + ['A', '2', 'second'], + ['A', '10', 'third'], + ['A', '10', 'fifth'], + ]) + + for args in (['-c', '~2,1'], ['--columns=~2,1'], ['-c~2,1']): + with self.subTest(args=args), stdin_as_string(io.BytesIO(data)): + self.assertRows(args, [ + ['group', 'value', 'label'], + ['A', '10', 'third'], + ['A', '10', 'fifth'], + ['B', '3', 'fourth'], + ['A', '2', 'second'], + ['B', '1', 'first'], + ]) + + def test_mixed_sort_directions_with_nulls_and_ignore_case(self): + data = b'group,value\nb,apple\nA,banana\na,Apple\nA,\n' + + with stdin_as_string(io.BytesIO(data)): + self.assertRows(['-i', '-c', '1,~2'], [ + ['group', 'value'], + ['A', ''], + ['A', 'banana'], + ['a', 'Apple'], + ['b', 'apple'], + ]) + + def test_mixed_sort_directions_with_range(self): + data = b'group,value,label\nA,2,b\nA,2,a\nA,10,c\nB,1,z\n' + + with stdin_as_string(io.BytesIO(data)): + self.assertRows(['-c', '1,~2-3'], [ + ['group', 'value', 'label'], + ['A', '10', 'c'], + ['A', '2', 'b'], + ['A', '2', 'a'], + ['B', '1', 'z'], + ]) + + def test_column_name_starting_with_tilde(self): + data = b'~priority,value\n2,second\n1,first\n' + + with stdin_as_string(io.BytesIO(data)): + self.assertRows(['-c', '~priority'], [ + ['~priority', 'value'], + ['1', 'first'], + ['2', 'second'], + ]) + def test_ignore_case(self): self.assertRows(['-i', 'examples/test_ignore_case.csv'], [ ['a', 'b', 'c'],