diff --git a/i2/__init__.py b/i2/__init__.py index 20723790..522f858d 100644 --- a/i2/__init__.py +++ b/i2/__init__.py @@ -45,6 +45,7 @@ include_exclude, rm_params, partialx, + curried, ch_names, arg_val_converter, ignore_extra_args, # Wrap a function to accept (and ignore) extra arguments diff --git a/i2/doc_mint.py b/i2/doc_mint.py index 517c8530..ed90a004 100644 --- a/i2/doc_mint.py +++ b/i2/doc_mint.py @@ -427,9 +427,9 @@ def convert_string(s: str, converters: list[dict | Callable]) -> object: DocStyle = Literal["numpy", "google", "rest"] -# Each style's tell-tale marker, checked in this order (most specific first): a numpy -# section header underlined with dashes, a reST field (``:param x:``), a google -# section header ending with a colon. +# Each style's tell-tale marker: a numpy section header underlined with dashes, a reST +# field (``:param x:``), a google section header ending with a colon. The style whose +# marker comes first in the docstring wins (ties go to the earlier style here). _doc_style_markers = ( ( "numpy", @@ -481,6 +481,18 @@ def detect_doc_style(docstring: str | Callable) -> DocStyle | None: ... :type x: int ... ''') 'rest' + + When several styles' markers appear (e.g. a google docstring whose example + shows reST fields), the one that comes first wins: + + >>> detect_doc_style(''' + ... Args: + ... x (int): An integer. + ... + ... Example: + ... :param y: shown as text, not a field. + ... ''') + 'google' >>> detect_doc_style('Just a summary line.') is None True >>> detect_doc_style(None) is None @@ -501,10 +513,23 @@ def detect_doc_style(docstring: str | Callable) -> DocStyle | None: docstring = docstring.__doc__ or "" if not isinstance(docstring, str): return None + docstring = _normalize_newlines(docstring) + first_match_positions = {} for style, marker in _doc_style_markers: - if marker.search(docstring): - return style - return None + if match := marker.search(docstring): + first_match_positions[style] = match.start() + if not first_match_positions: + return None + return min(first_match_positions, key=first_match_positions.get) + + +def _normalize_newlines(text: str) -> str: + r'''Turn CRLF (and lone CR) line endings into LF. + + >>> _normalize_newlines('a\r\nb\rc') + 'a\nb\nc' + ''' + return text.replace("\r\n", "\n").replace("\r", "\n") def docstring_to_params( @@ -584,11 +609,19 @@ def docstring_to_params( ... ''' >>> [p['name'] for p in docstring_to_params(f, doc_style='auto')] ['x'] + + The Sphinx typed-field form (``:param int x:``) and CRLF line endings are handled: + + >>> docstring_to_params(':param int x: X.\\r\\n:param y: Note: Y.', doc_style='auto') == [ + ... {"name": "x", "default": None, "annotation": int, "description": "X."}, + ... {"name": "y", "default": None, "annotation": None, "description": "Note: Y."}, + ... ] + True """ if not isinstance(docstring, str) and callable(docstring): docstring = docstring.__doc__ or "" # dedent, so the section headers of an indented ``__doc__`` are found - docstring = cleandoc(docstring or "") + docstring = cleandoc(_normalize_newlines(docstring or "")) if doc_style == "auto": doc_style = detect_doc_style(docstring) or "numpy" @@ -747,11 +780,10 @@ def _parse_rest_param_lines(param_lines: list[str]) -> list[dict]: i = 0 while i < len(param_lines): line = param_lines[i].strip() - # Match ':param name: description' - param_match = re.match(r"^:param (\w+):\s*(.+)", line) + # Match ':param name: description' or ':param type name: description' + param_match = re.match(r"^:param\s+(?:([^:]+?)\s+)?(\w+):\s*(.*)", line) if param_match: - name = param_match.group(1) - description = param_match.group(2) + annotation, name, description = param_match.groups() # Check for default value in description default_match = re.search(r"\(Default:\s*(.+)\)", description) default = default_match.group(1).strip() if default_match else None @@ -761,7 +793,7 @@ def _parse_rest_param_lines(param_lines: list[str]) -> list[dict]: params_dict[name] = { "name": name, "default": default, - "annotation": None, + "annotation": annotation, "description": description, } i += 1 diff --git a/i2/tests/test_partials_and_methodize.py b/i2/tests/test_partials_and_methodize.py new file mode 100644 index 00000000..867d1133 --- /dev/null +++ b/i2/tests/test_partials_and_methodize.py @@ -0,0 +1,115 @@ +"""Tests for partialx(_rm_partialize), curried, methodize(default_from) and the +docstring parser fixes of i2mint/i2#111, #112, #113 and #116.""" + +import asyncio +import inspect +from dataclasses import dataclass +from functools import partial + +import pytest + +from i2 import Sig, curried, methodize, partialx +from i2.doc_mint import detect_doc_style, docstring_to_params + + +def test_rm_partialize_keeps_a_partial_and_binds_positionals_once(): + def h(a, *args, **kw): + return a, args, kw + + g = partialx(h, 1, 2, _rm_partialize=True) + assert isinstance(g, partial) and g.func is h + assert g(3) == (1, (2, 3), {}) + + +def test_rm_partialize_makes_params_after_a_keyword_binding_keyword_only(): + def f(a, b, c=3): + return a, b, c + + g = partialx(f, b=2, _rm_partialize=True) + assert str(Sig(g)) == "(a, *, c=3)" + assert g(1, c=5) == (1, 2, 5) + + +def test_rm_partialize_keeps_var_params_named_by_a_keyword(): + def h(a, *args, **kw): + return a, args, kw + + g = partialx(h, kw=3, _rm_partialize=True) + assert str(Sig(g)) == "(a, *args, **kw)" + assert g(1) == (1, (), {"kw": 3}) + + +def test_rm_partialize_keeps_coroutine_functions(): + async def af(a, b): + return a + b + + g = partialx(af, b=2, _rm_partialize=True) + assert inspect.iscoroutinefunction(g) + assert asyncio.run(g(1)) == 3 + + +def test_curried(): + def f(a, b, *, c): + return a, b, c + + g = curried(f) + assert g(1)(2)(c=3) == g(1, 2, c=3) == g(c=3)(1)(2) == (1, 2, 3) + with pytest.raises(TypeError): + g(1, d=4) + + +def test_curried_async_awaits_on_the_full_call_only(): + async def af(a, b): + return a + b + + ga = curried(af) + assert not inspect.iscoroutinefunction(ga) + assert asyncio.run(ga(1)(2)) == 3 + + +def test_methodize_default_from(): + def foo(x, y, z): + return x + y * z + + @dataclass + class K: + x: int + zoo: int = 2 + bar = methodize(foo, bind="x", default_from=[("z", "zoo")]) + + k = K(1) + assert str(Sig(k.bar)) == "(y, z=self.zoo)" + assert (k.bar(3), k.bar(3, 10), k.bar(3, z=0)) == (7, 31, 1) + + +def test_methodize_default_from_rejects_variadic_and_unknown_params(): + def foo(x, *args, **kwargs): + return x + + with pytest.raises(ValueError, match="args"): + methodize(foo, default_from="args") + with pytest.raises(ValueError, match="not in the method"): + methodize(foo, default_from="nope") + + +@pytest.mark.parametrize( + "line, name, annotation, description", + [ + (":param x: Note: foo", "x", None, "Note: foo"), + (":param int x: X.", "x", int, "X."), + (":param dict[str, int] x: X.", "x", "dict[str, int]", "X."), + ], +) +def test_rest_param_lines(line, name, annotation, description): + [param] = docstring_to_params(line, doc_style="rest") + assert (param["name"], param["annotation"], param["description"]) == ( + name, + annotation, + description, + ) + + +def test_crlf_docstrings(): + doc = "Do it.\r\n\r\nArgs:\r\n x (int): An integer.\r\n" + assert detect_doc_style(doc) == "google" + assert [p["name"] for p in docstring_to_params(doc, doc_style="auto")] == ["x"] diff --git a/i2/wrapper.py b/i2/wrapper.py index 5e2c3149..f2e40bcc 100644 --- a/i2/wrapper.py +++ b/i2/wrapper.py @@ -63,6 +63,7 @@ """ from functools import wraps, partial +import inspect from inspect import Parameter, signature from typing import ( Any, @@ -86,6 +87,7 @@ PO, mk_sig_from_args, VK, + VP, parameter_to_dict, call_forgivingly, _call_forgivingly, @@ -1852,8 +1854,26 @@ def pairs(): return param_to_attr +class InstanceAttrDefault: + """The default of a method param that is sourced from an instance attribute. + + Shown in the method's signature as ``self.`` (see ``methodize``'s + ``default_from``). + + >>> InstanceAttrDefault('zoo') + self.zoo + """ + + def __init__(self, attr: str, *, instance_arg_name: str = "self"): + self.attr = attr + self.instance_arg_name = instance_arg_name + + def __repr__(self): + return f"{self.instance_arg_name}.{self.attr}" + + def kwargs_trans_to_extract_args_from_attrs( - outer_kwargs: dict, attr_names=(), obj_param="self" + outer_kwargs: dict, attr_names=(), obj_param="self", *, default_from=() ): """Pop the ``obj_param`` object out of ``outer_kwargs`` and source ``attr_names`` from its attributes. @@ -1869,9 +1889,22 @@ def kwargs_trans_to_extract_args_from_attrs( >>> kwargs_trans_to_extract_args_from_attrs({'self': o}, attr_names={'x': 'a'}) {'x': 1} + + ``default_from`` params are sourced from attributes only when they are missing + from ``outer_kwargs`` or hold an ``InstanceAttrDefault``: + + >>> kwargs_trans_to_extract_args_from_attrs( + ... {'self': o, 'x': InstanceAttrDefault('a'), 'y': 5}, default_from=[('x', 'a'), ('y', 'a'), ('z', 'b')]) + {'x': 1, 'z': 2, 'y': 5} """ self = outer_kwargs.pop(obj_param) param_to_attr = bind_spec_to_dict(attr_names) + for param, attr in bind_spec_to_dict(default_from).items(): + if param not in outer_kwargs or isinstance( + outer_kwargs[param], InstanceAttrDefault + ): + outer_kwargs.pop(param, None) + param_to_attr[param] = attr arguments_extracted_from_obj = { param: getattr(self, attr) for param, attr in param_to_attr.items() } @@ -1896,6 +1929,7 @@ def func_to_method_func( method_name=None, method_params=None, instance_arg_name="self", + default_from: BindSpec = (), ) -> MethodFunc: """Get a 'method function' from a 'normal function'. Also known as "methodize". @@ -1984,6 +2018,10 @@ def func_to_method_func( ... method_func = func_to_method_func(func, [('x', 'a'), ('z', 'c')]) >>> Klass(1, 3).method_func(2, d='hello') 'hello: 9' + + ``default_from`` (same spec as ``instance_params``) keeps params in the method's + signature, sourcing only their *default* from an instance attribute: see + ``methodize``. """ # get a signature object for func sig = Sig(func) @@ -1998,7 +2036,13 @@ def func_to_method_func( name for name in sig.names if name not in instance_params ) # the Sig object of the method: instance name followed with method_params - method_sig = instance_arg_name + Sig(func)[method_params] + method_sig = Sig(func)[method_params] + default_from = bind_spec_to_dict(default_from) + if default_from: + method_sig = _with_instance_attr_defaults( + method_sig, default_from, instance_arg_name=instance_arg_name + ) + method_sig = instance_arg_name + method_sig # make the ingress function that will map method_sig's interface to sig's. ingress = Ingress( inner_sig=sig, @@ -2008,6 +2052,7 @@ def func_to_method_func( kwargs_trans_to_extract_args_from_attrs, attr_names=instance_params, obj_param=instance_arg_name, + default_from=default_from, ), outer_sig=method_sig, # this is the signature we want at the interface ) @@ -2017,8 +2062,53 @@ def func_to_method_func( return method_func +def _with_instance_attr_defaults( + sig: Sig, default_from: dict, *, instance_arg_name: str = "self" +) -> Sig: + """Give the ``default_from`` params of ``sig`` an ``InstanceAttrDefault`` default. + + A param followed by a required positional param can't have a default where it is, + so it becomes keyword-only. + + >>> def f(x, y, z, w=1): ... + >>> _with_instance_attr_defaults(Sig(f), {'y': 'yy', 'w': 'ww'}) + + >>> _with_instance_attr_defaults(Sig(f), {'z': 'zz'}) + + """ + if missing := set(default_from) - set(sig.names): + raise ValueError(f"default_from params not in the method's signature: {missing}") + if variadic := {name for name in default_from if sig.kinds[name] in (VP, VK)}: + raise ValueError(f"default_from can't give a default to *args/**kwargs: {variadic}") + names = sig.names + required_positional = { + name + for name, kind in sig.kinds.items() + if kind in (PO, PK) and name not in default_from and name not in sig.defaults + } + to_keyword_only = { + name: KO + for name in default_from + if sig.kinds[name] in (PO, PK) + and required_positional & set(names[names.index(name) + 1 :]) + } + if to_keyword_only: + sig = sig.ch_kinds(**to_keyword_only) + return sig.ch_defaults( + **{ + param: InstanceAttrDefault(attr, instance_arg_name=instance_arg_name) + for param, attr in default_from.items() + } + ) + + def methodize( - func, bind: BindSpec = (), *, method_name=None, instance_arg_name="self" + func, + bind: BindSpec = (), + *, + method_name=None, + instance_arg_name="self", + default_from: BindSpec = (), ) -> MethodFunc: """Make a method from ``func``, sourcing the ``bind`` params from instance attributes. @@ -2042,9 +2132,34 @@ def methodize( >>> from inspect import signature >>> str(signature(c.bar)) '(y)' + + ``default_from`` (same spec as ``bind``) keeps params in the method's signature, + sourcing only their *default* from an instance attribute (i2mint/i2#112): + + >>> @dataclass + ... class Klass: + ... x: int + ... zoo: int = 2 + ... bar = methodize(foo, bind='x', default_from=[('z', 'zoo')]) + >>> c = Klass(x=1, zoo=2) + >>> str(signature(c.bar)) + '(y, z=self.zoo)' + >>> c.bar(3), c.bar(3, 10), c.bar(3, z=10) + (7, 31, 31) + >>> c.zoo = 5 + >>> c.bar(3) + 16 + + The signature default is an ``InstanceAttrDefault`` marker (shown ``self.zoo``), + not a value: code that serializes signature defaults (e.g. to JSON) should not + expect a JSON-able one there. """ return func_to_method_func( - func, bind, method_name=method_name, instance_arg_name=instance_arg_name + func, + bind, + method_name=method_name, + instance_arg_name=instance_arg_name, + default_from=default_from, ) @@ -2590,10 +2705,27 @@ def partialx( (1, 20, 3, 4, 5) >>> g = partialx(func, b=2, _rm_partialize=True) >>> str(Sig(g)) - '(a, c, d, e=5)' + '(a, *, c, d, e=5)' >>> g(1, c=3, d=4) (1, 2, 3, 4, 5) + With ``_rm_partialize=True``, the params after a keyword-bound one become + keyword-only (as in ``functools.partial``, which only takes them by keyword), + positional bindings are applied once, and the result is still a ``partial``: + + >>> def f(a, b, c=3): + ... return a, b, c + >>> g = partialx(f, b=2, _rm_partialize=True) + >>> str(Sig(g)) + '(a, *, c=3)' + >>> g(1, c=5) + (1, 2, 5) + >>> def h(a, *args, **kw): + ... return a, args, kw + >>> hh = partialx(h, 1, 2, _rm_partialize=True) + >>> hh(3), isinstance(hh, partial) + ((1, (2, 3), {}), True) + If ``func`` has a ``**kwargs``, a keyword named like a position-only param goes to that ``**kwargs``, as in Python (and ``functools.partial``): @@ -2610,10 +2742,13 @@ def partialx( func_ = _accept_position_only_by_keyword(func) f = partial(func_, *args, **kwargs) if _rm_partialize: - sig = Sig(func_) - partialized = list(sig.map_arguments(args, kwargs, allow_partial=True)) - sig = sig - partialized - f = sig(partial(f, *args, **kwargs)) + # The partial's own signature already drops the positional bindings and + # makes every param after a keyword-bound one keyword-only (see i2mint/i2#113); + # only the keyword-bound params remain to be removed. + sig = Sig(f) + # Only named params are removed: a keyword going to ``**kwargs`` stays there. + bound = [name for name in kwargs if sig.kinds.get(name) in (PK, KO)] + f = (sig - bound)(f) if _allow_reordering: # TODO: Instead of Sig(f).defaults, move only params that need to move # TODO: + Change signature so that the number of params that become keyword-only @@ -2623,6 +2758,57 @@ def partialx( return f +def curried(func: Callable) -> Callable: + """Make ``func`` callable with only some of its required arguments. + + A call that leaves required parameters unspecified returns a new curried + function with the given arguments bound (and removed from its signature, + see ``partialx(..., _rm_partialize=True)``); a call that specifies them all + calls ``func``. + + >>> def f(a, b, c=3): + ... return a + b * c + >>> g = curried(f) + >>> str(Sig(g)) + '(a, b, c=3)' + >>> g(1, 2) + 7 + >>> g(1)(2) + 7 + >>> g(1)(c=10)(2) + 21 + >>> h = g(b=2) + >>> str(Sig(h)) + '(a, *, c=3)' + >>> h(1, c=10) + 21 + + A curried function is never a coroutine function, even when ``func`` is: only + the call that specifies every required argument returns ``func``'s awaitable. + + Invalid arguments fail at the call that gives them, as they would for ``func``: + + >>> g(1, 2, 3, 4) + Traceback (most recent call last): + ... + TypeError: too many positional arguments + """ + sig = Sig(func) + required = [ + p.name for p in sig.params if p.default is p.empty and p.kind not in (VP, VK) + ] + + @wraps(func) + def curried_func(*args, **kwargs): + bound = sig.bind_partial(*args, **kwargs) + if all(name in bound.arguments for name in required): + return func(*args, **kwargs) + return curried(partialx(func, *args, _rm_partialize=True, **kwargs)) + + curried_func.__signature__ = sig + return curried_func + + def _accept_position_only_by_keyword(func: Callable) -> Callable: """Wrap ``func`` so its position-only parameters can also be given by keyword.