diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a85dbb..8e9695d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [0.4.7] 2026-02-01 +### Modified +- Disk cache functions now detect saving backend when custom load/dump functions are provided. + +### Fixed +- `Callable` in `isinstance_generic` check. + ## [0.4.6] 2026-01-09 ### Added - `cache_fname_fmt` can now be a custom callable formatter. diff --git a/CITATION.cff b/CITATION.cff index 7f89c5d..c9c71de 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -17,5 +17,5 @@ keywords: - tools - utilities license: MIT -version: 0.4.6 -date-released: '2026-01-09' +version: 0.4.7 +date-released: '2026-02-01' diff --git a/src/pythonwrench/__init__.py b/src/pythonwrench/__init__.py index 47bb55c..d9c9a81 100644 --- a/src/pythonwrench/__init__.py +++ b/src/pythonwrench/__init__.py @@ -9,7 +9,7 @@ __license__ = "MIT" __maintainer__ = "Étienne Labbé (Labbeti)" __status__ = "Development" -__version__ = "0.4.6" +__version__ = "0.4.7" # Re-import for language servers @@ -54,7 +54,7 @@ str_to_type, ) from .cast import as_builtin, register_as_builtin_fn -from .checksum import checksum_any, register_checksum_fn +from .checksum import checksum_any, checksum_object, register_checksum_fn from .collections import ( all_eq, all_ne, diff --git a/src/pythonwrench/checksum.py b/src/pythonwrench/checksum.py index 2b4d1e9..82b6e11 100644 --- a/src/pythonwrench/checksum.py +++ b/src/pythonwrench/checksum.py @@ -22,6 +22,7 @@ ) from pythonwrench._core import ClassOrTuple, Predicate, _FunctionRegistry +from pythonwrench.functools import function_alias from pythonwrench.inspect import get_fullname from pythonwrench.typing import ( DataclassInstance, @@ -85,10 +86,15 @@ def checksum_any( """Compute checksum integer value from an arbitrary object. Supports most builtin types. Checksum can be used to compare objects. + Not meant for security/cryptography. """ return _CHECKSUM_REGISTRY.apply(x, isinstance_fn=isinstance_fn, **kwargs) +@function_alias(checksum_any) +def checksum_object(*args, **kwargs): ... + + # Terminate functions @register_checksum_fn(bool) def checksum_bool(x: bool, **kwargs) -> int: @@ -253,6 +259,10 @@ def checksum_path(x: Path, **kwargs) -> int: kwargs["accumulator"] = kwargs.get("accumulator", 0) + _cached_checksum_str( get_fullname(x) ) + resolve_path = kwargs.get("resolve_path", False) + if isinstance(resolve_path, bool) and resolve_path: + x = x.expanduser().resolve() + return checksum_str(str(x), **kwargs) diff --git a/src/pythonwrench/disk_cache.py b/src/pythonwrench/disk_cache.py index b9c5c9c..9b748a8 100644 --- a/src/pythonwrench/disk_cache.py +++ b/src/pythonwrench/disk_cache.py @@ -61,7 +61,24 @@ def disk_cache_decorator( cache_force: bool = False, cache_verbose: int = 0, cache_checksum_fn: ChecksumFn = checksum_any, - cache_saving_backend: Optional[SavingBackend] = "pickle", + cache_saving_backend: Literal["custom"], + cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", + cache_dump_fn: Callable[[Any, Path], Any], + cache_load_fn: Callable[[Path], Any], + cache_enable: bool = True, + cache_store_mode: StoreMode = "outputs_metadata", +) -> Callable[[Callable[P, T]], Callable[P, T]]: ... + + +@overload +def disk_cache_decorator( + fn: None = None, + *, + cache_dpath: Union[str, Path, None] = None, + cache_force: bool = False, + cache_verbose: int = 0, + cache_checksum_fn: ChecksumFn = checksum_any, + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, cache_load_fn: Optional[Callable[[Path], Any]] = None, @@ -78,7 +95,24 @@ def disk_cache_decorator( cache_force: bool = False, cache_verbose: int = 0, cache_checksum_fn: ChecksumFn = checksum_any, - cache_saving_backend: Optional[SavingBackend] = "pickle", + cache_saving_backend: Literal["custom"], + cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", + cache_dump_fn: Callable[[Any, Path], Any], + cache_load_fn: Callable[[Path], Any], + cache_enable: bool = True, + cache_store_mode: StoreMode = "outputs_metadata", +) -> Callable[P, T]: ... + + +@overload +def disk_cache_decorator( + fn: Callable[P, T], + *, + cache_dpath: Union[str, Path, None] = None, + cache_force: bool = False, + cache_verbose: int = 0, + cache_checksum_fn: ChecksumFn = checksum_any, + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, cache_load_fn: Optional[Callable[[Path], Any]] = None, @@ -94,7 +128,7 @@ def disk_cache_decorator( cache_force: bool = False, cache_verbose: int = 0, cache_checksum_fn: ChecksumFn = checksum_any, - cache_saving_backend: Optional[SavingBackend] = "pickle", + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, cache_load_fn: Optional[Callable[[Path], Any]] = None, @@ -121,7 +155,7 @@ def disk_cache_decorator( cache_force: Force function call and overwrite cache. defaults to False. cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0. cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``. - cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle'). defaults to 'pickle'. + cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'. cache_fname_fmt: Cache filename format. defaults to "{fn_name}_{csum}{suffix}". cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None. cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None. @@ -146,6 +180,25 @@ def disk_cache_decorator( return impl_fn +@overload +def disk_cache_call( + fn: Callable[..., T], + *args, + cache_dpath: Union[str, Path, None] = None, + cache_force: bool = False, + cache_verbose: int = 0, + cache_checksum_fn: ChecksumFn = checksum_any, + cache_saving_backend: Literal["custom"], + cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", + cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, + cache_load_fn: Optional[Callable[[Path], Any]] = None, + cache_enable: bool = True, + cache_store_mode: StoreMode = "outputs_metadata", + **kwargs, +) -> T: ... + + +@overload def disk_cache_call( fn: Callable[..., T], *args, @@ -153,7 +206,24 @@ def disk_cache_call( cache_force: bool = False, cache_verbose: int = 0, cache_checksum_fn: ChecksumFn = checksum_any, - cache_saving_backend: Optional[SavingBackend] = "pickle", + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", + cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", + cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, + cache_load_fn: Optional[Callable[[Path], Any]] = None, + cache_enable: bool = True, + cache_store_mode: StoreMode = "outputs_metadata", + **kwargs, +) -> T: ... + + +def disk_cache_call( + fn: Callable[..., T], + *args, + cache_dpath: Union[str, Path, None] = None, + cache_force: bool = False, + cache_verbose: int = 0, + cache_checksum_fn: ChecksumFn = checksum_any, + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, cache_load_fn: Optional[Callable[[Path], Any]] = None, @@ -180,7 +250,7 @@ def disk_cache_call( cache_force: Force function call and overwrite cache. defaults to False. cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0. cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``. - cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle'). defaults to 'pickle'. + cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'. cache_fname_fmt: Cache filename format. defaults to '{fn_name}_{csum}{suffix}'. cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None. cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None. @@ -210,7 +280,7 @@ def _disk_cache_impl( cache_force: bool = False, cache_verbose: int = 0, cache_checksum_fn: ChecksumFn = checksum_any, - cache_saving_backend: Optional[SavingBackend] = "pickle", + cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto", cache_fname_fmt: Union[str, Callable[..., str]] = "{fn_name}_{csum}{suffix}", cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None, cache_load_fn: Optional[Callable[[Path], Any]] = None, @@ -219,11 +289,26 @@ def _disk_cache_impl( ) -> Callable[[Callable[P, T]], Callable[P, T]]: # for backward compatibility if cache_fname_fmt is None: + expected = "{fn_name}_{csum}{suffix}" + warnings.warn( + f"Deprecated argument value {cache_fname_fmt=}. (use {expected} instead)", + DeprecationWarning, + ) + cache_fname_fmt = expected + + if cache_saving_backend is None: + expected = "auto" warnings.warn( - f"Deprecated argument value {cache_fname_fmt=}. (use default instead)", + f"Deprecated argument value {cache_saving_backend=}. (use {expected} instead)", DeprecationWarning, ) - cache_fname_fmt = "{fn_name}_{csum}{suffix}" + cache_saving_backend = expected + + if cache_saving_backend == "auto": + if cache_dump_fn is not None and cache_load_fn is not None: + cache_saving_backend = "custom" + else: + cache_saving_backend = "pickle" if cache_saving_backend == "pickle": from pythonwrench.pickle import dump_pickle, load_pickle @@ -250,9 +335,9 @@ def _disk_cache_impl( cache_dump_fn = dump_csv cache_load_fn = load_csv - elif cache_saving_backend is None: - if cache_fname_fmt is None or cache_dump_fn is None or cache_load_fn is None: - msg = f"If {cache_saving_backend=}, arguments cache_fname_fmt, cache_dump_fn and cache_load_fn cannot be None. (found {cache_fname_fmt=}, {cache_dump_fn=} {cache_load_fn=})" + elif cache_saving_backend == "custom": + if cache_dump_fn is None or cache_load_fn is None: + msg = f"If {cache_saving_backend=}, arguments cache_dump_fn and cache_load_fn cannot be None. (found {cache_dump_fn=} {cache_load_fn=})" raise ValueError(msg) suffix = "" diff --git a/src/pythonwrench/entries.py b/src/pythonwrench/entries.py index b83b7fa..47b38a7 100644 --- a/src/pythonwrench/entries.py +++ b/src/pythonwrench/entries.py @@ -175,24 +175,29 @@ def main_safe_rmdir() -> None: ) parser.add_argument( "--rm_root", + "--rm-root", type=str_to_bool, default=True, help="If True, remove the root directory too if it is empty at the end. defaults to True.", ) parser.add_argument( "--error_on_non_empty_dir", + "--error-on-non-empty-dir", type=str_to_bool, default=True, help="If True, raises a RuntimeError if a subdirectory contains at least 1 file. Otherwise it will ignore non-empty directories. defaults to True.", ) parser.add_argument( "--followlinks", + "--follow_links", + "--follow-links", type=str_to_bool, default=False, help="Indicates whether or not symbolic links shound be followed. defaults to False.", ) parser.add_argument( "--dry_run", + "--dry-run", type=str_to_bool, default=False, help="If True, does not remove any directory and just output the list of directories which could be deleted. defaults to False.", diff --git a/src/pythonwrench/functools.py b/src/pythonwrench/functools.py index 54ac9df..50e1f90 100644 --- a/src/pythonwrench/functools.py +++ b/src/pythonwrench/functools.py @@ -89,7 +89,7 @@ def __init__(self, *fns) -> None: elif isinstance_generic(fns, Tuple[Callable, ...]): pass else: - msg = f"Invalid argument types {type(fns)=}." + msg = f"Invalid argument types {type(fns)=}. (with {fns=})" raise TypeError(msg) super().__init__() diff --git a/src/pythonwrench/pickle.py b/src/pythonwrench/pickle.py index ffadc41..63ddf9a 100644 --- a/src/pythonwrench/pickle.py +++ b/src/pythonwrench/pickle.py @@ -66,6 +66,16 @@ def dumps_pickle( to_builtins: bool = False, **pkl_dumps_kwds, ) -> bytes: + r"""Dump content to PICKLE format into bytes. + + Args: + data: Data to dump to PICKLE. + to_builtins: If True, converts data to builtin equivalent before saving. defaults to False. + \*\*pkl_dumps_kwds: Other args passed to `pickle.dumps`. + + Returns: + Dumped content as bytes. + """ with BytesIO() as buffer: _serialize_pickle( data, @@ -87,6 +97,16 @@ def save_pickle( to_builtins: bool = False, **pkl_dumps_kwds, ) -> None: + r"""Dump content to PICKLE format into file. + + Args: + data: Data to dump to PICKLE. + file: Filepath to save dumped data. + overwrite: If True, overwrite target filepath. defaults to True. + make_parents: Build intermediate directories to filepath. defaults to True. + to_builtins: If True, converts data to builtin equivalent before saving. defaults to False. + \*\*pkl_dumps_kwds: Other args passed to `pickle.dumps`. + """ if isinstance(file, (str, Path, PathLike)): file = _setup_output_fpath(file, overwrite=overwrite, make_parents=make_parents) file = open(file, "wb") @@ -125,6 +145,12 @@ def _serialize_pickle( def load_pickle(file: Union[str, Path, BinaryIO], /, **pkl_loads_kwds) -> Any: + r"""Load content from PICKLE file. + + Args: + file: Filepath file path. + \*\*pkl_loads_kwds: Other args passed to `pickle.loads`. + """ if isinstance(file, (str, Path, PathLike)): file = open(file, "rb") close = True @@ -138,6 +164,12 @@ def load_pickle(file: Union[str, Path, BinaryIO], /, **pkl_loads_kwds) -> Any: def loads_pickle(content: bytes, /, **pkl_loads_kwds) -> Any: + r"""Load content from raw bytes. + + Args: + content: Encoded elements bytes. + \*\*pkl_loads_kwds: Other args passed to `pickle.loads`. + """ with BytesIO(content) as buffer: return _parse_pickle(buffer, **pkl_loads_kwds) diff --git a/src/pythonwrench/typing/checks.py b/src/pythonwrench/typing/checks.py index 6294efd..b77ef6f 100644 --- a/src/pythonwrench/typing/checks.py +++ b/src/pythonwrench/typing/checks.py @@ -99,7 +99,7 @@ def _wrapper(*args: P.args, **kwargs: P.kwargs) -> T: def isinstance_generic( obj: Any, - class_or_tuple: Union[Type[T], None, Tuple[Type[T], ...]], + class_or_tuple: Union[Type[T], None, Tuple[Type[T], ...], Any], *, check_only_first: bool = False, ) -> TypeIs[T]: @@ -123,8 +123,6 @@ def isinstance_generic( ... True """ - if isinstance(obj, type): - return False if class_or_tuple is Any or class_or_tuple is typing_extensions.Any: return True if class_or_tuple is None: @@ -139,7 +137,7 @@ def isinstance_generic( origin = get_origin(class_or_tuple) if origin is None: - return isinstance(obj, class_or_tuple) + return isinstance(obj, class_or_tuple) # type: ignore # Special case for empty tuple because get_args(Tuple[()]) returns () and not ((),) in python >= 3.11 # More info at https://github.com/python/cpython/issues/91137 @@ -147,6 +145,14 @@ def isinstance_generic( return obj == () args = get_args(class_or_tuple) + if origin is Callable: + if len(args) == 0: + return callable(obj) + else: + # TODO: impl + msg = "Function `isinstance_generic` currently does not support parametrized Callable." + raise NotImplementedError(msg) + if len(args) == 0: return isinstance_generic(obj, origin) @@ -298,7 +304,7 @@ def is_dataclass_instance(x: Any) -> TypeIs[DataclassInstance]: Unlike function `dataclasses.is_dataclass`, this function returns False for a dataclass type. """ - return isinstance_generic(x, DataclassInstance) + return not isinstance(x, type) and isinstance_generic(x, DataclassInstance) def is_iterable_bool( @@ -369,7 +375,7 @@ def is_iterable_str( def is_namedtuple_instance(x: Any) -> TypeIs[NamedTupleInstance]: """Returns True if argument is a NamedTuple.""" - return isinstance_generic(x, NamedTupleInstance) + return not isinstance(x, type) and isinstance_generic(x, NamedTupleInstance) def is_sequence_str( diff --git a/src/pythonwrench/typing/classes.py b/src/pythonwrench/typing/classes.py index d8867be..04b10c5 100644 --- a/src/pythonwrench/typing/classes.py +++ b/src/pythonwrench/typing/classes.py @@ -67,37 +67,50 @@ def __len__(self) -> int: @runtime_checkable class SupportsAdd(Protocol[_T_Other]): + """Protocol that support `__add__` (+) method.""" + def __add__(self, other: _T_Other, /): raise NotImplementedError @runtime_checkable class SupportsAnd(Protocol[_T_Other]): + """Protocol that support `__and__` (&) method.""" + def __and__(self, other: _T_Other, /): raise NotImplementedError @runtime_checkable class SupportsBool(Protocol): + """Protocol that support `__bool__` method.""" + def __bool__(self) -> bool: raise NotImplementedError @runtime_checkable class SupportsDiv(Protocol[_T_Other]): + """Protocol that support `__div__` (/) method.""" + def __div__(self, other: _T_Other, /): raise NotImplementedError @runtime_checkable class SupportsGetitem(Protocol[_T_Item, _T_Index]): + """Protocol that support `__getitem__` method.""" + def __getitem__(self, idx: _T_Index, /) -> _T_Item: raise NotImplementedError @runtime_checkable class SupportsGetitem2(Protocol[_T_Index2, _T_Item]): - """Same than `SupportsGetitem` except that generic parameters are in reversed order: [T_Index, T_Item].""" + """Protocol that support `__getitem__` method. + + Same than `SupportsGetitem` except that generic parameters are in reversed order: [T_Index, T_Item]. + """ def __getitem__(self, idx: _T_Index2, /) -> _T_Item: raise NotImplementedError @@ -105,6 +118,8 @@ def __getitem__(self, idx: _T_Index2, /) -> _T_Item: @runtime_checkable class SupportsGetitemLen(Protocol[_T_Item, _T_Index]): + """Protocol that support `__getitem__` and `__len__` methods.""" + def __getitem__(self, idx: _T_Index, /) -> _T_Item: raise NotImplementedError @@ -114,7 +129,9 @@ def __len__(self) -> int: @runtime_checkable class SupportsGetitemLen2(Protocol[_T_Index2, _T_Item]): - """Same than `SupportsGetitemLen` except that generic parameters are in reversed order: [T_Index, T_Item].""" + """Protocol that support `__getitem__` and `__len__` methods. + + Same than `SupportsGetitemLen` except that generic parameters are in reversed order: [T_Index, T_Item].""" def __getitem__(self, idx: _T_Index2, /) -> _T_Item: raise NotImplementedError @@ -125,6 +142,8 @@ def __len__(self) -> int: @runtime_checkable class SupportsGetitemIterLen(Protocol[_T_Item, _T_Index]): + """Protocol that support `__getitem__`, `__iter__` and `__len__` methods.""" + def __getitem__(self, idx: _T_Index, /) -> _T_Item: raise NotImplementedError @@ -137,7 +156,10 @@ def __len__(self) -> int: @runtime_checkable class SupportsGetitemIterLen2(Protocol[_T_Index2, _T_Item]): - """Same than `SupportsGetitemIterLen` except that generic parameters are in reversed order: [T_Index, T_Item].""" + """Protocol that support `__getitem__`, `__iter__` and `__len__` methods. + + Same than `SupportsGetitemIterLen` except that generic parameters are in reversed order: [T_Index, T_Item]. + """ def __getitem__(self, idx: _T_Index2, /) -> _T_Item: raise NotImplementedError @@ -151,6 +173,8 @@ def __len__(self) -> int: @runtime_checkable class SupportsIterLen(Protocol[_T_Item]): + """Protocol that support `__iter__` and `__len__` methods.""" + def __iter__(self) -> Iterator[_T_Item]: raise NotImplementedError @@ -160,23 +184,31 @@ def __len__(self) -> int: @runtime_checkable class SupportsLen(Protocol): + """Protocol that support `__len__` method.""" + def __len__(self) -> int: raise NotImplementedError @runtime_checkable class SupportsMul(Protocol[_T_Other]): + """Protocol that support `__mul__` (*) method.""" + def __mul__(self, other: _T_Other, /): raise NotImplementedError @runtime_checkable class SupportsOr(Protocol[_T_Other]): + """Protocol that support `__or__` (|) method.""" + def __or__(self, other: _T_Other, /): raise NotImplementedError @runtime_checkable class SupportsMatmul(Protocol[_T_Other]): + """Protocol that support `__matmul__` (@) method.""" + def __matmul__(self, other: _T_Other, /): raise NotImplementedError diff --git a/tests/test_typing.py b/tests/test_typing.py index 9c6a2dc..536e7bd 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -4,6 +4,7 @@ import unittest from dataclasses import dataclass from numbers import Number +from pathlib import Path from typing import ( Any, Callable, @@ -280,6 +281,13 @@ def test_edges_cases(self) -> None: with self.assertRaises(TypeError): assert not isinstance_generic(1, Generator[int, None, None]) + def test_callable(self) -> None: + assert isinstance_generic(lambda x: x, Callable) + assert isinstance_generic(Path, Callable) + + with self.assertRaises(NotImplementedError): + assert isinstance_generic(Path, Callable[[str], Path]) + class TestCheckArgsType(TestCase): def test_example_1(self) -> None: