diff --git a/README.md b/README.md index efba1ef..83da283 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,41 @@ def is_newer_version(version_no: str) -> bool: print(is_newer_version("v0.2.0")) ``` +## Why use a lazy importer? ## + +One obvious use case is if you are creating a simple CLI application that you wish to feel fast. +If the application has multiple pathways a lazy importer can improve performance by avoiding +loading the modules that are only needed for heavier pathways. (It may also be worth looking +at what library you are using for CLI argument parsing.) + +I created this so I could use it on my own projects so here's an example of the performance +of `ducktools-env` with and without lazy imports. + +With lazy imports: +```commandline +hyperfine -w3 -r20 "python -m ducktools.env run examples\inline\empty_312_env.py" +``` +``` +Benchmark 1: python -m ducktools.env run examples\inline\empty_312_env.py + Time (mean ± σ): 87.1 ms ± 1.1 ms [User: 52.2 ms, System: 22.4 ms] + Range (min … max): 85.2 ms … 89.1 ms 20 runs +``` + +Without lazy imports (by setting `DUCKTOOLS_EAGER_IMPORT=true`): +```commandline +hyperfine -w3 -r20 "python -m ducktools.env run examples\inline\empty_312_env.py" +``` +``` +Benchmark 1: python -m ducktools.env run examples\inline\empty_312_env.py + Time (mean ± σ): 144.2 ms ± 1.4 ms [User: 84.8 ms, System: 45.3 ms] + Range (min … max): 141.0 ms … 146.7 ms 20 runs +``` + +In this case the module is searching for a matching python environment to run the script in, +the environment already exists and is cached so there is no need to load the code required +for constructing new environments. This timer includes the time to relaunch the correct +python environment and run the (empty) script. + ## Hasn't this already been done ## Yes. @@ -50,10 +85,10 @@ But... Most implementations rely on stdlib modules that are themselves slow to import (for example: typing, importlib.util, logging, inspect, ast). -By contrast `lazyimporter` only uses modules that python imports on launch. +By contrast `ducktools-lazyimporter` only uses modules that python imports on launch. -`lazyimporter` does not attempt to propagate laziness, only the modules provided -to `lazyimporter` directly will be imported lazily. Any subdependencies of those +`ducktools-lazyimporter` does not attempt to propagate laziness, only the modules provided +to `ducktools-lazyimporter` directly will be imported lazily. Any subdependencies of those modules will be imported eagerly as if the import statement is placed where the importer attribute is first accessed. @@ -217,6 +252,24 @@ except ImportError: when provided to a LazyImporter. +## Environment Variables ## + +There are two environment variables that can be used to modify the behaviour for +debugging purposes. + +If `DUCKTOOLS_EAGER_PROCESS` is set to any value other than 'False' (case insensitive) +the initial processing of imports will be done on instance creation. + +Similarly if `DUCKTOOLS_EAGER_IMPORT` is set to any value other than 'False' all imports +will be performed eagerly on instance creation (this will also force processing on import). + +If they are unset this is equivalent to being set to False. + +If there is a lazy importer where it is known this will not work +(for instance if it is managing a circular dependency issue) +these can be overridden for an importer by passing values to `eager_process` and/or +`eager_import` arguments to the `LazyImporter` constructer as keyword arguments. + ## How does it work ## The following lazy importer: diff --git a/docs/extending.md b/docs/extending.md index 801b78f..d9f0f9a 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -10,7 +10,7 @@ Subclasses of `ImportBase` require 3 things: `asname` or `asnames` must be either the identifier or a list of identifiers (respectively) to use to store attributes. This can be an attribute or a property. -`do_import` must be a method that takes 2 arguments `(self, globs=None)`, performs +`import_objects` must be a method that takes 2 arguments `(self, globs=None)`, performs the import and returns a dictionary of the form `{asname: , ...}` for all of the names defined in `asname`/`asnames`. @@ -42,7 +42,7 @@ class IfElseImporter(ImportBase): if not self.asname.isidentifier(): raise ValueError(f"{self.asname} is not a valid python identifier.") - def do_import(self, globs=None): + def import_objects(self, globs=None): if globs is not None: package = globs.get('__name__') else: diff --git a/docs/index.md b/docs/index.md index d82cdcf..3a10d55 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ Ducktools: Lazy Importer is a module intended to make it easier to defer imports until needed without requiring the import statement to be written in-line. -The goal of deferring imports is to avoid importing modules that is not guaranteed +The goal of deferring imports is to avoid importing modules that are not guaranteed to be used in the course of running an application. This can be done both on the side of the application, in deferring imports diff --git a/src/ducktools/lazyimporter/__init__.py b/src/ducktools/lazyimporter/__init__.py index ed1c1ab..9aa0fb8 100644 --- a/src/ducktools/lazyimporter/__init__.py +++ b/src/ducktools/lazyimporter/__init__.py @@ -84,7 +84,7 @@ def submodule_names(self): return self.module_name_noprefix.split(".")[1:] @abc.abstractmethod - def do_import(self, globs=None): + def import_objects(self, globs=None): """ Perform the imports defined and return a dictionary. @@ -138,7 +138,7 @@ def __eq__(self, other): return (self.module_name, self.asname) == (other.module_name, other.asname) return NotImplemented - def do_import(self, globs=None): + def import_objects(self, globs=None): mod = __import__( self.module_name_noprefix, globals=globs, @@ -193,7 +193,7 @@ def __eq__(self, other): ) return NotImplemented - def do_import(self, globs=None): + def import_objects(self, globs=None): # Perform the import mod = __import__( self.module_name_noprefix, @@ -260,7 +260,7 @@ def asnames(self): return names - def do_import(self, globs=None): + def import_objects(self, globs=None): from_imports = {} # Perform the import @@ -373,7 +373,7 @@ def __eq__(self, other): ) return NotImplemented - def do_import(self, globs=None): + def import_objects(self, globs=None): try: mod = __import__( self.module_name_noprefix, @@ -468,7 +468,7 @@ def __eq__(self, other): ) return NotImplemented - def do_import(self, globs=None): + def import_objects(self, globs=None): try: mod = __import__( self.module_name_noprefix, @@ -544,7 +544,7 @@ def __eq__(self, other): ) return NotImplemented - def do_import(self, globs=None): + def import_objects(self, globs=None): try: mod = __import__( self.module_name_noprefix, @@ -679,7 +679,7 @@ def __getattr__(self, name): f"{self.__class__.__name__!r} object has no attribute {name!r}" ) - import_data = importer.do_import(globs=self._globals) + import_data = importer.import_objects(globs=self._globals) for key, value in import_data.items(): setattr(self, key, value) diff --git a/src/ducktools/lazyimporter/__init__.pyi b/src/ducktools/lazyimporter/__init__.pyi index 1c17d17..5c7098b 100644 --- a/src/ducktools/lazyimporter/__init__.pyi +++ b/src/ducktools/lazyimporter/__init__.pyi @@ -37,7 +37,7 @@ class ImportBase(metaclass=abc.ABCMeta): @property def submodule_names(self) -> list[str]: ... @abc.abstractmethod - def do_import( + def import_objects( self, globs: dict[str, Any] | None = ... ) -> dict[str, types.ModuleType | Any]: ... @@ -48,7 +48,7 @@ class ModuleImport(ImportBase): def __init__(self, module_name: str, asname: str | None = ...) -> None: ... def __repr__(self) -> str: ... def __eq__(self, other) -> bool: ... - def do_import( + def import_objects( self, globs: dict[str, Any] | None = ... ) -> dict[str, types.ModuleType]: ... @@ -62,7 +62,7 @@ class FromImport(ImportBase): ) -> None: ... def __repr__(self) -> str: ... def __eq__(self, other) -> bool: ... - def do_import(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... + def import_objects(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... class MultiFromImport(ImportBase): module_name: str @@ -75,7 +75,7 @@ class MultiFromImport(ImportBase): def __eq__(self, other) -> bool: ... @property def asnames(self): ... - def do_import(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... + def import_objects(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... class _TryExceptImportMixin(metaclass=abc.ABCMeta): except_module: str @@ -96,7 +96,7 @@ class TryExceptImport(_TryExceptImportMixin, ImportBase): def __init__(self, module_name: str, except_module: str, asname: str) -> None: ... def __repr__(self) -> str: ... def __eq__(self, other) -> bool: ... - def do_import(self, globs: dict[str, Any] | None = ...): ... + def import_objects(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... class TryExceptFromImport(_TryExceptImportMixin, ImportBase): module_name: str @@ -114,7 +114,7 @@ class TryExceptFromImport(_TryExceptImportMixin, ImportBase): ) -> None: ... def __repr__(self) -> str: ... def __eq__(self, other) -> bool: ... - def do_import(self, globs: dict[str, Any] | None = ...): ... + def import_objects(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... class TryFallbackImport(ImportBase): module_name: str @@ -129,7 +129,7 @@ class TryFallbackImport(ImportBase): ) -> None: ... def __repr__(self) -> str: ... def __eq__(self, other) -> bool: ... - def do_import(self, globs: dict[str, Any] | None = ...): ... + def import_objects(self, globs: dict[str, Any] | None = ...) -> dict[str, Any]: ... class _ImporterGrouper: def __init__(self) -> None: ... diff --git a/tests/test_readme_example.py b/tests/test_readme_example.py index a3787df..d2540e9 100644 --- a/tests/test_readme_example.py +++ b/tests/test_readme_example.py @@ -15,7 +15,7 @@ def __init__(self, condition, module_name, else_module_name, asname): if not self.asname.isidentifier(): raise ValueError(f"{self.asname} is not a valid python identifier.") - def do_import(self, globs=None): + def import_objects(self, globs=None): if globs is not None: package = globs.get('__name__') else: