Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 26 additions & 11 deletions src/crawlee/_utils/try_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,44 @@


@contextmanager
def try_import(module_name: str, *symbol_names: str, extra_name: str | list[str]) -> Iterator[None]:
def try_import(
module_name: str,
*symbol_names: str,
extra_name: str | list[str] | None = None,
package_name: str = 'crawlee',
) -> Iterator[None]:
"""Context manager to attempt importing symbols into a module.

If an `ImportError` is raised during the import, the symbols are replaced with `FailedImport` objects. When the
error is a `ModuleNotFoundError`, the message also names the optional extra (or one of several) that installs the
missing dependency. Other import errors, including those raised by a nested guard, keep their message as is.
error is a `ModuleNotFoundError` and `extra_name` is given, the message also names the optional extra (or one of
several) that installs the missing dependency. Other import errors, including those raised by a nested guard,
keep their message as is.

Args:
module_name: The name of the module the symbols are imported into.
symbol_names: The names of the symbols being imported.
extra_name: The optional extra (or extras) providing the dependency. When omitted, the error message is
left as it is.
package_name: The distribution whose extras are named in the message. Defaults to this package, and exists
so that downstream packages reusing this helper can point users at their own extras.
"""
try:
yield
except ImportError as e:
message = e.args[0]
if isinstance(e, ModuleNotFoundError):
message = f'{message}. {_get_install_hint(extra_name)}'
message = str(e)
if isinstance(e, ModuleNotFoundError) and extra_name:
message = f'{message}. {_get_install_hint(extra_name, package_name)}'
for symbol_name in symbol_names:
setattr(sys.modules[module_name], symbol_name, FailedImport(message))


def _get_install_hint(extra_name: str | list[str]) -> str:
def _get_install_hint(extra_name: str | list[str], package_name: str) -> str:
"""Build the sentence telling the user which extra installs the missing optional dependency."""
if isinstance(extra_name, str):
return f"Install the optional '{extra_name}' extra to use it: pip install 'crawlee[{extra_name}]'"
extras = ', '.join(f"'{name}'" for name in extra_name)
return f"Install one of the optional extras {extras} to use it, e.g. pip install 'crawlee[{extra_name[0]}]'"
extras = [extra_name] if isinstance(extra_name, str) else list(extra_name)
if len(extras) == 1:
return f"Install the optional '{extras[0]}' extra to use it: pip install '{package_name}[{extras[0]}]'"
names = ', '.join(f"'{name}'" for name in extras)
return f"Install one of the optional extras {names} to use it, e.g. pip install '{package_name}[{extras[0]}]'"


def install_import_hook(module_name: str) -> None:
Expand Down
26 changes: 26 additions & 0 deletions tests/unit/_utils/test_try_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,29 @@ def test_nested_guard_keeps_the_inner_message(module_name: str, monkeypatch: pyt

assert sys.modules[module_name].OptionalSymbol.message == inner_message
assert inner_message.count('pip install') == 1


def test_omitted_extra_keeps_the_original_message(module_name: str) -> None:
"""Without `extra_name` the message is left as it is, so external callers of the helper keep working."""
with try_import(module_name, 'OptionalSymbol'):
raise ModuleNotFoundError("No module named 'parsel'", name='parsel')

assert sys.modules[module_name].OptionalSymbol.message == "No module named 'parsel'"


def test_install_hint_names_the_given_package(module_name: str) -> None:
"""Downstream packages reusing the helper can point users at their own extras."""
with try_import(module_name, 'OptionalSymbol', extra_name='scrapy', package_name='apify'):
raise ModuleNotFoundError("No module named 'scrapy'", name='scrapy')

assert sys.modules[module_name].OptionalSymbol.message == (
"No module named 'scrapy'. Install the optional 'scrapy' extra to use it: pip install 'apify[scrapy]'"
)


def test_import_error_without_arguments_is_handled(module_name: str) -> None:
"""An `ImportError` carrying no message does not break the guard itself."""
with try_import(module_name, 'OptionalSymbol', extra_name='parsel'):
raise ImportError

assert sys.modules[module_name].OptionalSymbol.message == ''
Loading