1#@!#!123s

: / opt / imh-python / lib / python3.9 / site-packages / astroid / interpreter / _import /
Filename : spec.py
back
# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt

from __future__ import annotations

import abc
import enum
import importlib
import importlib.machinery
import importlib.util
import os
import pathlib
import sys
import types
import warnings
import zipimport
from collections.abc import Iterable, Iterator, Sequence
from functools import lru_cache
from pathlib import Path
from typing import Any, Literal, NamedTuple, Protocol

from astroid.const import PY310_PLUS
from astroid.modutils import EXT_LIB_DIRS

from . import util


# The MetaPathFinder protocol comes from typeshed, which says:
# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
class _MetaPathFinder(Protocol):
    def find_spec(
        self,
        fullname: str,
        path: Sequence[str] | None,
        target: types.ModuleType | None = ...,
    ) -> importlib.machinery.ModuleSpec | None: ...  # pragma: no cover


class ModuleType(enum.Enum):
    """Python module types used for ModuleSpec."""

    C_BUILTIN = enum.auto()
    C_EXTENSION = enum.auto()
    PKG_DIRECTORY = enum.auto()
    PY_CODERESOURCE = enum.auto()
    PY_COMPILED = enum.auto()
    PY_FROZEN = enum.auto()
    PY_RESOURCE = enum.auto()
    PY_SOURCE = enum.auto()
    PY_ZIPMODULE = enum.auto()
    PY_NAMESPACE = enum.auto()


_MetaPathFinderModuleTypes: dict[str, ModuleType] = {
    # Finders created by setuptools editable installs
    "_EditableFinder": ModuleType.PY_SOURCE,
    "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE,
    # Finders create by six
    "_SixMetaPathImporter": ModuleType.PY_SOURCE,
}

_EditableFinderClasses: set[str] = {
    "_EditableFinder",
    "_EditableNamespaceFinder",
}


class ModuleSpec(NamedTuple):
    """Defines a class similar to PEP 420's ModuleSpec.

    A module spec defines a name of a module, its type, location
    and where submodules can be found, if the module is a package.
    """

    name: str
    type: ModuleType | None
    location: str | None = None
    origin: str | None = None
    submodule_search_locations: Sequence[str] | None = None


class Finder:
    """A finder is a class which knows how to find a particular module."""

    def __init__(self, path: Sequence[str] | None = None) -> None:
        self._path = path or sys.path

    @abc.abstractmethod
    def find_module(
        self,
        modname: str,
        module_parts: Sequence[str],
        processed: list[str],
        submodule_path: Sequence[str] | None,
    ) -> ModuleSpec | None:
        """Find the given module.

        Each finder is responsible for each protocol of finding, as long as
        they all return a ModuleSpec.

        :param modname: The module which needs to be searched.
        :param module_parts: It should be a list of strings,
                                  where each part contributes to the module's
                                  namespace.
        :param processed: What parts from the module parts were processed
                               so far.
        :param submodule_path: A list of paths where the module
                                    can be looked into.
        :returns: A ModuleSpec, describing how and where the module was found,
                  None, otherwise.
        """

    def contribute_to_path(
        self, spec: ModuleSpec, processed: list[str]
    ) -> Sequence[str] | None:
        """Get a list of extra paths where this finder can search."""


class ImportlibFinder(Finder):
    """A finder based on the importlib module."""

    _SUFFIXES: Sequence[tuple[str, ModuleType]] = (
        [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
        + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
        + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
    )

    def find_module(
        self,
        modname: str,
        module_parts: Sequence[str],
        processed: list[str],
        submodule_path: Sequence[str] | None,
    ) -> ModuleSpec | None:
        # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins.
        # Therefore, we use the `builtin_module_nams` heuristic for these.
        if submodule_path is None and modname in sys.builtin_module_names:
            return ModuleSpec(
                name=modname,
                location=None,
                type=ModuleType.C_BUILTIN,
            )

        if submodule_path is not None:
            search_paths = list(submodule_path)
        else:
            search_paths = sys.path

        suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0])
        for entry in search_paths:
            package_directory = os.path.join(entry, modname)
            for suffix in suffixes:
                package_file_name = "__init__" + suffix
                file_path = os.path.join(package_directory, package_file_name)
                if os.path.isfile(file_path):
                    return ModuleSpec(
                        name=modname,
                        location=package_directory,
                        type=ModuleType.PKG_DIRECTORY,
                    )
            for suffix, type_ in ImportlibFinder._SUFFIXES:
                file_name = modname + suffix
                file_path = os.path.join(entry, file_name)
                if os.path.isfile(file_path):
                    return ModuleSpec(name=modname, location=file_path, type=type_)

        # sys.stdlib_module_names was added in Python 3.10
        if PY310_PLUS:
            # If the module name matches a stdlib module name, check whether this is a frozen
            # module. Note that `find_spec` actually imports parent modules, so we want to make
            # sure we only run this code for stuff that can be expected to be frozen. For now
            # this is only stdlib.
            if modname in sys.stdlib_module_names or (
                processed and processed[0] in sys.stdlib_module_names
            ):
                try:
                    with warnings.catch_warnings():
                        warnings.filterwarnings("ignore", category=Warning)
                        spec = importlib.util.find_spec(".".join((*processed, modname)))
                except ValueError:
                    spec = None

                if (
                    spec
                    and spec.loader  # type: ignore[comparison-overlap] # noqa: E501
                    is importlib.machinery.FrozenImporter
                ):
                    return ModuleSpec(
                        name=modname,
                        location=getattr(spec.loader_state, "filename", None),
                        type=ModuleType.PY_FROZEN,
                    )
        else:
            # NOTE: This is broken code. It doesn't work on Python 3.13+ where submodules can also
            # be frozen. However, we don't want to worry about this and we don't want to break
            # support for older versions of Python. This is just copy-pasted from the old non
            # working version to at least have no functional behaviour change on <=3.10.
            # It can be removed after 3.10 is no longer supported in favour of the logic above.
            if submodule_path is None:  # pylint: disable=else-if-used
                try:
                    with warnings.catch_warnings():
                        warnings.filterwarnings("ignore", category=UserWarning)
                        spec = importlib.util.find_spec(modname)
                    if (
                        spec
                        and spec.loader  # type: ignore[comparison-overlap] # noqa: E501
                        is importlib.machinery.FrozenImporter
                    ):
                        # No need for BuiltinImporter; builtins handled above
                        return ModuleSpec(
                            name=modname,
                            location=getattr(spec.loader_state, "filename", None),
                            type=ModuleType.PY_FROZEN,
                        )
                except ValueError:
                    pass

        return None

    def contribute_to_path(
        self, spec: ModuleSpec, processed: list[str]
    ) -> Sequence[str] | None:
        if spec.location is None:
            # Builtin.
            return None

        if _is_setuptools_namespace(Path(spec.location)):
            # extend_path is called, search sys.path for module/packages
            # of this name see pkgutil.extend_path documentation
            path = [
                os.path.join(p, *processed)
                for p in sys.path
                if os.path.isdir(os.path.join(p, *processed))
            ]
        elif spec.name == "distutils" and not any(
            spec.location.lower().startswith(ext_lib_dir.lower())
            for ext_lib_dir in EXT_LIB_DIRS
        ):
            # virtualenv below 20.0 patches distutils in an unexpected way
            # so we just find the location of distutils that will be
            # imported to avoid spurious import-error messages
            # https://github.com/pylint-dev/pylint/issues/5645
            # A regression test to create this scenario exists in release-tests.yml
            # and can be triggered manually from GitHub Actions
            distutils_spec = importlib.util.find_spec("distutils")
            if distutils_spec and distutils_spec.origin:
                origin_path = Path(
                    distutils_spec.origin
                )  # e.g. .../distutils/__init__.py
                path = [str(origin_path.parent)]  # e.g. .../distutils
            else:
                path = [spec.location]
        else:
            path = [spec.location]
        return path


class ExplicitNamespacePackageFinder(ImportlibFinder):
    """A finder for the explicit namespace packages."""

    def find_module(
        self,
        modname: str,
        module_parts: Sequence[str],
        processed: list[str],
        submodule_path: Sequence[str] | None,
    ) -> ModuleSpec | None:
        if processed:
            modname = ".".join([*processed, modname])
        if util.is_namespace(modname) and modname in sys.modules:
            return ModuleSpec(
                name=modname,
                location="",
                origin="namespace",
                type=ModuleType.PY_NAMESPACE,
                submodule_search_locations=sys.modules[modname].__path__,
            )
        return None

    def contribute_to_path(
        self, spec: ModuleSpec, processed: list[str]
    ) -> Sequence[str] | None:
        return spec.submodule_search_locations


class ZipFinder(Finder):
    """Finder that knows how to find a module inside zip files."""

    def __init__(self, path: Sequence[str]) -> None:
        super().__init__(path)
        for entry_path in path:
            if entry_path not in sys.path_importer_cache:
                try:
                    sys.path_importer_cache[entry_path] = zipimport.zipimporter(  # type: ignore[assignment]
                        entry_path
                    )
                except zipimport.ZipImportError:
                    continue

    def find_module(
        self,
        modname: str,
        module_parts: Sequence[str],
        processed: list[str],
        submodule_path: Sequence[str] | None,
    ) -> ModuleSpec | None:
        try:
            file_type, filename, path = _search_zip(module_parts)
        except ImportError:
            return None

        return ModuleSpec(
            name=modname,
            location=filename,
            origin="egg",
            type=file_type,
            submodule_search_locations=path,
        )


class PathSpecFinder(Finder):
    """Finder based on importlib.machinery.PathFinder."""

    def find_module(
        self,
        modname: str,
        module_parts: Sequence[str],
        processed: list[str],
        submodule_path: Sequence[str] | None,
    ) -> ModuleSpec | None:
        spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
        if spec is not None:
            is_namespace_pkg = spec.origin is None
            location = spec.origin if not is_namespace_pkg else None
            module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
            return ModuleSpec(
                name=spec.name,
                location=location,
                origin=spec.origin,
                type=module_type,
                submodule_search_locations=list(spec.submodule_search_locations or []),
            )
        return spec

    def contribute_to_path(
        self, spec: ModuleSpec, processed: list[str]
    ) -> Sequence[str] | None:
        if spec.type == ModuleType.PY_NAMESPACE:
            return spec.submodule_search_locations
        return None


_SPEC_FINDERS = (
    ImportlibFinder,
    ZipFinder,
    PathSpecFinder,
    ExplicitNamespacePackageFinder,
)


def _is_setuptools_namespace(location: pathlib.Path) -> bool:
    try:
        with open(location / "__init__.py", "rb") as stream:
            data = stream.read(4096)
    except OSError:
        return False
    extend_path = b"pkgutil" in data and b"extend_path" in data
    declare_namespace = (
        b"pkg_resources" in data and b"declare_namespace(__name__)" in data
    )
    return extend_path or declare_namespace


def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]:
    for filepath, importer in sys.path_importer_cache.items():
        if importer is not None and isinstance(importer, zipimport.zipimporter):
            yield filepath, importer


def _search_zip(
    modpath: Sequence[str],
) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]:
    for filepath, importer in _get_zipimporters():
        if PY310_PLUS:
            found: Any = importer.find_spec(modpath[0])
        else:
            found = importer.find_module(modpath[0])
        if found:
            if PY310_PLUS:
                if not importer.find_spec(os.path.sep.join(modpath)):
                    raise ImportError(
                        "No module named {} in {}/{}".format(
                            ".".join(modpath[1:]), filepath, modpath
                        )
                    )
            elif not importer.find_module(os.path.sep.join(modpath)):
                raise ImportError(
                    "No module named {} in {}/{}".format(
                        ".".join(modpath[1:]), filepath, modpath
                    )
                )
            return (
                ModuleType.PY_ZIPMODULE,
                os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
                filepath,
            )
    raise ImportError(f"No module named {'.'.join(modpath)}")


def _find_spec_with_path(
    search_path: Sequence[str],
    modname: str,
    module_parts: list[str],
    processed: list[str],
    submodule_path: Sequence[str] | None,
) -> tuple[Finder | _MetaPathFinder, ModuleSpec]:
    for finder in _SPEC_FINDERS:
        finder_instance = finder(search_path)
        spec = finder_instance.find_module(
            modname, module_parts, processed, submodule_path
        )
        if spec is None:
            continue
        return finder_instance, spec

    # Support for custom finders
    for meta_finder in sys.meta_path:
        # See if we support the customer import hook of the meta_finder
        meta_finder_name = meta_finder.__class__.__name__
        if meta_finder_name not in _MetaPathFinderModuleTypes:
            # Setuptools>62 creates its EditableFinders dynamically and have
            # "type" as their __class__.__name__. We check __name__ as well
            # to see if we can support the finder.
            try:
                meta_finder_name = meta_finder.__name__  # type: ignore[attr-defined]
            except AttributeError:
                continue
            if meta_finder_name not in _MetaPathFinderModuleTypes:
                continue

        module_type = _MetaPathFinderModuleTypes[meta_finder_name]

        # Meta path finders are supposed to have a find_spec method since
        # Python 3.4. However, some third-party finders do not implement it.
        # PEP302 does not refer to find_spec as well.
        # See: https://github.com/pylint-dev/astroid/pull/1752/
        if not hasattr(meta_finder, "find_spec"):
            continue

        spec = meta_finder.find_spec(modname, submodule_path)
        if spec:
            return (
                meta_finder,
                ModuleSpec(
                    spec.name,
                    module_type,
                    spec.origin,
                    spec.origin,
                    spec.submodule_search_locations,
                ),
            )

    raise ImportError(f"No module named {'.'.join(module_parts)}")


def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec:
    """Find a spec for the given module.

    :type modpath: list or tuple
    :param modpath:
      split module's name (i.e name of a module or package split
      on '.'), with leading empty strings for explicit relative import

    :type path: list or None
    :param path:
      optional list of path where the module or package should be
      searched (use sys.path if nothing or None is given)

    :rtype: ModuleSpec
    :return: A module spec, which describes how the module was
             found and where.
    """
    return _find_spec(tuple(modpath), tuple(path) if path else None)


@lru_cache(maxsize=1024)
def _find_spec(module_path: tuple, path: tuple) -> ModuleSpec:
    _path = path or sys.path

    # Need a copy for not mutating the argument.
    modpath = list(module_path)

    submodule_path = None
    module_parts = modpath[:]
    processed: list[str] = []

    while modpath:
        modname = modpath.pop(0)
        finder, spec = _find_spec_with_path(
            _path, modname, module_parts, processed, submodule_path or path
        )
        processed.append(modname)
        if modpath:
            if isinstance(finder, Finder):
                submodule_path = finder.contribute_to_path(spec, processed)
            # If modname is a package from an editable install, update submodule_path
            # so that the next module in the path will be found inside of it using importlib.
            # Existence of __name__ is guaranteed by _find_spec_with_path.
            elif finder.__name__ in _EditableFinderClasses:  # type: ignore[attr-defined]
                submodule_path = spec.submodule_search_locations

        if spec.type == ModuleType.PKG_DIRECTORY:
            spec = spec._replace(submodule_search_locations=submodule_path)

    return spec