Mercurial > repos > rliterman > csp2

comparison CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/setuptools/discovery.py @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
comparison
equal deleted inserted replaced
67:0e9998148a16 68:5028fdace37b
1 """Automatic discovery of Python modules and packages (for inclusion in the
2 distribution) and other config values.
3
4 For the purposes of this module, the following nomenclature is used:
5
6 - "src-layout": a directory representing a Python project that contains a "src"
7 folder. Everything under the "src" folder is meant to be included in the
8 distribution when packaging the project. Example::
9
10 .
11 ├── tox.ini
12 ├── pyproject.toml
13 └── src/
14 └── mypkg/
15 ├── __init__.py
16 ├── mymodule.py
17 └── my_data_file.txt
18
19 - "flat-layout": a Python project that does not use "src-layout" but instead
20 have a directory under the project root for each package::
21
22 .
23 ├── tox.ini
24 ├── pyproject.toml
25 └── mypkg/
26 ├── __init__.py
27 ├── mymodule.py
28 └── my_data_file.txt
29
30 - "single-module": a project that contains a single Python script direct under
31 the project root (no directory used)::
32
33 .
34 ├── tox.ini
35 ├── pyproject.toml
36 └── mymodule.py
37
38 """
39
40 from __future__ import annotations
41
42 import itertools
43 import os
44 from collections.abc import Iterator
45 from fnmatch import fnmatchcase
46 from glob import glob
47 from pathlib import Path
48 from typing import TYPE_CHECKING, Iterable, Mapping
49
50 import _distutils_hack.override # noqa: F401
51
52 from ._path import StrPath
53
54 from distutils import log
55 from distutils.util import convert_path
56
57 if TYPE_CHECKING:
58 from setuptools import Distribution
59
60 chain_iter = itertools.chain.from_iterable
61
62
63 def _valid_name(path: StrPath) -> bool:
64 # Ignore invalid names that cannot be imported directly
65 return os.path.basename(path).isidentifier()
66
67
68 class _Filter:
69 """
70 Given a list of patterns, create a callable that will be true only if
71 the input matches at least one of the patterns.
72 """
73
74 def __init__(self, *patterns: str):
75 self._patterns = dict.fromkeys(patterns)
76
77 def __call__(self, item: str) -> bool:
78 return any(fnmatchcase(item, pat) for pat in self._patterns)
79
80 def __contains__(self, item: str) -> bool:
81 return item in self._patterns
82
83
84 class _Finder:
85 """Base class that exposes functionality for module/package finders"""
86
87 ALWAYS_EXCLUDE: tuple[str, ...] = ()
88 DEFAULT_EXCLUDE: tuple[str, ...] = ()
89
90 @classmethod
91 def find(
92 cls,
93 where: StrPath = '.',
94 exclude: Iterable[str] = (),
95 include: Iterable[str] = ('*',),
96 ) -> list[str]:
97 """Return a list of all Python items (packages or modules, depending on
98 the finder implementation) found within directory 'where'.
99
100 'where' is the root directory which will be searched.
101 It should be supplied as a "cross-platform" (i.e. URL-style) path;
102 it will be converted to the appropriate local path syntax.
103
104 'exclude' is a sequence of names to exclude; '*' can be used
105 as a wildcard in the names.
106 When finding packages, 'foo.*' will exclude all subpackages of 'foo'
107 (but not 'foo' itself).
108
109 'include' is a sequence of names to include.
110 If it's specified, only the named items will be included.
111 If it's not specified, all found items will be included.
112 'include' can contain shell style wildcard patterns just like
113 'exclude'.
114 """
115
116 exclude = exclude or cls.DEFAULT_EXCLUDE
117 return list(
118 cls._find_iter(
119 convert_path(str(where)),
120 _Filter(*cls.ALWAYS_EXCLUDE, *exclude),
121 _Filter(*include),
122 )
123 )
124
125 @classmethod
126 def _find_iter(
127 cls, where: StrPath, exclude: _Filter, include: _Filter
128 ) -> Iterator[str]:
129 raise NotImplementedError
130
131
132 class PackageFinder(_Finder):
133 """
134 Generate a list of all Python packages found within a directory
135 """
136
137 ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__")
138
139 @classmethod
140 def _find_iter(
141 cls, where: StrPath, exclude: _Filter, include: _Filter
142 ) -> Iterator[str]:
143 """
144 All the packages found in 'where' that pass the 'include' filter, but
145 not the 'exclude' filter.
146 """
147 for root, dirs, files in os.walk(str(where), followlinks=True):
148 # Copy dirs to iterate over it, then empty dirs.
149 all_dirs = dirs[:]
150 dirs[:] = []
151
152 for dir in all_dirs:
153 full_path = os.path.join(root, dir)
154 rel_path = os.path.relpath(full_path, where)
155 package = rel_path.replace(os.path.sep, '.')
156
157 # Skip directory trees that are not valid packages
158 if '.' in dir or not cls._looks_like_package(full_path, package):
159 continue
160
161 # Should this package be included?
162 if include(package) and not exclude(package):
163 yield package
164
165 # Early pruning if there is nothing else to be scanned
166 if f"{package}*" in exclude or f"{package}.*" in exclude:
167 continue
168
169 # Keep searching subdirectories, as there may be more packages
170 # down there, even if the parent was excluded.
171 dirs.append(dir)
172
173 @staticmethod
174 def _looks_like_package(path: StrPath, _package_name: str) -> bool:
175 """Does a directory look like a package?"""
176 return os.path.isfile(os.path.join(path, '__init__.py'))
177
178
179 class PEP420PackageFinder(PackageFinder):
180 @staticmethod
181 def _looks_like_package(_path: StrPath, _package_name: str) -> bool:
182 return True
183
184
185 class ModuleFinder(_Finder):
186 """Find isolated Python modules.
187 This function will **not** recurse subdirectories.
188 """
189
190 @classmethod
191 def _find_iter(
192 cls, where: StrPath, exclude: _Filter, include: _Filter
193 ) -> Iterator[str]:
194 for file in glob(os.path.join(where, "*.py")):
195 module, _ext = os.path.splitext(os.path.basename(file))
196
197 if not cls._looks_like_module(module):
198 continue
199
200 if include(module) and not exclude(module):
201 yield module
202
203 _looks_like_module = staticmethod(_valid_name)
204
205
206 # We have to be extra careful in the case of flat layout to not include files
207 # and directories not meant for distribution (e.g. tool-related)
208
209
210 class FlatLayoutPackageFinder(PEP420PackageFinder):
211 _EXCLUDE = (
212 "ci",
213 "bin",
214 "debian",
215 "doc",
216 "docs",
217 "documentation",
218 "manpages",
219 "news",
220 "newsfragments",
221 "changelog",
222 "test",
223 "tests",
224 "unit_test",
225 "unit_tests",
226 "example",
227 "examples",
228 "scripts",
229 "tools",
230 "util",
231 "utils",
232 "python",
233 "build",
234 "dist",
235 "venv",
236 "env",
237 "requirements",
238 # ---- Task runners / Build tools ----
239 "tasks", # invoke
240 "fabfile", # fabric
241 "site_scons", # SCons
242 # ---- Other tools ----
243 "benchmark",
244 "benchmarks",
245 "exercise",
246 "exercises",
247 "htmlcov", # Coverage.py
248 # ---- Hidden directories/Private packages ----
249 "[._]*",
250 )
251
252 DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE))
253 """Reserved package names"""
254
255 @staticmethod
256 def _looks_like_package(_path: StrPath, package_name: str) -> bool:
257 names = package_name.split('.')
258 # Consider PEP 561
259 root_pkg_is_valid = names[0].isidentifier() or names[0].endswith("-stubs")
260 return root_pkg_is_valid and all(name.isidentifier() for name in names[1:])
261
262
263 class FlatLayoutModuleFinder(ModuleFinder):
264 DEFAULT_EXCLUDE = (
265 "setup",
266 "conftest",
267 "test",
268 "tests",
269 "example",
270 "examples",
271 "build",
272 # ---- Task runners ----
273 "toxfile",
274 "noxfile",
275 "pavement",
276 "dodo",
277 "tasks",
278 "fabfile",
279 # ---- Other tools ----
280 "[Ss][Cc]onstruct", # SCons
281 "conanfile", # Connan: C/C++ build tool
282 "manage", # Django
283 "benchmark",
284 "benchmarks",
285 "exercise",
286 "exercises",
287 # ---- Hidden files/Private modules ----
288 "[._]*",
289 )
290 """Reserved top-level module names"""
291
292
293 def _find_packages_within(root_pkg: str, pkg_dir: StrPath) -> list[str]:
294 nested = PEP420PackageFinder.find(pkg_dir)
295 return [root_pkg] + [".".join((root_pkg, n)) for n in nested]
296
297
298 class ConfigDiscovery:
299 """Fill-in metadata and options that can be automatically derived
300 (from other metadata/options, the file system or conventions)
301 """
302
303 def __init__(self, distribution: Distribution):
304 self.dist = distribution
305 self._called = False
306 self._disabled = False
307 self._skip_ext_modules = False
308
309 def _disable(self):
310 """Internal API to disable automatic discovery"""
311 self._disabled = True
312
313 def _ignore_ext_modules(self):
314 """Internal API to disregard ext_modules.
315
316 Normally auto-discovery would not be triggered if ``ext_modules`` are set
317 (this is done for backward compatibility with existing packages relying on
318 ``setup.py`` or ``setup.cfg``). However, ``setuptools`` can call this function
319 to ignore given ``ext_modules`` and proceed with the auto-discovery if
320 ``packages`` and ``py_modules`` are not given (e.g. when using pyproject.toml
321 metadata).
322 """
323 self._skip_ext_modules = True
324
325 @property
326 def _root_dir(self) -> StrPath:
327 # The best is to wait until `src_root` is set in dist, before using _root_dir.
328 return self.dist.src_root or os.curdir
329
330 @property
331 def _package_dir(self) -> dict[str, str]:
332 if self.dist.package_dir is None:
333 return {}
334 return self.dist.package_dir
335
336 def __call__(
337 self, force: bool = False, name: bool = True, ignore_ext_modules: bool = False
338 ):
339 """Automatically discover missing configuration fields
340 and modifies the given ``distribution`` object in-place.
341
342 Note that by default this will only have an effect the first time the
343 ``ConfigDiscovery`` object is called.
344
345 To repeatedly invoke automatic discovery (e.g. when the project
346 directory changes), please use ``force=True`` (or create a new
347 ``ConfigDiscovery`` instance).
348 """
349 if force is False and (self._called or self._disabled):
350 # Avoid overhead of multiple calls
351 return
352
353 self._analyse_package_layout(ignore_ext_modules)
354 if name:
355 self.analyse_name() # depends on ``packages`` and ``py_modules``
356
357 self._called = True
358
359 def _explicitly_specified(self, ignore_ext_modules: bool) -> bool:
360 """``True`` if the user has specified some form of package/module listing"""
361 ignore_ext_modules = ignore_ext_modules or self._skip_ext_modules
362 ext_modules = not (self.dist.ext_modules is None or ignore_ext_modules)
363 return (
364 self.dist.packages is not None
365 or self.dist.py_modules is not None
366 or ext_modules
367 or hasattr(self.dist, "configuration")
368 and self.dist.configuration
369 # ^ Some projects use numpy.distutils.misc_util.Configuration
370 )
371
372 def _analyse_package_layout(self, ignore_ext_modules: bool) -> bool:
373 if self._explicitly_specified(ignore_ext_modules):
374 # For backward compatibility, just try to find modules/packages
375 # when nothing is given
376 return True
377
378 log.debug(
379 "No `packages` or `py_modules` configuration, performing "
380 "automatic discovery."
381 )
382
383 return (
384 self._analyse_explicit_layout()
385 or self._analyse_src_layout()
386 # flat-layout is the trickiest for discovery so it should be last
387 or self._analyse_flat_layout()
388 )
389
390 def _analyse_explicit_layout(self) -> bool:
391 """The user can explicitly give a package layout via ``package_dir``"""
392 package_dir = self._package_dir.copy() # don't modify directly
393 package_dir.pop("", None) # This falls under the "src-layout" umbrella
394 root_dir = self._root_dir
395
396 if not package_dir:
397 return False
398
399 log.debug(f"`explicit-layout` detected -- analysing {package_dir}")
400 pkgs = chain_iter(
401 _find_packages_within(pkg, os.path.join(root_dir, parent_dir))
402 for pkg, parent_dir in package_dir.items()
403 )
404 self.dist.packages = list(pkgs)
405 log.debug(f"discovered packages -- {self.dist.packages}")
406 return True
407
408 def _analyse_src_layout(self) -> bool:
409 """Try to find all packages or modules under the ``src`` directory
410 (or anything pointed by ``package_dir[""]``).
411
412 The "src-layout" is relatively safe for automatic discovery.
413 We assume that everything within is meant to be included in the
414 distribution.
415
416 If ``package_dir[""]`` is not given, but the ``src`` directory exists,
417 this function will set ``package_dir[""] = "src"``.
418 """
419 package_dir = self._package_dir
420 src_dir = os.path.join(self._root_dir, package_dir.get("", "src"))
421 if not os.path.isdir(src_dir):
422 return False
423
424 log.debug(f"`src-layout` detected -- analysing {src_dir}")
425 package_dir.setdefault("", os.path.basename(src_dir))
426 self.dist.package_dir = package_dir # persist eventual modifications
427 self.dist.packages = PEP420PackageFinder.find(src_dir)
428 self.dist.py_modules = ModuleFinder.find(src_dir)
429 log.debug(f"discovered packages -- {self.dist.packages}")
430 log.debug(f"discovered py_modules -- {self.dist.py_modules}")
431 return True
432
433 def _analyse_flat_layout(self) -> bool:
434 """Try to find all packages and modules under the project root.
435
436 Since the ``flat-layout`` is more dangerous in terms of accidentally including
437 extra files/directories, this function is more conservative and will raise an
438 error if multiple packages or modules are found.
439
440 This assumes that multi-package dists are uncommon and refuse to support that
441 use case in order to be able to prevent unintended errors.
442 """
443 log.debug(f"`flat-layout` detected -- analysing {self._root_dir}")
444 return self._analyse_flat_packages() or self._analyse_flat_modules()
445
446 def _analyse_flat_packages(self) -> bool:
447 self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir)
448 top_level = remove_nested_packages(remove_stubs(self.dist.packages))
449 log.debug(f"discovered packages -- {self.dist.packages}")
450 self._ensure_no_accidental_inclusion(top_level, "packages")
451 return bool(top_level)
452
453 def _analyse_flat_modules(self) -> bool:
454 self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir)
455 log.debug(f"discovered py_modules -- {self.dist.py_modules}")
456 self._ensure_no_accidental_inclusion(self.dist.py_modules, "modules")
457 return bool(self.dist.py_modules)
458
459 def _ensure_no_accidental_inclusion(self, detected: list[str], kind: str):
460 if len(detected) > 1:
461 from inspect import cleandoc
462
463 from setuptools.errors import PackageDiscoveryError
464
465 msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.
466
467 To avoid accidental inclusion of unwanted files or directories,
468 setuptools will not proceed with this build.
469
470 If you are trying to create a single distribution with multiple {kind}
471 on purpose, you should not rely on automatic discovery.
472 Instead, consider the following options:
473
474 1. set up custom discovery (`find` directive with `include` or `exclude`)
475 2. use a `src-layout`
476 3. explicitly set `py_modules` or `packages` with a list of names
477
478 To find more information, look for "package discovery" on setuptools docs.
479 """
480 raise PackageDiscoveryError(cleandoc(msg))
481
482 def analyse_name(self):
483 """The packages/modules are the essential contribution of the author.
484 Therefore the name of the distribution can be derived from them.
485 """
486 if self.dist.metadata.name or self.dist.name:
487 # get_name() is not reliable (can return "UNKNOWN")
488 return
489
490 log.debug("No `name` configuration, performing automatic discovery")
491
492 name = (
493 self._find_name_single_package_or_module()
494 or self._find_name_from_packages()
495 )
496 if name:
497 self.dist.metadata.name = name
498
499 def _find_name_single_package_or_module(self) -> str | None:
500 """Exactly one module or package"""
501 for field in ('packages', 'py_modules'):
502 items = getattr(self.dist, field, None) or []
503 if items and len(items) == 1:
504 log.debug(f"Single module/package detected, name: {items[0]}")
505 return items[0]
506
507 return None
508
509 def _find_name_from_packages(self) -> str | None:
510 """Try to find the root package that is not a PEP 420 namespace"""
511 if not self.dist.packages:
512 return None
513
514 packages = remove_stubs(sorted(self.dist.packages, key=len))
515 package_dir = self.dist.package_dir or {}
516
517 parent_pkg = find_parent_package(packages, package_dir, self._root_dir)
518 if parent_pkg:
519 log.debug(f"Common parent package detected, name: {parent_pkg}")
520 return parent_pkg
521
522 log.warn("No parent package detected, impossible to derive `name`")
523 return None
524
525
526 def remove_nested_packages(packages: list[str]) -> list[str]:
527 """Remove nested packages from a list of packages.
528
529 >>> remove_nested_packages(["a", "a.b1", "a.b2", "a.b1.c1"])
530 ['a']
531 >>> remove_nested_packages(["a", "b", "c.d", "c.d.e.f", "g.h", "a.a1"])
532 ['a', 'b', 'c.d', 'g.h']
533 """
534 pkgs = sorted(packages, key=len)
535 top_level = pkgs[:]
536 size = len(pkgs)
537 for i, name in enumerate(reversed(pkgs)):
538 if any(name.startswith(f"{other}.") for other in top_level):
539 top_level.pop(size - i - 1)
540
541 return top_level
542
543
544 def remove_stubs(packages: list[str]) -> list[str]:
545 """Remove type stubs (:pep:`561`) from a list of packages.
546
547 >>> remove_stubs(["a", "a.b", "a-stubs", "a-stubs.b.c", "b", "c-stubs"])
548 ['a', 'a.b', 'b']
549 """
550 return [pkg for pkg in packages if not pkg.split(".")[0].endswith("-stubs")]
551
552
553 def find_parent_package(
554 packages: list[str], package_dir: Mapping[str, str], root_dir: StrPath
555 ) -> str | None:
556 """Find the parent package that is not a namespace."""
557 packages = sorted(packages, key=len)
558 common_ancestors = []
559 for i, name in enumerate(packages):
560 if not all(n.startswith(f"{name}.") for n in packages[i + 1 :]):
561 # Since packages are sorted by length, this condition is able
562 # to find a list of all common ancestors.
563 # When there is divergence (e.g. multiple root packages)
564 # the list will be empty
565 break
566 common_ancestors.append(name)
567
568 for name in common_ancestors:
569 pkg_path = find_package_path(name, package_dir, root_dir)
570 init = os.path.join(pkg_path, "__init__.py")
571 if os.path.isfile(init):
572 return name
573
574 return None
575
576
577 def find_package_path(
578 name: str, package_dir: Mapping[str, str], root_dir: StrPath
579 ) -> str:
580 """Given a package name, return the path where it should be found on
581 disk, considering the ``package_dir`` option.
582
583 >>> path = find_package_path("my.pkg", {"": "root/is/nested"}, ".")
584 >>> path.replace(os.sep, "/")
585 './root/is/nested/my/pkg'
586
587 >>> path = find_package_path("my.pkg", {"my": "root/is/nested"}, ".")
588 >>> path.replace(os.sep, "/")
589 './root/is/nested/pkg'
590
591 >>> path = find_package_path("my.pkg", {"my.pkg": "root/is/nested"}, ".")
592 >>> path.replace(os.sep, "/")
593 './root/is/nested'
594
595 >>> path = find_package_path("other.pkg", {"my.pkg": "root/is/nested"}, ".")
596 >>> path.replace(os.sep, "/")
597 './other/pkg'
598 """
599 parts = name.split(".")
600 for i in range(len(parts), 0, -1):
601 # Look backwards, the most specific package_dir first
602 partial_name = ".".join(parts[:i])
603 if partial_name in package_dir:
604 parent = package_dir[partial_name]
605 return os.path.join(root_dir, parent, *parts[i:])
606
607 parent = package_dir.get("") or ""
608 return os.path.join(root_dir, *parent.split("/"), *parts)
609
610
611 def construct_package_dir(packages: list[str], package_path: StrPath) -> dict[str, str]:
612 parent_pkgs = remove_nested_packages(packages)
613 prefix = Path(package_path).parts
614 return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}