1import os
2from pathlib import Path
3
4from ignore import WalkBuilder
5from ignore.overrides import OverrideBuilder
6
7from sphinx_codelinks.logger import get_logger
8from sphinx_codelinks.source_discover.config import (
9 COMMENT_FILETYPE,
10 CommentType,
11 SourceDiscoverConfig,
12)
13
14logger = get_logger(__name__)
15
16
17def lies_within(path: Path, directory: Path) -> bool:
18 """Whether ``path`` lies below ``directory``, both resolved.
19
20 Strings, not ``Path.is_relative_to``, which costs as much as the walk itself over
21 thousands of files. ``normcase`` folds case on Windows; on a case-insensitive POSIX
22 file system (APFS) a differently-spelled ancestor still counts as outside --
23 harmless, the file is traced under its own spelling.
24 """
25 prefix = os.path.join(os.path.normcase(str(directory)), "")
26 return os.path.normcase(str(path)).startswith(prefix)
27
28
29def warn_outside_src_dir(walked: Path, resolved: Path, src_dir: Path) -> None:
30 """Say that ``walked`` -- the path discovery reached, the link -- is not traced:
31 it resolves to outside ``src_dir``. POSIX on every platform.
32
33 The walked path leads the message rather than being its location: the CLI prints
34 the message alone, and Sphinx reads a location without a ``:`` as a document
35 name (``<path>.rst``) -- while a Windows path has one. It is normalised for the
36 message only (``src/../x.cpp`` reads ``x.cpp``).
37 """
38 shown = Path(os.path.normpath(walked)).as_posix()
39 logger.warning(
40 f"{shown} resolves to {resolved.as_posix()}, outside src_dir "
41 f"{src_dir.as_posix()}: not traced (widen src_dir to cover it, or exclude "
42 "the link)",
43 subtype="outside_src_dir",
44 )
45
46
47def _json_starts_with_comment(filepath: Path, sample_size: int = 256) -> bool:
48 """Return True if a ``.json`` file's first non-whitespace content is a comment.
49
50 Used to decide whether a ``.json`` file should be treated as JSONC. Per
51 https://jsonc.org/#filename-extension a ``.json`` file should only be treated as
52 JSONC when it opens with a comment (e.g. the mode line ``// -*- mode: jsonc -*-``).
53 """
54 try:
55 with filepath.open("rb") as f:
56 chunk = f.read(sample_size)
57 except OSError:
58 return False
59 # strip a leading UTF-8 BOM, then leading whitespace
60 text = chunk.removeprefix(b"\xef\xbb\xbf").lstrip()
61 return text.startswith((b"//", b"/*"))
62
63
[docs] 64# @Source code file discovery with gitignore support, IMPL_DISC_1, impl, [FE_DISCOVERY, FE_CLI_DISCOVER]
65class SourceDiscover:
66 """The source files below ``src_discover_config.src_dir``, as ``source_paths``.
67
68 A file that resolves to outside ``boundary`` -- a symbolic link out of the tree, or
69 a file below a followed directory link -- is not listed, and a warning names
70 its walked path (``codelinks.outside_src_dir``) unless ``warn`` is false: every
71 record and copy is relative to the source directory, which cannot hold it.
72
73 :param src_discover_config: What to walk, and how.
74 :param boundary: The directory every listed file lies below; the walked directory
75 when ``None``. A ``src-trace`` scope passes its project's ``src_dir``, which
76 its ``:directory:`` lies below.
77 :param warn: Whether to warn about a file outside ``boundary``.
78 """
79
80 def __init__(
81 self,
82 src_discover_config: SourceDiscoverConfig,
83 *,
84 boundary: Path | None = None,
85 warn: bool = True,
86 ):
87 self.src_discover_config = src_discover_config
88 self.boundary = boundary
89 self.warn = warn
90 # normalize the file types to lower case with leading dot
91 self.file_types = {
92 f".{ext}" for ext in COMMENT_FILETYPE[src_discover_config.comment_type]
93 }
94
95 self.source_paths = self._discover()
96
97 def _build_overrides(self) -> OverrideBuilder | None:
98 """Build an OverrideBuilder for include/exclude patterns.
99
100 Include patterns are added as whitelist globs.
101 Exclude patterns are added as negated globs (prefixed with ``!``).
102 """
103 include = self.src_discover_config.include
104 # ``SourceDiscoverConfig.__post_init__`` always resolves ``exclude``
105 # to a concrete list (never leaves it ``None``); the ``| None`` on
106 # the field itself only exists to detect "not explicitly set".
107 exclude = self.src_discover_config.exclude
108
109 if not include and not exclude:
110 return None
111
112 ob = OverrideBuilder(self.src_discover_config.src_dir)
113
114 if include:
115 for pattern in include:
116 ob.add(pattern)
117
118 if exclude:
119 for pattern in exclude:
120 ob.add(f"!{pattern}")
121
122 return ob
123
124 def _discover(self) -> list[Path]:
125 """Discover source files recursively in the given directory."""
126 src_dir = self.src_discover_config.src_dir
127 if not src_dir.is_dir():
128 return []
129
130 gitignore = self.src_discover_config.gitignore
131
132 builder = WalkBuilder(src_dir)
133 # Replicate the Rust ignore crate's standard_filters(gitignore)
134 # followed by hidden(false), matching ubc_codelinks behaviour.
135 builder.ignore(gitignore)
136 builder.parents(gitignore)
137 builder.git_ignore(gitignore)
138 builder.git_global(gitignore)
139 builder.git_exclude(gitignore)
140 builder.hidden(False)
141 builder.follow_links(self.src_discover_config.follow_links)
142
143 override_builder = self._build_overrides()
144 if override_builder is not None:
145 builder.overrides(override_builder.build())
146
147 boundary = (self.boundary or src_dir).resolve()
148 discovered_files = []
149 for entry in builder.build():
150 filepath = entry.path()
151 if not filepath.is_file():
152 continue
153 if self.file_types and filepath.suffix.lower() not in self.file_types:
154 continue
[docs]155 # @JSONC .json files require a leading comment, IMPL_JSONC_3, impl, [FE_JSONC]
156 # A plain ``.json`` file is only treated as JSONC when it opens with a
157 # comment; otherwise it is skipped under the ``jsonc`` comment type.
158 if (
159 self.src_discover_config.comment_type == CommentType.jsonc
160 and filepath.suffix.lower() == ".json"
161 and not _json_starts_with_comment(filepath)
162 ):
163 continue
164 # resolve() produces canonical absolute paths; follow_links only
165 # controls whether the walker descends into symlinked directories
166 resolved = filepath.resolve()
167 if not lies_within(resolved, boundary):
168 # a link out of the tree: no root-relative path, no copy (#2062)
169 if self.warn:
170 warn_outside_src_dir(filepath, resolved, boundary)
171 continue
172 discovered_files.append(resolved)
173
174 # a file reached through a symbolic link inside the tree resolves to a path
175 # already found: list each file once, or every caller analyses it once per
176 # path to it
177 sorted_filepaths = sorted(
178 dict.fromkeys(discovered_files),
179 key=lambda x: os.path.normcase(os.path.normpath(x)),
180 )
181 return sorted_filepaths
Back to top