[docs]  1# @Test that line numbers stay exact far past line 256, TEST_ANA_2, test, [IMPL_LNK_1, IMPL_ONE_1, IMPL_MLN_1, IMPL_JSONC_2]
  2"""Line numbers far down a long source file, with the analysis in a subprocess.
  3
  4tree-sitter 0.26.0's ``Point.row`` and ``Point.column`` return a borrowed reference
  5(py-tree-sitter#472, fixed by #466). Ints up to 256 are CPython's cached small ints, so a
  6short file never notices; a row above that is freed early and the heap is corrupted. The
  7analysis then dies with SIGSEGV or reports a wrong line, and every fixture elsewhere in
  8this suite is too short to reach it. This module is what makes the suite see it: it is the
  9reason ``tree-sitter`` excludes 0.26.0 in ``pyproject.toml``.
 10
 11The analysis runs in a child interpreter so that a native crash is a clean failure of one
 12test, naming the exit code, rather than the death of the whole pytest process.
 13
 14Two paths read tree-sitter points, and each case below reaches one or both of them:
 15``analyse.py`` reads ``start_point.row`` for every one-line need, need-id reference and
 16multi-line need, whatever the language; ``utils.find_prev_sibling_on_same_row`` reads
 17``start_point.row`` and ``end_point.row`` to tie an inline comment to its structure, and
 18only YAML and JSONC reach it.
 19"""
 20
 21from __future__ import annotations
 22
 23import json
 24import signal
 25import subprocess
 26import sys
 27from dataclasses import dataclass
 28from pathlib import Path
 29
 30import pytest
 31
 32# Well past the small-int cache (-5..256) at both ends of every marker: the first marker
 33# sits at row FIRST_ROW, and the file runs to a few thousand lines so that there are
 34# hundreds of them, each of whose rows is read more than once.
 35FIRST_ROW = 300
 36NUM_BLOCKS = 300
 37UPSTREAM = (
 38    "tree-sitter's Point getters return a borrowed reference in 0.26.0 "
 39    "(https://github.com/tree-sitter/py-tree-sitter/issues/472, "
 40    "fixed by https://github.com/tree-sitter/py-tree-sitter/pull/466)"
 41)
 42
 43# The child: one SourceAnalyse over one file, every extractor on, the result written as
 44# JSON to a file rather than stdout so that nothing the analysis logs can corrupt it.
 45CHILD = """
 46import json
 47import sys
 48from pathlib import Path
 49
 50from sphinx_codelinks.analyse.analyse import SourceAnalyse
 51from sphinx_codelinks.config import SourceAnalyseConfig
 52from sphinx_codelinks.source_discover.config import CommentType
 53
 54src, comment_type, out = Path(sys.argv[1]), CommentType(sys.argv[2]), Path(sys.argv[3])
 55analyse = SourceAnalyse(
 56    SourceAnalyseConfig(
 57        src_files=[src],
 58        src_dir=src.parent,
 59        get_need_id_refs=True,
 60        get_oneline_needs=True,
 61        get_multiline_needs=True,
 62        comment_type=comment_type,
 63    )
 64)
 65analyse.git_remote_url = None
 66analyse.git_commit_rev = None
 67analyse.run()
 68out.write_text(json.dumps([m.to_dict() for m in analyse.all_marked_content]))
 69"""
 70
 71
 72@dataclass(frozen=True)
 73class Marker:
 74    """One marker the analysis must report, as ``(kind, 0-based row, key, scope)``."""
 75
 76    kind: str
 77    row: int
 78    key: str
 79    scope: str | None = None
 80
 81
 82def _pad(lines: list[str], filler: str) -> None:
 83    """Pad ``lines`` with ``filler`` (formatted with the 0-based row) up to FIRST_ROW."""
 84    while len(lines) < FIRST_ROW:
 85        lines.append(filler.format(n=len(lines)))
 86
 87
 88def _cpp_source() -> tuple[str, list[Marker]]:
 89    """A C++ file with a one-line need, a need-id reference and a multi-line need per
 90    stanza, some on the first line of their comment and some further down it."""
 91    lines: list[str] = []
 92    expected: list[Marker] = []
 93    _pad(lines, "int filler_{n} = {n};")
 94    for i in range(NUM_BLOCKS):
 95        row = len(lines)
 96        lines.append(f"// @Title {i}, IMPL_CPP_{i}, impl")
 97        expected.append(Marker("need", row, f"IMPL_CPP_{i}"))
 98        lines.append(f"int value_{i} = {i};")
 99        # a block comment: the need-id reference is on its second line, so the row is
100        # the comment's start row plus an offset
101        lines.append("/* block")
102        expected.append(Marker("need-id-refs", len(lines), f"REQ_CPP_{i}"))
103        lines.append(f"   @need-ids: REQ_CPP_{i}")
104        lines.append("*/")
105        # a multi-line need, reported at the row of its open line
106        lines.append("/*")
107        expected.append(Marker("multiline-need", len(lines), f"MLN_CPP_{i}"))
108        lines.append(f"@need impl: Block {i}")
109        lines.append(f":id: MLN_CPP_{i}")
110        lines.append("@endneed")
111        lines.append("*/")
112        lines.append(f"void func_{i}() {{}}")
113    return "\n".join(lines) + "\n", expected
114
115
116def _yaml_source() -> tuple[str, list[Marker]]:
117    """A YAML mapping with an inline marker on every other key, and a marker on a line of
118    its own between them, so that both the same-row match and the early exit of the
119    sibling walk run."""
120    lines: list[str] = []
121    expected: list[Marker] = []
122    _pad(lines, "filler_{n}: {n}")
123    for i in range(NUM_BLOCKS):
124        row = len(lines)
125        lines.append(f"key_{i}: value_{i}  # @Title {i}, IMPL_YAML_{i}, impl")
126        expected.append(Marker("need", row, f"IMPL_YAML_{i}", f"key_{i}: value_{i}"))
127        expected.append(
128            Marker("need", row + 1, f"IMPL_YAML_OWN_{i}", f"other_{i}: {i}")
129        )
130        lines.append(f"# @Own line {i}, IMPL_YAML_OWN_{i}, impl")
131        lines.append(f"other_{i}: {i}")
132    return "\n".join(lines) + "\n", expected
133
134
135def _jsonc_source() -> tuple[str, list[Marker]]:
136    """A JSONC object with an inline marker after every other pair, and a marker on a
137    line of its own between them."""
138    lines: list[str] = ["{"]
139    expected: list[Marker] = []
140    _pad(lines, '  "filler_{n}": {n},')
141    for i in range(NUM_BLOCKS):
142        row = len(lines)
143        lines.append(f'  "key_{i}": {i}, // @Title {i}, IMPL_JSONC_{i}, impl')
144        expected.append(Marker("need", row, f"IMPL_JSONC_{i}", f'"key_{i}": {i}'))
145        expected.append(
146            Marker("need", row + 1, f"IMPL_JSONC_OWN_{i}", f'"other_{i}": {i}')
147        )
148        lines.append(f"  // @Own line {i}, IMPL_JSONC_OWN_{i}, impl")
149        lines.append(f'  "other_{i}": {i},')
150    lines.append('  "last": 0')
151    lines.append("}")
152    return "\n".join(lines) + "\n", expected
153
154
155def _describe_exit(returncode: int) -> str:
156    """Name a child's exit: a negative code is the signal that killed it (POSIX)."""
157    if returncode < 0:
158        try:
159            return f"exit code {returncode} ({signal.Signals(-returncode).name})"
160        except ValueError:
161            pass
162    return f"exit code {returncode} ({returncode & 0xFFFFFFFF:#010x})"
163
164
165def _reported(record: dict, with_scope: bool) -> Marker:
166    """The Marker a ``to_dict()`` record from the child describes; the scope only when
167    ``with_scope``, since it is tree-sitter point arithmetic only for YAML and JSONC."""
168    start, end = record["source_map"]["start"], record["source_map"]["end"]
169    kind = record["type"]
170    if kind == "multiline-need":
171        # open line, one option line, close line
172        assert end["row"] == start["row"] + 2, record
173        assert record["source"]["start"]["line"] == start["row"] + 1, record
174        key = record["need"]["id"]
175    elif kind == "need":
176        assert start["row"] == end["row"], record
177        key = record["need"]["id"]
178    else:
179        assert start["row"] == end["row"], record
180        (key,) = record["need_ids"]
181    scope = record["tagged_scope"] if with_scope else None
182    return Marker(kind, start["row"], key, scope)
183
184
185@pytest.mark.parametrize(
186    ("comment_type", "suffix", "make_source"),
187    [
188        pytest.param("cpp", "cpp", _cpp_source, id="cpp"),
189        pytest.param("yaml", "yaml", _yaml_source, id="yaml"),
190        pytest.param("jsonc", "jsonc", _jsonc_source, id="jsonc"),
191    ],
192)
193def test_line_numbers_past_row_256(
194    tmp_path: Path, comment_type: str, suffix: str, make_source
195) -> None:
196    text, expected = make_source()
197    assert min(marker.row for marker in expected) > 256
198    source = tmp_path / f"long.{suffix}"
199    source.write_text(text, encoding="utf-8")
200    out = tmp_path / "marked_content.json"
201
202    result = subprocess.run(
203        [sys.executable, "-c", CHILD, str(source), comment_type, str(out)],
204        capture_output=True,
205        text=True,
206        timeout=120,
207        check=False,
208    )
209
210    if result.returncode != 0:
211        pytest.fail(
212            f"the {comment_type} analysis of a {len(text.splitlines())}-line file "
213            f"crashed with {_describe_exit(result.returncode)}; the likeliest cause is "
214            f"that {UPSTREAM}.\nstderr (tail):\n{result.stderr[-2000:]}"
215        )
216    with_scope = comment_type != "cpp"
217    reported = [_reported(record, with_scope) for record in json.loads(out.read_text())]
218    wrong = sorted(set(reported) ^ set(expected), key=lambda m: (m.row, m.key))
219    assert len(reported) == len(expected) and not wrong, (
220        f"the {comment_type} analysis reported {len(reported)} markers for "
221        f"{len(expected)} expected; the ones that differ (reported XOR expected, first "
222        f"10): {wrong[:10]}. If the rows are wrong, the likeliest cause is that {UPSTREAM}."
223    )