diff --git a/CHANGELOG.md b/CHANGELOG.md index 076d602c8..bbe3f85b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.16 (2026-07-15) + +- Add: R (`.r`/`.R`) support — bespoke `extract_r` handles all five function-assignment forms (`<-`, `<<-`, `=`, `->`, `->>`), nested function scoping beneath the enclosing function, top-level vs function-level call attribution, member calls (`$`/`@`), package-qualified calls (`::`/`:::`), and pipes (`|>`/`%>%`). Named functions are assignments whose right- or left-hand side is an anonymous `function_definition`, so the generic extractor is not a good fit; the new bespoke extractor lives under `graphify/extractors/`, the established home for language-specific extractors (not the `extract.py` god node). `library()`/`require()`/`requireNamespace()` with a literal identifier or string emit `imports` edges; `pkg::fn`/`pkg:::fn` double as import evidence for `pkg`. `source("helper.r")` with a static `.r` path resolved against the caller file's directory emits `imports_from` when the target exists on disk (URLs, connections, variables, computed paths, and missing files are skipped — mirroring the Bash static-`source` policy). Unqualified bare calls funnel through the shared cross-file `raw_calls` resolver; an R language-family guard blocks binding to non-R definitions, so a cross-file candidate resolves `EXTRACTED` with `source()` import evidence or `INFERRED` without it, and ambiguous names remain unresolved. Ships under the optional `[r]` extra via `tree-sitter-language-pack` (the pack may fetch/cache the R grammar on first use; no source is uploaded and R itself is not executed). Install: `uv tool install "graphifyy[r]"`. R package metadata (`DESCRIPTION`/`NAMESPACE`/exports) and package-qualified call resolution are deferred to a follow-up (#9). +- Fix: `.R` files no longer trigger the "no AST extractor for `.r`/`.R`" warning (#1689) now that an `extract_r` dispatch exists; the missing-dependency path reuses the existing `#1745` install-extra warning with `pip install "graphifyy[r]"`. `Rscript`-shebang scripts (already detected as code) now dispatch to `extract_r` instead of falling through to the no-extractor warning. + ## 0.9.15 (2026-07-13) - Fix: detection now honors nested `.gitignore`/`.graphifyignore` files below the scan root, not just those at the scan root and above (#1847, thanks @Mohak-Agrawal). git applies a `.gitignore` to everything under its own directory, but graphify only loaded ignore files from the VCS-root-down-to-scan-root chain — so a `vendor/sub/.gitignore` deeper in the tree was never read and its exclusions leaked into the graph. Each directory's own ignore files are now read during the walk and anchored to that directory, preserving last-match-wins precedence (nearer files win, including over `.git/info/exclude`) and the parent-exclusion rule. diff --git a/README.md b/README.md index 0a883e75b..64765dbcd 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | +| `r` | R `.r`/`.R` AST extraction (functions, calls, imports, static `source()`); ships via `tree-sitter-language-pack` which may fetch/cache the R grammar on first use (no source uploaded, R not executed) | `uv tool install "graphifyy[r]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -327,7 +328,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .r .R .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.r`/`.R` requires `uv tool install "graphifyy[r]"`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | diff --git a/graphify/extract.py b/graphify/extract.py index a4ac6cbbd..6b4334275 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -138,6 +138,7 @@ from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 from graphify.extractors.julia import extract_julia # noqa: E402,F401 +from graphify.extractors.r import extract_r # noqa: E402,F401 _RECURSION_LIMIT = 10_000 @@ -1786,6 +1787,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: ".zig": "zig", ".ex": "elixir", ".exs": "elixir", ".jl": "julia", + ".r": "r", ".dart": "dart", ".sh": "shell", ".bash": "shell", ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell", @@ -3832,6 +3834,7 @@ def add_existing_edge(edge: dict) -> None: ".m": extract_objc, ".mm": extract_objc, ".jl": extract_julia, + ".r": extract_r, ".f": extract_fortran, ".F": extract_fortran, ".f90": extract_fortran, @@ -3897,6 +3900,7 @@ def add_existing_edge(edge: dict) -> None: ".hcl": "terraform", ".dm": "dm", ".dme": "dm", + ".r": "r", } @@ -3905,7 +3909,7 @@ def add_existing_edge(edge: dict) -> None: # routes them to the CODE path via _shebang_interpreter; _get_extractor must # honor the same signal or these files are classified as code and then silently # dropped by extraction. Only interpreters with a real extractor are mapped — -# detect's wider set (perl, fish, tcsh, Rscript) stays unmapped and skipped. +# detect's wider set (perl, fish, tcsh) stays unmapped and skipped. _SHEBANG_DISPATCH: dict[str, Any] = { "python": extract_python, "python2": extract_python, @@ -3921,6 +3925,7 @@ def add_existing_edge(edge: dict) -> None: "lua": extract_lua, "php": extract_php, "julia": extract_julia, + "Rscript": extract_r, } @@ -4357,7 +4362,7 @@ def extract( ) # #1689: a file counted as code (extension in CODE_EXTENSIONS) but with no AST - # extractor wired up (e.g. .r/.R — there is no tree-sitter-r dispatch) silently + # extractor wired up (e.g. .ejs) silently # contributes zero nodes. The #1666 warning above deliberately skips these (it # only fires when an extractor exists), so surface them explicitly, grouped by # extension, rather than reporting success as if the language were mapped. diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..ffe6b5f09 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -23,6 +23,7 @@ from graphify.extractors.markdown import extract_markdown from graphify.extractors.objc import extract_objc from graphify.extractors.pascal import extract_pascal +from graphify.extractors.r import extract_r from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest from graphify.extractors.razor import extract_razor @@ -52,6 +53,7 @@ "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, + "r": extract_r, "powershell": extract_powershell, "powershell_manifest": extract_powershell_manifest, "razor": extract_razor, diff --git a/graphify/extractors/r.py b/graphify/extractors/r.py new file mode 100644 index 000000000..55d8f03d4 --- /dev/null +++ b/graphify/extractors/r.py @@ -0,0 +1,281 @@ +"""R extractor. + +R's named functions are assignments whose left- or right-hand side is an +anonymous ``function_definition`` (``name <- function(...) ...`` and the +right-assignment forms ``function(...) -> name``), so a bespoke extractor is +required rather than the generic one. The contract mirrors the other +extractors: file/function nodes, ``contains``/``calls``/``imports`` edges and +``raw_calls`` for the shared cross-file bare-call resolver. + +Package metadata (DESCRIPTION/NAMESPACE/exports) is intentionally out of scope +for this initial integration; package-qualified ``pkg::fn`` calls are recorded +as member raw facts only. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id + + +_ASSIGN_OPS_LEFT = frozenset({"<-", "<<-", "="}) +_ASSIGN_OPS_RIGHT = frozenset({"->", "->>"}) +_PKG_LOADERS = frozenset({"library", "require", "requireNamespace"}) + + +def extract_r(path: Path) -> dict: + """Extract functions, calls, imports, and static source() from a .r/.R file.""" + try: + from tree_sitter_language_pack import get_language + from tree_sitter import Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-language-pack not installed"} + + try: + language = get_language("r") + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = _file_stem(path) + str_path = str(path) + del stem # current contract builds function ids from scope nids, not the file stem + + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, Any]] = [] + pkg_imports_seen: set[tuple[str, str]] = set() + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + if not src or not tgt or src == tgt: + return + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) + + file_nid = _make_id(str(path)) + add_node(file_nid, path.name, 1) + + def _op_of(binop) -> str: + for child in binop.children: + if child.type in _ASSIGN_OPS_LEFT or child.type in _ASSIGN_OPS_RIGHT: + return child.type + return "" + + def _parent_type(node) -> str: + parent = node.parent + return parent.type if parent is not None else "" + + def _named_children(node): + return [c for c in node.children if c.is_named] + + def _function_body(fn_def): + for c in _named_children(fn_def): + if c.type != "parameters": + return c + return None + + def walk(node, scope_nid: str) -> None: + if node.type == "binary_operator": + op = _op_of(node) + named = _named_children(node) + if op in _ASSIGN_OPS_LEFT and len(named) >= 2: + lhs, rhs = named[0], named[1] + in_call_arg = _parent_type(node) == "argument" + if (not in_call_arg and lhs.type == "identifier" + and rhs.type == "function_definition"): + name = _text(lhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + body = _function_body(rhs) + if body is not None: + function_bodies.append((func_nid, body)) + walk(rhs, func_nid) + return + elif op in _ASSIGN_OPS_RIGHT and len(named) >= 2: + lhs, rhs = named[0], named[1] + if rhs.type == "identifier" and lhs.type == "function_definition": + name = _text(rhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + body = _function_body(lhs) + if body is not None: + function_bodies.append((func_nid, body)) + walk(lhs, func_nid) + return + for c in node.children: + walk(c, scope_nid) + return + + if node.type == "function_definition": + parent_op = _op_of(node.parent) if node.parent is not None and node.parent.type == "binary_operator" else "" + if parent_op in _ASSIGN_OPS_LEFT or parent_op in _ASSIGN_OPS_RIGHT: + for c in node.children: + walk(c, scope_nid) + return + body = None + for c in _named_children(node): + if c.type not in ("parameters",): + body = c + break + if (body is not None and body.type == "binary_operator" + and _op_of(body) in _ASSIGN_OPS_RIGHT + and _named_children(body)[-1].type == "identifier"): + body_named = _named_children(body) + rhs = body_named[-1] + name = _text(rhs) + line = node.start_point[0] + 1 + func_nid = _make_id(scope_nid, name) + add_node(func_nid, f"{name}()", line) + add_edge(scope_nid, func_nid, "contains", line) + fbody = _function_body(node) + if fbody is not None: + function_bodies.append((func_nid, fbody)) + if len(body_named) >= 2: + walk(body_named[0], func_nid) + return + for c in node.children: + walk(c, scope_nid) + return + + for c in node.children: + walk(c, scope_nid) + + walk(root, file_nid) + + label_to_nid: dict[str, str] = {} + for n in nodes: + normalised = n["label"].strip("()").lstrip(".") + label_to_nid[normalised] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + raw_calls: list[dict] = [] + + def _first_string(node) -> str | None: + for c in node.children: + if c.type == "string": + inner = c.children[1] if len(c.children) > 1 else c + return _text(inner) + if c.type == "argument": + return _first_string(c) + return None + + def _first_identifier(node) -> str | None: + for c in node.children: + if c.type == "identifier": + return _text(c) + if c.type == "argument": + return _first_identifier(c) + return None + + def _emit_pkg_import(scope_nid: str, pkg: str, line: int) -> None: + key = (scope_nid, pkg) + if pkg and key not in pkg_imports_seen: + pkg_imports_seen.add(key) + add_edge(scope_nid, _make_id(pkg), "imports", line, context="import") + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "function_definition": + return + if node.type != "call": + for c in node.children: + walk_calls(c, caller_nid) + return + + line = node.start_point[0] + 1 + fn_node = None + arguments_node = None + for c in node.children: + if c.type in ("identifier", "namespace_operator", "extract_operator"): + fn_node = c + elif c.type == "arguments": + arguments_node = c + + if fn_node is not None and fn_node.type == "identifier": + callee = _text(fn_node) + if callee in _PKG_LOADERS: + if arguments_node is not None: + pkg = _first_identifier(arguments_node) or _first_string(arguments_node) + if pkg: + _emit_pkg_import(caller_nid, pkg, line) + for c in node.children: + walk_calls(c, caller_nid) + return + if callee == "source": + if arguments_node is not None: + raw = _first_string(arguments_node) + if raw and raw.lower().endswith(".r"): + if not raw.startswith(("http://", "https://", "ftp://")): + resolved = (path.parent / raw).resolve() + if resolved.exists(): + add_edge(file_nid, _make_id(str(resolved)), "imports_from", line, context="import") + for c in node.children: + walk_calls(c, caller_nid) + return + is_member_call = False + elif fn_node is not None and fn_node.type == "namespace_operator": + named = _named_children(fn_node) + pkg = _text(named[0]) if named else "" + callee = _text(named[-1]) if named else "" + is_member_call = True + if pkg: + _emit_pkg_import(caller_nid, pkg, line) + elif fn_node is not None and fn_node.type == "extract_operator": + named = _named_children(fn_node) + callee = _text(named[-1]) if named else "" + is_member_call = True + else: + callee = "" + is_member_call = False + + if callee and callee not in _LANGUAGE_BUILTIN_GLOBALS: + tgt_nid = label_to_nid.get(callee) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", line, + confidence="EXTRACTED", weight=1.0, context="call") + else: + raw_calls.append({ + "caller_nid": caller_nid, + "callee": callee, + "is_member_call": is_member_call, + "source_file": str_path, + "source_location": f"L{line}", + }) + + for c in node.children: + walk_calls(c, caller_nid) + + walk_calls(root, file_nid) + for caller_nid, body in function_bodies: + walk_calls(body, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} \ No newline at end of file diff --git a/graphify/serve.py b/graphify/serve.py index 33db0108f..f7218d76b 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -6,6 +6,7 @@ import sys from array import array from pathlib import Path +from typing import NamedTuple import networkx as nx from networkx.readwrite import json_graph from graphify.security import sanitize_label, check_graph_file_size_cap @@ -283,8 +284,63 @@ def _trigram_candidates(G: nx.Graph, needles: list[str], *, guard_frac: float = return [ids[i] for i in sorted(cand)] +class _QueryScores(NamedTuple): + """Per-query scoring result, returned by the private `_score_query` helper. + + `ranked` is the existing ordered `(score, node_id)` ranking produced by the + combined query scorer (the value `_score_nodes` always returned). When the + caller asks for it via `collect_per_term_seeds=True`, `best_seed_by_term` + additionally carries the winning node id for each normalized search token — + the seed `_pick_seeds` would have picked for that token via the now-retired + per-token `_score_nodes([token])` rescoring pass — computed in the *same* + per-node traversal so the query path makes exactly one graph scoring pass + regardless of query length. Empty when `collect_per_term_seeds=False`. + """ + ranked: list[tuple[float, str]] + best_seed_by_term: dict[str, str] + + def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: - scored = [] + """Combined query scorer returning the existing ranked `(score, node_id)` list. + + Backwards-compatible thin wrapper around `_score_query` for path, explain, + tests, and every other caller that only needs the combined ranking. The + per-term seed metadata computed by `_score_query` (when requested) is + discarded here so existing callers see no API or runtime-cost change. + """ + return _score_query(G, terms, collect_per_term_seeds=False).ranked + + +def _score_query( + G: nx.Graph, terms: list[str], *, collect_per_term_seeds: bool +) -> _QueryScores: + """Single-pass combined scorer that optionally also records the best seed + for each normalized query token. + + The combined ranking is byte-identical to what `_score_nodes` produced + before the refactor; `_score_nodes` is now a thin wrapper that asks for + `collect_per_term_seeds=False` and returns only `.ranked`. + + When `collect_per_term_seeds=True`, the per-token singleton winner is + computed alongside the combined score in the *same* per-node visit (it + reuses the same `norm_label` / `label_tokens` / `source` already evaluated + for the combined tier), so `_query_graph_text` can feed `best_seed_by_term` + straight into `_pick_seeds` and skip the T additional whole-graph rescoring + passes the old per-token `_score_nodes([token])` loop ran. + + Singleton-winner semantics match the legacy per-token path exactly. The + score itself mirrors `_score_nodes([token])` with `n_terms == 1` (so the + coverage term is 1 and the per-token tier is unscaled) plus the broader + joined-singlet tier (which also checks `label_tokens` and `nid_lower`). + Tie-break order is (1) highest singleton score, (2) highest graph degree, + (3) shortest displayed label, (4) lexicographically smallest node id — + exactly what `max(tied, key=degree)` over a sort by `(-score, label_len, + nid)` produced in the legacy `_pick_seeds` per-token loop. The combined + trigram candidate set (needles `norm_terms + [joined]`) is a superset of + each per-token `[t]` candidate set, so iterating combined candidates + discovers every non-zero singleton-score node for every term. + """ + scored: list[tuple[float, str]] = [] # Dedupe tokens, order-preserving (as _pick_seeds already does): a repeated # query word must not double-count every tier, and with coverage scaling # below it would also inflate the matched-term ratio (#1602). @@ -305,6 +361,16 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: G.nodes(data=True) if candidate_ids is None else ((nid, G.nodes[nid]) for nid in candidate_ids) ) + # Per-token best tracking, only when the caller (the query path) wants the + # seed metadata. The key tuple is the full multi-key tie-break + # (`(-singleton_score, -degree, label_len, nid)`), so `min` over the + # stored key mirrors the legacy `max(tied, key=degree)` over a + # (-score, label_len, nid)-sorted term_scored list. `None` is comparable + # as "smaller" than every tuple, so the first non-zero candidate seeds the + # entry without a separate `if t not in best_by_term` branch. + best_by_term: dict[str, tuple[tuple, str]] | None = ( + {} if collect_per_term_seeds else None + ) for nid, data in node_iter: norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() bare_label = norm_label.rstrip("()") @@ -315,6 +381,11 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # driver". label_tokens = " ".join(_search_tokens(data.get("label") or "")) source = (data.get("source_file") or "").lower() + # `nid_lower` is needed both by the full-query tier (`if joined`) and by + # the per-token singleton tier (joined-singlet exact-match check). When + # neither runs (`joined` empty AND not collecting seeds) skip the call; + # this preserves the single-query-time perf where nid_lower was lazy. + nid_lower = nid.lower() if (joined or collect_per_term_seeds) else "" score = 0.0 # Full-query tier: a multi-word query that equals (or prefixes) the whole # label must dominate the per-token bag-of-words sums below, so `path`/ @@ -323,7 +394,6 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # tier never fires, and every node sharing the token set ties -> arbitrary # node-id sort -> wrong/disconnected endpoint -> false "No path found". if joined: - nid_lower = nid.lower() if joined in (norm_label, bare_label, label_tokens, nid_lower): score += _EXACT_MATCH_BONUS * 10 * joined_w elif ( @@ -349,19 +419,55 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: tiered = 0.0 for t in norm_terms: w = idf.get(t, 1.0) - # Three-tier precedence: exact > prefix > substring (take the - # strongest tier per term so a single term cannot double-count). + # Per-tier contributions for this token, kept separate so the + # singleton tracking below can reuse them without re-evaluating + # the same predicates. Three-tier precedence: exact > prefix > + # substring (take the strongest tier per term so a single term + # cannot double-count). + tier_value = 0.0 + substr_value = 0.0 + source_value = 0.0 if t == norm_label or t == bare_label: - tiered += _EXACT_MATCH_BONUS * w + tier_value = _EXACT_MATCH_BONUS * w matched += 1 elif norm_label.startswith(t) or bare_label.startswith(t): - tiered += _PREFIX_MATCH_BONUS * w + tier_value = _PREFIX_MATCH_BONUS * w matched += 1 elif t in norm_label: - score += _SUBSTRING_MATCH_BONUS * w + substr_value = _SUBSTRING_MATCH_BONUS * w + score += substr_value matched += 1 if t in source: - score += _SOURCE_MATCH_BONUS * w + source_value = _SOURCE_MATCH_BONUS * w + score += source_value + tiered += tier_value + if collect_per_term_seeds and best_by_term is not None: + # Singleton score for [t] on this node, mirroring + # `_score_nodes(G, [t])` exactly (n_terms == 1, no coverage + # scaling). The joined-singlet tier is broader than the per- + # token tier: it also checks `label_tokens` and `nid_lower`, + # matching the legacy single-token `_score_nodes([t])` call + # (where `joined == t`). + if t in (norm_label, bare_label, label_tokens, nid_lower): + singleton = _EXACT_MATCH_BONUS * 10 * w + elif ( + norm_label.startswith(t) + or bare_label.startswith(t) + or label_tokens.startswith(t) + ): + singleton = _PREFIX_MATCH_BONUS * 10 * w + else: + singleton = 0.0 + singleton += tier_value + substr_value + source_value + if singleton > 0: + # Tie-break key mirrors the legacy sort+max(degree): + # (-singleton, -degree, label_len, nid) — the minimum + # tuple wins, exactly matching max(tied, key=degree) + # over (label_len asc, nid asc)-sorted ties. + key = (-singleton, -G.degree(nid), len(data.get("label") or nid), nid) + cur = best_by_term.get(t) + if cur is None or key < cur[0]: + best_by_term[t] = (key, nid) if tiered: score += tiered * (matched / n_terms) ** 2 if score > 0: @@ -369,7 +475,10 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) - return scored + best_seed_by_term: dict[str, str] = {} + if collect_per_term_seeds and best_by_term: + best_seed_by_term = {t: nid for t, (_key, nid) in best_by_term.items()} + return _QueryScores(ranked=scored, best_seed_by_term=best_seed_by_term) def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: str) -> str: @@ -402,7 +511,7 @@ def _pick_seeds( gap_ratio: float = 0.2, *, G: "nx.Graph | None" = None, - terms: list[str] | None = None, + best_seed_by_term: dict[str, str] | None = None, ) -> list[str]: """Select BFS seed nodes, stopping when score drops too far below the top. @@ -421,11 +530,12 @@ def _pick_seeds( seeds, so the BFS traversal only ever explores the neighborhood of the one unrelated exact match — see #1445. - When `G` and `terms` are supplied, this guarantees at least one seed per - distinct query term that has any match at all, so one term's incidental - collision cannot starve out the others. Ties within a term are broken by - graph degree (structural centrality), so an isolated incidental match - doesn't out-rank a real, well-connected hub for that term. + When `G` and `best_seed_by_term` are supplied, this guarantees at least one + seed per distinct query term that has any match at all, so one term's + incidental collision cannot starve out the others. The per-token winners + in `best_seed_by_term` are precomputed by `_score_query` (during the same + traversal that produced `scored`) so this function no longer rescores the + graph per term — see #1445 and the `_score_query` docstring. Coverage scaling in _score_nodes (#1602) now dampens a lone collision's exact tier on multi-term queries, which brings label-matching relevant @@ -464,15 +574,20 @@ def _seed_label_key(nid: str) -> str: seen_labels.add(key) seeds.append(nid) - if G is not None and terms: - norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) - for term in norm_terms: - term_scored = _score_nodes(G, [term]) - if not term_scored: - continue - best_score = term_scored[0][0] - tied = [nid for s, nid in term_scored if s == best_score] - best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + if G is not None and best_seed_by_term: + # Guarantee one seed per distinct query term that has any match at all, + # so an incidental exact match on one term cannot starve matches on + # other terms (#1445). Iterate tokens in a deterministic sorted order + # so seeds added by this loop have a stable order independent of dict + # iteration — preserving the legacy `_pick_seeds(terms=...)` behavior + # which iterated `sorted({tok ...})`. Per-token winners arrive + # precomputed in `best_seed_by_term` from `_score_query`'s single + # traversal, so `_pick_seeds` no longer rescoring the graph per term. + # The per-label dedup cap also gates these additions, so the guarantee + # cannot reintroduce a second copy of an already-seeded generic label + # (#1766). + for term in sorted(best_seed_by_term): + best_nid = best_seed_by_term[term] # Honor the same per-label cap so the per-term guarantee can't # reintroduce a second copy of an already-seeded generic label. key = _seed_label_key(best_nid) @@ -716,8 +831,14 @@ def _query_graph_text( context_filters: list[str] | None = None, ) -> str: terms = _query_terms(question) - scored = _score_nodes(G, terms) - start_nodes = _pick_seeds(scored, G=G, terms=terms) + # One graph scoring pass produces both the combined ranking (used to drive + # the gap-based seed selection below) and the per-token singleton winners + # (used by _pick_seeds' per-term guarantee). Previously this was T+1 passes + # — one combined + one per query token — re-walking the whole graph each + # time; on a 100k-node, three-term benchmark ~71% of scoring time was + # spent in those redundant per-term passes. + qs = _score_query(G, terms, collect_per_term_seeds=True) + start_nodes = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) diff --git a/pyproject.toml b/pyproject.toml index 1b2b818d1..34d276d8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,12 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +# No maintained tree-sitter-r wheel fits the repo's per-language wheel model, +# so the R grammar ships via tree-sitter-language-pack (lazy-loaded in +# extract_r). The pack may fetch/cache the R grammar on first use; no source +# is uploaded and R itself is never executed (#9). +r = ["tree-sitter-language-pack>=1.12,<2"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-language-pack>=1.12,<2"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/bench_query_scoring.py b/tests/bench_query_scoring.py new file mode 100644 index 000000000..375c9449b --- /dev/null +++ b/tests/bench_query_scoring.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Non-CI microbenchmark: single-pass scoring vs legacy per-term rescoring. + +Verifies the single-pass refactor eliminates the T+1 graph-scoring passes a +T-term query used to run, without changing the result. Reports median/min +latency for: + + * legacy — one combined `_score_nodes(G, terms)` call PLUS one + `_score_nodes(G, [token])` call per distinct query token + (the old `_pick_seeds(terms=...)` per-term guarantee loop). + * optimized — one `_score_query(G, terms, collect_per_term_seeds=True)` + call, with the per-token best computed inline in the same + traversal. `_pick_seeds` then consumes `best_seed_by_term` + directly — no rescoring. + +Equality is asserted (ranked, best_seed_by_term, seeds) before timing. The +optimized path's traversal count is invariant in the number of query terms. + +Run it manually; do NOT wire this into CI (wall-clock assertions are flaky): + + uv run python tests/bench_query_scoring.py \\ + --nodes 100000 --term-counts 3,10 --repeats 5 + + uv run python tests/bench_query_scoring.py \\ + --graph graphify-out/graph.json \\ + --query "what calls extract" --query "symbol resolution" \\ + --repeats 10 +""" +from __future__ import annotations + +import argparse +import json +import random +import statistics +import sys +import time +from pathlib import Path + +import networkx as nx + +from graphify.serve import ( + _get_trigram_index, + _pick_seeds, + _query_terms, + _score_nodes, + _score_query, + _search_tokens, +) + + +SYLLABLES = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", +] + +QUERIES_BY_TERM_COUNT: dict[int, list[str]] = { + 1: ["foo"], + 2: ["foo", "bar"], + 3: ["router", "service", "handler"], + 5: ["get", "user", "run", "name", "path"], + 10: ["extract", "build", "report", "router", "config", + "service", "token", "rate", "limit", "widget"], +} + + +def _build_random_graph(n: int, *, seed: int) -> nx.DiGraph: + """Reproducible broad-match DiGraph: short constructed labels + edge noise. + + Labels draw from a small syllable pool so tokens collide across nodes, + forcing the trigram prefilter to be selective and exercising score ties + on common tokens. Edge noise provides degree variance so the legacy + tie-break (`max(tied, key=degree)`) is actually exercised against the + new `(-singleton, -degree, label_len, nid)` key tuple.""" + rng = random.Random(seed) + G: nx.DiGraph = nx.DiGraph() + for i in range(n): + label = "_".join(rng.sample(SYLLABLES, rng.randint(1, 3))) + G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + for _ in range(n * 2): + a, b = rng.randrange(n), rng.randrange(n) + if a != b: + G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") + return G + + +def _load_real_graph(path: str) -> nx.Graph: + """Light wrapper that just builds a NetworkX graph from a real + `graphify-out/graph.json`, skipping the size cap and work-memory overlay + that `serve._load_graph` enforces (the bench is read-only).""" + data = json.loads(Path(path).read_text(encoding="utf-8")) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data = {**data, "directed": True} + try: + return nx.readwrite.json_graph.node_link_graph(data, edges="links") + except TypeError: + return nx.readwrite.json_graph.node_link_graph(data) + + +def _legacy_score_and_pick(G, terms): + """Recreates the pre-refactor flow: combined scoring plus one + `_score_nodes([token])` call per distinct query token, with the legacy + tie-break (`max(tied, key=degree)` over a (-score, label_len, nid)-sorted + list) to derive `best_seed_by_term`. Returns the same triple as the + optimized path so they can be compared for equality.""" + ranked = _score_nodes(G, terms) + norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) + best_seed_by_term: dict[str, str] = {} + for term in norm_terms: + term_scored = _score_nodes(G, [term]) + if not term_scored: + continue + best_score = term_scored[0][0] + tied = [nid for s, nid in term_scored if s == best_score] + best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best_seed_by_term[term] = best_nid + seeds = _pick_seeds(ranked, G=G, best_seed_by_term=best_seed_by_term) + return ranked, best_seed_by_term, seeds + + +def _optimized_score_and_pick(G, terms): + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + return qs.ranked, qs.best_seed_by_term, seeds + + +def _warm_caches(G, terms): + """Pre-populate the trigram index and IDF cache so the first timed + iteration doesn't pay the amortized build cost on either path. Both + legacy and optimized share these caches via the graph object, so warming + once is fair to both.""" + _get_trigram_index(G) + # Touch the idf cache for the combined terms and every per-token singleton, + # matching exactly the calls the legacy path will make. + _score_nodes(G, terms) + for term in {tok for t in terms for tok in _search_tokens(t)}: + _score_nodes(G, [term]) + + +def _bench(fn, *, repeats: int) -> list[float]: + # One uncounted warm-up — `_warm_caches` already populated caches, but + # this also amortizes any per-call code-path setup unique to `fn`. + fn() + times: list[float] = [] + for _ in range(repeats): + t0 = time.perf_counter() + fn() + times.append(time.perf_counter() - t0) + return times + + +def _verify_equality(G, terms) -> tuple[int, int]: + leg_rank, leg_best, leg_seeds = _legacy_score_and_pick(G, terms) + opt_rank, opt_best, opt_seeds = _optimized_score_and_pick(G, terms) + assert leg_rank == opt_rank, ( + f"ranked diverged for terms={terms}: legacy[:5]={leg_rank[:5]} opt[:5]={opt_rank[:5]}" + ) + assert leg_best == opt_best, ( + f"best_seed_by_term diverged for terms={terms}: legacy={leg_best} opt={opt_best}" + ) + assert leg_seeds == opt_seeds, ( + f"seeds diverged for terms={terms}: legacy={leg_seeds} opt={opt_seeds}" + ) + return len(leg_rank), len(leg_seeds) + + +def _row(label: str, n_nodes: int, n_terms: int, times: list[float], + traversal_count: int, n_ranked: int, n_seeds: int) -> str: + med = statistics.median(times) * 1000 + mn = min(times) * 1000 + return (f"{label:<10} | n={n_nodes:<7} | terms={n_terms:<3} | " + f"median={med:7.2f}ms | min={mn:7.2f}ms | " + f"passes={traversal_count:<3} | ranked={n_ranked:<6} seeds={n_seeds}") + + +def _legacy_traversal_count(terms) -> int: + # 1 combined pass + one per-token singleton pass. + return 1 + len({tok for t in terms for tok in _search_tokens(t)}) + + +def _run_scenario(G, terms, *, repeats: int) -> tuple[float, float]: + _warm_caches(G, terms) + n_ranked, n_seeds = _verify_equality(G, terms) + + legacy_times = _bench(lambda: _legacy_score_and_pick(G, terms), repeats=repeats) + opt_times = _bench(lambda: _optimized_score_and_pick(G, terms), repeats=repeats) + + n_nodes = G.number_of_nodes() + n_terms = len(set(tok for t in terms for tok in _search_tokens(t))) + print(_row("legacy", n_nodes, n_terms, legacy_times, + _legacy_traversal_count(terms), n_ranked, n_seeds)) + print(_row("optimized", n_nodes, n_terms, opt_times, + 1, n_ranked, n_seeds)) + + med_legacy = statistics.median(legacy_times) + med_opt = statistics.median(opt_times) + speedup = med_legacy / med_opt if med_opt > 0 else float("inf") + print(f"speedup | median: {speedup:.2f}x | " + f"min: {min(legacy_times) / min(opt_times):.2f}x") + return med_legacy, med_opt + + +def _resolve_scenarios(args) -> list[list[str]]: + if args.graph: + # Real-graph mode: each --query is a natural-language sentence, tokenized + # using the same helper the production path uses. + sentences = args.query or ["what calls extract"] + scenarios = [_query_terms(s) for s in sentences] + # Dedupe identical token sets (multiple --query args may tokenize the same). + seen: list[list[str]] = [] + for q in scenarios: + if q not in seen: + seen.append(q) + return seen + term_counts = [int(s.strip()) for s in args.term_counts.split(",") if s.strip()] + scenarios: list[list[str]] = [] + for tc in term_counts: + if tc in QUERIES_BY_TERM_COUNT: + scenarios.append(QUERIES_BY_TERM_COUNT[tc]) + elif tc > 0: + scenarios.append([SYLLABLES[i % len(SYLLABLES)] for i in range(tc)]) + return scenarios + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser( + description="Microbenchmark single-pass query scoring vs legacy per-term rescoring.", + ) + p.add_argument("--nodes", type=int, default=100_000, + help="node count for the synthetic benchmark graph (default: 100000)") + p.add_argument("--seed", type=int, default=20260714, help="RNG seed for the synthetic graph") + p.add_argument("--term-counts", default="3,10", + help="comma-separated list of term counts to benchmark (synthetic mode)") + p.add_argument("--repeats", type=int, default=5, + help="timed iterations per scenario (after one warm-up)") + p.add_argument("--graph", default=None, + help="optional path to a real graphify-out/graph.json; overrides --nodes") + p.add_argument("--query", action="append", default=None, + help="natural-language query sentence (real-graph mode; repeat for multiple)") + args = p.parse_args(argv) + + if args.graph: + print(f"loading real graph: {args.graph} ...", file=sys.stderr) + t0 = time.perf_counter() + G = _load_real_graph(args.graph) + print(f" loaded in {time.perf_counter() - t0:.2f}s", file=sys.stderr) + else: + print(f"building synthetic graph: n={args.nodes} seed={args.seed} ...", + file=sys.stderr) + t0 = time.perf_counter() + G = _build_random_graph(args.nodes, seed=args.seed) + print(f" built in {time.perf_counter() - t0:.2f}s", file=sys.stderr) + + print() + print(f"graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") + scenarios = _resolve_scenarios(args) + print(f"scenarios: {len(scenarios)} | repeats per scenario: {args.repeats}") + print("-" * 110) + + summaries = [] + for terms in scenarios: + print() + med_legacy, med_opt = _run_scenario(G, terms, repeats=args.repeats) + summaries.append((terms, med_legacy, med_opt)) + + print() + print("-" * 110) + print("summary:") + for terms, med_legacy, med_opt in summaries: + speedup = med_legacy / med_opt if med_opt > 0 else float("inf") + print(f" terms={len(set(tok for t in terms for tok in _search_tokens(t))):>3} | " + f"median legacy={med_legacy*1000:>8.2f}ms | " + f"median optimized={med_opt*1000:>8.2f}ms | " + f"speedup={speedup:>5.2f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/fixtures/r_caller.r b/tests/fixtures/r_caller.r new file mode 100644 index 000000000..99d6ba043 --- /dev/null +++ b/tests/fixtures/r_caller.r @@ -0,0 +1,5 @@ +# R fixture: top-level bare calls with no own definition and no source() link, +# so `dup()` is ambiguous (defined in r_other.R and r_extra.R) and `helper()` +# resolves INFERRED to a single unique cross-file candidate. +dup() +helper() \ No newline at end of file diff --git a/tests/fixtures/r_extra.r b/tests/fixtures/r_extra.r new file mode 100644 index 000000000..72c3604ea --- /dev/null +++ b/tests/fixtures/r_extra.r @@ -0,0 +1,9 @@ +# R fixture: defines a second `dup` (ambiguity with r_other.R) and a no-source +# caller scope, so `helper()` resolves INFERRED unless source() linkage exists. +extra_fn <- function() { + helper() +} + +dup <- function() { + NA +} \ No newline at end of file diff --git a/tests/fixtures/r_other.r b/tests/fixtures/r_other.r new file mode 100644 index 000000000..dd8232809 --- /dev/null +++ b/tests/fixtures/r_other.r @@ -0,0 +1,8 @@ +# R fixture: helper + dup definitions for cross-file resolution tests. +helper <- function() { + invisible() +} + +dup <- function() { + NA +} \ No newline at end of file diff --git a/tests/fixtures/sample.r b/tests/fixtures/sample.r new file mode 100644 index 000000000..8c85741b1 --- /dev/null +++ b/tests/fixtures/sample.r @@ -0,0 +1,43 @@ +# Graphify R fixture: exercises every construct the extractor handles. +# Function assignment forms (left, super, equals, right, super-right): +library(dplyr) +requireNamespace("utils") +library(installed.packages()) # dynamic arg -> NO import edge +source("r_other.r") # static source() -> imports_from edge + +foo <- function(x) { # `<-` form + helper() # cross-file call (defined in r_other.R) + print(x) # builtin in raw_calls +} + +bar <<- function(y) y * 2 # `<<-` super-assignment form + +baz = function(z) { # `=` form + inner <- function(w) { # nested function + helper() # attributed to inner, not baz + z + w + } + inner(z) # same-file call -> EXTRACTED +} + +function(a) a + 1 -> qux # `->` right-assignment form +function(b) b * 2 ->> quux # `->>` super-right-assignment form + +# Member calls (do NOT resolve via the bare-name resolver): +obj$method() +obj@field() + +# Package-qualified calls (recorded as member raw_calls; pkg import edge emitted): +base::summary() +dplyr::filter() +utils:::deep_fn() + +# Pipes (traversed so contained calls are captured): +1 |> qux() |> foo() +"data" %>% process() %>% save() + +# Named call argument with `=` is NOT a function definition: +options(config = function() 42) + +# Top-level call (attributed to the file node): +foo(1) \ No newline at end of file diff --git a/tests/test_extract.py b/tests/test_extract.py index dfb28d6d1..5d5b5774e 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1937,18 +1937,19 @@ def test_case_insensitive_suffix_filtering(tmp_path): def test_extract_warns_on_code_files_with_no_ast_extractor(tmp_path, capsys): - # #1689: .r/.R is in CODE_EXTENSIONS (counted as code) but has no AST extractor, - # so R files silently contribute nothing. extract() must surface that instead of - # reporting success as if the language were mapped. - r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") - r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + # #1689: .ejs is in CODE_EXTENSIONS (counted as code) but has no AST extractor, + # so such files silently contribute nothing. extract() must surface that instead + # of reporting success as if the language were mapped. (.R was the original + # motivating case but now has extract_r; .ejs is the canonical no-extractor one.) + r1 = tmp_path / "view.ejs"; r1.write_text("

<%= name %>

\n") + r2 = tmp_path / "page.ejs"; r2.write_text("

Title

\n") py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") result = extract([r1, r2, py], cache_root=tmp_path) err = capsys.readouterr().err assert "no AST extractor" in err - assert ".r (2)" in err # both R files grouped under the lowercased ext + assert ".ejs (2)" in err assert "#1689" in err # the Python file still extracts normally labels = [n.get("label") for n in result["nodes"]] @@ -1992,6 +1993,38 @@ def test_extract_no_missing_dep_warning_when_sql_installed(tmp_path, capsys): assert "#1745" not in err +def test_extract_warns_when_r_extra_missing(tmp_path, capsys, monkeypatch): + # #1745: .r HAS a dispatch entry, so the #1689 warning can't fire, and + # extract_r returns an "error" result when tree-sitter-language-pack is + # absent, so the files must not vanish silently — extract() surfaces them + # with the [r] extra named. + monkeypatch.setitem(sys.modules, "tree_sitter_language_pack", None) + r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") + r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") + + result = extract([r1, r2, py], cache_root=tmp_path) + err = capsys.readouterr().err + + assert "2 .r file(s)" in err + assert "tree-sitter-language-pack not installed" in err + assert 'graphifyy[r]' in err + assert "#1745" in err + assert "#1689" not in err # .r/.R is now mapped; no #1689 no-extractor warning + # the Python file still extracts normally + labels = [n.get("label") for n in result["nodes"]] + assert any(str(l).startswith("main") for l in labels) + + +def test_extract_no_warning_when_r_extra_installed(tmp_path, capsys): + pytest.importorskip("tree_sitter_language_pack") + r = tmp_path / "module.R"; r.write_text("f <- function(x) x + 1\n") + extract([r], cache_root=tmp_path) + err = capsys.readouterr().err + assert "#1745" not in err + assert "no AST extractor" not in err + + def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsys): # #1693: intermediate progress lines count against uncached_work; the final # "100%" line must NOT switch to total_files (which includes cached hits and @@ -1999,8 +2032,8 @@ def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsy for i in range(100): (tmp_path / f"m{i}.py").write_text(f"def f{i}():\n return {i}\n") for i in range(5): - (tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n") # no extractor - paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r")) # total 105 + (tmp_path / f"s{i}.ejs").write_text(f"hello {i}\n") # no extractor + paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.ejs")) # total 105 extract(paths, cache_root=tmp_path, parallel=False) out = capsys.readouterr().out diff --git a/tests/test_languages.py b/tests/test_languages.py index f610fbc40..d9c31e430 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -9,7 +9,7 @@ extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, extract_powershell, extract_apex, extract_verilog, - extract_powershell_manifest, + extract_powershell_manifest, extract_r, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -22,6 +22,10 @@ _ilu.find_spec("tree_sitter_dm") is None, reason="tree-sitter-dm not installed (optional [dm] extra)", ) +_needs_r = pytest.mark.skipif( + _ilu.find_spec("tree_sitter_language_pack") is None, + reason="tree-sitter-language-pack not installed (optional [r] extra)", +) def _labels(r): @@ -2945,3 +2949,229 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + + +# --- R --------------------------------------------------------------- + +def _r_assert_no_dangling(r): + """R-aware variant of _assert_no_dangling: `imports` targets may be + external package ids (e.g. dplyr) with no matching node, so they're + allowed to dangle like other languages' external imports.""" + ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + assert e["source"] in ids, f"dangling source: {e}" + if e["relation"] not in ("imports", "imports_from"): + assert e["target"] in ids, f"dangling target: {e}" + + +def test_r_dispatch_lower(): + from graphify.extract import _get_extractor + assert _get_extractor(Path("foo.r")) is extract_r + + +def test_r_dispatch_upper_uses_case_insensitive_fallback(): + # .R is not in _DISPATCH directly; _get_extractor lowercases the suffix + # via the existing fallback (extract.py ~4019-4020). + from graphify.extract import _get_extractor + assert _get_extractor(Path("FOO.R")) is extract_r + + +def test_r_script_shebang_dispatch(tmp_path): + from graphify.extract import _get_extractor + p = tmp_path / "rscript_no_ext" + p.write_text("#!/usr/bin/env Rscript\nx <- 1\n") + assert _get_extractor(p) is extract_r + + +def test_r_missing_dependency_returns_error(tmp_path, monkeypatch): + import sys as _sys + monkeypatch.setitem(_sys.modules, "tree_sitter_language_pack", None) + p = tmp_path / "x.r"; p.write_text("f <- function() 1\n") + r = extract_r(p) + assert "error" in r and "not installed" in r["error"] + + +@_needs_r +def test_r_no_error(): + r = extract_r(FIXTURES / "sample.r") + assert "error" not in r + + +@_needs_r +def test_r_left_assignment_forms_create_functions(): + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "foo()" in labels # <- + assert "bar()" in labels # <<- + assert "baz()" in labels # = + + +@_needs_r +def test_r_right_assignment_forms_create_functions(): + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "qux()" in labels # -> + assert "quux()" in labels # ->> + + +@_needs_r +def test_r_named_call_argument_with_equals_not_a_function(): + # options(config = function() 42) — the `=` is a named call arg, not an + # assignment binding a function to a name; must NOT produce a `config()` node. + r = extract_r(FIXTURES / "sample.r") + labels = _labels(r) + assert "config()" not in labels + + +@_needs_r +def test_r_nested_functions_get_scoped_ids_and_contains(): + r = extract_r(FIXTURES / "sample.r") + baz = _node_by_label(r, "baz") + inner_nodes = _nodes_with_label(r, "inner()") + assert len(inner_nodes) == 1 + inner = inner_nodes[0] + assert baz["id"] != inner["id"] + contains = _edge_labels(r, "contains") + assert (baz["id"], inner["id"]) in contains or \ + (_normalize_symbol_label("baz()"), _normalize_symbol_label("inner()")) in \ + {(_normalize_symbol_label(a), _normalize_symbol_label(b)) for a, b in contains} + + +@_needs_r +def test_r_nested_call_not_attributed_to_outer_function(): + # inner()'s body calls helper(); this call must attribute to `inner`, not `baz` + r = extract_r(FIXTURES / "sample.r") + baz_id = _node_by_label(r, "baz")["id"] + inner_id = _node_by_label(r, "inner")["id"] + # find a raw_call to helper whose caller is inner (not baz) + helper_rcs = [rc for rc in r["raw_calls"] if rc["callee"] == "helper"] + assert any(rc["caller_nid"] == inner_id for rc in helper_rcs), \ + "inner()'s call to helper must appear in raw_calls from inner, not baz" + assert not any(rc["caller_nid"] == baz_id for rc in helper_rcs), \ + "helper() call must not be attributed to the enclosing baz()" + + +@_needs_r +def test_r_same_file_calls_extracted(): + r = extract_r(FIXTURES / "sample.r") + calls = _calls(r) # set of (raw_src_label, raw_tgt_label) pairs + file_label = "sample.r" + # top-level call to foo() attributed to the file node + assert (file_label, "foo()") in calls + # baz() contains inner(); nested call to inner() attributed to baz + assert ("baz()", "inner()") in calls + # pipe chain captures qux() call at top level + assert (file_label, "qux()") in calls + + +@_needs_r +def test_r_member_calls_recorded_as_member_raw_calls(): + r = extract_r(FIXTURES / "sample.r") + member_callees = {rc["callee"] for rc in r["raw_calls"] if rc.get("is_member_call")} + # $ and @ receivers are member calls + assert "method" in member_callees # obj$method() + assert "field" in member_callees # obj@field + # pkg::fn / pkg:::fn are qualified-member (is_member_call=True) so the + # shared resolver doesn't bind them to a same-named def + assert "summary" in member_callees # pkg::summary + assert "deep_fn" in member_callees # base:::deep_fn + # NOTE: `dplyr::filter` — filter is a Python builtin, filtered from raw_calls + # by _LANGUAGE_BUILTIN_GLOBALS, so it does NOT appear here. + + +@_needs_r +def test_r_member_calls_never_bind_to_same_named_def(): + r = extract_r(FIXTURES / "sample.r") + calls = _calls(r) + # No calls edges to method/field/summary/deep_fn — they're member raw_calls + for callee in ("method", "field", "summary", "deep_fn"): + assert not any(tgt == f"{callee}()" for _, tgt in calls), \ + f"member call {callee}() must not bind to a same-named def" + + +@_needs_r +def test_r_imports_from_static_loaders(): + r = extract_r(FIXTURES / "sample.r") + imports_targets = {e["target"] for e in r["edges"] if e["relation"] == "imports"} + assert "dplyr" in imports_targets # library(dplyr) + assert "utils" in imports_targets # requireNamespace("utils") + assert "base" in imports_targets # pkg::fn emits pkg import evidence + + +@_needs_r +def test_r_imports_skipped_for_dynamic_loader_arg(): + # library(installed.packages()) — the arg is a call expression, not a + # literal identifier/string; must NOT produce an imports edge. + r = extract_r(FIXTURES / "sample.r") + imports_targets = {e["target"] for e in r["edges"] if e["relation"] == "imports"} + assert "installed.packages" not in imports_targets + assert "installed" not in imports_targets + + +@_needs_r +def test_r_static_source_emits_imports_from_to_existing_target(): + r = extract_r(FIXTURES / "sample.r") + ifs = [e for e in r["edges"] if e["relation"] == "imports_from"] + assert len(ifs) == 1 + assert "r_other" in ifs[0]["target"] # sibling r_other.r exists on disk + + +@_needs_r +def test_r_source_ignores_url_missing_and_non_r_targets(tmp_path): + src = tmp_path / "src.r" + src.write_text( + 'source("https://example.com/remote.R")\n' # URL — skip + 'source("missing.r")\n' # missing — skip + 'source("helper.py")\n' # not .r — skip + ) + r = extract_r(src) + assert not [e for e in r["edges"] if e["relation"] == "imports_from"] + + +@_needs_r +def test_r_no_dangling_edges(): + r = extract_r(FIXTURES / "sample.r") + _r_assert_no_dangling(r) + + +@_needs_r +def test_r_pipes_capture_contained_calls(): + r = extract_r(FIXTURES / "sample.r") + callees = {rc["callee"] for rc in r["raw_calls"]} + # pipe chain `1 |> process() |> save()` captures process & save + assert "process" in callees + assert "save" in callees + + +@_needs_r +def test_r_cross_file_call_resolves_with_source_link_extracted(): + # sample.r sources r_other.r, creating imports_from evidence; the cross-file + # helper() call resolves EXTRACTED (single candidate + import evidence). + r = _corpus("sample.r", "r_other.r") + calls = _calls(r) + assert ("foo()", "helper()") in calls or ("inner()", "helper()") in calls, \ + "cross-file helper() must resolve via source() import evidence" + + +@_needs_r +def test_r_cross_file_call_resolves_inferred_without_source(): + # r_caller.r has no source() linkage; helper() in r_other.r has a single + # cross-file candidate → INFERRED (confidence 0.8). + r = _corpus("r_caller.r", "r_other.r") + # Find any calls edge whose normalised target label is "helper" + helper_calls = [e for e in r["edges"] if e["relation"] == "calls" + and "helper" in _normalize_symbol_label(e.get("target", ""))] + assert helper_calls, "unique cross-file bare call must resolve as INFERRED" + # Verify the resolved target is in r_other.r + r_other_helper = [n for n in r["nodes"] if n.get("label") == "helper()"][0] + assert Path(r_other_helper["source_file"]).name == "r_other.r" + + +@_needs_r +def test_r_cross_file_ambiguous_call_remains_unresolved(): + # r_caller.r calls dup(); r_other.r AND r_extra.r both define dup(). + # Ambiguity guard must keep this unresolved (no calls edge to dup()). + r = _corpus("r_caller.r", "r_other.r", "r_extra.r") + dup_calls = [e for e in r["edges"] if e["relation"] == "calls" + and "dup" in _normalize_symbol_label(e.get("target", ""))] + assert len(dup_calls) == 0, f"ambiguous dup() must not resolve, got {dup_calls}" diff --git a/tests/test_serve.py b/tests/test_serve.py index 859cde026..dc4229c24 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -7,6 +7,7 @@ from graphify.serve import ( _communities_from_graph, _score_nodes, + _score_query, _compute_idf, _EXACT_MATCH_BONUS, _SOURCE_MATCH_BONUS, @@ -26,6 +27,7 @@ _subgraph_to_text, _load_graph, _community_header, + _search_tokens, ) @@ -734,8 +736,8 @@ def test_pick_seeds_respects_max_k(): def test_pick_seeds_without_diversity_args_is_unchanged(): - """G/terms are optional and default to None: existing callers see identical - behavior to before this change.""" + """G/best_seed_by_term are optional and default to None: existing callers + see identical behavior to before this change.""" scored = [(1000.0, "fbs"), (1.0, "err1"), (0.9, "err2")] assert _pick_seeds(scored) == ["fbs"] @@ -744,9 +746,9 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): """Reproduces #1445: a vague natural-language query where one term's incidental EXACT match on an unrelated node (e.g. a common word also used as an unrelated field/identifier) outscores every SUBSTRING match on the - query's other, actually-relevant terms by ~1000x. Without G/terms, the - 20%-gap cutoff discards the relevant candidate entirely; with them, it is - recovered as a guaranteed per-term seed. + query's other, actually-relevant terms by ~1000x. Without + G/best_seed_by_term, the 20%-gap cutoff discards the relevant candidate + entirely; with them, it is recovered as a guaranteed per-term seed. """ G = nx.DiGraph() # "unrelated" is an exact label match for the query term "unrelated" and @@ -758,13 +760,17 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): G.add_edge("other", "target") terms = ["unrelated", "widget"] - scored = _score_nodes(G, terms) + # `_score_query` does the combined scoring and the per-term singleton + # winner tracking in one traversal; `_pick_seeds` consumes its + # `best_seed_by_term` to satisfy the per-term guarantee without rescoring. + qs = _score_query(G, terms, collect_per_term_seeds=True) + scored = qs.ranked # Sanity check the premise: without diversity, only the exact match survives. seeds_before = _pick_seeds(scored) assert seeds_before == ["noise"] - seeds_after = _pick_seeds(scored, G=G, terms=terms) + seeds_after = _pick_seeds(scored, G=G, best_seed_by_term=qs.best_seed_by_term) assert "noise" in seeds_after assert "target" in seeds_after @@ -807,8 +813,9 @@ def test_pick_seeds_per_term_guarantee_does_not_reintroduce_generic_dupe(monkeyp G.add_node(f"get{i}", label="GET", source_file=f"r{i}.py") G.add_node("um", label="users_model", source_file="users.py") G.add_edge("um", "get0") - scored = _score_nodes(G, ["get", "users"]) - seeds = _pick_seeds(scored, G=G, terms=["get", "users"]) + terms = ["get", "users"] + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) get_seeds = [s for s in seeds if s.startswith("get")] assert len(get_seeds) == 1, f"per-term guarantee reintroduced a GET dupe: {seeds}" @@ -974,3 +981,229 @@ def test_community_header_sanitizes_name(): out = _community_header(3, "Pay\x00ments\x1b[31m") assert out.startswith("Community 3 — ") assert "\x00" not in out and "\x1b" not in out + + +# --- single-pass scoring refactor: reference-impl equality + one-traversal --- + + +def _reference_best_seed_by_term(G: nx.Graph, terms: list[str]) -> dict[str, str]: + """Test-only oracle for the legacy per-term `_pick_seeds(terms=...)` loop. + + Re-creates what `_pick_seeds` did before the single-pass refactor: rescore + the whole graph per token via `_score_nodes(G, [token])`, take the top- + scoring ties, and break them by `max(tied, key=degree)` (which, over a + list sorted by `(-score, label_len, nid)`, returns the highest-degree node + with ties broken toward the shortest label then the smallest node id). + This is the semantics `_score_query(..., collect_per_term_seeds=True)` now + produces inline during its single traversal. + """ + norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) + best: dict[str, str] = {} + for term in norm_terms: + term_scored = _score_nodes(G, [term]) + if not term_scored: + continue + best_score = term_scored[0][0] + tied = [nid for s, nid in term_scored if s == best_score] + best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best[term] = best_nid + return best + + +def _make_random_scoring_graph(n: int, *, seed: int) -> nx.DiGraph: + """Reproducible broad-match DiGraph: short constructed labels + edge noise. + + Labels draw from a small syllable pool so tokens collide across nodes, + forcing the trigram prefilter to be selective and exercising score ties + on common tokens. Edge noise provides degree variance so the legacy + tie-break (`max(tied, key=degree)`) is exercised against the new + `(-singleton, -degree, label_len, nid)` key tuple. + """ + import random + + rng = random.Random(seed) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + G: nx.DiGraph = nx.DiGraph() + for i in range(n): + label = "_".join(rng.sample(syllables, rng.randint(1, 3))) + G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + for _ in range(n * 2): + a, b = rng.randrange(n), rng.randrange(n) + if a != b: + G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") + return G + + +SYLLABLE_QUERIES = [ + ["get"], # single token, exact-match + ["get", "user"], # two distinct tokens + ["router", "service", "handler"], # multi-token identifier + ["extract", "build", "report", "path"], # broad term + ["nonexistent"], # no matches + ["nonexistent", "get"], # one missing term + match + ["bar", "bar"], # repeated token (must dedupe) + ["baz", "run", "set", "auth", "rate", "limit"], # many tokens +] + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_ranked_matches_score_nodes_byte_identical(terms): + """`_score_query(..., collect_per_term_seeds=False).ranked` is the byte-for- + byte match of `_score_nodes(G, terms)` — guaranteeing path/explain/tests see + no behavior change from the refactor.""" + G = _make_random_scoring_graph(80, seed=7) + assert _score_query(G, terms, collect_per_term_seeds=False).ranked == _score_nodes(G, terms) + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_best_seed_by_term_matches_legacy_singleton_scoring(terms): + """Per-token winner the single-pass scorer records matches the legacy + `_score_nodes([token])` + `max(tied, key=degree)` oracle exactly.""" + G = _make_random_scoring_graph(80, seed=7) + ref = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True).best_seed_by_term + assert ref == opt, f"terms={terms}: legacy={ref} optimized={opt}" + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_pick_seeds_with_optimized_best_seed_matches_legacy_semantics(terms): + """The seeds produced by `_pick_seeds(qs.ranked, G=G, best_seed_by_term= + qs.best_seed_by_term)` exactly match what the legacy `_pick_seeds(terms=...)` + loop would have produced (recreated via the reference oracle).""" + G = _make_random_scoring_graph(80, seed=7) + qs = _score_query(G, terms, collect_per_term_seeds=True) + ref_best = _reference_best_seed_by_term(G, terms) + # Legacy `_pick_seeds(terms=...)` ran `_score_nodes(G, [term])` per token + # to build ref_best, then deduped by label key. The new `_pick_seeds( + # best_seed_by_term=...)` only swaps the source of the per-token winners, + # so it must produce the same seeds given equivalent inputs. + opt_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + ref_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=ref_best) + assert opt_seeds == ref_seeds, f"terms={terms}: ref={ref_seeds} opt={opt_seeds}" + # Per-term guarantee: every legacy winner with a non-empty seed slot is + # accounted for — either it appears in the seed list or another node with + # the same normalized label already claimed the slot (#1766 label dedup). + ref_seed_set = set(ref_seeds) + for term, nid in ref_best.items(): + if nid in ref_seed_set: + continue + nid_label = (G.nodes[nid].get("norm_label") + or G.nodes[nid].get("label") + or nid) + seeded_with_same_label = any( + (G.nodes[s].get("norm_label") or G.nodes[s].get("label") or s) == nid_label + for s in ref_seeds + ) + assert seeded_with_same_label, ( + f"term {term!r} winner {nid!r} dropped without label-dedup reason" + ) + + +def test_score_query_matches_legacy_across_random_deterministic_graphs(): + """Across many deterministic random graphs and many random multi-term + queries, the single-pass scorer's combined ranking, per-token winners, + and resulting seed list all match the legacy semantics. Exercises label + collisions, ties, broad terms, missing terms, and graph size variance.""" + import random + + rng = random.Random(42) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + for trial in range(30): + n = rng.randint(20, 200) + G = _make_random_scoring_graph(n, seed=rng.randint(0, 10**9)) + nq = rng.randint(1, 5) + terms = [rng.choice(syllables) for _ in range(nq)] + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + # (a) Combined ranking unchanged. + assert opt.ranked == _score_nodes(G, terms), ( + f"trial {trial}: combined ranking diverged for terms={terms}" + ) + # (b) Per-token winners match the legacy per-term rescoring loop. + assert opt.best_seed_by_term == ref_best, ( + f"trial {trial}: best_seed_by_term diverged; ref={ref_best} opt={opt.best_seed_by_term}" + ) + # (c) Final seed list is identical under the legacy semantics. + ref_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=ref_best) + opt_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=opt.best_seed_by_term) + assert opt_seeds == ref_seeds, ( + f"trial {trial}: seeds diverged; ref={ref_seeds} opt={opt_seeds}" + ) + + +def test_score_query_matches_legacy_under_full_scan_fallback(monkeypatch): + """When the trigram prefilter falls back to a full-graph scan, the + single-pass path still produces identical rankings and per-term winners. + + Forces `_trigram_candidates` to return None so the combined iterates the + whole graph — mirroring per-token `_score_nodes([token])` which would also + full-scan when its own trigram search isn't selective.""" + monkeypatch.setattr( + "graphify.serve._trigram_candidates", lambda G, needles: None + ) + terms = ["router", "service", "handler"] + G = _make_random_scoring_graph(80, seed=19) + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + assert opt.ranked == _score_nodes(G, terms) + assert opt.best_seed_by_term == ref_best + + +def test_query_graph_text_makes_exactly_one_score_query_call(monkeypatch): + """`_query_graph_text` must invoke `_score_query` exactly once per query, + regardless of how many tokens the query has — eliminating the legacy + T+1-pass rescoring. `_score_nodes` must NOT be called from the query path + (only path/explain still call it).""" + G = _make_random_scoring_graph(60, seed=23) + original_sq = _score_query + original_sn = _score_nodes + + state = {"sq": 0, "sn": 0} + + def counting_sq(*a, **k): + state["sq"] += 1 + return original_sq(*a, **k) + + def counting_sn(*a, **k): + state["sn"] += 1 + return original_sn(*a, **k) + + monkeypatch.setattr("graphify.serve._score_query", counting_sq) + monkeypatch.setattr("graphify.serve._score_nodes", counting_sn) + + queries = [ + "foo", # one term + "foo bar", # two + "router service handler", # three (the scenario the RFC targets) + "get user run name path", # five + "extract build report router config service token rate limit widget", # ten + ] + for q in queries: + state["sq"] = 0 + state["sn"] = 0 + _query_graph_text(G, q, mode="bfs", depth=1) + assert state["sq"] == 1, ( + f"expected exactly one _score_query call for {q!r}, got {state['sq']}" + ) + assert state["sn"] == 0, ( + f"query path must not call _score_nodes; got {state['sn']} call(s) for {q!r}" + ) + + +def test_score_query_collect_per_term_seeds_false_omits_tracking(monkeypatch): + """`collect_per_term_seeds=False` returns empty `best_seed_by_term` and + does not pay for per-token best tracking — preserving the cost contract + for path/explain/tests callers that only want the combined ranking.""" + G = _make_random_scoring_graph(50, seed=29) + qs = _score_query(G, ["foo", "bar", "baz"], collect_per_term_seeds=False) + assert qs.best_seed_by_term == {} + # And the combined output is still byte-identical to _score_nodes. + assert qs.ranked == _score_nodes(G, ["foo", "bar", "baz"])