From c4d8cee952ff04fb295f585c801ddb572cd93c47 Mon Sep 17 00:00:00 2001 From: AmitMY Date: Fri, 26 Jun 2026 15:20:18 +0200 Subject: [PATCH] perf: skip merge on subtrees the memo proves are unaffected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When get_merges is memoized, _merges lists every mergeable subsequence in the subtree. If the chosen merge isn't among them, this subtree contains it nowhere, so merge() returns self immediately — skipping the scan, the child recursion, and the rebuild. Falls back to the plain scan when the node isn't memoized. Co-Authored-By: Claude Opus 4.8 (1M context) --- complex_tokenization/graph.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/complex_tokenization/graph.py b/complex_tokenization/graph.py index e58adf9..20cb08c 100644 --- a/complex_tokenization/graph.py +++ b/complex_tokenization/graph.py @@ -132,16 +132,22 @@ def node_count(self) -> int: return sum(n.node_count() for n in self.nodes) def merge(self, token: Node, merge: tuple["GraphVertex", ...]): - m = len(merge) nodes = self.nodes + + # _merges (when memoized) lists every mergeable subsequence in this + # subtree, so if the merge isn't among them nothing here changes. + cached = getattr(self, "_merges", None) + if cached is not None and merge not in cached: + return self + + m = len(merge) n = len(nodes) + first = merge[0] out: list[GraphVertex] = [] i = 0 - - first = merge[0] while i <= n - m: - # Check the first node before slicing: it fails at most positions, - # so we avoid allocating the nodes[i:i + m] tuple in the common case. + # First-node guard before slicing (see #29): skips the nodes[i:i + m] + # allocation at the many positions that can't start the merge. if nodes[i] == first and nodes[i:i + m] == merge: out.append(token) i += m @@ -152,8 +158,7 @@ def merge(self, token: Node, merge: tuple["GraphVertex", ...]): if len(out) == 1: return out[0] - - merged_nodes = tuple(n.merge(token, merge) for n in out) + merged_nodes = tuple(node.merge(token, merge) for node in out) if merged_nodes == nodes: return self return NodesSequence(merged_nodes)