From ece701ef5736c2633a9d5b07458ac7eabb4134dd Mon Sep 17 00:00:00 2001 From: AmitMY Date: Fri, 26 Jun 2026 16:36:11 +0200 Subject: [PATCH] perf: build merge candidates with itertools.chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UnconnectedGraphs.get_merges yielded candidates with a Python-level `for sg: yield from sg.get_merges()` loop, run every training step to feed the Counter. itertools.chain.from_iterable does the same flattening in C. Same candidates, same order — output identical. ~2-3% faster across BPE/BNE/Boundless; memory unchanged. 137 tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- complex_tokenization/graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/complex_tokenization/graph.py b/complex_tokenization/graph.py index 20cb08c..82f372d 100644 --- a/complex_tokenization/graph.py +++ b/complex_tokenization/graph.py @@ -1,5 +1,6 @@ from collections.abc import Iterable, Iterator from dataclasses import dataclass, field +from itertools import chain from complex_tokenization.graphs.settings import GraphSettings from complex_tokenization.languages.chinese.ideographic_description_sequences import get_character_for_ids @@ -341,8 +342,7 @@ def merge(self, token: Node, merge: tuple): return UnconnectedGraphs(subgraphs=new) def get_merges(self) -> Iterator[tuple]: - for subgraph in self.subgraphs: - yield from subgraph.get_merges() + return chain.from_iterable(sg.get_merges() for sg in self.subgraphs) def dot(self, level=0) -> Iterable[str]: for subgraph in self.subgraphs: