From 10fe875c78c2f4ccaf3a9f78e54d46f945fb766f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 21 Aug 2026 21:21:07 +0200 Subject: [PATCH] shellpattern: avoid exponential backtracking for runs of * wildcards, see #2624 sh: patterns were translated 1:1 into SRE regexes ("[^/]*" per "*"), so patterns with a handful of wildcards like "input/a*a*a*a*a*a*b" took exponential time to not match - the issue's reproducer never finished. fm: patterns already do not have this problem since Python 3.11, because fnmatch.translate() emits atomic groups. Do the same for sh: patterns: "* FIXED *" is now emitted as "(?>[^/]*?FIXED)[^/]*", i.e. FIXED is matched at its leftmost occurrence and the regex engine can not backtrack into it. This is equivalent as long as both neighbours are plain "*" (even if FIXED contains path separators), but not if the right neighbour is "**/" or FIXED is a "{,}" group, so the atomic group is only used for plain-star neighbours. Adjacent "*" and adjacent "**/" are collapsed, they are equivalent to a single one. "input/" + "a*" * 50 + "b" against 200 "a"s: never finished -> 0 ms. 60000 random patterns/paths give identical results with old and new translate(). Also update the note in "borg help patterns": re: patterns (and sh: patterns with many "**/" or "{}" alternatives) can still be slow. --- docs/man/borg-patterns.1 | 11 ++--- docs/usage/help.rst.inc | 9 ++-- src/borg/archiver/help_cmd.py | 9 ++-- src/borg/helpers/shellpattern.py | 44 ++++++++++++++----- .../testsuite/helpers/shellpattern_test.py | 32 ++++++++++++++ 5 files changed, 80 insertions(+), 25 deletions(-) diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 59f34caeaa..c1dc17dbd3 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-patterns" "1" "2026-07-21" "" "borg backup tool" +.TH "borg-patterns" "1" "2026-08-21" "" "borg backup tool" .SH Name borg-patterns \- Details regarding patterns .SH DESCRIPTION @@ -141,11 +141,12 @@ Same logic applies for exclude. .INDENT 0.0 .INDENT 3.5 \fBre:\fP, \fBsh:\fP and \fBfm:\fP patterns are all implemented on top of -the Python SRE engine. It is very easy to formulate patterns for each -of these types which requires an inordinate amount of time to match +the Python SRE engine. \fBsh:\fP and \fBfm:\fP patterns are translated so +that any number of \fB*\fP wildcards can be matched in reasonable time, +but \fBre:\fP patterns (and \fBsh:\fP patterns with many \fB**/\fP or \fB{}\fP +alternatives) can still require an inordinate amount of time to match paths. If untrusted users are able to supply patterns, ensure they -cannot supply \fBre:\fP patterns. Further, ensure that \fBsh:\fP and -\fBfm:\fP patterns only contain a handful of wildcards at most. +cannot supply \fBre:\fP patterns and keep the other patterns simple. .UNINDENT .UNINDENT .sp diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index d7877b84e4..36f9c08221 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -98,11 +98,12 @@ Path full-match, selector ``pf:`` .. note:: ``re:``, ``sh:`` and ``fm:`` patterns are all implemented on top of - the Python SRE engine. It is very easy to formulate patterns for each - of these types which requires an inordinate amount of time to match + the Python SRE engine. ``sh:`` and ``fm:`` patterns are translated so + that any number of ``*`` wildcards can be matched in reasonable time, + but ``re:`` patterns (and ``sh:`` patterns with many ``**/`` or ``{}`` + alternatives) can still require an inordinate amount of time to match paths. If untrusted users are able to supply patterns, ensure they - cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and - ``fm:`` patterns only contain a handful of wildcards at most. + cannot supply ``re:`` patterns and keep the other patterns simple. .. note:: diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py index cc679431f7..1412ec3ea5 100644 --- a/src/borg/archiver/help_cmd.py +++ b/src/borg/archiver/help_cmd.py @@ -102,11 +102,12 @@ class HelpMixIn: .. note:: ``re:``, ``sh:`` and ``fm:`` patterns are all implemented on top of - the Python SRE engine. It is very easy to formulate patterns for each - of these types which requires an inordinate amount of time to match + the Python SRE engine. ``sh:`` and ``fm:`` patterns are translated so + that any number of ``*`` wildcards can be matched in reasonable time, + but ``re:`` patterns (and ``sh:`` patterns with many ``**/`` or ``{}`` + alternatives) can still require an inordinate amount of time to match paths. If untrusted users are able to supply patterns, ensure they - cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and - ``fm:`` patterns only contain a handful of wildcards at most. + cannot supply ``re:`` patterns and keep the other patterns simple. .. note:: diff --git a/src/borg/helpers/shellpattern.py b/src/borg/helpers/shellpattern.py index 00e9237bdd..fa126b28b1 100644 --- a/src/borg/helpers/shellpattern.py +++ b/src/borg/helpers/shellpattern.py @@ -23,7 +23,7 @@ def translate(pat, match_end=r"\Z"): sep = os.path.sep n = len(pat) i = 0 - res = "" + tokens = [] # (kind, regex) with kind "star" (*), "dstar" (**), "group" (alternatives) or "fixed" while i < n: c = pat[i] @@ -32,15 +32,16 @@ def translate(pat, match_end=r"\Z"): if c == "*": if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep: # **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands - # for the platform-specific path separator - res += rf"(?:[^\{sep}]*\{sep})*" + # for the platform-specific path separator. Adjacent **/ are collapsed into one. + if not (tokens and tokens[-1][0] == "dstar"): + tokens.append(("dstar", rf"(?:[^\{sep}]*\{sep})*")) i += 2 - else: - # * == wildcard for name parts (does not cross path separator) - res += r"[^\%s]*" % sep + elif not (tokens and tokens[-1][0] == "star"): + # * == wildcard for name parts (does not cross path separator). Adjacent * are collapsed into one. + tokens.append(("star", r"[^\%s]*" % sep)) elif c == "?": # ? == any single character excluding path separator - res += r"[^\%s]" % sep + tokens.append(("fixed", r"[^\%s]" % sep)) elif c == "[": j = i if j < n and pat[j] == "!": @@ -50,7 +51,7 @@ def translate(pat, match_end=r"\Z"): while j < n and pat[j] != "]": j += 1 if j >= n: - res += "\\[" + tokens.append(("fixed", "\\[")) else: stuff = pat[i:j].replace("\\", "\\\\") i = j + 1 @@ -58,14 +59,33 @@ def translate(pat, match_end=r"\Z"): stuff = "^" + stuff[1:] elif stuff[0] == "^": stuff = "\\" + stuff - res += "[%s]" % stuff + tokens.append(("fixed", "[%s]" % stuff)) elif c in "(|)": if i > 0 and pat[i - 1] != "\\": - res += c + tokens.append(("group", c)) else: - res += re.escape(c) + tokens.append(("fixed", re.escape(c))) + + # Join the tokens. A "* FIXED *" sequence (FIXED: one or more "fixed" tokens) is emitted as an atomic group + # matching FIXED at its leftmost occurrence, so that runs of wildcards can not cause exponential backtracking, + # see #2624. This is only correct for "star" neighbours on both sides, not for "dstar" or "group" neighbours. + res = [] + i = 0 + n = len(tokens) + while i < n: + kind, regex = tokens[i] + if kind == "star": + j = i + 1 + while j < n and tokens[j][0] == "fixed": + j += 1 + if j > i + 1 and j < n and tokens[j][0] == "star": + res.append("(?>" + regex + "?" + "".join(r for _, r in tokens[i + 1 : j]) + ")") + i = j + continue + res.append(regex) + i += 1 - return "(?ms)" + res + match_end + return "(?ms)" + "".join(res) + match_end def _parse_braces(pat): diff --git a/src/borg/testsuite/helpers/shellpattern_test.py b/src/borg/testsuite/helpers/shellpattern_test.py index 123116ffff..5a4b20d0e0 100644 --- a/src/borg/testsuite/helpers/shellpattern_test.py +++ b/src/borg/testsuite/helpers/shellpattern_test.py @@ -1,4 +1,5 @@ import re +import time import pytest @@ -76,6 +77,17 @@ def check(path, pattern): ("{foobar", ["{foo{,bar}"]), ("{foo},bar}", ["{foo},bar}"]), ("bar/foobar", ["**/foo{ba[!z]*,[0-9]}"]), + # Adjacent wildcards are equivalent to one + ("foo/bar", ["foo/**/**/bar", "foo/***/bar", "f**o/bar"]), + ("foo/1/22/bar", ["foo/**/**/bar", "foo/**/**/**/bar"]), + # Star-fixed-star sequences are matched atomically (leftmost fixed occurrence), see #2624. + # These must still match, the leftmost occurrence is not the right one if the right neighbour is + # not a plain star, or if the fixed part is an alternative group. + ("aab", ["*a**/b", "*a*b"]), + ("a/ab", ["**/a*b"]), + ("abc", ["*{a,ab}c"]), + ("xaab", ["*a*b", "*a*a*b"]), + ("xa/by", ["*a/b*"]), ], ) def test_match(path, patterns): @@ -114,6 +126,10 @@ def test_match(path, patterns): ("foo", ["foo{1,2}"]), ("foo{1,2}", ["foo{1,2}"]), ("bar/foobaz", ["**/foo{ba[!z]*,[0-9]}"]), + # Star-fixed-star sequences, see #2624 + ("a/ab", ["*a*b"]), + ("aab", ["*a*b*c"]), + ("ab", ["*a*a*b"]), ], ) def test_mismatch(path, patterns): @@ -130,3 +146,19 @@ def test_match_end(): regex = shellpattern.translate("*-home", match_end=match_end) assert re.match(regex, "2017-07-03-home") assert re.match(regex, "2017-07-03-home.xxx") + + +def test_translate_atomic_wildcards(): + # "* FIXED *" becomes an atomic group, so that many wildcards can not cause exponential backtracking, see #2624 + assert shellpattern.translate("a*b*c*d") == r"(?ms)a(?>[^\/]*?b)(?>[^\/]*?c)[^\/]*d\Z" + # Not for "**/" or alternative-group neighbours, where the leftmost occurrence is not always the right one. + assert shellpattern.translate("*a**/b") == r"(?ms)[^\/]*a(?:[^\/]*\/)*b\Z" + assert shellpattern.translate("*{a,ab}c*") == r"(?ms)[^\/]*(a|ab)c[^\/]*\Z" + + +def test_no_exponential_backtracking(): + # see #2624: this took "forever" before wildcard runs were matched atomically + regex = re.compile(shellpattern.translate("input/" + "a*" * 50 + "b")) + start = time.monotonic() + assert not regex.match("input/" + "a" * 200) + assert time.monotonic() - start < 10