shellpattern: avoid exponential backtracking for runs of * wildcards, see #2624

docs/man/borg-patterns.1

@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-patterns" "1" "2026-07-21" "" "borg backup tool"
+.TH "borg-patterns" "1" "2026-08-21" "" "borg backup tool"
 .SH Name
 borg-patterns \- Details regarding patterns
 .SH DESCRIPTION
@@ -141,11 +141,12 @@ Same logic applies for exclude.
 .INDENT 0.0
 .INDENT 3.5
 \fBre:\fP, \fBsh:\fP and \fBfm:\fP patterns are all implemented on top of
-the Python SRE engine. It is very easy to formulate patterns for each
-of these types which requires an inordinate amount of time to match
+the Python SRE engine. \fBsh:\fP and \fBfm:\fP patterns are translated so
+that any number of \fB*\fP wildcards can be matched in reasonable time,
+but \fBre:\fP patterns (and \fBsh:\fP patterns with many \fB**/\fP or \fB{}\fP
+alternatives) can still require an inordinate amount of time to match
 paths. If untrusted users are able to supply patterns, ensure they
-cannot supply \fBre:\fP patterns. Further, ensure that \fBsh:\fP and
-\fBfm:\fP patterns only contain a handful of wildcards at most.
+cannot supply \fBre:\fP patterns and keep the other patterns simple.
 .UNINDENT
 .UNINDENT
 .sp

docs/usage/help.rst.inc

@@ -98,11 +98,12 @@ Path full-match, selector ``pf:``
 .. note::
 
     ``re:``, ``sh:`` and ``fm:`` patterns are all implemented on top of
-    the Python SRE engine. It is very easy to formulate patterns for each
-    of these types which requires an inordinate amount of time to match
+    the Python SRE engine. ``sh:`` and ``fm:`` patterns are translated so
+    that any number of ``*`` wildcards can be matched in reasonable time,
+    but ``re:`` patterns (and ``sh:`` patterns with many ``**/`` or ``{}``
+    alternatives) can still require an inordinate amount of time to match
     paths. If untrusted users are able to supply patterns, ensure they
-    cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and
-    ``fm:`` patterns only contain a handful of wildcards at most.
+    cannot supply ``re:`` patterns and keep the other patterns simple.
 
 .. note::
 

src/borg/archiver/help_cmd.py

@@ -102,11 +102,12 @@ class HelpMixIn:
         .. note::
 
             ``re:``, ``sh:`` and ``fm:`` patterns are all implemented on top of
-            the Python SRE engine. It is very easy to formulate patterns for each
-            of these types which requires an inordinate amount of time to match
+            the Python SRE engine. ``sh:`` and ``fm:`` patterns are translated so
+            that any number of ``*`` wildcards can be matched in reasonable time,
+            but ``re:`` patterns (and ``sh:`` patterns with many ``**/`` or ``{}``
+            alternatives) can still require an inordinate amount of time to match
             paths. If untrusted users are able to supply patterns, ensure they
-            cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and
-            ``fm:`` patterns only contain a handful of wildcards at most.
+            cannot supply ``re:`` patterns and keep the other patterns simple.
 
         .. note::
 

src/borg/helpers/shellpattern.py

@@ -23,7 +23,7 @@ def translate(pat, match_end=r"\Z"):
     sep = os.path.sep
     n = len(pat)
     i = 0
-    res = ""
+    tokens = []  # (kind, regex) with kind "star" (*), "dstar" (**<sep>), "group" (alternatives) or "fixed"
 
     while i < n:
         c = pat[i]
@@ -32,15 +32,16 @@ def translate(pat, match_end=r"\Z"):
         if c == "*":
             if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep:
                 # **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands
-                # for the platform-specific path separator
-                res += rf"(?:[^\{sep}]*\{sep})*"
+                # for the platform-specific path separator. Adjacent **/ are collapsed into one.
+                if not (tokens and tokens[-1][0] == "dstar"):
+                    tokens.append(("dstar", rf"(?:[^\{sep}]*\{sep})*"))
                 i += 2
-            else:
-                # * == wildcard for name parts (does not cross path separator)
-                res += r"[^\%s]*" % sep
+            elif not (tokens and tokens[-1][0] == "star"):
+                # * == wildcard for name parts (does not cross path separator). Adjacent * are collapsed into one.
+                tokens.append(("star", r"[^\%s]*" % sep))
         elif c == "?":
             # ? == any single character excluding path separator
-            res += r"[^\%s]" % sep
+            tokens.append(("fixed", r"[^\%s]" % sep))
         elif c == "[":
             j = i
             if j < n and pat[j] == "!":
@@ -50,22 +51,41 @@ def translate(pat, match_end=r"\Z"):
             while j < n and pat[j] != "]":
                 j += 1
             if j >= n:
-                res += "\\["
+                tokens.append(("fixed", "\\["))
             else:
                 stuff = pat[i:j].replace("\\", "\\\\")
                 i = j + 1
                 if stuff[0] == "!":
                     stuff = "^" + stuff[1:]
                 elif stuff[0] == "^":
                     stuff = "\\" + stuff
-                res += "[%s]" % stuff
+                tokens.append(("fixed", "[%s]" % stuff))
         elif c in "(|)":
             if i > 0 and pat[i - 1] != "\\":
-                res += c
+                tokens.append(("group", c))
         else:
-            res += re.escape(c)
+            tokens.append(("fixed", re.escape(c)))
+
+    # Join the tokens. A "* FIXED *" sequence (FIXED: one or more "fixed" tokens) is emitted as an atomic group
+    # matching FIXED at its leftmost occurrence, so that runs of wildcards can not cause exponential backtracking,
+    # see #2624. This is only correct for "star" neighbours on both sides, not for "dstar" or "group" neighbours.
+    res = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        kind, regex = tokens[i]
+        if kind == "star":
+            j = i + 1
+            while j < n and tokens[j][0] == "fixed":
+                j += 1
+            if j > i + 1 and j < n and tokens[j][0] == "star":
+                res.append("(?>" + regex + "?" + "".join(r for _, r in tokens[i + 1 : j]) + ")")
+                i = j
+                continue
+        res.append(regex)
+        i += 1
 
-    return "(?ms)" + res + match_end
+    return "(?ms)" + "".join(res) + match_end
 
 
 def _parse_braces(pat):

src/borg/testsuite/helpers/shellpattern_test.py

@@ -1,4 +1,5 @@
 import re
+import time
 
 import pytest
 
@@ -76,6 +77,17 @@ def check(path, pattern):
         ("{foobar", ["{foo{,bar}"]),
         ("{foo},bar}", ["{foo},bar}"]),
         ("bar/foobar", ["**/foo{ba[!z]*,[0-9]}"]),
+        # Adjacent wildcards are equivalent to one
+        ("foo/bar", ["foo/**/**/bar", "foo/***/bar", "f**o/bar"]),
+        ("foo/1/22/bar", ["foo/**/**/bar", "foo/**/**/**/bar"]),
+        # Star-fixed-star sequences are matched atomically (leftmost fixed occurrence), see #2624.
+        # These must still match, the leftmost occurrence is not the right one if the right neighbour is
+        # not a plain star, or if the fixed part is an alternative group.
+        ("aab", ["*a**/b", "*a*b"]),
+        ("a/ab", ["**/a*b"]),
+        ("abc", ["*{a,ab}c"]),
+        ("xaab", ["*a*b", "*a*a*b"]),
+        ("xa/by", ["*a/b*"]),
     ],
 )
 def test_match(path, patterns):
@@ -114,6 +126,10 @@ def test_match(path, patterns):
         ("foo", ["foo{1,2}"]),
         ("foo{1,2}", ["foo{1,2}"]),
         ("bar/foobaz", ["**/foo{ba[!z]*,[0-9]}"]),
+        # Star-fixed-star sequences, see #2624
+        ("a/ab", ["*a*b"]),
+        ("aab", ["*a*b*c"]),
+        ("ab", ["*a*a*b"]),
     ],
 )
 def test_mismatch(path, patterns):
@@ -130,3 +146,19 @@ def test_match_end():
     regex = shellpattern.translate("*-home", match_end=match_end)
     assert re.match(regex, "2017-07-03-home")
     assert re.match(regex, "2017-07-03-home.xxx")
+
+
+def test_translate_atomic_wildcards():
+    # "* FIXED *" becomes an atomic group, so that many wildcards can not cause exponential backtracking, see #2624
+    assert shellpattern.translate("a*b*c*d") == r"(?ms)a(?>[^\/]*?b)(?>[^\/]*?c)[^\/]*d\Z"
+    # Not for "**/" or alternative-group neighbours, where the leftmost occurrence is not always the right one.
+    assert shellpattern.translate("*a**/b") == r"(?ms)[^\/]*a(?:[^\/]*\/)*b\Z"
+    assert shellpattern.translate("*{a,ab}c*") == r"(?ms)[^\/]*(a|ab)c[^\/]*\Z"
+
+
+def test_no_exponential_backtracking():
+    # see #2624: this took "forever" before wildcard runs were matched atomically
+    regex = re.compile(shellpattern.translate("input/" + "a*" * 50 + "b"))
+    start = time.monotonic()
+    assert not regex.match("input/" + "a" * 200)
+    assert time.monotonic() - start < 10