[3.14] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate (GH-153405) (GH-153831)

str.isdigit() returned True for non-decimal Unicode digits such as
U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError.
(cherry picked from commit 597ec2d)

Co-authored-by: tonghuaroot (童话) <tonghuaroot@gmail.com>

Author: miss-islington

Lib/test/test_robotparser.py

@@ -188,6 +188,8 @@ def test_request_rate(self):
                         parsed_request_rate.seconds,
                         self.request_rate.seconds
                     )
+                else:
+                    self.assertIsNone(parsed_request_rate)
 
 
 class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
@@ -246,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase):
     bad = []
 
 
+class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase):
+    # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+    # them, so the directive must be silently ignored, not raise.
+    robots_txt = """\
+User-Agent: *
+Disallow: /tmp/
+Crawl-delay: ²
+Request-rate: ²/5
+    """
+    good = ['/foo.html']
+    bad = ['/tmp/']
+    crawl_delay = None
+    request_rate = None
+
+
+class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase):
+    robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: 5/²
+    """
+    good = ['/foo.html']
+    request_rate = None
+    bad = ['/tmp/']
+
+
 class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
     # also test that Allow and Diasallow works well with each other
     robots_txt = """\

Lib/urllib/robotparser.py

@@ -143,15 +143,15 @@ def parse(self, lines):
                         # before trying to convert to int we need to make
                         # sure that robots.txt has valid syntax otherwise
                         # it will crash
-                        if line[1].strip().isdigit():
+                        if line[1].strip().isdecimal():
                             entry.delay = int(line[1])
                         state = 2
                 elif line[0] == "request-rate":
                     if state != 0:
                         numbers = line[1].split('/')
                         # check if all values are sane
-                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
-                            and numbers[1].strip().isdigit()):
+                        if (len(numbers) == 2 and numbers[0].strip().isdecimal()
+                            and numbers[1].strip().isdecimal()):
                             entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
                         state = 2
                 elif line[0] == "sitemap":

Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst

@@ -0,0 +1,4 @@
+:class:`urllib.robotparser.RobotFileParser` now silently ignores a
+``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits
+(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError`
+and aborting the parse of the whole ``robots.txt`` file.