diff --git a/HISTORY.rst b/HISTORY.rst index f463918..a275e19 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,24 @@ History ------- +3.2.0 ++++++ + +* Fixed a denial-of-service issue in the pure Python decoder. A crafted database + could nest data-section pointers to shared targets so that decoding one record + could cost exponential time and memory from a small file. The decoder now + limits the number of values it decodes for a single record and rejects a + database that exceeds it, along with pointer cycles and over-deep data, with an + ``InvalidDatabaseError``. See GHSA-hj94-g986-h9r7. +* Fixed a related payload-amplification denial of service in the pure Python + decoder. A crafted database could point many times at one large string or + bytes value, so a record with few values still materialized far more data than + the file holds. The decoder now also limits the total string and bytes payload + it produces for a single record to 2 MiB, and rejects an oversized + variable-length integer, with an ``InvalidDatabaseError``. The same limit + applies to the metadata read when a database is opened. See + GHSA-hj94-g986-h9r7. + 3.1.1 (2026-03-05) ++++++++++++++++++ diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index 8f67a7d..99a73b3 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -18,7 +18,36 @@ from maxminddb.file import FileBuffer from maxminddb.types import Record - DecoderFunc = Callable[["Decoder", int, int], tuple[Record, int]] + DecoderFunc = Callable[["Decoder", int, int, list[int]], tuple[Record, int]] + + +# Per-lookup limit on the number of values decoded, recommended by the MaxMind +# DB specification. It stops a pointer fan-out, where nested pointers to shared +# targets would otherwise cost 2**depth decode operations. The largest real +# records decode a few hundred values, so the limit leaves a wide margin. +# Pointer cycles and over-deep data are caught separately by an explicit, +# call-local depth limit (see ``decode``). +_MAX_VALUES = 1 << 16 +_MAX_DEPTH = 512 +# Per-lookup limit on the total string and bytes payload materialized, matching +# libmaxminddb and the Go reader. It stops a payload amplification, where many +# pointers to one large value would otherwise materialize N * size bytes from a +# small file. Each string or bytes value is charged its length wherever it is +# decoded, so re-decoding a shared target through another pointer recharges. +_MAX_PAYLOAD_BYTES = 1 << 21 +# The widest fixed-width integer the format defines is the 16-byte uint128; a +# declared size past that is malformed and could copy attacker-controlled bytes. +_MAX_UINT_BYTES = 16 +_MAX_INT32_BYTES = 4 +_TOO_MANY_VALUES = ( + "The MaxMind DB file's data section exceeds the maximum number of values" +) +_TOO_DEEP = "The MaxMind DB file's data section exceeds the maximum depth" +_TOO_LARGE = "The MaxMind DB file's data section exceeds the maximum payload size" +_BAD_DATA = ( + "The MaxMind DB file's data section contains bad data " + "(unknown data type or corrupt data)" +) class Decoder: @@ -42,35 +71,79 @@ def __init__( self._buffer = database_buffer self._pointer_base = pointer_base - def _decode_array(self, size: int, offset: int) -> tuple[list[Record], int]: + def _decode_array( + self, + size: int, + offset: int, + budget: list[int], + ) -> tuple[list[Record], int]: + budget[0] -= size + if budget[0] < 0: + raise InvalidDatabaseError(_TOO_MANY_VALUES) + budget[1] += 1 + if budget[1] > _MAX_DEPTH: + raise InvalidDatabaseError(_TOO_DEEP) array = [] for _ in range(size): - (value, offset) = self.decode(offset) + (value, offset) = self._decode(offset, budget) array.append(value) + budget[1] -= 1 return array, offset - def _decode_boolean(self, size: int, offset: int) -> tuple[bool, int]: + def _decode_boolean( + self, + size: int, + offset: int, + _budget: list[int], + ) -> tuple[bool, int]: return size != 0, offset - def _decode_bytes(self, size: int, offset: int) -> tuple[bytes, int]: + def _decode_bytes( + self, + size: int, + offset: int, + budget: list[int], + ) -> tuple[bytes, int]: + # Charge the payload before copying so a crafted size cannot force a + # large allocation, and so pointers reusing one target recharge. + budget[2] -= size + if budget[2] < 0: + raise InvalidDatabaseError(_TOO_LARGE) new_offset = offset + size return self._buffer[offset:new_offset], new_offset - def _decode_double(self, size: int, offset: int) -> tuple[float, int]: + def _decode_double( + self, + size: int, + offset: int, + _budget: list[int], + ) -> tuple[float, int]: self._verify_size(size, 8) new_offset = offset + size packed_bytes = self._buffer[offset:new_offset] (value,) = struct.unpack(b"!d", packed_bytes) return value, new_offset - def _decode_float(self, size: int, offset: int) -> tuple[float, int]: + def _decode_float( + self, + size: int, + offset: int, + _budget: list[int], + ) -> tuple[float, int]: self._verify_size(size, 4) new_offset = offset + size packed_bytes = self._buffer[offset:new_offset] (value,) = struct.unpack(b"!f", packed_bytes) return value, new_offset - def _decode_int32(self, size: int, offset: int) -> tuple[int, int]: + def _decode_int32( + self, + size: int, + offset: int, + _budget: list[int], + ) -> tuple[int, int]: + if size > _MAX_INT32_BYTES: + raise InvalidDatabaseError(_BAD_DATA) if size == 0: return 0, offset new_offset = offset + size @@ -81,15 +154,33 @@ def _decode_int32(self, size: int, offset: int) -> tuple[int, int]: (value,) = struct.unpack(b"!i", packed_bytes) return value, new_offset - def _decode_map(self, size: int, offset: int) -> tuple[dict[str, Record], int]: + def _decode_map( + self, + size: int, + offset: int, + budget: list[int], + ) -> tuple[dict[str, Record], int]: + # A map entry decodes a key and a value, so it costs two values. + budget[0] -= size * 2 + if budget[0] < 0: + raise InvalidDatabaseError(_TOO_MANY_VALUES) + budget[1] += 1 + if budget[1] > _MAX_DEPTH: + raise InvalidDatabaseError(_TOO_DEEP) container: dict[str, Record] = {} for _ in range(size): - (key, offset) = self.decode(offset) - (value, offset) = self.decode(offset) + (key, offset) = self._decode(offset, budget) + (value, offset) = self._decode(offset, budget) container[cast("str", key)] = value + budget[1] -= 1 return container, offset - def _decode_pointer(self, size: int, offset: int) -> tuple[Record, int]: + def _decode_pointer( + self, + size: int, + offset: int, + budget: list[int], + ) -> tuple[Record, int]: pointer_size = (size >> 3) + 1 buf = self._buffer[offset : offset + pointer_size] @@ -109,15 +200,46 @@ def _decode_pointer(self, size: int, offset: int) -> tuple[Record, int]: if self._pointer_test: return pointer, new_offset - (value, _) = self.decode(pointer) + + # The pointer itself was charged by its containing array or map. Charge + # its target separately because it is decoded separately each time the + # pointer is followed. + remaining = budget[0] - 1 + if remaining < 0: + raise InvalidDatabaseError(_TOO_MANY_VALUES) + budget[0] = remaining + budget[1] += 1 + if budget[1] > _MAX_DEPTH: + raise InvalidDatabaseError(_TOO_DEEP) + (value, _) = self._decode(pointer, budget) + budget[1] -= 1 return value, new_offset - def _decode_uint(self, size: int, offset: int) -> tuple[int, int]: + def _decode_uint( + self, + size: int, + offset: int, + _budget: list[int], + ) -> tuple[int, int]: + # Reject a declared size past the widest defined unsigned integer before + # copying, so a crafted size cannot force a large allocation. + if size > _MAX_UINT_BYTES: + raise InvalidDatabaseError(_BAD_DATA) new_offset = offset + size uint_bytes = self._buffer[offset:new_offset] return int.from_bytes(uint_bytes, "big"), new_offset - def _decode_utf8_string(self, size: int, offset: int) -> tuple[str, int]: + def _decode_utf8_string( + self, + size: int, + offset: int, + budget: list[int], + ) -> tuple[str, int]: + # Charge the payload before copying so a crafted size cannot force a + # large allocation, and so pointers reusing one target recharge. + budget[2] -= size + if budget[2] < 0: + raise InvalidDatabaseError(_TOO_LARGE) new_offset = offset + size return self._buffer[offset:new_offset].decode("utf-8"), new_offset @@ -144,6 +266,19 @@ def decode(self, offset: int) -> tuple[Record, int]: offset: the location of the data structure to decode """ + # Bound the work per lookup so a crafted database cannot exhaust CPU or + # memory. ``budget`` carries the remaining value count, the current + # nested decode depth, and the remaining string and bytes payload, so + # all three are shared across the recursion. It is call-local, which + # keeps the decoder safe for concurrent reads. The explicit depth limit + # is independent of Python's process-wide recursion limit; RecursionError + # remains a fallback on interpreters whose stack limit is reached first. + try: + return self._decode(offset, [_MAX_VALUES, 0, _MAX_PAYLOAD_BYTES]) + except RecursionError as ex: + raise InvalidDatabaseError(_TOO_DEEP) from ex + + def _decode(self, offset: int, budget: list[int]) -> tuple[Record, int]: new_offset = offset + 1 ctrl_byte = self._buffer[offset] type_num = ctrl_byte >> 5 @@ -160,7 +295,7 @@ def decode(self, offset: int) -> tuple[Record, int]: ) from ex (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num) - return decoder(self, size, new_offset) + return decoder(self, size, new_offset, budget) def _read_extended(self, offset: int) -> tuple[int, int]: next_byte = self._buffer[offset] @@ -178,13 +313,7 @@ def _read_extended(self, offset: int) -> tuple[int, int]: @staticmethod def _verify_size(expected: int, actual: int) -> None: if expected != actual: - msg = ( - "The MaxMind DB file's data section contains bad data " - "(unknown data type or corrupt data)" - ) - raise InvalidDatabaseError( - msg, - ) + raise InvalidDatabaseError(_BAD_DATA) def _size_from_ctrl_byte( self, diff --git a/tests/data b/tests/data index 7fc6aa0..d692a4b 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 7fc6aa0aa1ad9ccafcb54eee163949e65b6146aa +Subproject commit d692a4b74c68c6e856d0bd85a38ee405b65c816f diff --git a/tests/decoder_test.py b/tests/decoder_test.py index b755b5d..90dc3a0 100644 --- a/tests/decoder_test.py +++ b/tests/decoder_test.py @@ -1,14 +1,32 @@ from __future__ import annotations import mmap +import sys import unittest -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, cast +from maxminddb import MODE_MEMORY, MODE_MMAP_EXT, open_database from maxminddb.decoder import Decoder +from maxminddb.errors import InvalidDatabaseError + +try: + import maxminddb.extension as _extension +except ImportError: + _extension = None # type: ignore[assignment] if TYPE_CHECKING: from _typeshed import SizedBuffer +# Each structural level uses about two Python frames. This lets the 513-level +# cases reach the decoder's explicit limit with ample test-harness headroom. +_DEPTH_TEST_RECURSION_LIMIT = 2_000 + +# Directory holding the shared MaxMind DB test fixtures. +_TEST_DATA_DIR = "tests/data/test-data" +_PAYLOAD_TOO_LARGE = ( + "^The MaxMind DB file's data section exceeds the maximum payload size$" +) + class TestDecoder(unittest.TestCase): def test_arrays(self) -> None: @@ -232,3 +250,249 @@ def test_real_pointers(self) -> None: self.assertEqual(({"long_key2": "long_value2"}, 59), decoder.decode(57)) mm.close() + + @staticmethod + def _pointer(target: int) -> bytes: + # One-byte-payload pointer (type 1, pointer_size 1) with base 0. + return bytes([(1 << 5) | ((target >> 8) & 0x7), target & 0xFF]) + + def test_pointer_fan_out_is_bounded(self) -> None: + # A data section of nested arrays, each holding two pointers to the + # node below, would cost 2**depth decode operations. The decoder bounds + # the number of values it decodes per lookup and rejects the database. + depth = 100 + buf = bytearray([0xA0]) # leaf: uint16 with value 0 + prev = 0 + for _ in range(depth): + offset = len(buf) + buf += bytes([0x02, 0x04]) + self._pointer(prev) + self._pointer(prev) + prev = offset + + with self.assertRaises(InvalidDatabaseError): + Decoder(bytes(buf), pointer_base=0).decode(prev) + + def test_flat_scalar_pointer_fan_out_is_bounded(self) -> None: + # The array's 32,769 pointer fields and their separately decoded scalar + # targets exceed the 65,536-value limit even without nested containers. + # 0x1e: extended type with size code 30; 0x04: array; 0x7ee4: + # 32,769 - 285. + data = bytes([0xA0, 0x1E, 0x04, 0x7E, 0xE4]) + self._pointer(0) * 32_769 + + with self.assertRaisesRegex( + InvalidDatabaseError, + "^The MaxMind DB file's data section exceeds the maximum number of values$", + ): + Decoder(data, pointer_base=0).decode(1) + + def test_cyclic_pointer_raises(self) -> None: + # A pointer to itself must hit the decoder's own depth limit even when + # Python's process-wide recursion limit is much higher. + cyclic = bytes([0x20, 0x00]) # pointer (base 0) to offset 0, itself + old_recursion_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(_DEPTH_TEST_RECURSION_LIMIT) + with self.assertRaisesRegex( + InvalidDatabaseError, + "^The MaxMind DB file's data section exceeds the maximum depth$", + ): + Decoder(cyclic, pointer_base=0).decode(0) + finally: + sys.setrecursionlimit(old_recursion_limit) + + def test_container_depth_is_bounded_independently_of_recursion_limit(self) -> None: + # Each prefix is an array with one element. Raising Python's global + # recursion limit proves that the decoder's call-local limit is what + # accepts 512 containers and rejects the 513th. + at_limit = bytes([0x01, 0x04]) * 512 + bytes([0xA0]) + over_limit = bytes([0x01, 0x04]) * 513 + bytes([0xA0]) + old_recursion_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(_DEPTH_TEST_RECURSION_LIMIT) + Decoder(at_limit, pointer_base=0).decode(0) + with self.assertRaisesRegex( + InvalidDatabaseError, + "^The MaxMind DB file's data section exceeds the maximum depth$", + ): + Decoder(over_limit, pointer_base=0).decode(0) + finally: + sys.setrecursionlimit(old_recursion_limit) + + def test_oversized_map_is_bounded(self) -> None: + # A map entry decodes a key and a value, so a map of N entries costs + # 2N values. A map that declares 32,769 entries reaches 65,538 values, + # just past the 65,536 limit, and is rejected before any entry is read. + # 0xfe: map with size code 30, then the two size bytes for + # 32,769 - 285 = 32,484 (0x7ee4). + oversized_map = bytes([0xFE, 0x7E, 0xE4]) + with self.assertRaises(InvalidDatabaseError): + Decoder(oversized_map, pointer_base=0).decode(0) + + def test_oversized_string_payload_is_bounded(self) -> None: + # A single string that declares one byte more than the 2 MiB payload + # limit is rejected before its bytes are copied. This also covers the + # wrapped-scalar variant: the charge is applied wherever a string is + # decoded, not only for a direct pointer target. 0x5f: string with size + # code 31; 0x1efee4: 2,097,153 - 65,821, one byte over 2 MiB. + oversized_string = bytes([0x5F, 0x1E, 0xFE, 0xE4]) + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + Decoder(oversized_string, pointer_base=0).decode(0) + + def test_oversized_bytes_payload_is_bounded(self) -> None: + # As above for the bytes type. 0x9f: bytes with size code 31. + oversized_bytes = bytes([0x9F, 0x1E, 0xFE, 0xE4]) + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + Decoder(oversized_bytes, pointer_base=0).decode(0) + + def test_oversized_uint_is_bounded(self) -> None: + # A uint128 that declares 17 bytes exceeds the 16-byte format maximum + # and is rejected before the declared bytes are copied. 0x11: extended + # type, size 17; 0x03: extended type number 10 (uint128). + oversized_uint = bytes([0x11, 0x03]) + with self.assertRaises(InvalidDatabaseError): + Decoder(oversized_uint, pointer_base=0).decode(0) + + def test_oversized_int32_is_bounded(self) -> None: + # An int32 that declares 5 bytes exceeds its 4-byte maximum and is + # rejected before the declared bytes are copied. 0x05: extended type, + # size 5; 0x01: extended type number 8 (int32). + oversized_int32 = bytes([0x05, 0x01]) + with self.assertRaises(InvalidDatabaseError): + Decoder(oversized_int32, pointer_base=0).decode(0) + + +class TestDecoderResourceLimits(unittest.TestCase): + """Fixture-backed checks for the pure-Python decoder resource limits.""" + + @staticmethod + def _lookup(filename: str, ip: str = "0.0.0.1") -> object: + # MODE_MEMORY forces the pure-Python decoder. Each DoS fixture resolves + # any IPv4 address to its single crafted record. + with open_database(f"{_TEST_DATA_DIR}/{filename}", mode=MODE_MEMORY) as reader: + return reader.get(ip) + + def test_payload_amplification_is_rejected(self) -> None: + # An array of 8,192 pointers to one 65,535-byte value. The value count + # stays low, but copying each target would materialize about 512 MiB. + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + self._lookup("MaxMind-DB-test-payload-amplification-dos.mmdb") + + def test_payload_amplification_string_is_rejected(self) -> None: + # The UTF-8 string variant, so the decode path for strings is exercised. + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + self._lookup("MaxMind-DB-test-payload-amplification-dos-string.mmdb") + + def test_payload_amplification_worst_case_is_rejected(self) -> None: + # 65,534 pointers to one 65,535-byte value. This is the largest fan-out + # that stays under the value limit for a reader that counts each array + # element once; either the value counter (this decoder also charges each + # pointer target) or the payload budget must reject it. + with self.assertRaises(InvalidDatabaseError): + self._lookup("MaxMind-DB-test-payload-amplification-dos-worst-case.mmdb") + + def test_pointer_fan_out_fixture_is_rejected(self) -> None: + # A full database whose record nests arrays of pointers to the level + # below, the classic 2**depth fan-out. + with self.assertRaises(InvalidDatabaseError): + self._lookup("MaxMind-DB-test-pointer-decoder-dos.mmdb") + + def test_payload_at_limit_is_accepted(self) -> None: + # References totaling exactly 2 MiB of payload decode successfully, so + # the limit does not reject a record at the boundary. + self.assertIsInstance( + self._lookup("MaxMind-DB-test-decoder-payload-limit.mmdb"), + list, + ) + + def test_payload_one_over_limit_is_rejected(self) -> None: + # One byte more than 2 MiB is rejected, catching an off-by-one. + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + self._lookup("MaxMind-DB-test-decoder-payload-limit-over.mmdb") + + def test_metadata_payload_limit_is_enforced_on_open(self) -> None: + # The same decoder reads metadata on open, so an over-limit metadata + # structure is rejected there too. + with self.assertRaisesRegex(InvalidDatabaseError, _PAYLOAD_TOO_LARGE): + open_database( + f"{_TEST_DATA_DIR}/MaxMind-DB-test-metadata-payload-limit.mmdb", + mode=MODE_MEMORY, + ) + + def test_normal_record_still_decodes(self) -> None: + # A record with ordinary string and bytes values, which the payload + # budget also charges, decodes unchanged. + record = cast( + "dict", + self._lookup("MaxMind-DB-test-decoder.mmdb", "::1.1.1.0"), + ) + self.assertEqual(record["utf8_string"], "unicode! ☯ - ♫") + self.assertEqual(record["bytes"], b"\x00\x00\x00*") + + +def _has_extension() -> bool: + return _extension is not None and hasattr(_extension, "Reader") + + +# The patched libmaxminddb reports its decoder resource limits through this +# text (MMDB_DECODER_LIMIT_ERROR). A libmaxminddb without the fix decodes the +# DoS fixtures instead, so the tests below skip rather than run the extension's +# decoder out of memory. +_EXTENSION_LIMIT_MESSAGE = "exceeds the configured resource limits" + + +@unittest.skipUnless(_has_extension(), "C extension not available") +class TestExtensionResourceLimits(unittest.TestCase): + """DoS-fixture checks for the C extension's libmaxminddb decoder. + + The extension decodes through libmaxminddb, so these limits live in that + library, not in the pure-Python decoder that TestDecoderResourceLimits + covers. The checks run only when the linked libmaxminddb enforces the + limits and skip otherwise; see setUp. + """ + + @staticmethod + def _lookup(filename: str, ip: str = "0.0.0.1") -> object: + # MODE_MMAP_EXT forces the C extension. Each DoS fixture resolves any + # IPv4 address to its single crafted record. + with open_database( + f"{_TEST_DATA_DIR}/{filename}", + mode=MODE_MMAP_EXT, + ) as reader: + return reader.get(ip) + + def setUp(self) -> None: + # Probe with a fixture one byte over the 2 MiB payload limit. A patched + # libmaxminddb rejects it with the decoder-limit message. An older one + # decodes it, which is only about 2 MiB and so safe, but means the large + # DoS fixtures below would exhaust memory, so skip instead of running + # them. + try: + self._lookup("MaxMind-DB-test-decoder-payload-limit-over.mmdb") + except InvalidDatabaseError as exc: + if _EXTENSION_LIMIT_MESSAGE in str(exc): + return + self.skipTest( + "linked libmaxminddb predates the decoder resource limits " + "(needs the fix that adds MMDB_DECODER_LIMIT_ERROR)", + ) + + def test_pointer_fan_out_fixture_is_rejected(self) -> None: + # A full database whose record nests arrays of pointers to the level + # below, the classic 2**depth fan-out. + with self.assertRaisesRegex(InvalidDatabaseError, _EXTENSION_LIMIT_MESSAGE): + self._lookup("MaxMind-DB-test-pointer-decoder-dos.mmdb") + + def test_payload_amplification_is_rejected(self) -> None: + # An array of 8,192 pointers to one 65,535-byte value. + with self.assertRaisesRegex(InvalidDatabaseError, _EXTENSION_LIMIT_MESSAGE): + self._lookup("MaxMind-DB-test-payload-amplification-dos.mmdb") + + def test_payload_amplification_string_is_rejected(self) -> None: + # The UTF-8 string variant, so the string decode path is exercised. + with self.assertRaisesRegex(InvalidDatabaseError, _EXTENSION_LIMIT_MESSAGE): + self._lookup("MaxMind-DB-test-payload-amplification-dos-string.mmdb") + + def test_payload_amplification_worst_case_is_rejected(self) -> None: + # 65,534 pointers to one 65,535-byte value, the largest fan-out that + # stays under the value limit. + with self.assertRaisesRegex(InvalidDatabaseError, _EXTENSION_LIMIT_MESSAGE): + self._lookup("MaxMind-DB-test-payload-amplification-dos-worst-case.mmdb")