Bound decoder work to prevent a pointer fan-out DoS (STF-1488)

HISTORY.rst

@@ -3,6 +3,23 @@
 History
 -------
 
+3.2.0
++++++
+
+* Fixed two denial-of-service issues in the pure Python decoder. A crafted
+  database could nest data-section pointers to shared targets so that decoding
+  one record cost exponential time and memory from a small file, or point many
+  times at one large string or bytes value so that a record with few values
+  materialized far more data than the file holds. The decoder now applies the
+  limits recommended by the MaxMind DB specification to each record it decodes
+  and to the metadata read when a database is opened. A database that exceeds
+  a limit raises ``InvalidDatabaseError``. The limits are:
+
+  * 65,536 decoded values.
+  * 2 MiB of string and bytes payload. An oversized variable-length integer is
+    also rejected.
+  * 512 levels of nesting, which also stops pointer cycles.
+
 3.1.1 (2026-03-05)
 ++++++++++++++++++
 

maxminddb/decoder.py

@@ -18,7 +18,39 @@
     from maxminddb.file import FileBuffer
     from maxminddb.types import Record
 
-    DecoderFunc = Callable[["Decoder", int, int], tuple[Record, int]]
+    DecoderFunc = Callable[["Decoder", int, int, list[int]], tuple[Record, int]]
+
+
+# Per-lookup limit on the number of values decoded, recommended by the MaxMind
+# DB specification. It stops a pointer fan-out, where nested pointers to shared
+# targets would otherwise cost 2**depth decode operations. The count follows
+# the specification's flat rule: the root is one value, each array and map
+# charges its declared children, and a pointer costs nothing beyond the value
+# it resolves to, which its container already charged. The largest real
+# records decode a few hundred values, so the limit leaves a wide margin.
+# Pointer cycles and over-deep data are caught separately by an explicit,
+# call-local depth limit (see ``decode``).
+_MAX_VALUES = 1 << 16
+_MAX_DEPTH = 512
+# Per-lookup limit on the total string and bytes payload materialized, matching
+# libmaxminddb and the Go reader. It stops a payload amplification, where many
+# pointers to one large value would otherwise materialize N * size bytes from a
+# small file. Each string or bytes value is charged its length wherever it is
+# decoded, so re-decoding a shared target through another pointer recharges.
+_MAX_PAYLOAD_BYTES = 1 << 21
+# The widest fixed-width integer the format defines is the 16-byte uint128; a
+# declared size past that is malformed and could copy attacker-controlled bytes.
+_MAX_UINT_BYTES = 16
+_MAX_INT32_BYTES = 4
+_TOO_MANY_VALUES = (
+    "The MaxMind DB file's data section exceeds the maximum number of values"
+)
+_TOO_DEEP = "The MaxMind DB file's data section exceeds the maximum depth"
+_TOO_LARGE = "The MaxMind DB file's data section exceeds the maximum payload size"
+_BAD_DATA = (
+    "The MaxMind DB file's data section contains bad data "
+    "(unknown data type or corrupt data)"
+)
 
 
 class Decoder:
@@ -42,35 +74,79 @@ def __init__(
         self._buffer = database_buffer
         self._pointer_base = pointer_base
 
-    def _decode_array(self, size: int, offset: int) -> tuple[list[Record], int]:
+    def _decode_array(
+        self,
+        size: int,
+        offset: int,
+        budget: list[int],
+    ) -> tuple[list[Record], int]:
+        budget[0] -= size
+        if budget[0] < 0:
+            raise InvalidDatabaseError(_TOO_MANY_VALUES)
+        budget[1] += 1
+        if budget[1] > _MAX_DEPTH:
+            raise InvalidDatabaseError(_TOO_DEEP)
         array = []
         for _ in range(size):
-            (value, offset) = self.decode(offset)
+            (value, offset) = self._decode(offset, budget)
             array.append(value)
+        budget[1] -= 1
         return array, offset
 
-    def _decode_boolean(self, size: int, offset: int) -> tuple[bool, int]:
+    def _decode_boolean(
+        self,
+        size: int,
+        offset: int,
+        _budget: list[int],
+    ) -> tuple[bool, int]:
         return size != 0, offset
 
-    def _decode_bytes(self, size: int, offset: int) -> tuple[bytes, int]:
+    def _decode_bytes(
+        self,
+        size: int,
+        offset: int,
+        budget: list[int],
+    ) -> tuple[bytes, int]:
+        # Charge the payload before copying so a crafted size cannot force a
+        # large allocation, and so pointers reusing one target recharge.
+        budget[2] -= size
+        if budget[2] < 0:
+            raise InvalidDatabaseError(_TOO_LARGE)
         new_offset = offset + size
         return self._buffer[offset:new_offset], new_offset
 
-    def _decode_double(self, size: int, offset: int) -> tuple[float, int]:
+    def _decode_double(
+        self,
+        size: int,
+        offset: int,
+        _budget: list[int],
+    ) -> tuple[float, int]:
         self._verify_size(size, 8)
         new_offset = offset + size
         packed_bytes = self._buffer[offset:new_offset]
         (value,) = struct.unpack(b"!d", packed_bytes)
         return value, new_offset
 
-    def _decode_float(self, size: int, offset: int) -> tuple[float, int]:
+    def _decode_float(
+        self,
+        size: int,
+        offset: int,
+        _budget: list[int],
+    ) -> tuple[float, int]:
         self._verify_size(size, 4)
         new_offset = offset + size
         packed_bytes = self._buffer[offset:new_offset]
         (value,) = struct.unpack(b"!f", packed_bytes)
         return value, new_offset
 
-    def _decode_int32(self, size: int, offset: int) -> tuple[int, int]:
+    def _decode_int32(
+        self,
+        size: int,
+        offset: int,
+        _budget: list[int],
+    ) -> tuple[int, int]:
+        if size > _MAX_INT32_BYTES:
+            raise InvalidDatabaseError(_BAD_DATA)
         if size == 0:
             return 0, offset
         new_offset = offset + size
@@ -81,15 +157,33 @@ def _decode_int32(self, size: int, offset: int) -> tuple[int, int]:
         (value,) = struct.unpack(b"!i", packed_bytes)
         return value, new_offset
 
-    def _decode_map(self, size: int, offset: int) -> tuple[dict[str, Record], int]:
+    def _decode_map(
+        self,
+        size: int,
+        offset: int,
+        budget: list[int],
+    ) -> tuple[dict[str, Record], int]:
+        # A map entry decodes a key and a value, so it costs two values.
+        budget[0] -= size * 2
+        if budget[0] < 0:
+            raise InvalidDatabaseError(_TOO_MANY_VALUES)
+        budget[1] += 1
+        if budget[1] > _MAX_DEPTH:
+            raise InvalidDatabaseError(_TOO_DEEP)
         container: dict[str, Record] = {}
         for _ in range(size):
-            (key, offset) = self.decode(offset)
-            (value, offset) = self.decode(offset)
+            (key, offset) = self._decode(offset, budget)
+            (value, offset) = self._decode(offset, budget)
             container[cast("str", key)] = value
+        budget[1] -= 1
         return container, offset
 
-    def _decode_pointer(self, size: int, offset: int) -> tuple[Record, int]:
+    def _decode_pointer(
+        self,
+        size: int,
+        offset: int,
+        budget: list[int],
+    ) -> tuple[Record, int]:
         pointer_size = (size >> 3) + 1
 
         buf = self._buffer[offset : offset + pointer_size]
@@ -109,15 +203,41 @@ def _decode_pointer(self, size: int, offset: int) -> tuple[Record, int]:
 
         if self._pointer_test:
             return pointer, new_offset
-        (value, _) = self.decode(pointer)
+
+        # The value at the pointer's position was charged by its containing
+        # array or map, so the target costs nothing more. Only the depth changes.
+        budget[1] += 1
+        if budget[1] > _MAX_DEPTH:
+            raise InvalidDatabaseError(_TOO_DEEP)
+        (value, _) = self._decode(pointer, budget)
+        budget[1] -= 1
         return value, new_offset
 
-    def _decode_uint(self, size: int, offset: int) -> tuple[int, int]:
+    def _decode_uint(
+        self,
+        size: int,
+        offset: int,
+        _budget: list[int],
+    ) -> tuple[int, int]:
+        # Reject a declared size past the widest defined unsigned integer before
+        # copying, so a crafted size cannot force a large allocation.
+        if size > _MAX_UINT_BYTES:
+            raise InvalidDatabaseError(_BAD_DATA)
         new_offset = offset + size
         uint_bytes = self._buffer[offset:new_offset]
         return int.from_bytes(uint_bytes, "big"), new_offset
 
-    def _decode_utf8_string(self, size: int, offset: int) -> tuple[str, int]:
+    def _decode_utf8_string(
+        self,
+        size: int,
+        offset: int,
+        budget: list[int],
+    ) -> tuple[str, int]:
+        # Charge the payload before copying so a crafted size cannot force a
+        # large allocation, and so pointers reusing one target recharge.
+        budget[2] -= size
+        if budget[2] < 0:
+            raise InvalidDatabaseError(_TOO_LARGE)
         new_offset = offset + size
         return self._buffer[offset:new_offset].decode("utf-8"), new_offset
 
@@ -144,6 +264,20 @@ def decode(self, offset: int) -> tuple[Record, int]:
             offset: the location of the data structure to decode
 
         """
+        # Bound the work per lookup so a crafted database cannot exhaust CPU or
+        # memory. ``budget`` carries the remaining value count, the current
+        # nested decode depth, and the remaining string and bytes payload, so
+        # all three are shared across the recursion. It is call-local, which
+        # keeps the decoder safe for concurrent reads. The root value is charged
+        # here; containers charge their children. The explicit depth limit
+        # is independent of Python's process-wide recursion limit; RecursionError
+        # remains a fallback on interpreters whose stack limit is reached first.
+        try:
+            return self._decode(offset, [_MAX_VALUES - 1, 0, _MAX_PAYLOAD_BYTES])
+        except RecursionError as ex:
+            raise InvalidDatabaseError(_TOO_DEEP) from ex
+
+    def _decode(self, offset: int, budget: list[int]) -> tuple[Record, int]:
         new_offset = offset + 1
         ctrl_byte = self._buffer[offset]
         type_num = ctrl_byte >> 5
@@ -160,7 +294,7 @@ def decode(self, offset: int) -> tuple[Record, int]:
             ) from ex
 
         (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num)
-        return decoder(self, size, new_offset)
+        return decoder(self, size, new_offset, budget)
 
     def _read_extended(self, offset: int) -> tuple[int, int]:
         next_byte = self._buffer[offset]
@@ -178,13 +312,7 @@ def _read_extended(self, offset: int) -> tuple[int, int]:
     @staticmethod
     def _verify_size(expected: int, actual: int) -> None:
         if expected != actual:
-            msg = (
-                "The MaxMind DB file's data section contains bad data "
-                "(unknown data type or corrupt data)"
-            )
-            raise InvalidDatabaseError(
-                msg,
-            )
+            raise InvalidDatabaseError(_BAD_DATA)
 
     def _size_from_ctrl_byte(
         self,