From 3e10215b0bfa3078b480741df4fca28d83554184 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Wed, 12 Aug 2026 15:46:23 -0700 Subject: [PATCH 1/7] Detect region reorder during Blob decryption --- .../azure/storage/blob/_encryption.py | 49 +++++++++++++++---- .../tests/test_blob_encryption_v2.py | 41 +++++++++++++++- .../tests/test_blob_encryption_v2_async.py | 41 +++++++++++++++- 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 2153d1da1da6..3b17c1c9cb32 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -839,6 +839,21 @@ def generate_blob_encryption_data( return content_encryption_key, initialization_vector, encryption_data +def _parse_content_range(content_range: str) -> Tuple[int, int, int]: + """ + Parses a Content-Range header of the form 'bytes x-y/size' into its + start, end, and total size components. + + :param str content_range: The Content-Range header value. + :return: A tuple of (start, end, total size). + :rtype: Tuple[int, int, int] + """ + # Format: 'bytes x-y/size' -- ignore the leading 'bytes' word. + byte_range, size = content_range.split(" ")[1].split("/") + start, end = byte_range.split("-") + return int(start), int(end), int(size) + + def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements require_encryption: bool, key_encryption_key: Optional[KeyEncryptionKey], @@ -904,16 +919,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements iv: Optional[bytes] = None unpad = False if "content-range" in response_headers: - content_range = response_headers["content-range"] - # Format: 'bytes x-y/size' - - # Ignore the word 'bytes' - content_range = content_range.split(" ") - - content_range = content_range[1].split("-") - content_range = content_range[1].split("/") - end_range = int(content_range[0]) - blob_size = int(content_range[1]) + _, end_range, blob_size = _parse_content_range(response_headers["content-range"]) if start_offset >= 16: iv = content[:16] @@ -957,6 +963,22 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements tag_length = encryption_data.encrypted_region_info.tag_length region_length = nonce_length + data_length + tag_length + # During encryption the nonce for each region is a counter of the region's index + # within the blob. The downloaded content always begins on an encryption region + # boundary, so derive the index of the first region from the download range. This + # lets us validate each region's nonce and detect if regions have been reordered. + # When there is no content-range header the whole blob was downloaded, so the first + # region is index 0. + start_range = 0 + if "content-range" in response_headers: + start_range, _, _ = _parse_content_range(response_headers["content-range"]) + nonce_counter = start_range // region_length + + # The nonce validation can be bypassed via an environment variable to allow for + # data recovery scenarios where the encryption regions have been reordered. This + # is not recommended for normal usage as it can allow tampered data to be decrypted. + validate_nonce = not os.environ.get("AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS") + decrypted_content = bytearray() while offset < total_size: # Process one encryption region at a time @@ -965,6 +987,12 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # First bytes are the nonce nonce = encrypted_region[:nonce_length] + # Validate the nonce matches the expected counter value for this region. A + # mismatch indicates the encryption regions have been reordered or tampered with. + if validate_nonce: + expected_nonce = nonce_counter.to_bytes(nonce_length, "big") + if nonce != expected_nonce: + raise ValueError("The encryption metadata is not valid and may have been modified.") ciphertext_with_tag = encrypted_region[nonce_length:] aesgcm = AESGCM(content_encryption_key) @@ -972,6 +1000,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements decrypted_content.extend(decrypted_data) offset += process_size + nonce_counter += 1 # Read the caller requested data from the decrypted content return decrypted_content[start_offset:end_offset] diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 05fa020d065c..49f00514a9ea 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -23,7 +23,7 @@ from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceExistsError -from azure.storage.blob import BlobServiceClient, BlobType, ContentSettings +from azure.storage.blob import BlobBlock, BlobServiceClient, BlobType, ContentSettings from azure.storage.blob._encryption import ( _dict_to_encryption_data, _GCM_NONCE_LENGTH, @@ -422,6 +422,45 @@ def test_encryption_modify_cek(self, **kwargs): assert "Decryption failed." in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + def test_encryption_reordered_regions(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_put_size=1024, + max_block_size=4 * MiB, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions + + # Upload with the block size equal to the encryption region size so each + # committed block corresponds to a single encryption region. + blob.upload_blob(content, overwrite=True) + + # Reorder the committed blocks so the encryption regions are out of order. + plain_blob = self.bsc.get_blob_client(self.container_name, self._get_blob_reference()) + metadata = plain_blob.get_blob_properties().metadata + committed, _ = plain_blob.get_block_list(block_list_type="committed") + reordered = committed[:-2] + committed[-1:] + committed[-2:-1] + reordered = [BlobBlock(block_id=block.id) for block in reordered] + plain_blob.commit_block_list(reordered, metadata=metadata) + + # Act / Assert -- a region's nonce no longer matches its position + with pytest.raises(HttpResponseError) as e: + blob.download_blob().readall() + + assert "Decryption failed." in str(e.value) + @BlobPreparer() @recorded_by_proxy @mock.patch("os.urandom", mock_urandom) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py index e3979cc3b27c..2a95e4c285d1 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py @@ -24,7 +24,7 @@ from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceExistsError -from azure.storage.blob import BlobType, ContentSettings +from azure.storage.blob import BlobBlock, BlobType, ContentSettings from azure.storage.blob._encryption import ( _dict_to_encryption_data, _GCM_NONCE_LENGTH, @@ -427,6 +427,45 @@ async def test_encryption_modify_cek(self, **kwargs): assert "Decryption failed." in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + async def test_encryption_reordered_regions(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_put_size=1024, + max_block_size=4 * MiB, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions + + # Upload with the block size equal to the encryption region size so each + # committed block corresponds to a single encryption region. + await blob.upload_blob(content, overwrite=True) + + # Reorder the committed blocks so the encryption regions are out of order. + plain_blob = self.bsc.get_blob_client(self.container_name, self._get_blob_reference()) + metadata = (await plain_blob.get_blob_properties()).metadata + committed, _ = await plain_blob.get_block_list(block_list_type="committed") + reordered = committed[:-2] + committed[-1:] + committed[-2:-1] + reordered = [BlobBlock(block_id=block.id) for block in reordered] + await plain_blob.commit_block_list(reordered, metadata=metadata) + + # Act / Assert -- a region's nonce no longer matches its position + with pytest.raises(HttpResponseError) as e: + await (await blob.download_blob()).readall() + + assert "Decryption failed." in str(e.value) + @BlobPreparer() @recorded_by_proxy_async async def test_case_insensitive_metadata_key(self, **kwargs): From f0964b13b52b68f5e7257c6993a4936dc669a250 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Wed, 12 Aug 2026 16:05:44 -0700 Subject: [PATCH 2/7] Fix test per Copilot --- .../azure-storage-blob/tests/test_blob_encryption_v2.py | 8 ++++---- .../tests/test_blob_encryption_v2_async.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 49f00514a9ea..081483f8c5d8 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -430,11 +430,14 @@ def test_encryption_reordered_regions(self, **kwargs): self._setup(storage_account_name, storage_account_key) kek = KeyWrapper("key1") + # Each encrypted region is the plaintext region plus a nonce and tag. Size each + # block to a full encrypted region so every committed block is exactly one region. + region_length = 4 * MiB + _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH bsc = BlobServiceClient( self.account_url(storage_account_name, "blob"), credential=storage_account_key.secret, max_single_put_size=1024, - max_block_size=4 * MiB, + max_block_size=region_length, require_encryption=True, encryption_version="2.0", key_encryption_key=kek, @@ -442,9 +445,6 @@ def test_encryption_reordered_regions(self, **kwargs): blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions - - # Upload with the block size equal to the encryption region size so each - # committed block corresponds to a single encryption region. blob.upload_blob(content, overwrite=True) # Reorder the committed blocks so the encryption regions are out of order. diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py index 2a95e4c285d1..f26558d6613e 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py @@ -435,11 +435,14 @@ async def test_encryption_reordered_regions(self, **kwargs): await self._setup(storage_account_name, storage_account_key) kek = KeyWrapper("key1") + # Each encrypted region is the plaintext region plus a nonce and tag. Size each + # block to a full encrypted region so every committed block is exactly one region. + region_length = 4 * MiB + _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH bsc = BlobServiceClient( self.account_url(storage_account_name, "blob"), credential=storage_account_key.secret, max_single_put_size=1024, - max_block_size=4 * MiB, + max_block_size=region_length, require_encryption=True, encryption_version="2.0", key_encryption_key=kek, @@ -447,9 +450,6 @@ async def test_encryption_reordered_regions(self, **kwargs): blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions - - # Upload with the block size equal to the encryption region size so each - # committed block corresponds to a single encryption region. await blob.upload_blob(content, overwrite=True) # Reorder the committed blocks so the encryption regions are out of order. From b635b322b92987386bcf0b1beb6c08b90a26e859 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Wed, 12 Aug 2026 16:46:33 -0700 Subject: [PATCH 3/7] Fix for .NET nonce counter --- .../azure/storage/blob/_encryption.py | 48 ++++++++++---- .../tests/test_blob_encryption_v2.py | 65 +++++++++++++++++++ 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 3b17c1c9cb32..8af29cdd237f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -15,7 +15,7 @@ dumps, loads, ) -from typing import Any, Callable, Dict, IO, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, IO, List, Optional, Tuple, TYPE_CHECKING from typing import OrderedDict as TypedOrderedDict from typing_extensions import Protocol @@ -854,6 +854,34 @@ def _parse_content_range(content_range: str) -> Tuple[int, int, int]: return int(start), int(end), int(size) +def _region_nonce_candidates(region_index: int, nonce_length: int) -> List[bytes]: + """ + Returns the valid nonce encodings for the encryption region at the given zero-based index. + + The per-region nonce is a counter of the region's position, but SDKs encode it + differently, so all supported encodings must be accepted for interoperability: + Python/Java use a zero-based big-endian counter; .NET uses a one-based little-endian + counter in the trailing 8 bytes. Each encoding is a bijection of the index, so accepting all + of them does not weaken reordering detection. + + :param int region_index: The zero-based index of the region within the blob. + :param int nonce_length: The length of the nonce in bytes. + :return: The list of valid nonce byte encodings for the region. + :rtype: List[bytes] + """ + candidates = [region_index.to_bytes(nonce_length, "big")] + + # .NET: one-based little-endian counter in the trailing 8 bytes, zero-padded. + dotnet_counter_length = 8 + if nonce_length >= dotnet_counter_length: + candidates.append( + b"\x00" * (nonce_length - dotnet_counter_length) + + (region_index + 1).to_bytes(dotnet_counter_length, "little") + ) + + return candidates + + def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements require_encryption: bool, key_encryption_key: Optional[KeyEncryptionKey], @@ -963,20 +991,17 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements tag_length = encryption_data.encrypted_region_info.tag_length region_length = nonce_length + data_length + tag_length - # During encryption the nonce for each region is a counter of the region's index - # within the blob. The downloaded content always begins on an encryption region - # boundary, so derive the index of the first region from the download range. This - # lets us validate each region's nonce and detect if regions have been reordered. - # When there is no content-range header the whole blob was downloaded, so the first - # region is index 0. + # The per-region nonce is a counter of the region's index within the blob. The + # downloaded content always begins on a region boundary, so derive the first + # region's index from the download range (0 when the whole blob was downloaded). + # This lets us validate each nonce and detect reordered regions. start_range = 0 if "content-range" in response_headers: start_range, _, _ = _parse_content_range(response_headers["content-range"]) nonce_counter = start_range // region_length - # The nonce validation can be bypassed via an environment variable to allow for - # data recovery scenarios where the encryption regions have been reordered. This - # is not recommended for normal usage as it can allow tampered data to be decrypted. + # Bypass nonce validation via an environment variable for data-recovery scenarios + # where regions were reordered. Not recommended: it can allow tampered data through. validate_nonce = not os.environ.get("AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS") decrypted_content = bytearray() @@ -990,8 +1015,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # Validate the nonce matches the expected counter value for this region. A # mismatch indicates the encryption regions have been reordered or tampered with. if validate_nonce: - expected_nonce = nonce_counter.to_bytes(nonce_length, "big") - if nonce != expected_nonce: + if nonce not in _region_nonce_candidates(nonce_counter, nonce_length): raise ValueError("The encryption metadata is not valid and may have been modified.") ciphertext_with_tag = encrypted_region[nonce_length:] diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 081483f8c5d8..0d36bd487dd7 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -29,6 +29,7 @@ _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, _validate_and_unwrap_cek, + decrypt_blob, ) TEST_CONTAINER_PREFIX = "encryptionv2_container" @@ -461,6 +462,70 @@ def test_encryption_reordered_regions(self, **kwargs): assert "Decryption failed." in str(e.value) + def test_decrypt_dotnet_v2_1_nonce_encoding(self): + # Regression test for cross-SDK interoperability. Blobs produced by the .NET + # Storage SDK encode each region's GCM nonce as a one-based counter written + # little-endian into the final 8 nonce bytes, whereas Python/Java use a zero-based + # big-endian counter. Python must still decrypt these .NET-produced V2.1 blobs while + # continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + + region_data_length = 32 + num_regions = 3 + plaintext_regions = [bytes([i]) * region_data_length for i in range(num_regions)] + + def dotnet_nonce(region_index): + # 4 zero bytes + one-based counter, little-endian, 8 bytes -- see .NET + # GcmAuthenticatedCryptographicTransform.GetNewNonce(). + return b"\x00\x00\x00\x00" + (region_index + 1).to_bytes(8, "little") + + aesgcm = AESGCM(cek) + encrypted_regions = [ + dotnet_nonce(i) + aesgcm.encrypt(dotnet_nonce(i), region, None) + for i, region in enumerate(plaintext_regions) + ] + + # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). + wrapped_cek = kek.wrap_key(b"2.1".ljust(8, b"\x00") + cek) + encryption_data = { + "WrappedContentKey": { + "KeyId": kek.get_kid(), + "EncryptedKey": base64.b64encode(wrapped_cek).decode(), + "Algorithm": kek.get_key_wrap_algorithm(), + }, + "EncryptionAgent": {"Protocol": "2.1", "EncryptionAlgorithm": "AES_GCM_256"}, + "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, + "KeyWrappingMetadata": {"EncryptionLibrary": "Dotnet"}, + } + headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the .NET nonce encoding is accepted and the content round-trips. + decrypted = decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=b"".join(encrypted_regions), + start_offset=0, + end_offset=len(plaintext), + response_headers=headers, + ) + assert decrypted == plaintext + + # Reordering the .NET-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=reordered, + start_offset=0, + end_offset=len(plaintext), + response_headers=headers, + ) + @BlobPreparer() @recorded_by_proxy @mock.patch("os.urandom", mock_urandom) From e16b52c237ee74bb46c4c3f7751281868794e888 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Wed, 12 Aug 2026 17:01:45 -0700 Subject: [PATCH 4/7] Fix Java, fix env var reading --- .../azure/storage/blob/_encryption.py | 29 +++++--- .../tests/test_blob_encryption_v2.py | 70 +++++++++++++++++-- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 8af29cdd237f..2b74499ea334 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -858,11 +858,15 @@ def _region_nonce_candidates(region_index: int, nonce_length: int) -> List[bytes """ Returns the valid nonce encodings for the encryption region at the given zero-based index. - The per-region nonce is a counter of the region's position, but SDKs encode it + The per-region nonce is a counter of the region's position, but each SDK encodes it differently, so all supported encodings must be accepted for interoperability: - Python/Java use a zero-based big-endian counter; .NET uses a one-based little-endian - counter in the trailing 8 bytes. Each encoding is a bijection of the index, so accepting all - of them does not weaken reordering detection. + + * Python: zero-based counter, big-endian across the whole nonce (value in trailing bytes). + * Java: zero-based counter, big-endian in the leading 8 bytes, trailing bytes zeroed. + * .NET: one-based counter, little-endian in the trailing 8 bytes, leading bytes zeroed. + + Each encoding is a bijection of the index, so accepting all of them does not weaken + reordering detection. :param int region_index: The zero-based index of the region within the blob. :param int nonce_length: The length of the nonce in bytes. @@ -871,12 +875,15 @@ def _region_nonce_candidates(region_index: int, nonce_length: int) -> List[bytes """ candidates = [region_index.to_bytes(nonce_length, "big")] - # .NET: one-based little-endian counter in the trailing 8 bytes, zero-padded. - dotnet_counter_length = 8 - if nonce_length >= dotnet_counter_length: + counter_length = 8 + if nonce_length >= counter_length: + # Java + candidates.append( + region_index.to_bytes(counter_length, "big") + b"\x00" * (nonce_length - counter_length) + ) + # .NET candidates.append( - b"\x00" * (nonce_length - dotnet_counter_length) - + (region_index + 1).to_bytes(dotnet_counter_length, "little") + b"\x00" * (nonce_length - counter_length) + (region_index + 1).to_bytes(counter_length, "little") ) return candidates @@ -1002,7 +1009,9 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # Bypass nonce validation via an environment variable for data-recovery scenarios # where regions were reordered. Not recommended: it can allow tampered data through. - validate_nonce = not os.environ.get("AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS") + validate_nonce = os.environ.get( + "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS", "" + ).strip().lower() not in ("true", "1") decrypted_content = bytearray() while offset < total_size: diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 0d36bd487dd7..a5412535a1c2 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -463,10 +463,9 @@ def test_encryption_reordered_regions(self, **kwargs): assert "Decryption failed." in str(e.value) def test_decrypt_dotnet_v2_1_nonce_encoding(self): - # Regression test for cross-SDK interoperability. Blobs produced by the .NET - # Storage SDK encode each region's GCM nonce as a one-based counter written - # little-endian into the final 8 nonce bytes, whereas Python/Java use a zero-based - # big-endian counter. Python must still decrypt these .NET-produced V2.1 blobs while + # Regression test for cross-SDK interoperability. The .NET Storage SDK encodes each + # region's GCM nonce as a one-based counter written little-endian into the final 8 + # nonce bytes. Python must still decrypt these .NET-produced V2.1 blobs while # continuing to detect reordered regions. kek = KeyWrapper("key1") cek = os.urandom(32) @@ -526,6 +525,69 @@ def dotnet_nonce(region_index): response_headers=headers, ) + def test_decrypt_java_v2_nonce_encoding(self): + # Regression test for cross-SDK interoperability. The Java Storage SDK encodes each + # region's GCM nonce as a zero-based counter written big-endian into the leading 8 + # nonce bytes (ByteBuffer.allocate(12).putLong(index)), which differs from Python's + # full-width big-endian counter. Python must still decrypt Java-produced V2 blobs + # while continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + + region_data_length = 32 + num_regions = 3 + plaintext_regions = [bytes([i]) * region_data_length for i in range(num_regions)] + + def java_nonce(region_index): + # Zero-based counter, big-endian, in the leading 8 bytes; trailing bytes zeroed. + return region_index.to_bytes(8, "big") + b"\x00" * (_GCM_NONCE_LENGTH - 8) + + aesgcm = AESGCM(cek) + encrypted_regions = [ + java_nonce(i) + aesgcm.encrypt(java_nonce(i), region, None) + for i, region in enumerate(plaintext_regions) + ] + + # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). + wrapped_cek = kek.wrap_key(b"2.0".ljust(8, b"\x00") + cek) + encryption_data = { + "WrappedContentKey": { + "KeyId": kek.get_kid(), + "EncryptedKey": base64.b64encode(wrapped_cek).decode(), + "Algorithm": kek.get_key_wrap_algorithm(), + }, + "EncryptionAgent": {"Protocol": "2.0", "EncryptionAlgorithm": "AES_GCM_256"}, + "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, + "KeyWrappingMetadata": {"EncryptionLibrary": "Java"}, + } + headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the Java nonce encoding is accepted and the content round-trips. + decrypted = decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=b"".join(encrypted_regions), + start_offset=0, + end_offset=len(plaintext), + response_headers=headers, + ) + assert decrypted == plaintext + + # Reordering the Java-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=reordered, + start_offset=0, + end_offset=len(plaintext), + response_headers=headers, + ) + @BlobPreparer() @recorded_by_proxy @mock.patch("os.urandom", mock_urandom) From 2ab30d6e714be5ce4ba2b851a73e7b66fd33b140 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Wed, 12 Aug 2026 17:44:12 -0700 Subject: [PATCH 5/7] Small nit --- .../azure/storage/blob/_encryption.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 2b74499ea334..3b9d1b6b591d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -876,15 +876,12 @@ def _region_nonce_candidates(region_index: int, nonce_length: int) -> List[bytes candidates = [region_index.to_bytes(nonce_length, "big")] counter_length = 8 - if nonce_length >= counter_length: - # Java - candidates.append( - region_index.to_bytes(counter_length, "big") + b"\x00" * (nonce_length - counter_length) - ) - # .NET - candidates.append( - b"\x00" * (nonce_length - counter_length) + (region_index + 1).to_bytes(counter_length, "little") - ) + # Java + candidates.append(region_index.to_bytes(counter_length, "big") + b"\x00" * (nonce_length - counter_length)) + # .NET + candidates.append( + b"\x00" * (nonce_length - counter_length) + (region_index + 1).to_bytes(counter_length, "little") + ) return candidates From 99abd7d5564a9d4f37c094ac7edcd6922af9e22e Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Thu, 13 Aug 2026 11:37:08 -0700 Subject: [PATCH 6/7] Pick one algorithm --- .../azure/storage/blob/_encryption.py | 48 +++++++++++-------- .../tests/test_blob_encryption_v2.py | 48 +++++++++++++++++++ 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 3b9d1b6b591d..8778b205a71b 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -15,7 +15,7 @@ dumps, loads, ) -from typing import Any, Callable, Dict, IO, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, IO, Optional, Tuple, TYPE_CHECKING from typing import OrderedDict as TypedOrderedDict from typing_extensions import Protocol @@ -854,36 +854,36 @@ def _parse_content_range(content_range: str) -> Tuple[int, int, int]: return int(start), int(end), int(size) -def _region_nonce_candidates(region_index: int, nonce_length: int) -> List[bytes]: +def _region_nonce_encodings(nonce_length: int) -> Dict[str, Callable[[int], bytes]]: """ - Returns the valid nonce encodings for the encryption region at the given zero-based index. + Returns the supported per-region nonce encodings, keyed by the SDK that produces them. The per-region nonce is a counter of the region's position, but each SDK encodes it - differently, so all supported encodings must be accepted for interoperability: + differently, so all supported encodings must be understood for interoperability: * Python: zero-based counter, big-endian across the whole nonce (value in trailing bytes). * Java: zero-based counter, big-endian in the leading 8 bytes, trailing bytes zeroed. * .NET: one-based counter, little-endian in the trailing 8 bytes, leading bytes zeroed. - Each encoding is a bijection of the index, so accepting all of them does not weaken - reordering detection. + These encodings share the same value space, so they must not be accepted independently + per region (for example Java's nonce for region 1 is identical to .NET's nonce for + region 16,777,215). A single encoding is instead selected and enforced across the whole + download; see ``decrypt_blob``. - :param int region_index: The zero-based index of the region within the blob. :param int nonce_length: The length of the nonce in bytes. - :return: The list of valid nonce byte encodings for the region. - :rtype: List[bytes] + :return: A mapping of SDK name to a function returning that SDK's nonce for a region index. + :rtype: Dict[str, Callable[[int], bytes]] """ - candidates = [region_index.to_bytes(nonce_length, "big")] + encodings: Dict[str, Callable[[int], bytes]] = { + "python": lambda index: index.to_bytes(nonce_length, "big"), + } counter_length = 8 - # Java - candidates.append(region_index.to_bytes(counter_length, "big") + b"\x00" * (nonce_length - counter_length)) - # .NET - candidates.append( - b"\x00" * (nonce_length - counter_length) + (region_index + 1).to_bytes(counter_length, "little") - ) + pad = nonce_length - counter_length + encodings["java"] = lambda index: index.to_bytes(counter_length, "big") + b"\x00" * pad + encodings["dotnet"] = lambda index: b"\x00" * pad + (index + 1).to_bytes(counter_length, "little") - return candidates + return encodings def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements @@ -1010,6 +1010,8 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS", "" ).strip().lower() not in ("true", "1") + candidate_encodings = _region_nonce_encodings(nonce_length) + decrypted_content = bytearray() while offset < total_size: # Process one encryption region at a time @@ -1018,10 +1020,16 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # First bytes are the nonce nonce = encrypted_region[:nonce_length] - # Validate the nonce matches the expected counter value for this region. A - # mismatch indicates the encryption regions have been reordered or tampered with. + # Validate the nonce matches the expected counter for this region under a single + # consistent encoding. A mismatch (empty candidate set) indicates the regions were + # reordered or tampered with. if validate_nonce: - if nonce not in _region_nonce_candidates(nonce_counter, nonce_length): + candidate_encodings = { + name: encode + for name, encode in candidate_encodings.items() + if encode(nonce_counter) == nonce + } + if not candidate_encodings: raise ValueError("The encryption metadata is not valid and may have been modified.") ciphertext_with_tag = encrypted_region[nonce_length:] diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index a5412535a1c2..306a704bc495 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -28,6 +28,7 @@ _dict_to_encryption_data, _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, + _region_nonce_encodings, _validate_and_unwrap_cek, decrypt_blob, ) @@ -588,6 +589,53 @@ def java_nonce(region_index): response_headers=headers, ) + def test_decrypt_rejects_mixed_nonce_encodings(self): + # Regression test: the supported SDK nonce encodings share a value space, so accepting + # them independently per region would weaken reorder detection. For example Java's + # nonce for region 1 is identical to .NET's nonce for region 16,777,215, so a Java + # region could be moved to that position and still pass a per-region union check. + # decrypt_blob must instead select a single encoding and enforce it consistently. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + # Document the overlap that motivates single-encoding enforcement. + assert encodings["java"](1) == encodings["dotnet"](16_777_215) + + kek = KeyWrapper("key1") + cek = os.urandom(32) + region_data_length = 32 + aesgcm = AESGCM(cek) + + # Region 0 uses the Java/Python encoding (all zeros); region 1 uses the .NET encoding. + # A per-region union check would accept both; single-encoding enforcement rejects the mix. + region0_nonce = encodings["java"](0) + region1_nonce = encodings["dotnet"](1) + region0 = region0_nonce + aesgcm.encrypt(region0_nonce, b"\x00" * region_data_length, None) + region1 = region1_nonce + aesgcm.encrypt(region1_nonce, b"\x11" * region_data_length, None) + + wrapped_cek = kek.wrap_key(b"2.0".ljust(8, b"\x00") + cek) + encryption_data = { + "WrappedContentKey": { + "KeyId": kek.get_kid(), + "EncryptedKey": base64.b64encode(wrapped_cek).decode(), + "Algorithm": kek.get_key_wrap_algorithm(), + }, + "EncryptionAgent": {"Protocol": "2.0", "EncryptionAlgorithm": "AES_GCM_256"}, + "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, + "KeyWrappingMetadata": {"EncryptionLibrary": "Mixed"}, + } + headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} + + # Act / Assert -- the mixed encoding is rejected rather than silently accepted. + with pytest.raises(ValueError): + decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=region0 + region1, + start_offset=0, + end_offset=2 * region_data_length, + response_headers=headers, + ) + @BlobPreparer() @recorded_by_proxy @mock.patch("os.urandom", mock_urandom) From 348274a775dbeb17276f4cffa6736ff2b4902bdd Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 14 Aug 2026 13:34:24 -0700 Subject: [PATCH 7/7] Feedback --- .../azure/storage/blob/_download.py | 4 + .../azure/storage/blob/_encryption.py | 69 +++- .../azure/storage/blob/aio/_download_async.py | 11 +- .../tests/test_blob_encryption_v2.py | 340 +++++++++--------- 4 files changed, 238 insertions(+), 186 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py index d704d13bccaa..40ac3d521745 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py @@ -41,6 +41,7 @@ get_adjusted_download_range_and_offset, is_encryption_v2, parse_encryption_data, + _GCMRegionNonceValidator, ) if TYPE_CHECKING: @@ -85,6 +86,7 @@ def process_content(data: Any, start_offset: int, end_offset: int, encryption: D start_offset, end_offset, data.response.headers, + encryption.get("gcm_nonce_validator"), ) except Exception as error: raise HttpResponseError(message="Decryption failed.", response=data.response, error=error) from error @@ -388,6 +390,8 @@ def __init__( if self._encryption_options.get("key") is not None or self._encryption_options.get("resolver") is not None: self._get_encryption_data_request() + if is_encryption_v2(self._encryption_data): + self._encryption_options["gcm_nonce_validator"] = _GCMRegionNonceValidator() # The service only provides transactional MD5s for chunks under 4MB. # If validate_content is using MD5, get only self.MAX_CHUNK_GET_SIZE for the first diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 8778b205a71b..b0854340d6ac 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -8,6 +8,7 @@ import math import os import sys +import threading import warnings from collections import OrderedDict from io import BytesIO @@ -53,6 +54,8 @@ "The require_encryption flag is set, but encryption is not supported for this method." ) +_ERROR_INVALID_ENCRYPTION_METADATA = "The encryption metadata is not valid and may have been modified." + class KeyEncryptionKey(Protocol): @@ -665,7 +668,7 @@ def _validate_and_unwrap_cek( version_2_bytes = encryption_data.encryption_agent.protocol.encode().ljust(8, b"\0") cek_version_bytes = content_encryption_key[: len(version_2_bytes)] if cek_version_bytes != version_2_bytes: - raise ValueError("The encryption metadata is not valid and may have been modified.") + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) # Remove version from the start of the cek. content_encryption_key = content_encryption_key[len(version_2_bytes) :] @@ -886,6 +889,48 @@ def _region_nonce_encodings(nonce_length: int) -> Dict[str, Callable[[int], byte return encodings +class _GCMRegionNonceValidator: + """ + Enforces that every region across a whole download uses a single nonce encoding. + + ``decrypt_blob`` runs once per HTTP chunk, so the candidate encodings are shared and + intersected across all chunks (including concurrent ones) rather than reset per call. + Otherwise the encoding could change at a chunk boundary and, at an encoding collision, + let a relocated region pass validation. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._candidates: Optional[Dict[str, Callable[[int], bytes]]] = None + # Set once the candidates collapse to a single encoding; read lock-free thereafter. + self._encoding: Optional[Callable[[int], bytes]] = None + + def validate_region(self, region_index: int, nonce: bytes, nonce_length: int) -> None: + """ + Narrows the shared candidate encodings to those consistent with this region. + + :param int region_index: The zero-based index of the region within the blob. + :param bytes nonce: The nonce read from the region. + :param int nonce_length: The length of the nonce in bytes. + """ + encoding = self._encoding + if encoding is not None: + if encoding(region_index) != nonce: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + return + + with self._lock: + if self._candidates is None: + self._candidates = _region_nonce_encodings(nonce_length) + self._candidates = { + name: encode for name, encode in self._candidates.items() if encode(region_index) == nonce + } + if not self._candidates: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + if len(self._candidates) == 1: + self._encoding = next(iter(self._candidates.values())) + + def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements require_encryption: bool, key_encryption_key: Optional[KeyEncryptionKey], @@ -894,6 +939,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements start_offset: int, end_offset: int, response_headers: Dict[str, Any], + nonce_validator: Optional["_GCMRegionNonceValidator"] = None, ) -> bytes: """ Decrypts the given blob contents and returns only the requested range. @@ -921,6 +967,10 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements :param Dict[str, Any] response_headers: A dictionary of response headers from the download request. Expected to include the 'x-ms-meta-encryptiondata' header if the blob was encrypted. + :param Optional[_GCMRegionNonceValidator] nonce_validator: + Shared state used to enforce a single V2 nonce encoding across every chunk of a + download. Required for V2 decryption unless nonce validation is bypassed via the + AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS environment variable. :return: The decrypted blob content. :rtype: bytes """ @@ -1010,7 +1060,9 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS", "" ).strip().lower() not in ("true", "1") - candidate_encodings = _region_nonce_encodings(nonce_length) + # A validator is required to enforce a single nonce encoding across the whole download. + if validate_nonce and nonce_validator is None: + raise ValueError("A nonce validator is required to decrypt Encryption V2 content.") decrypted_content = bytearray() while offset < total_size: @@ -1021,16 +1073,9 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # First bytes are the nonce nonce = encrypted_region[:nonce_length] # Validate the nonce matches the expected counter for this region under a single - # consistent encoding. A mismatch (empty candidate set) indicates the regions were - # reordered or tampered with. - if validate_nonce: - candidate_encodings = { - name: encode - for name, encode in candidate_encodings.items() - if encode(nonce_counter) == nonce - } - if not candidate_encodings: - raise ValueError("The encryption metadata is not valid and may have been modified.") + # consistent encoding. A mismatch indicates the regions were reordered or tampered with. + if nonce_validator is not None and validate_nonce: + nonce_validator.validate_region(nonce_counter, nonce, nonce_length) ciphertext_with_tag = encrypted_region[nonce_length:] aesgcm = AESGCM(content_encryption_key) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index 609f5efb3e1e..3445a8f0d2dc 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -38,7 +38,13 @@ from .._shared.validation import is_md5_validation, CV_TYPE_PARSED from .._deserialize import deserialize_blob_properties, get_page_ranges_result from .._download import process_range_and_offset, _ChunkDownloader -from .._encryption import adjust_blob_size_for_encryption, decrypt_blob, is_encryption_v2, parse_encryption_data +from .._encryption import ( + adjust_blob_size_for_encryption, + decrypt_blob, + is_encryption_v2, + parse_encryption_data, + _GCMRegionNonceValidator, +) if TYPE_CHECKING: from codecs import IncrementalDecoder @@ -68,6 +74,7 @@ async def process_content(data: Any, start_offset: int, end_offset: int, encrypt start_offset, end_offset, data.response.headers, + encryption.get("gcm_nonce_validator"), ) except Exception as error: raise HttpResponseError(message="Decryption failed.", response=data.response, error=error) from error @@ -322,6 +329,8 @@ async def _get_encryption_data_request(self) -> None: async def _setup(self) -> None: if self._encryption_options.get("key") is not None or self._encryption_options.get("resolver") is not None: await self._get_encryption_data_request() + if is_encryption_v2(self._encryption_data): + self._encryption_options["gcm_nonce_validator"] = _GCMRegionNonceValidator() # The service only provides transactional MD5s for chunks under 4MB. # If validate_content is using MD5, get only self.MAX_CHUNK_GET_SIZE for the first diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 306a704bc495..eba0ffa545ef 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -28,6 +28,7 @@ _dict_to_encryption_data, _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, + _GCMRegionNonceValidator, _region_nonce_encodings, _validate_and_unwrap_cek, decrypt_blob, @@ -463,179 +464,6 @@ def test_encryption_reordered_regions(self, **kwargs): assert "Decryption failed." in str(e.value) - def test_decrypt_dotnet_v2_1_nonce_encoding(self): - # Regression test for cross-SDK interoperability. The .NET Storage SDK encodes each - # region's GCM nonce as a one-based counter written little-endian into the final 8 - # nonce bytes. Python must still decrypt these .NET-produced V2.1 blobs while - # continuing to detect reordered regions. - kek = KeyWrapper("key1") - cek = os.urandom(32) - - region_data_length = 32 - num_regions = 3 - plaintext_regions = [bytes([i]) * region_data_length for i in range(num_regions)] - - def dotnet_nonce(region_index): - # 4 zero bytes + one-based counter, little-endian, 8 bytes -- see .NET - # GcmAuthenticatedCryptographicTransform.GetNewNonce(). - return b"\x00\x00\x00\x00" + (region_index + 1).to_bytes(8, "little") - - aesgcm = AESGCM(cek) - encrypted_regions = [ - dotnet_nonce(i) + aesgcm.encrypt(dotnet_nonce(i), region, None) - for i, region in enumerate(plaintext_regions) - ] - - # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). - wrapped_cek = kek.wrap_key(b"2.1".ljust(8, b"\x00") + cek) - encryption_data = { - "WrappedContentKey": { - "KeyId": kek.get_kid(), - "EncryptedKey": base64.b64encode(wrapped_cek).decode(), - "Algorithm": kek.get_key_wrap_algorithm(), - }, - "EncryptionAgent": {"Protocol": "2.1", "EncryptionAlgorithm": "AES_GCM_256"}, - "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, - "KeyWrappingMetadata": {"EncryptionLibrary": "Dotnet"}, - } - headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} - plaintext = b"".join(plaintext_regions) - - # Act / Assert -- the .NET nonce encoding is accepted and the content round-trips. - decrypted = decrypt_blob( - require_encryption=True, - key_encryption_key=kek, - key_resolver=None, - content=b"".join(encrypted_regions), - start_offset=0, - end_offset=len(plaintext), - response_headers=headers, - ) - assert decrypted == plaintext - - # Reordering the .NET-produced regions must still be detected. - reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] - with pytest.raises(ValueError): - decrypt_blob( - require_encryption=True, - key_encryption_key=kek, - key_resolver=None, - content=reordered, - start_offset=0, - end_offset=len(plaintext), - response_headers=headers, - ) - - def test_decrypt_java_v2_nonce_encoding(self): - # Regression test for cross-SDK interoperability. The Java Storage SDK encodes each - # region's GCM nonce as a zero-based counter written big-endian into the leading 8 - # nonce bytes (ByteBuffer.allocate(12).putLong(index)), which differs from Python's - # full-width big-endian counter. Python must still decrypt Java-produced V2 blobs - # while continuing to detect reordered regions. - kek = KeyWrapper("key1") - cek = os.urandom(32) - - region_data_length = 32 - num_regions = 3 - plaintext_regions = [bytes([i]) * region_data_length for i in range(num_regions)] - - def java_nonce(region_index): - # Zero-based counter, big-endian, in the leading 8 bytes; trailing bytes zeroed. - return region_index.to_bytes(8, "big") + b"\x00" * (_GCM_NONCE_LENGTH - 8) - - aesgcm = AESGCM(cek) - encrypted_regions = [ - java_nonce(i) + aesgcm.encrypt(java_nonce(i), region, None) - for i, region in enumerate(plaintext_regions) - ] - - # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). - wrapped_cek = kek.wrap_key(b"2.0".ljust(8, b"\x00") + cek) - encryption_data = { - "WrappedContentKey": { - "KeyId": kek.get_kid(), - "EncryptedKey": base64.b64encode(wrapped_cek).decode(), - "Algorithm": kek.get_key_wrap_algorithm(), - }, - "EncryptionAgent": {"Protocol": "2.0", "EncryptionAlgorithm": "AES_GCM_256"}, - "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, - "KeyWrappingMetadata": {"EncryptionLibrary": "Java"}, - } - headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} - plaintext = b"".join(plaintext_regions) - - # Act / Assert -- the Java nonce encoding is accepted and the content round-trips. - decrypted = decrypt_blob( - require_encryption=True, - key_encryption_key=kek, - key_resolver=None, - content=b"".join(encrypted_regions), - start_offset=0, - end_offset=len(plaintext), - response_headers=headers, - ) - assert decrypted == plaintext - - # Reordering the Java-produced regions must still be detected. - reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] - with pytest.raises(ValueError): - decrypt_blob( - require_encryption=True, - key_encryption_key=kek, - key_resolver=None, - content=reordered, - start_offset=0, - end_offset=len(plaintext), - response_headers=headers, - ) - - def test_decrypt_rejects_mixed_nonce_encodings(self): - # Regression test: the supported SDK nonce encodings share a value space, so accepting - # them independently per region would weaken reorder detection. For example Java's - # nonce for region 1 is identical to .NET's nonce for region 16,777,215, so a Java - # region could be moved to that position and still pass a per-region union check. - # decrypt_blob must instead select a single encoding and enforce it consistently. - encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) - # Document the overlap that motivates single-encoding enforcement. - assert encodings["java"](1) == encodings["dotnet"](16_777_215) - - kek = KeyWrapper("key1") - cek = os.urandom(32) - region_data_length = 32 - aesgcm = AESGCM(cek) - - # Region 0 uses the Java/Python encoding (all zeros); region 1 uses the .NET encoding. - # A per-region union check would accept both; single-encoding enforcement rejects the mix. - region0_nonce = encodings["java"](0) - region1_nonce = encodings["dotnet"](1) - region0 = region0_nonce + aesgcm.encrypt(region0_nonce, b"\x00" * region_data_length, None) - region1 = region1_nonce + aesgcm.encrypt(region1_nonce, b"\x11" * region_data_length, None) - - wrapped_cek = kek.wrap_key(b"2.0".ljust(8, b"\x00") + cek) - encryption_data = { - "WrappedContentKey": { - "KeyId": kek.get_kid(), - "EncryptedKey": base64.b64encode(wrapped_cek).decode(), - "Algorithm": kek.get_key_wrap_algorithm(), - }, - "EncryptionAgent": {"Protocol": "2.0", "EncryptionAlgorithm": "AES_GCM_256"}, - "EncryptedRegionInfo": {"DataLength": region_data_length, "NonceLength": _GCM_NONCE_LENGTH}, - "KeyWrappingMetadata": {"EncryptionLibrary": "Mixed"}, - } - headers = {"x-ms-meta-encryptiondata": dumps(encryption_data)} - - # Act / Assert -- the mixed encoding is rejected rather than silently accepted. - with pytest.raises(ValueError): - decrypt_blob( - require_encryption=True, - key_encryption_key=kek, - key_resolver=None, - content=region0 + region1, - start_offset=0, - end_offset=2 * region_data_length, - response_headers=headers, - ) - @BlobPreparer() @recorded_by_proxy @mock.patch("os.urandom", mock_urandom) @@ -1471,3 +1299,169 @@ def assert_user_agent(request): blob.upload_blob(content, overwrite=True, raw_request_hook=assert_user_agent) blob.download_blob(raw_request_hook=assert_user_agent).readall() + + +class TestGCMRegionNonceValidation: + REGION_DATA_LENGTH = 32 + + @staticmethod + def _encryption_headers(kek, cek, protocol, library, data_length=REGION_DATA_LENGTH): + # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). + wrapped_cek = kek.wrap_key(protocol.encode().ljust(8, b"\x00") + cek) + encryption_data = { + "WrappedContentKey": { + "KeyId": kek.get_kid(), + "EncryptedKey": base64.b64encode(wrapped_cek).decode(), + "Algorithm": kek.get_key_wrap_algorithm(), + }, + "EncryptionAgent": {"Protocol": protocol, "EncryptionAlgorithm": "AES_GCM_256"}, + "EncryptedRegionInfo": {"DataLength": data_length, "NonceLength": _GCM_NONCE_LENGTH}, + "KeyWrappingMetadata": {"EncryptionLibrary": library}, + } + return {"x-ms-meta-encryptiondata": dumps(encryption_data)} + + @staticmethod + def _encrypt_regions(cek, nonce_for_region, plaintext_regions): + aesgcm = AESGCM(cek) + return [ + nonce_for_region(i) + aesgcm.encrypt(nonce_for_region(i), region, None) + for i, region in enumerate(plaintext_regions) + ] + + @staticmethod + def _decrypt(kek, headers, content, end_offset, nonce_validator): + return decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=content, + start_offset=0, + end_offset=end_offset, + response_headers=headers, + nonce_validator=nonce_validator, + ) + + def test_decrypt_dotnet_v2_1_nonce_encoding(self): + # Regression test for cross-SDK interoperability. The .NET Storage SDK encodes each + # region's GCM nonce as a one-based counter written little-endian into the final 8 + # nonce bytes. Python must still decrypt these .NET-produced V2.1 blobs while + # continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + num_regions = 3 + plaintext_regions = [bytes([i]) * self.REGION_DATA_LENGTH for i in range(num_regions)] + + def dotnet_nonce(region_index): + # 4 zero bytes + one-based counter, little-endian, 8 bytes -- see .NET + # GcmAuthenticatedCryptographicTransform.GetNewNonce(). + return b"\x00\x00\x00\x00" + (region_index + 1).to_bytes(8, "little") + + encrypted_regions = self._encrypt_regions(cek, dotnet_nonce, plaintext_regions) + headers = self._encryption_headers(kek, cek, "2.1", "Dotnet") + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the .NET nonce encoding is accepted and the content round-trips. + decrypted = self._decrypt(kek, headers, b"".join(encrypted_regions), len(plaintext), _GCMRegionNonceValidator()) + assert decrypted == plaintext + + # Reordering the .NET-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + self._decrypt(kek, headers, reordered, len(plaintext), _GCMRegionNonceValidator()) + + def test_decrypt_java_v2_nonce_encoding(self): + # Regression test for cross-SDK interoperability. The Java Storage SDK encodes each + # region's GCM nonce as a zero-based counter written big-endian into the leading 8 + # nonce bytes (ByteBuffer.allocate(12).putLong(index)), which differs from Python's + # full-width big-endian counter. Python must still decrypt Java-produced V2 blobs + # while continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + num_regions = 3 + plaintext_regions = [bytes([i]) * self.REGION_DATA_LENGTH for i in range(num_regions)] + + def java_nonce(region_index): + # Zero-based counter, big-endian, in the leading 8 bytes; trailing bytes zeroed. + return region_index.to_bytes(8, "big") + b"\x00" * (_GCM_NONCE_LENGTH - 8) + + encrypted_regions = self._encrypt_regions(cek, java_nonce, plaintext_regions) + headers = self._encryption_headers(kek, cek, "2.0", "Java") + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the Java nonce encoding is accepted and the content round-trips. + decrypted = self._decrypt(kek, headers, b"".join(encrypted_regions), len(plaintext), _GCMRegionNonceValidator()) + assert decrypted == plaintext + + # Reordering the Java-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + self._decrypt(kek, headers, reordered, len(plaintext), _GCMRegionNonceValidator()) + + def test_decrypt_rejects_mixed_nonce_encodings(self): + # Regression test: the supported SDK nonce encodings share a value space, so accepting + # them independently per region would weaken reorder detection. For example Java's + # nonce for region 1 is identical to .NET's nonce for region 16,777,215, so a Java + # region could be moved to that position and still pass a per-region union check. + # decrypt_blob must instead select a single encoding and enforce it consistently. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + # Document the overlap that motivates single-encoding enforcement. + assert encodings["java"](1) == encodings["dotnet"](16_777_215) + + kek = KeyWrapper("key1") + cek = os.urandom(32) + aesgcm = AESGCM(cek) + + # Region 0 uses the Java/Python encoding (all zeros); region 1 uses the .NET encoding. + # A per-region union check would accept both; single-encoding enforcement rejects the mix. + region0_nonce = encodings["java"](0) + region1_nonce = encodings["dotnet"](1) + region0 = region0_nonce + aesgcm.encrypt(region0_nonce, b"\x00" * self.REGION_DATA_LENGTH, None) + region1 = region1_nonce + aesgcm.encrypt(region1_nonce, b"\x11" * self.REGION_DATA_LENGTH, None) + headers = self._encryption_headers(kek, cek, "2.0", "Mixed") + + # Act / Assert -- the mixed encoding is rejected rather than silently accepted. + with pytest.raises(ValueError): + self._decrypt(kek, headers, region0 + region1, 2 * self.REGION_DATA_LENGTH, _GCMRegionNonceValidator()) + + def test_nonce_validator_enforces_single_encoding_across_chunks(self): + # decrypt_blob runs once per download chunk, so a shared validator must intersect the + # candidate encodings across chunks; otherwise the encoding could change at a chunk + # boundary and, at a collision, let a relocated region pass. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + # The collision that makes per-chunk validation unsound: Java region 1 == .NET region 16,777,215. + assert encodings["java"](1) == encodings["dotnet"](16_777_215) + + validator = _GCMRegionNonceValidator() + # First chunk: two Java regions resolve the encoding to Java. + validator.validate_region(0, encodings["java"](0), _GCM_NONCE_LENGTH) + validator.validate_region(1, encodings["java"](1), _GCM_NONCE_LENGTH) + + # Later chunk: a region relocated to the colliding .NET index carries Java's region-1 + # nonce. Its only consistent encoding is .NET, which conflicts with the resolved Java + # encoding, so the shared validator rejects it. + with pytest.raises(ValueError): + validator.validate_region(16_777_215, encodings["java"](1), _GCM_NONCE_LENGTH) + + def test_env_var_bypasses_nonce_validation(self): + # The AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS escape hatch disables nonce + # validation for data-recovery scenarios. With it set, a mix of nonce encodings that + # would normally be rejected must decrypt, and no validator is required. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + kek = KeyWrapper("key1") + cek = os.urandom(32) + aesgcm = AESGCM(cek) + + # Two regions using incompatible encodings (Java for region 0, .NET for region 1). + region0_plaintext = b"\x00" * self.REGION_DATA_LENGTH + region1_plaintext = b"\x11" * self.REGION_DATA_LENGTH + region0_nonce = encodings["java"](0) + region1_nonce = encodings["dotnet"](1) + region0 = region0_nonce + aesgcm.encrypt(region0_nonce, region0_plaintext, None) + region1 = region1_nonce + aesgcm.encrypt(region1_nonce, region1_plaintext, None) + headers = self._encryption_headers(kek, cek, "2.0", "Mixed") + plaintext = region0_plaintext + region1_plaintext + + # Act / Assert -- with the bypass set, decryption succeeds without a validator. + with mock.patch.dict(os.environ, {"AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS": "true"}): + decrypted = self._decrypt(kek, headers, region0 + region1, 2 * self.REGION_DATA_LENGTH, nonce_validator=None) + assert decrypted == plaintext