From a39b99c48152b1d6f01b09b7b21dca0e23815f9e Mon Sep 17 00:00:00 2001 From: sujan kota Date: Thu, 27 Aug 2026 17:30:13 -0400 Subject: [PATCH] feat(sdk): remove 64GiB TDF limit, use counter-based payload IVs (DSPX-4495) Removes MAX_TDF_INPUT_SIZE (68719476736) and the size check in createTDF. That constant was GCM's per-invocation plaintext limit (2^39-256 bits) misapplied to the whole TDF input; each segment is its own invocation and is capped at 4MiB by Config.MAX_SEGMENT_SIZE, so the bound was never relevant. SDK.DataSizeNotSupported is retained as public API but is no longer thrown. Replaces the random per-segment AES-GCM nonce with a deterministic unsigned 96-bit big-endian counter (TDF.IvCounter). NIST SP 800-38D prefers the deterministic construction; the RBG-based one it replaces carries a birthday bound that a counter does not have. Metadata is encrypted with the per-split symKey while the payload uses the XOR of all split keys, so with a single key split the two are the same key. IV 0 is therefore reserved for the metadata and payload segments start at IV 1. The counter enforces the SP 800-38D 8.3 cap of 2^32 invocations per key and refuses to wrap, so an IV can never be issued twice. Neither limit is reachable in practice - at the 16KiB minimum segment size the cap is 64TiB of input - but the invariant now holds by construction rather than by assumption. Every segment is still prefixed with its 12-byte IV, so the wire format is unchanged and existing TDFs continue to decrypt. Signed-off-by: sujan kota --- .../java/io/opentdf/platform/sdk/SDK.java | 4 +- .../java/io/opentdf/platform/sdk/TDF.java | 136 ++++++++--- .../java/io/opentdf/platform/sdk/TDFTest.java | 220 ++++++++++++++---- 3 files changed, 285 insertions(+), 75 deletions(-) diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index 16c7a35a..818a9b90 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -390,8 +390,8 @@ public SplitKeyException(String errorMessage) { } /** - * {@link DataSizeNotSupported} is thrown when the user attempts to create - * a TDF with a size larger than the maximum size (currently 64GiB). + * Legacy exception type retained for compatibility. TDF creation no longer + * imposes a fixed input-size limit. */ public static class DataSizeNotSupported extends SDKException { public DataSizeNotSupported(String errorMessage) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index 8827cb85..959d5fd7 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -55,35 +55,14 @@ private static byte[] tdfECKeySaltCompute() { */ public static final String TDF_SPEC_VERSION = "4.3.0"; private static final String KEY_ACCESS_SCHEMA_VERSION = "1.0"; - private final long maximumSize; - private final SDK.Services services; - /** - * Constructs a new TDF instance using the default maximum input size defined by - * MAX_TDF_INPUT_SIZE. - *

- * This constructor is primarily used to initialize the TDF object with the - * standard maximum - * input size, which controls the maximum size of the input data that can be - * processed. - * For test purposes, an alternative constructor allows for setting a custom - * maximum input size. - */ TDF(SDK.Services services) { - this(MAX_TDF_INPUT_SIZE, services); - } - - // constructor for tests so that we can set a maximum size that's tractable for - // tests - TDF(long maximumInputSize, SDK.Services services) { - this.maximumSize = maximumInputSize; this.services = services; } private static final Logger logger = LoggerFactory.getLogger(TDF.class); - private static final long MAX_TDF_INPUT_SIZE = 68719476736L; private static final int GCM_KEY_SIZE = 32; private static final String kSplitKeyType = "split"; private static final String kWrapped = "wrapped"; @@ -103,6 +82,100 @@ private static byte[] tdfECKeySaltCompute() { private static final Gson gson = new GsonBuilder().create(); + /** + * NIST SP 800-38D section 8.3 caps the total number of AES-GCM + * authenticated-encryption invocations under a single key at 2^32. One + * invocation is spent on the metadata (IV 0), leaving 2^32 - 1 for payload + * segments. + *

+ * This is not reachable in practice — at the smallest permitted segment size + * ({@link Config#MIN_SEGMENT_SIZE}, 16 KiB) it would take 64 TiB of input — but + * it is enforced so the invariant holds by construction rather than by + * assumption. + */ + static final long MAX_GCM_INVOCATIONS_PER_KEY = 1L << 32; + + /** + * A deterministic, unsigned 96-bit big-endian AES-GCM IV counter. + *

+ * A TDF encrypts its metadata and its payload segments under keys that are + * identical when there is a single key split, so the two must never share an + * IV. IV 0 is reserved for the metadata and payload segments start at IV 1, + * incrementing once per segment. + *

+ * The counter refuses to issue an IV once its invocation budget is spent, and + * refuses to wrap past its maximum value, so an IV can never be handed out + * twice. + *

+ * Precondition: this is safe only because the key is freshly generated + * for every TDF ({@code AesGcm.generateKey()} in {@code prepareManifest}). + * Reusing a key across two TDFs would repeat this IV sequence, which is + * catastrophic for AES-GCM — it leaks the XOR of the plaintexts and enables + * authentication-key recovery. Do not add a way to supply or reuse a payload + * key without also changing this construction. + */ + static final class IvCounter { + private final byte[] nextIv; + private long remainingInvocations; + private boolean wrapped; + + /** + * The IV reserved for encrypting the TDF metadata. + * + * @return twelve zero bytes + */ + static byte[] metadataIv() { + return new byte[kGcmIvSize]; + } + + /** + * A payload IV counter whose first value is 1, leaving IV 0 for the metadata + * and the remainder of the per-key invocation budget for payload segments. + */ + static IvCounter forPayload() { + byte[] initialIv = new byte[kGcmIvSize]; + initialIv[initialIv.length - 1] = 1; + return new IvCounter(initialIv, MAX_GCM_INVOCATIONS_PER_KEY - 1); + } + + IvCounter(byte[] initialIv, long invocationBudget) { + Objects.requireNonNull(initialIv, "initial IV"); + if (initialIv.length != kGcmIvSize) { + throw new IllegalArgumentException("invalid IV size: " + initialIv.length); + } + if (invocationBudget < 0) { + throw new IllegalArgumentException("invalid invocation budget: " + invocationBudget); + } + this.nextIv = initialIv.clone(); + this.remainingInvocations = invocationBudget; + } + + byte[] next() { + if (remainingInvocations <= 0) { + throw new SDKException("exceeded the maximum of " + MAX_GCM_INVOCATIONS_PER_KEY + + " AES-GCM invocations for a single key"); + } + if (wrapped) { + throw new SDKException("AES-GCM IV counter exhausted"); + } + + byte[] currentIv = nextIv.clone(); + remainingInvocations--; + wrapped = increment(nextIv); + return currentIv; + } + + private static boolean increment(byte[] iv) { + for (int index = iv.length - 1; index >= 0; index--) { + iv[index]++; + if (iv[index] != 0) { + return false; + } + } + return true; + } + } + static class EncryptedMetadata { private String ciphertext; private String iv; @@ -176,12 +249,17 @@ private void prepareManifest(Config.TDFConfig tdfConfig, Map(); - long totalSize = 0; boolean finished; try (var payloadOutput = tdfWriter.payload()) { do { @@ -420,18 +498,14 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo readThisLoop += nRead; } finished = nRead < 0; - totalSize += readThisLoop; - - if (totalSize > maximumSize) { - throw new SDK.DataSizeNotSupported("can't create tdf larger than 64gb"); - } byte[] cipherData; byte[] segmentSig; Manifest.Segment segmentInfo = new Manifest.Segment(); // encrypt - cipherData = tdfObject.aesGcm.encrypt(readBuf, 0, readThisLoop).asBytes(); + cipherData = tdfObject.aesGcm.encrypt(payloadIv.next(), kAesBlockSize, + readBuf, 0, readThisLoop); payloadOutput.write(cipherData); segmentSig = calculateSignature(cipherData, tdfObject.payloadKey, tdfConfig.segmentIntegrityAlgorithm); diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java index 74ef151e..b2803773 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFTest.java @@ -6,6 +6,7 @@ import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.jwk.JWK; import com.google.gson.Gson; +import com.google.gson.JsonObject; import io.opentdf.platform.policy.KeyAccessServer; import io.opentdf.platform.policy.kasregistry.KeyAccessServerRegistryServiceClient; import io.opentdf.platform.policy.kasregistry.ListKeyAccessServersRequest; @@ -21,22 +22,24 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.Map; import java.util.List; import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import static io.opentdf.platform.sdk.TDF.GLOBAL_KEY_SALT; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -697,7 +700,26 @@ public void testCreatingTDFWithMultipleSegments() throws Exception { var tdf = new TDF( new FakeServicesBuilder().setKas(kas) .setKeyAccessServerRegistryService(kasRegistryService).build()); - tdf.createTDF(plainTextInputStream, tdfOutputStream, config); + var tdfObject = tdf.createTDF(plainTextInputStream, tdfOutputStream, config); + + var segments = tdfObject.getManifest().encryptionInformation.integrityInformation.segments; + assertThat(segments.size()) + .withFailMessage("test needs more than one segment to be meaningful") + .isGreaterThan(1); + + // payload segments start at IV 1 (IV 0 is reserved for the metadata) and + // increment by one for every segment + var seenIvs = new ArrayList(); + var encryptedReader = new TDFReader(new SeekableInMemoryByteChannel(tdfOutputStream.toByteArray())); + for (int segmentIndex = 0; segmentIndex < segments.size(); segmentIndex++) { + byte[] encryptedSegment = new byte[(int) segments.get(segmentIndex).encryptedSegmentSize]; + assertThat(encryptedReader.readPayloadBytes(encryptedSegment)).isEqualTo(encryptedSegment.length); + byte[] iv = Arrays.copyOf(encryptedSegment, AesGcm.GCM_NONCE_LENGTH); + assertThat(iv).containsExactly(bigEndianIv(segmentIndex + 1)); + seenIvs.add(Base64.getEncoder().encodeToString(iv)); + } + assertThat(seenIvs).doesNotHaveDuplicates(); + var unwrappedData = new ByteArrayOutputStream(); var reader = tdf.loadTDF(new SeekableInMemoryByteChannel(tdfOutputStream.toByteArray()), platformUrl); reader.readPayload(unwrappedData); @@ -708,52 +730,166 @@ public void testCreatingTDFWithMultipleSegments() throws Exception { } - @Test - public void testCreatingTooLargeTDF() { - var random = new Random(); - var maxSize = random.nextInt(1024); - var numReturned = new AtomicInteger(0); - - // return 1 more byte than the maximum size - var is = new InputStream() { - @Override - public int read() { - if (numReturned.get() > maxSize) { - return -1; - } - numReturned.incrementAndGet(); - return 1; - } + /** + * The unsigned 96-bit big-endian encoding of {@code value}, for asserting on + * expected IVs. + */ + private static byte[] bigEndianIv(long value) { + byte[] iv = new byte[AesGcm.GCM_NONCE_LENGTH]; + for (int index = iv.length - 1; index >= 0 && value != 0; index--) { + iv[index] = (byte) value; + value >>>= 8; + } + return iv; + } - @Override - public int read(byte[] b, int off, int len) { - var numToReturn = Math.min(len, maxSize - numReturned.get() + 1); - numReturned.addAndGet(numToReturn); - return numToReturn; - } - }; + @Test + public void testMetadataUsesIvZero() throws Exception { + Config.TDFConfig config = Config.newTDFConfig( + Config.withAutoconfigure(false), + Config.withKasInformation(getRSAKASInfos()), + Config.withMetaData("here is some metadata")); - var os = new OutputStream() { - @Override - public void write(int b) { - } + var tdfOutputStream = new ByteArrayOutputStream(); + var tdf = new TDF( + new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + var tdfObject = tdf.createTDF(new ByteArrayInputStream("some data".getBytes(StandardCharsets.UTF_8)), + tdfOutputStream, config); + + var keyAccessObjects = tdfObject.getManifest().encryptionInformation.keyAccessObj; + assertThat(keyAccessObjects).isNotEmpty(); + for (Manifest.KeyAccess keyAccess : keyAccessObjects) { + var encryptedMetadata = new Gson().fromJson( + new String(Base64.getDecoder().decode(keyAccess.encryptedMetadata), StandardCharsets.UTF_8), + JsonObject.class); + + assertThat(Base64.getDecoder().decode(encryptedMetadata.get("iv").getAsString())) + .withFailMessage("metadata IV is not zero") + .containsExactly(new byte[AesGcm.GCM_NONCE_LENGTH]); + // the ciphertext field carries the IV as a prefix as well + assertThat(Arrays.copyOf( + Base64.getDecoder().decode(encryptedMetadata.get("ciphertext").getAsString()), + AesGcm.GCM_NONCE_LENGTH)) + .containsExactly(new byte[AesGcm.GCM_NONCE_LENGTH]); + } - @Override - public void write(byte[] b, int off, int len) { - } - }; + var reader = tdf.loadTDF(new SeekableInMemoryByteChannel(tdfOutputStream.toByteArray()), platformUrl); + assertThat(reader.getMetadata()).isEqualTo("here is some metadata"); + } - var tdf = new TDF(maxSize, new FakeServicesBuilder().setKas(kas).build()); - var tdfConfig = Config.newTDFConfig( + @Test + public void testFirstPayloadSegmentUsesIvOne() throws Exception { + Config.TDFConfig config = Config.newTDFConfig( Config.withAutoconfigure(false), Config.withKasInformation(getRSAKASInfos()), - Config.withSegmentSize(Config.MIN_SEGMENT_SIZE)); - assertThrows(SDK.DataSizeNotSupported.class, - () -> tdf.createTDF(is, os, tdfConfig), - "didn't throw an exception when we created TDF that was too large"); - assertThat(numReturned.get()) - .withFailMessage("test returned the wrong number of bytes") - .isEqualTo(maxSize + 1); + Config.withMetaData("here is some metadata")); + + var tdfOutputStream = new ByteArrayOutputStream(); + var tdf = new TDF( + new FakeServicesBuilder().setKas(kas) + .setKeyAccessServerRegistryService(kasRegistryService).build()); + var tdfObject = tdf.createTDF(new ByteArrayInputStream("some data".getBytes(StandardCharsets.UTF_8)), + tdfOutputStream, config); + + var segments = tdfObject.getManifest().encryptionInformation.integrityInformation.segments; + var encryptedReader = new TDFReader(new SeekableInMemoryByteChannel(tdfOutputStream.toByteArray())); + byte[] firstSegment = new byte[(int) segments.get(0).encryptedSegmentSize]; + assertThat(encryptedReader.readPayloadBytes(firstSegment)).isEqualTo(firstSegment.length); + + assertThat(Arrays.copyOf(firstSegment, AesGcm.GCM_NONCE_LENGTH)) + .withFailMessage("first payload segment must use IV 1, leaving IV 0 for the metadata") + .containsExactly(bigEndianIv(1)); + } + + @Test + public void testPayloadIvCounterStartsAtOne() { + var counter = TDF.IvCounter.forPayload(); + + assertThat(TDF.IvCounter.metadataIv()).containsExactly(bigEndianIv(0)); + assertThat(counter.next()).containsExactly(bigEndianIv(1)); + assertThat(counter.next()).containsExactly(bigEndianIv(2)); + assertThat(counter.next()).containsExactly(bigEndianIv(3)); + } + + @Test + public void testPayloadIvCounterIncrementsWithCarry() { + // one below a two-byte carry boundary + var counter = new TDF.IvCounter(new byte[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, (byte) 0xff, (byte) 0xff + }, 3); + + assertThat(counter.next()).containsExactly( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, (byte) 0xff, (byte) 0xff); + assertThat(counter.next()).containsExactly( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 0, 0); + assertThat(counter.next()).containsExactly( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 0, 1); + } + + @Test + public void testPayloadIvCounterRejectsReuseAfterOverflow() { + byte[] finalIv = new byte[AesGcm.GCM_NONCE_LENGTH]; + Arrays.fill(finalIv, (byte) 0xff); + var counter = new TDF.IvCounter(finalIv, Long.MAX_VALUE); + + assertThat(counter.next()).containsExactly(finalIv); + // it must refuse rather than wrap around to zero, which would collide with the + // metadata IV + assertThrows(SDKException.class, counter::next); + assertThrows(SDKException.class, counter::next); + } + + @Test + public void testPayloadIvCounterStopsAtInvocationBudget() { + var counter = new TDF.IvCounter(bigEndianIv(1), 2); + + assertThat(counter.next()).containsExactly(bigEndianIv(1)); + assertThat(counter.next()).containsExactly(bigEndianIv(2)); + + var e = assertThrows(SDKException.class, counter::next); + assertThat(e).hasMessageContaining("AES-GCM invocations for a single key"); + // and it stays refused + assertThrows(SDKException.class, counter::next); + } + + @Test + public void testPayloadIvBudgetLeavesOneInvocationForMetadata() { + // NIST SP 800-38D 8.3 caps a key at 2^32 invocations; IV 0 is the metadata, so + // the payload gets 2^32 - 1 of them + assertThat(TDF.MAX_GCM_INVOCATIONS_PER_KEY).isEqualTo(4294967296L); + + var counter = TDF.IvCounter.forPayload(); + assertThat(counter.next()).containsExactly(bigEndianIv(1)); + + var budget = assertDoesNotThrow(() -> { + var field = TDF.IvCounter.class.getDeclaredField("remainingInvocations"); + field.setAccessible(true); + return (long) field.get(counter); + }); + assertThat(budget).isEqualTo(TDF.MAX_GCM_INVOCATIONS_PER_KEY - 2); + } + + @Test + public void testNoTdfInputSizeLimit() { + for (var constructor : TDF.class.getDeclaredConstructors()) { + assertThat(constructor.getParameterTypes()) + .withFailMessage("the maximum-input-size constructor should have been removed") + .doesNotContain(long.class); + } + + for (var field : TDF.class.getDeclaredFields()) { + boolean isNumericConstant = Modifier.isStatic(field.getModifiers()) + && (field.getType().equals(long.class) || field.getType().equals(int.class)); + if (!isNumericConstant) { + continue; + } + field.setAccessible(true); + long value = assertDoesNotThrow(() -> ((Number) field.get(null)).longValue()); + assertThat(value) + .withFailMessage("TDF still declares a size limit in %s", field.getName()) + .isNotIn(68719476736L /* 64 GiB */, 10485760L /* 10 MiB */); + } } @Test