From 464ccb2b54f1f335b289f10c1b9716c991d5f484 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Thu, 11 Jun 2026 16:42:31 +1000 Subject: [PATCH] UID2-7105: parse salt files line-by-line to avoid String[] buffer Replace reader.lines().toArray(String[]::new) in readInputStream with a streaming BufferedReader approach via a new SaltFileParser.parseLines method. Eliminates the ~147 MB intermediate String[] allocated per snapshot load. EncryptedRotatingSaltProvider also updated to use parseLines via StringReader, avoiding the split("\n") allocation in the old parseFile path. Co-Authored-By: Claude Sonnet 4.6 --- .../store/salt/EncryptedRotatingSaltProvider.java | 6 +++++- .../shared/store/salt/RotatingSaltProvider.java | 3 +-- .../uid2/shared/store/salt/SaltFileParser.java | 14 ++++++++++++++ .../shared/store/salt/SaltFileParserTest.java | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java index b0322852..d28abf99 100644 --- a/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/EncryptedRotatingSaltProvider.java @@ -5,8 +5,10 @@ import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.StoreScope; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; @@ -21,6 +23,8 @@ public EncryptedRotatingSaltProvider(DownloadCloudStorage fileStreamProvider, Ro @Override protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { String decrypted = decryptInputStream(inputStream, cloudEncryptionKeyProvider, "salts"); - return saltFileParser.parseFile(decrypted, size); + try (BufferedReader reader = new BufferedReader(new StringReader(decrypted))) { + return saltFileParser.parseLines(reader, size); + } } } diff --git a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java index ccaed217..a6dc5c00 100644 --- a/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java +++ b/src/main/java/com/uid2/shared/store/salt/RotatingSaltProvider.java @@ -141,8 +141,7 @@ private SaltSnapshot loadSnapshot(JsonObject spec, String firstLevelSalt, SaltFi protected SaltEntry[] readInputStream(InputStream inputStream, SaltFileParser saltFileParser, Integer size) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String[] saltFileLines = reader.lines().toArray(String[]::new); - return saltFileParser.parseFileLines(saltFileLines, size); + return saltFileParser.parseLines(reader, size); } } diff --git a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java index dfe905a3..f4f1fd85 100644 --- a/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java +++ b/src/main/java/com/uid2/shared/store/salt/SaltFileParser.java @@ -2,6 +2,9 @@ import com.uid2.shared.model.SaltEntry; +import java.io.BufferedReader; +import java.io.IOException; + public class SaltFileParser { private final IdHashingScheme idHashingScheme; @@ -9,6 +12,17 @@ public SaltFileParser(IdHashingScheme idHashingScheme) { this.idHashingScheme = idHashingScheme; } + public SaltEntry[] parseLines(BufferedReader reader, Integer size) throws IOException { + SaltEntry[] entries = new SaltEntry[size]; + int lineNumber = 0; + String line; + while ((line = reader.readLine()) != null) { + entries[lineNumber] = parseLine(line, lineNumber); + lineNumber++; + } + return entries; + } + public SaltEntry[] parseFile(String saltFileContent, Integer size) { var lines = saltFileContent.split("\n"); return parseFileLines(lines, size); diff --git a/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java index b4dfbecd..601cd233 100644 --- a/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java +++ b/src/test/java/com/uid2/shared/store/salt/SaltFileParserTest.java @@ -4,6 +4,9 @@ import com.uid2.shared.model.SaltEntry.KeyMaterial; import org.junit.jupiter.api.Test; +import java.io.BufferedReader; +import java.io.StringReader; + import static org.assertj.core.api.AssertionsForClassTypes.assertThat; class SaltFileParserTest { @@ -17,6 +20,18 @@ class SaltFileParserTest { private final String hashed4 = hashingScheme.encode(4); private final String hashed5 = hashingScheme.encode(5); + @Test + void parseLinesMatchesParseFile() throws Exception { + var file = """ +1,100,salt1,1000,old_salt1,10,key_1,key_salt_1,100,old_key_1,old_key_1_salt +2,200,salt2,2000,old_salt2,20,key_2,key_salt_2,200,old_key_2,old_key_2_salt +"""; + SaltEntry[] viaParseFile = parser.parseFile(file, 2); + SaltEntry[] viaParseLines = parser.parseLines(new BufferedReader(new StringReader(file)), 2); + + assertThat(viaParseLines).isEqualTo(viaParseFile); + } + @Test void parsesSaltFileWithAllFields() { var file = """