diff --git a/.gitignore b/.gitignore index 130724f..a627f88 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ _deps/ # macOS .DS_Store +**/.!*.DS_Store # Editor/IDE config .vscode/ diff --git a/src/client/dbps_api_client_test.cpp b/src/client/dbps_api_client_test.cpp index dd43a10..a51a690 100644 --- a/src/client/dbps_api_client_test.cpp +++ b/src/client/dbps_api_client_test.cpp @@ -23,6 +23,7 @@ #include "tcb/span.hpp" #include "dbps_api_client.h" #include "http_client_base.h" +#include "../common/bytes_utils.h" #include "../common/enums.h" #include #include @@ -30,12 +31,6 @@ using namespace dbps::external; using namespace dbps::enum_utils; -// TODO: Move this to a common test utility file. -// Helper function to convert string to binary data -std::vector StringToBytes(const std::string& str) { - return std::vector(str.begin(), str.end()); -} - // Utility function to compare JSON strings, ignoring specified fields bool CompareJsonStrings(const std::string& json1, const std::string& json2, const std::vector& ignore_fields = {}) { try { diff --git a/src/common/bytes_utils.h b/src/common/bytes_utils.h index 6b08aeb..8aa3013 100644 --- a/src/common/bytes_utils.h +++ b/src/common/bytes_utils.h @@ -81,6 +81,13 @@ inline void write_u32_le_at(std::vector& buf, size_t offset, uint32_t v buf[offset + 3] = static_cast((v >> 24) & 0xFF); } +inline void write_u32_le(uint8_t* p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} + inline uint32_t read_u32_le(const std::vector& in, size_t offset) { return static_cast(in[offset]) | (static_cast(in[offset + 1]) << 8) | @@ -95,6 +102,12 @@ inline uint32_t read_u32_le(tcb::span in, size_t offset) { (static_cast(in[offset + 3]) << 24); } +inline uint32_t read_u32_le(const uint8_t* p) { + uint32_t v; + std::memcpy(&v, p, sizeof(v)); + return v; +} + // Utility functions for splitting and joining byte vectors. struct BytesPair { @@ -289,3 +302,8 @@ inline std::string AddStringAttribute( out[key] = value; return value; } + +// Helper function to convert string to binary data +inline std::vector StringToBytes(const std::string& str) { + return std::vector(str.begin(), str.end()); +} diff --git a/src/common/bytes_utils_test.cpp b/src/common/bytes_utils_test.cpp index e4a8a0e..87a6a38 100644 --- a/src/common/bytes_utils_test.cpp +++ b/src/common/bytes_utils_test.cpp @@ -239,4 +239,49 @@ TEST(BytesUtils, AttributesMap_AddBool) { std::map bad_attrs{{"page_v2_is_compressed", "maybe"}}; EXPECT_THROW(AddBoolAttribute(out, bad_attrs, "page_v2_is_compressed"), InvalidInputException); +} + +TEST(BytesUtils, StringToBytes_AsciiText) { + const std::string input = "dbps"; + const std::vector result = StringToBytes(input); + + EXPECT_EQ((std::vector{'d', 'b', 'p', 's'}), result); +} + +TEST(BytesUtils, StringToBytes_EmptyString) { + const std::string input; + const std::vector result = StringToBytes(input); + + EXPECT_TRUE(result.empty()); +} + +TEST(BytesUtils, StringToBytes_PreservesRawBytesAndNulls) { + std::string input; + input.push_back('D'); + input.push_back('B'); + input.push_back('P'); + input.push_back('S'); + input.push_back('\0'); + input.push_back('X'); + input.push_back('Y'); + input.push_back(static_cast(0xFF)); + input.push_back(static_cast(0x80)); + input.push_back('\0'); + input.push_back('Z'); + + const std::vector result = StringToBytes(input); + const std::vector expected = { + static_cast('D'), + static_cast('B'), + static_cast('P'), + static_cast('S'), + static_cast(0x00), + static_cast('X'), + static_cast('Y'), + static_cast(0xFF), + static_cast(0x80), + static_cast(0x00), + static_cast('Z')}; + + EXPECT_EQ(expected, result); } \ No newline at end of file diff --git a/src/common/dbpa_local_test.cpp b/src/common/dbpa_local_test.cpp index a094650..0904f93 100644 --- a/src/common/dbpa_local_test.cpp +++ b/src/common/dbpa_local_test.cpp @@ -50,7 +50,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryption) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt)); std::vector test_data = BuildByteArrayValueBytesForTesting("test_ABC"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -74,7 +74,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryptionCompressedDictiona 0x03, 0x32, 0x92, 0x12, 0xF3, 0x80, 0x10, 0x00, 0xC7, 0xB8, 0x50, 0xFC, 0x13, 0x00, 0x00, 0x00 }; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data_gzip, encoding_attributes); ASSERT_NE(result, nullptr); @@ -93,7 +93,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulDecryption) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA)); std::vector test_data = BuildByteArrayValueBytesForTesting("test_EFG"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Decrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -113,7 +113,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) { // Original data to encrypt std::vector original_data = BuildByteArrayValueBytesForTesting("roundtrip_XYZ"); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; // Encrypt the data auto encrypt_result = encrypt_agent.Encrypt(original_data, encoding_attributes); @@ -156,7 +156,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, EncryptWithoutInit) { LocalDataBatchProtectionAgent agent; std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -171,7 +171,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, DecryptWithoutInit) { LocalDataBatchProtectionAgent agent; std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Decrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); @@ -203,7 +203,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, MissingPageEncoding) { Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt)); std::vector test_data = {1, 2, 3, 4}; - std::map encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent.Encrypt(test_data, encoding_attributes); ASSERT_NE(result, nullptr); diff --git a/src/processing/encryption_sequencer.cpp b/src/processing/encryption_sequencer.cpp index c1d728e..0bee26e 100644 --- a/src/processing/encryption_sequencer.cpp +++ b/src/processing/encryption_sequencer.cpp @@ -23,7 +23,6 @@ #include "../common/exceptions.h" #include "encryptors/basic_xor_encryptor.h" #include -#include #include #include #include @@ -138,19 +137,17 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span pla */ try { // Decompress and split plaintext into level and value bytes - auto [level_bytes, value_bytes] = DecompressAndSplit( + auto [level_bytes, value_bytes, num_elements] = DecompressAndSplit( plaintext, compression_, encoding_attributes_converted_); // Parse value bytes into typed values buffer - auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(value_bytes, datatype_, datatype_length_, encoding_); + auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer( + value_bytes, num_elements, datatype_, datatype_length_, encoding_); // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector. auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer); auto encrypted_level_bytes = encryptor_->EncryptBlock(level_bytes); - auto joined_encrypted_bytes = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); - - // Compress the joined encrypted bytes - encrypted_result_ = Compress(joined_encrypted_bytes, encrypted_compression_); + encrypted_result_ = JoinWithLengthPrefix(encrypted_level_bytes, encrypted_value_bytes); // Set the encryption type to per-value encryption_metadata_[encryption_mode_key] = ENCRYPTION_MODE_PER_VALUE; @@ -226,16 +223,13 @@ bool DataBatchEncryptionSequencer::DecryptAndEncode(tcb::span cip error_message_ = "Failed to get encryption_mode from encryption_metadata"; return false; } - std::string encryption_mode = encryption_mode_opt.value(); + const std::string& encryption_mode = encryption_mode_opt.value(); // Per-value encryption if (encryption_mode == ENCRYPTION_MODE_PER_VALUE) { - // Decompress the encrypted bytes - auto decompressed_encrypted_bytes = Decompress(ciphertext, encrypted_compression_); - + // Split the joined encrypted bytes, then decrypt the level and value bytes separately. - auto [encrypted_level_bytes, encrypted_value_bytes] = - SplitWithLengthPrefix(tcb::span(decompressed_encrypted_bytes)); + auto [encrypted_level_bytes, encrypted_value_bytes] = SplitWithLengthPrefix(ciphertext); auto level_bytes = encryptor_->DecryptBlock(encrypted_level_bytes); auto typed_buffer = encryptor_->DecryptValueList(encrypted_value_bytes); @@ -294,7 +288,7 @@ bool DataBatchEncryptionSequencer::ConvertEncodingAttributesToValues() { add_int("page_v2_num_nulls"); add_bool("page_v2_is_compressed"); } else if (page_type == "DICTIONARY_PAGE") { - // DICTIONARY_PAGE has no specific encoding attributes + add_int("dict_page_num_values"); } else { throw InvalidInputException("Unexpected page type: " + page_type); } diff --git a/src/processing/encryption_sequencer_test.cpp b/src/processing/encryption_sequencer_test.cpp index 4073639..cf9559d 100644 --- a/src/processing/encryption_sequencer_test.cpp +++ b/src/processing/encryption_sequencer_test.cpp @@ -81,7 +81,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { std::nullopt, // datatype_length CompressionCodec::UNCOMPRESSED, // compression Encoding::PLAIN, // encoding - {{"page_type", "DICTIONARY_PAGE"}}, // encoding_attributes (mostly empty for basic test) + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, // encoding_attributes (mostly empty for basic test) CompressionCodec::UNCOMPRESSED, // encrypted_compression "test_key_123", // key_id "test_user", // user_id @@ -97,11 +97,11 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 2: Different key_id produces different encryption { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} ); @@ -115,11 +115,11 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 3: Same key_id produces consistent encryption { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "same_key", "test_user", "{}", {} ); @@ -133,7 +133,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 4: Empty data encryption { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // This should fail because empty input is rejected @@ -144,7 +144,7 @@ TEST(EncryptionSequencer, EncryptionDecryption) { // Test 5: Binary data encryption { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // Binary data: 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 @@ -160,7 +160,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 1: Valid parameters, should succeed { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Valid parameters test failed: " << sequencer.error_stage_ << " - " << sequencer.error_message_; @@ -169,7 +169,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 2: Invalid compression (should succeed with warning) { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::GZIP, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::GZIP, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result); @@ -179,7 +179,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 3: Undefined encoding is supported { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::UNDEFINED, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::UNDEFINED, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Encoding UNDEFINED should be supported: " << sequencer.error_message_; @@ -189,7 +189,7 @@ TEST(EncryptionSequencer, ParameterValidation) { // Test 4: All encodings now supported (including RLE) { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); ASSERT_TRUE(result) << "Encoding RLE should now be supported: " << sequencer.error_message_; @@ -203,7 +203,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 1: Empty plaintext { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(EMPTY_DATA); EXPECT_FALSE(result) << "Empty plaintext test should have failed"; @@ -213,7 +213,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 2: Empty ciphertext { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecryptAndEncode(EMPTY_DATA); EXPECT_FALSE(result) << "Empty ciphertext test should have failed"; @@ -223,7 +223,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 3: Empty key_id { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); EXPECT_FALSE(result) << "Empty key_id test should have failed"; @@ -233,7 +233,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 4: Missing encryption_metadata { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} // encryption_metadata, setting it to empty map. ); bool result = sequencer.DecryptAndEncode(HELLO_WORLD_DATA); @@ -245,7 +245,7 @@ TEST(EncryptionSequencer, InputValidation) { // Test 5: Incorrect encryption_metadata version { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {{"dbps_agent_version", "v0.09"}} // encryption_metadata, setting it to incorrect version. ); bool result = sequencer.DecryptAndEncode(HELLO_WORLD_DATA); @@ -261,7 +261,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 1: Basic round trip - "Hello, World!" { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} ); // Encrypt @@ -280,7 +280,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 2: Binary data round trip { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "binary_test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "binary_test_key", "test_user", "{}", {} ); // Encrypt @@ -299,7 +299,7 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 3: Single character round trip { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "single_char_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "single_char_key", "test_user", "{}", {} ); // "A" @@ -320,11 +320,11 @@ TEST(EncryptionSequencer, RoundTripEncryption) { // Test 4: Different keys produce different encrypted results { DataBatchEncryptionSequencer sequencer1( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key1", "test_user", "{}", {} ); DataBatchEncryptionSequencer sequencer2( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "key2", "test_user", "{}", {} ); bool result1 = sequencer1.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -355,7 +355,7 @@ TEST(EncryptionSequencer, ResultStorage) { // Test 1: Verify encrypted result is stored { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -370,7 +370,7 @@ TEST(EncryptionSequencer, ResultStorage) { // Test 2: Verify decrypted result is stored { DataBatchEncryptionSequencer sequencer( - "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} + "test_column", Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", "{}", {} ); // First encrypt something @@ -401,7 +401,7 @@ TEST(EncryptionSequencer, BooleanTypeUsesPerBlockEncryption) { std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, - {{"page_type", "DICTIONARY_PAGE"}}, + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", @@ -433,7 +433,7 @@ TEST(EncryptionSequencer, RleDictionaryEncodingUsesPerBlockEncryption) { std::nullopt, CompressionCodec::UNCOMPRESSED, Encoding::RLE_DICTIONARY, - {{"page_type", "DICTIONARY_PAGE"}}, + {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key", "test_user", @@ -460,7 +460,7 @@ TEST(EncryptionSequencer, FixedLenByteArrayValidation) { // Helper function to test validation failure auto testValidationFailure = [&](const std::optional& datatype_length, const std::string& expected_msg) -> bool { DataBatchEncryptionSequencer sequencer( - "test_column", Type::FIXED_LEN_BYTE_ARRAY, datatype_length, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} + "test_column", Type::FIXED_LEN_BYTE_ARRAY, datatype_length, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {} ); bool result = sequencer.DecodeAndEncrypt(HELLO_WORLD_DATA); @@ -484,7 +484,7 @@ TEST(EncryptionSequencer, FixedLenByteArrayValidation) { EXPECT_TRUE(testValidationFailure(0, "FIXED_LEN_BYTE_ARRAY datatype_length must be positive")); // Test valid case (should pass parameter validation) - DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); + DataBatchEncryptionSequencer sequencer("test_column", Type::FIXED_LEN_BYTE_ARRAY, 16, CompressionCodec::UNCOMPRESSED, Encoding::PLAIN, {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "2"}}, CompressionCodec::UNCOMPRESSED, "test_key_123", "test_user", "{}", {}); bool result = sequencer.DecodeAndEncrypt(FIXED_LEN_BYTE_ARRAY_DATA); if (!result && sequencer.error_stage_ == "parameter_validation") { @@ -597,3 +597,234 @@ TEST(EncryptionSequencer, ConvertEncodingAttributesToValues_Negative) { EXPECT_FALSE(sequencer3.TestConvertEncodingAttributesToValues()); EXPECT_EQ(sequencer3.error_stage_, "encoding_attribute_conversion"); } + +// ----------------------------------------------------------------------------- +// DATA_PAGE end-to-end coverage through DataBatchEncryptionSequencer. +// ----------------------------------------------------------------------------- + +TEST(EncryptionSequencer, DataPageV1NullableByteArray_PerValueRoundTrip) { + std::vector byte_array_elements = { + {'a', 'l', 'p', 'h', 'a'}, + {'b', 'e', 't', 'a'}, + {'g', 'a', 'm', 'm', 'a', '3'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // Nullable DATA_PAGE_V1 levels: one RLE run with 3 present values. + // level bytes = [u32 len=2][0x06, 0x01] + std::vector level_bytes; + append_u32_le(level_bytes, 2u); + level_bytes.push_back(0x06); // run_len = 3 + level_bytes.push_back(0x01); // def level value = 1 (present) + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_v1", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV2NullableByteArray_PerValueRoundTrip) { + std::vector byte_array_elements = { + {'x', 'x'}, + {'y', 'y', 'y'}, + {'z', 'z', 'z', 'z'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // DATA_PAGE_V2: logical rows=5, nulls=2, so num_elements in value bytes is 3. + std::vector level_bytes = {0x00}; // one byte of definition-level section + auto plaintext = Join(level_bytes, value_bytes); + + std::map attribs = { + {"page_type", "DATA_PAGE_V2"}, + {"data_page_num_values", "5"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v2_definition_levels_byte_length", "1"}, + {"page_v2_repetition_levels_byte_length", "0"}, + {"page_v2_num_nulls", "2"}, + {"page_v2_is_compressed", "false"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_v2", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::UNCOMPRESSED, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_v2", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV1NullableFloat_PerValueRoundTrip) { + std::vector float_values = {1.25f, -2.5f, 100.75f}; + std::vector value_bytes; + value_bytes.reserve(float_values.size() * sizeof(float)); + for (float v : float_values) { + append_f32_le(value_bytes, v); + } + + // Nullable DATA_PAGE_V1 levels: one RLE run with 3 present values. + std::vector level_bytes; + append_u32_le(level_bytes, 2u); + level_bytes.push_back(0x06); // run_len = 3 + level_bytes.push_back(0x01); // def level value = 1 (present) + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "float_col_v1", + Type::FLOAT, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_float_v1", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV2NullableFloat_PerValueRoundTrip) { + std::vector float_values = {9.5f, 0.0f, -7.25f}; + std::vector value_bytes; + value_bytes.reserve(float_values.size() * sizeof(float)); + for (float v : float_values) { + append_f32_le(value_bytes, v); + } + + // DATA_PAGE_V2: logical rows=5, nulls=2, so num_elements in value bytes is 3. + std::vector level_bytes = {0x00}; + auto plaintext = Join(level_bytes, value_bytes); + + std::map attribs = { + {"page_type", "DATA_PAGE_V2"}, + {"data_page_num_values", "5"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v2_definition_levels_byte_length", "1"}, + {"page_v2_repetition_levels_byte_length", "0"}, + {"page_v2_num_nulls", "2"}, + {"page_v2_is_compressed", "false"}}; + + DataBatchEncryptionSequencer sequencer( + "float_col_v2", + Type::FLOAT, + std::nullopt, + CompressionCodec::UNCOMPRESSED, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_float_v2", + "test_user", + "{}", + {}); + + ASSERT_TRUE(sequencer.DecodeAndEncrypt(plaintext)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + ASSERT_TRUE(sequencer.encryption_metadata_.count("encrypt_mode_data_page") == 1); + EXPECT_EQ(sequencer.encryption_metadata_.at("encrypt_mode_data_page"), "per_value"); + + ASSERT_TRUE(sequencer.DecryptAndEncode(sequencer.encrypted_result_)) + << sequencer.error_stage_ << " - " << sequencer.error_message_; + EXPECT_EQ(sequencer.decrypted_result_, plaintext); +} + +TEST(EncryptionSequencer, DataPageV1MalformedDefinitionPayload_Throws) { + std::vector byte_array_elements = { + {'a', 'l', 'p', 'h', 'a'}, + {'b', 'e', 't', 'a'}, + {'g', 'a', 'm', 'm', 'a', '3'}}; + auto value_bytes = CombineRawBytesIntoValueBytesForTesting( + byte_array_elements, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + + // Malformed DATA_PAGE_V1 definition payload: truncated varint run header. + std::vector level_bytes; + append_u32_le(level_bytes, 1u); + level_bytes.push_back(0x80); // continuation bit set, missing next byte + + auto page_payload = Join(level_bytes, value_bytes); + auto plaintext = Compress(page_payload, CompressionCodec::SNAPPY); + + std::map attribs = { + {"page_type", "DATA_PAGE_V1"}, + {"data_page_num_values", "3"}, + {"data_page_max_definition_level", "1"}, + {"data_page_max_repetition_level", "0"}, + {"page_v1_repetition_level_encoding", "RLE"}, + {"page_v1_definition_level_encoding", "RLE"}}; + + DataBatchEncryptionSequencer sequencer( + "byte_array_col_bad_v1", + Type::BYTE_ARRAY, + std::nullopt, + CompressionCodec::SNAPPY, + Encoding::PLAIN, + attribs, + CompressionCodec::UNCOMPRESSED, + "test_key_bad_v1", + "test_user", + "{}", + {}); + + EXPECT_THROW((void)sequencer.DecodeAndEncrypt(plaintext), InvalidInputException); +} diff --git a/src/processing/encryptors/basic_xor_encryptor.cpp b/src/processing/encryptors/basic_xor_encryptor.cpp index 8ac3386..2f9a580 100644 --- a/src/processing/encryptors/basic_xor_encryptor.cpp +++ b/src/processing/encryptors/basic_xor_encryptor.cpp @@ -19,9 +19,6 @@ #include "encryptor_utils.h" #include "../../common/exceptions.h" #include "../../common/enum_utils.h" -#include -#include -#include using namespace dbps::processing; using namespace dbps::external; @@ -30,21 +27,26 @@ using namespace dbps::external; // Functions for encrypting and decrypting byte arrays. // --------------------------------------------------------------------------- -std::vector BasicXorEncryptor::XorEncrypt(tcb::span data) { - if (data.empty()) { - return {}; +// XorEncryptInto uses a writable span `out` to encrypt the data in-place. +// This is a performance optimization to avoid copying the data to a buffer and then returning it. +void BasicXorEncryptor::XorEncryptInto(tcb::span data, tcb::span out) { + size_t data_size = data.size(); + size_t out_size = out.size(); + if (data_size != out_size) { + throw InvalidInputException("XorEncryptInto: input and output sizes must match"); } + const size_t n = data_size; + const uint8_t* src = data.data(); + uint8_t* dst = out.data(); size_t key_hash = key_id_hash_; - std::vector out(data.size()); - for (size_t i = 0; i < data.size(); ++i) { - out[i] = data[i] ^ (key_hash & 0xFF); + for (size_t i = 0; i < n; ++i) { + dst[i] = src[i] ^ (key_hash & 0xFF); key_hash = (key_hash << 1) | (key_hash >> 31); } - return out; } -std::vector BasicXorEncryptor::XorDecrypt(tcb::span data) { - return XorEncrypt(data); +void BasicXorEncryptor::XorDecryptInto(tcb::span data, tcb::span out) { + XorEncryptInto(data, out); } // --------------------------------------------------------------------------- @@ -52,11 +54,21 @@ std::vector BasicXorEncryptor::XorDecrypt(tcb::span data // --------------------------------------------------------------------------- std::vector BasicXorEncryptor::EncryptBlock(tcb::span data) { - return XorEncrypt(data); + if (data.empty()) { + return {}; + } + std::vector out(data.size()); + XorEncryptInto(data, tcb::span(out.data(), out.size())); + return out; } std::vector BasicXorEncryptor::DecryptBlock(tcb::span data) { - return XorDecrypt(data); + if (data.empty()) { + return {}; + } + std::vector out(data.size()); + XorDecryptInto(data, tcb::span(out.data(), out.size())); + return out; } // --------------------------------------------------------------------------- @@ -104,7 +116,7 @@ std::vector BasicXorEncryptor::EncryptTypedElements( // Encrypt the elements by traversing the typed input buffer and encrypt each element separately: // - Create an output raw-bytes buffer to capture the encrypted elements as bytes. - // - For each element: read its raw bytes, encrypt them, write into the output buffer. + // - For each element: read its raw bytes, get a writable span on the output, and encrypt in-place. // - Finalize the output buffer into a contiguous byte vector. std::vector final_buffer; @@ -115,24 +127,30 @@ std::vector BasicXorEncryptor::EncryptTypedElements( element_size = input_buffer.GetElementSize(); TypedBufferRawBytesFixedSized output_buffer{ num_elements, prefix_length, RawBytesFixedSizedCodec{element_size}}; + size_t output_index = 0; - for (const auto raw_bytes : input_buffer.raw_elements()) { - auto encrypted = XorEncrypt(raw_bytes); - output_buffer.SetElement(output_index, tcb::span(encrypted)); + tcb::span raw_bytes; + + while (input_buffer.ElementsIteratorNext(raw_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, element_size); + XorEncryptInto(raw_bytes, write_span); output_index++; } final_buffer = output_buffer.FinalizeAndTakeBuffer(); - } + } // Encrypt variable-size elements else { auto reserved_bytes_hint = input_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{ num_elements, reserved_bytes_hint, true, prefix_length}; + size_t output_index = 0; - for (const auto raw_bytes : input_buffer.raw_elements()) { - auto encrypted = XorEncrypt(raw_bytes); - output_buffer.SetElement(output_index, tcb::span(encrypted)); + tcb::span raw_bytes; + + while (input_buffer.ElementsIteratorNext(raw_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, raw_bytes.size()); + XorEncryptInto(raw_bytes, write_span); output_index++; } final_buffer = output_buffer.FinalizeAndTakeBuffer(); @@ -142,12 +160,12 @@ std::vector BasicXorEncryptor::EncryptTypedElements( WriteHeader(final_buffer, {is_fixed, static_cast(num_elements), static_cast(element_size)}); + return final_buffer; } std::vector BasicXorEncryptor::EncryptValueList( const TypedValuesBuffer& typed_buffer) { - // Printable context string for logging // std::string context_str = std::string("Context parameters:") // + "\n column_name: " + column_name_ @@ -178,9 +196,11 @@ template TypedBuffer BasicXorEncryptor::DecryptFixedSizedElementsIntoTypedBuffer( const TypedBufferRawBytesFixedSized& encrypted_buffer, TypedBuffer output_buffer) { size_t output_index = 0; - for (const auto raw_bytes : encrypted_buffer.raw_elements()) { - auto decrypted_bytes = XorDecrypt(raw_bytes); - output_buffer.SetRawElement(output_index, tcb::span(decrypted_bytes)); + tcb::span element_bytes; + size_t element_size = encrypted_buffer.GetElementSize(); + while (encrypted_buffer.ElementsIteratorNext(element_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, element_size); + XorDecryptInto(element_bytes, write_span); output_index++; } return output_buffer; @@ -197,7 +217,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Create a fixed-sized byte buffer for reading the encrypted elements. TypedBufferRawBytesFixedSized encrypted_buffer{ - encrypted_bytes, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; + encrypted_bytes, num_elements, kFixedHeaderLength, RawBytesFixedSizedCodec{header.element_size}}; // Populate a typed buffer with the decrypted elements in the corresponding type. switch (datatype_) { @@ -230,7 +250,7 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( // Decrypt variable-size elements else { // Create a variable-sized byte buffer for reading the encrypted elements. - TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, kVariableHeaderLength}; + TypedBufferRawBytesVariableSized encrypted_buffer{ encrypted_bytes, num_elements, kVariableHeaderLength}; switch (datatype_) { // Create a BYTE-ARRAY typed buffer for storing the decrypted elements. @@ -238,11 +258,12 @@ TypedValuesBuffer BasicXorEncryptor::DecryptValueList( auto reserved_bytes_hint = encrypted_buffer.GetRawBufferSize(); TypedBufferRawBytesVariableSized output_buffer{num_elements, reserved_bytes_hint, true}; size_t output_index = 0; - for (const auto element : encrypted_buffer) { - auto decrypted_bytes = XorDecrypt(element); - output_buffer.SetElement(output_index, tcb::span(decrypted_bytes)); + tcb::span element_bytes; + while (encrypted_buffer.ElementsIteratorNext(element_bytes)) { + auto write_span = output_buffer.GetWritableRawElement(output_index, element_bytes.size()); + XorDecryptInto(element_bytes, write_span); output_index++; - } + } return output_buffer; } default: diff --git a/src/processing/encryptors/basic_xor_encryptor.h b/src/processing/encryptors/basic_xor_encryptor.h index 2e30354..c2f55f0 100644 --- a/src/processing/encryptors/basic_xor_encryptor.h +++ b/src/processing/encryptors/basic_xor_encryptor.h @@ -66,8 +66,8 @@ class DBPS_EXPORT BasicXorEncryptor : public DBPSEncryptor { private: const size_t key_id_hash_; - std::vector XorEncrypt(tcb::span data); - std::vector XorDecrypt(tcb::span data); + void XorEncryptInto(tcb::span data, tcb::span out); + void XorDecryptInto(tcb::span data, tcb::span out); template std::vector EncryptTypedElements(const InputBuffer& input_buffer); diff --git a/src/processing/encryptors/basic_xor_encryptor_test.cpp b/src/processing/encryptors/basic_xor_encryptor_test.cpp index e5e7322..43c285b 100644 --- a/src/processing/encryptors/basic_xor_encryptor_test.cpp +++ b/src/processing/encryptors/basic_xor_encryptor_test.cpp @@ -71,7 +71,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_INT32) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferI32 input_buffer_read{input_span}; + TypedBufferI32 input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); @@ -99,7 +99,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_DOUBLE) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferDouble input_buffer_read{input_span}; + TypedBufferDouble input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); @@ -149,7 +149,7 @@ TEST(BasicXorEncryptor, EncryptDecryptValueList_RoundTrip_BYTE_ARRAY) { // re-wrap bytes as a read buffer to match production read path behavior. std::vector input_buffer_bytes = input_buffer_write.FinalizeAndTakeBuffer(); const auto input_span = tcb::span(input_buffer_bytes.data(), input_buffer_bytes.size()); - TypedBufferRawBytesVariableSized input_buffer_read{input_span}; + TypedBufferRawBytesVariableSized input_buffer_read{input_span, values.size()}; TypedValuesBuffer typed_buffer = std::move(input_buffer_read); std::vector encrypted_blob = encryptor.EncryptValueList(typed_buffer); diff --git a/src/processing/parquet_utils.cpp b/src/processing/parquet_utils.cpp index 9b5930b..794e844 100644 --- a/src/processing/parquet_utils.cpp +++ b/src/processing/parquet_utils.cpp @@ -28,31 +28,208 @@ using namespace dbps::compression; using namespace dbps::processing; // ----------------------------------------------------------------------------- -// Process Parquet formatted Dictionary and Data pages +// Helper functions for Parquet DATA_PAGE_V1 definition level bytes parsing to count present values. // ----------------------------------------------------------------------------- -int CalculateLevelBytesLength(tcb::span raw, - const AttributesMap& encoding_attribs) { - - // Helper function to skip V1 RLE level data in raw bytes - // Returns number of bytes consumed: [4-byte len] + [level bytes indicated by `len`] - auto SkipV1RLELevel = [&raw](size_t& offset) -> int { - if (offset + 4 > raw.size()) { - throw InvalidInputException( - "Invalid RLE level data: offset + 4 exceeds data size (offset=" + - std::to_string(offset) + ", size=" + std::to_string(raw.size()) + ")"); +// Decodes one unsigned LEB128 (base-128 varint) run header from `bytes`, +// starting at `offset`, and advances `offset` past the decoded header bytes. +// +// A "run header" is a variable-length encoded integer that indicates the length of a run. +// +// In Parquet V1 hybrid RLE/bit-packed streams, this run header indicates: +// - RLE run when (header & 1) == 0, with run_length = header >> 1 +// - Bit-packed run when (header & 1) == 1, with num_groups = header >> 1 +// and run_length = num_groups * 8 values +// +// This is used by V1 definition-level decoding to iterate runs and compute +// the count of present (non-null) values in nullable data pages. +// +uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset) { + uint32_t value = 0; + int shift = 0; + while (true) { + if (offset >= bytes.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 level stream: truncated varint header"); } - uint32_t len = read_u32_le(raw, offset); - if (offset + 4 + len > raw.size()) { - throw InvalidInputException( - "Invalid RLE level data: length field overflows (offset=" + - std::to_string(offset) + ", len=" + std::to_string(len) + ", size=" + - std::to_string(raw.size()) + ")"); + uint8_t b = bytes[offset++]; + value |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; } - offset += 4 + len; - return 4 + len; - }; - + shift += 7; + if (shift > 28) { + throw InvalidInputException("Invalid DATA_PAGE_V1 level stream: varint header too large"); + } + } +} + +// Decodes a DATA_PAGE_V1 definition-level payload (hybrid RLE/bit-packed) and +// returns the number of present (non-null) values in the page. +// +// Reference: https://parquet.apache.org/docs/file-format/data-pages/encodings/#RLE +// +// Inputs: +// - def_payload: bytes of the V1 definition-level stream payload only +// (without the outer [u32 length] prefix). +// - num_values: total number of logical values in the page (includes nulls). +// - max_def_level: maximum definition level for the column in this page. +// +// Output: +// - present_count = number of decoded definition levels equal to max_def_level. +// +size_t CountPresentValuesFromDefinitionLevelsV1(tcb::span def_payload, int32_t num_values, int32_t max_def_level) { + if (num_values < 0) { + throw InvalidInputException("Invalid V1 definition levels: num_values must be non-negative, got " + + std::to_string(num_values)); + } + if (max_def_level <= 0) { + throw InvalidInputException("Invalid V1 definition levels: max_def_level must be positive, got " + + std::to_string(max_def_level)); + } + + // Definition level bit width is ceil(log2(max_def_level + 1)). + // Computes the minimum number of bits needed to represent definition levels from 0..max_def_level. + int bit_width = 0; + uint32_t def_level_domain = static_cast(max_def_level); + while (def_level_domain > 0) { + ++bit_width; + def_level_domain >>= 1; + } + if (bit_width <= 0) { + throw InvalidInputException("Invalid V1 definition levels: computed bit_width must be positive"); + } + + size_t present_count = 0; + size_t decoded_values = 0; + size_t def_offset = 0; + + // Hybrid RLE/bit-packed decode loop. + while (decoded_values < static_cast(num_values)) { + uint32_t header = ReadV1RunHeaderUleb128(def_payload, def_offset); + + if ((header & 1u) == 0u) { + // RLE run: header = (run_len << 1), then repeated value in ceil(bit_width/8) bytes. + const size_t run_len = static_cast(header >> 1); + const size_t remaining = static_cast(num_values) - decoded_values; + if (run_len == 0 || run_len > remaining) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: invalid RLE run length"); + } + + const size_t byte_width = static_cast((bit_width + 7) / 8); + if (def_offset + byte_width > def_payload.size()) { + throw InvalidInputException("Invalid V1 definition levels: truncated RLE run value"); + } + + uint32_t level = 0; + for (size_t i = 0; i < byte_width; ++i) { + level |= static_cast(def_payload[def_offset + i]) << (8 * i); + } + def_offset += byte_width; + if (level > static_cast(max_def_level)) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: decoded level exceeds max_def_level"); + } + + if (level == static_cast(max_def_level)) { + present_count += run_len; + } + decoded_values += run_len; + } else { + // Bit-packed run: header = (num_groups << 1) | 1, each group has 8 values. + const size_t num_groups = static_cast(header >> 1); + const size_t run_len = num_groups * 8; + const size_t remaining = static_cast(num_values) - decoded_values; + if (num_groups == 0) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: invalid bit-packed run length"); + } + + const size_t total_bits = run_len * static_cast(bit_width); + const size_t byte_len = (total_bits + 7) / 8; + if (def_offset + byte_len > def_payload.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: truncated bit-packed run payload"); + } + auto packed = tcb::span(def_payload.data() + def_offset, byte_len); + def_offset += byte_len; + + auto ReadPacked = [&](size_t bit_offset) -> uint32_t { + uint32_t v = 0; + for (int b = 0; b < bit_width; ++b) { + size_t abs_bit = bit_offset + static_cast(b); + size_t byte_index = abs_bit / 8; + size_t bit_index = abs_bit % 8; + uint8_t bit = static_cast((packed[byte_index] >> bit_index) & 0x01); + v |= static_cast(bit) << b; + } + return v; + }; + + // A final bit-packed run may include padded trailing values to complete + // 8-value groups. Decode only the logical values still remaining. + const size_t values_to_decode = std::min(run_len, remaining); + for (size_t i = 0; i < values_to_decode; ++i) { + uint32_t level = ReadPacked(i * static_cast(bit_width)); + if (level > static_cast(max_def_level)) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: decoded level exceeds max_def_level"); + } + if (level == static_cast(max_def_level)) { + ++present_count; + } + } + decoded_values += values_to_decode; + } + } + if (def_offset != def_payload.size()) { + throw InvalidInputException("Invalid DATA_PAGE_V1 definition levels: trailing bytes after decoding"); + } + return present_count; +} + +// ----------------------------------------------------------------------------- +// Helper functions to read/split DATA_PAGE_V1 level bytes -- Read length-prefixed level bytes +// ----------------------------------------------------------------------------- + +// Function to read a length-prefixed payload from the level bytes. +tcb::span ReadV1LengthPrefixedPayload(tcb::span bytes, size_t& offset) { + if (offset + 4 > bytes.size()) { + throw InvalidInputException( + "Invalid Parquet DATA_PAGE_V1 level bytes: missing 4-byte length prefix"); + } + uint32_t len = read_u32_le(bytes, offset); + const size_t payload_offset = offset + 4; + if (len > bytes.size() - payload_offset) { + throw InvalidInputException( + "Invalid Parquet DATA_PAGE_V1 level bytes: length-prefixed block exceeds bounds"); + } + offset = payload_offset + static_cast(len); + return tcb::span(bytes.data() + payload_offset, static_cast(len)); +} + +// Function to skip the repetition levels bytes and return the definition levels bytes payload. +tcb::span ReadDefinitionLevelBytesV1(tcb::span level_bytes, int32_t max_rep_level) { + // V1 level bytes are [rep_levels?][def_levels], each as [u32 len][payload]. + size_t level_offset = 0; + + // Skip the repetition levels bytes if any. + if (max_rep_level > 0) { + (void) ReadV1LengthPrefixedPayload(level_bytes, level_offset); + } + + // Read the definition levels bytes. + auto def_payload = ReadV1LengthPrefixedPayload(level_bytes, level_offset); + if (level_offset != level_bytes.size()) { + throw InvalidInputException("Invalid Parquet DATA_PAGE_V1 level bytes: trailing bytes after definition levels block"); + } + return def_payload; +} + +// ----------------------------------------------------------------------------- +// Helper function to calculate level bytes length +// ----------------------------------------------------------------------------- + +// Calculates the total length of level bytes based on encoding attributes. +// Assumes the input encoding attributes are already validated with the required keys and expected value types. +int CalculateLevelBytesLength(tcb::span raw, + const AttributesMap& encoding_attribs) { + // Get page_type from the converted attributes const std::string& page_type = std::get(encoding_attribs.at("page_type")); int total_level_bytes = 0; @@ -72,17 +249,21 @@ int CalculateLevelBytesLength(tcb::span raw, ", definition_level_encoding=" + def_encoding + " (only RLE is expected)"); } - // if max_rep_level > 0, there are repetition levels bytes. Same for definition levels. + // Read and skip the repetition/definition level bytes to calculate the final offset where + // the level bytes end and the value bytes start. + // - If max_rep_level > 0, there are repetition levels bytes. Same for definition levels. int32_t max_rep_level = std::get(encoding_attribs.at("data_page_max_repetition_level")); int32_t max_def_level = std::get(encoding_attribs.at("data_page_max_definition_level")); size_t offset = 0; if (max_rep_level > 0) { - int bytes_skipped = SkipV1RLELevel(offset); - total_level_bytes += bytes_skipped; + size_t start_offset = offset; + (void) ReadV1LengthPrefixedPayload(raw, offset); + total_level_bytes += static_cast(offset - start_offset); } if (max_def_level > 0) { - int bytes_skipped = SkipV1RLELevel(offset); - total_level_bytes += bytes_skipped; + size_t start_offset = offset; + (void) ReadV1LengthPrefixedPayload(raw, offset); + total_level_bytes += static_cast(offset - start_offset); } } else if (page_type == "DICTIONARY_PAGE") { @@ -107,6 +288,10 @@ int CalculateLevelBytesLength(tcb::span raw, return total_level_bytes; } +// ----------------------------------------------------------------------------- +// Public functions to process Parquet formatted Dictionary and Data pages +// ----------------------------------------------------------------------------- + LevelAndValueBytes DecompressAndSplit( tcb::span plaintext, CompressionCodec::type compression, @@ -123,7 +308,25 @@ LevelAndValueBytes DecompressAndSplit( int leading_bytes_to_strip = CalculateLevelBytesLength( decompressed_bytes, encoding_attributes); auto [level_bytes, value_bytes] = Split(decompressed_bytes, leading_bytes_to_strip); - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + + // For DATA_PAGE_V1, data_page_num_values is the count of logical rows (includes nulls). + // The V1 header does not carry num_nulls, so we cannot derive present values as in V2. + // To get the number of encoded physical values in value_bytes, we must parse definition levels. + size_t num_elements = 0; + const int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); + int32_t max_def_level = std::get(encoding_attributes.at("data_page_max_definition_level")); + int32_t max_rep_level = std::get(encoding_attributes.at("data_page_max_repetition_level")); + if (max_def_level == 0) { + // All values are present in the value bytes section. + num_elements = static_cast(num_values); + } + // If max_def_level > 0, there are definition levels bytes. So parse it and count the present values. + else { + auto def_bytes_payload = ReadDefinitionLevelBytesV1(level_bytes, max_rep_level); + num_elements = CountPresentValuesFromDefinitionLevelsV1(def_bytes_payload, num_values, max_def_level); + } + + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } // On DATA_PAGE_V2, only the value bytes are compressed. @@ -143,14 +346,28 @@ LevelAndValueBytes DecompressAndSplit( } else { value_bytes = std::vector(compressed_value_bytes_span.begin(), compressed_value_bytes_span.end()); } - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + + // For DATA_PAGE_V2, data_page_num_values is the count of logical rows, not present values. data_page_num_values includes nulls. + // num_nulls is the count of nulls in the page. + // So num_elements (the present values) is num_values - num_nulls. + int32_t num_values = std::get(encoding_attributes.at("data_page_num_values")); + int32_t num_nulls = std::get(encoding_attributes.at("page_v2_num_nulls")); + if (num_nulls > num_values) { + throw InvalidInputException( + "Invalid num_nulls: " + std::to_string(num_nulls) + " > num_values: " + + std::to_string(num_values) + " in DATA_PAGE_V2 encoding attributes"); + } + size_t num_elements = static_cast(num_values - num_nulls); + + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } // DICTIONARY_PAGE has no level bytes. if (page_type == "DICTIONARY_PAGE") { auto level_bytes = std::vector(); auto value_bytes = Decompress(plaintext, compression); - return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes)}; + size_t num_elements = static_cast( std::get(encoding_attributes.at("dict_page_num_values"))); + return LevelAndValueBytes{std::move(level_bytes), std::move(value_bytes), num_elements}; } throw InvalidInputException("Unexpected page type: " + page_type); @@ -196,10 +413,12 @@ std::vector CompressAndJoin( } // ----------------------------------------------------------------------------- -// Build Parquet formatted value bytes into TypedValuesBuffer +// Public functions to build Parquet formatted value bytes into TypedValuesBuffer // ----------------------------------------------------------------------------- -TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer(tcb::span value_bytes, +TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( + tcb::span value_bytes, + size_t num_elements, Type::type datatype, const std::optional& datatype_length, Encoding::type encoding) { @@ -223,24 +442,24 @@ TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer(tcb::span(datatype_length.value())}}; + value_bytes, num_elements, 0, RawBytesFixedSizedCodec{static_cast(datatype_length.value())}}; } case Type::BYTE_ARRAY: - return TypedBufferRawBytesVariableSized{value_bytes}; + return TypedBufferRawBytesVariableSized{value_bytes, num_elements}; default: throw InvalidInputException( "Invalid datatype: " + std::string(to_string(datatype))); @@ -254,3 +473,5 @@ std::vector GetTypedValuesBufferAsValueBytes(TypedValuesBuffer&& buffer return buf.FinalizeAndTakeBuffer(); }, buffer); } + +// ----------------------------------------------------------------------------- diff --git a/src/processing/parquet_utils.h b/src/processing/parquet_utils.h index 48f53e9..dd48290 100644 --- a/src/processing/parquet_utils.h +++ b/src/processing/parquet_utils.h @@ -33,25 +33,15 @@ struct LevelAndValueBytes { std::vector level_bytes; std::vector value_bytes; + size_t num_elements; }; using namespace dbps::external; // ----------------------------------------------------------------------------- -// Helper functions for processing Parquet formatted data pages and dictionary pages. +// Functions to decompress and split a Parquet page into level and value bytes. // ----------------------------------------------------------------------------- -/** - * Calculates the total length of level bytes based on encoding attributes. - * Assumes the input encoding attributes are already validated with the required keys and expected value types. - * - * @param raw Raw binary data (currently unused but kept for future V1 implementation) - * @param encoding_attribs Converted encoding attributes map - * @return Total length of level bytes. Throws exceptions if calculation fails or page type is unsupported - */ -int CalculateLevelBytesLength(tcb::span raw, - const AttributesMap& encoding_attribs); - /** * Decompresses and splits a Parquet page into level and value bytes. * Handles DATA_PAGE_V1, DATA_PAGE_V2 (including optional compression on value bytes), @@ -74,7 +64,7 @@ std::vector CompressAndJoin( // ----------------------------------------------------------------------------- -// Helper functions for zero-copy reinterpretation of raw value bytes into a typed buffer. +// Functions for zero-copy reinterpretation of raw value bytes into a typed buffer. // ----------------------------------------------------------------------------- /** @@ -94,6 +84,7 @@ std::vector CompressAndJoin( */ dbps::processing::TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( tcb::span value_bytes, + size_t num_elements, Type::type datatype, const std::optional& datatype_length, Encoding::type encoding); @@ -106,3 +97,5 @@ dbps::processing::TypedValuesBuffer ReinterpretValueBytesAsTypedValuesBuffer( */ std::vector GetTypedValuesBufferAsValueBytes( dbps::processing::TypedValuesBuffer&& buffer); + +// ----------------------------------------------------------------------------- diff --git a/src/processing/parquet_utils_test.cpp b/src/processing/parquet_utils_test.cpp index 1c4f8b5..f98e88d 100644 --- a/src/processing/parquet_utils_test.cpp +++ b/src/processing/parquet_utils_test.cpp @@ -32,6 +32,97 @@ using namespace dbps::external; using namespace dbps::compression; using namespace dbps::processing; +// Forward declaration for internal helper tested here without exposing it in parquet_utils.h. +uint32_t ReadV1RunHeaderUleb128(tcb::span bytes, size_t& offset); + +size_t CountPresentValuesFromDefinitionLevelsV1( + tcb::span def_payload, + int32_t num_values, + int32_t max_def_level); + +int CalculateLevelBytesLength(tcb::span raw, + const AttributesMap& encoding_attribs); + +// ----------------------------------------------------------------------------- +// Helper functions to generate Parquet DATA_PAGE_V1 level bytes payloads for testing. +// ----------------------------------------------------------------------------- + +namespace { + +// Encodes an unsigned integer as ULEB128 bytes (base-128 varint) for test payload construction. +// Each output byte stores 7 data bits; the MSB is a continuation flag. +// - Example: 6 -> {0x06} (single byte, continuation flag not set) +// - Example: 300 -> {0xAC, 0x02}: +// 300 = 2*128 + 44, so the first 7-bit chunk is 44 (0x2C) and the remaining value is 2. +// First byte is 0x2C | 0x80 = 0xAC (set continuation bit because more chunks follow), +// second byte is 0x02 (final chunk, continuation bit cleared). +std::vector EncodeUleb128(uint32_t value) { + std::vector out; + do { + uint8_t byte = static_cast(value & 0x7F); + value >>= 7; + if (value != 0) { + byte |= 0x80; + } + out.push_back(byte); + } while (value != 0); + return out; +} + +// Builds a DATA_PAGE_V1 RLE run payload: varint run header + repeated level value bytes. +std::vector MakeRleDefPayload(uint32_t run_len, uint32_t level, int bit_width) { + std::vector out = EncodeUleb128(run_len << 1); // RLE run + const size_t byte_width = static_cast((bit_width + 7) / 8); + for (size_t i = 0; i < byte_width; ++i) { + out.push_back(static_cast((level >> (8 * i)) & 0xFF)); + } + return out; +} + +// Packs level values into Parquet's LSB-first bit-packed byte layout for a given bit width. +std::vector PackBitPackedLevels( + const std::vector& levels, + int bit_width) { + const size_t total_bits = levels.size() * static_cast(bit_width); + std::vector out((total_bits + 7) / 8, 0); + size_t bit_offset = 0; + for (uint32_t level : levels) { + for (int b = 0; b < bit_width; ++b) { + uint8_t bit = static_cast((level >> b) & 0x01); + size_t abs_bit = bit_offset + static_cast(b); + out[abs_bit / 8] |= static_cast(bit << (abs_bit % 8)); + } + bit_offset += static_cast(bit_width); + } + return out; +} + +// Builds a DATA_PAGE_V1 bit-packed run payload: varint run header + packed level bytes. +std::vector MakeBitPackedDefPayload( + const std::vector& levels, + int bit_width) { + EXPECT_EQ(levels.size() % 8, 0u); + const uint32_t num_groups = static_cast(levels.size() / 8); + std::vector out = EncodeUleb128((num_groups << 1) | 1u); // bit-packed run + auto packed = PackBitPackedLevels(levels, bit_width); + out.insert(out.end(), packed.begin(), packed.end()); + return out; +} + +// Prepends a 4-byte little-endian length prefix used by DATA_PAGE_V1 level streams. +std::vector WrapLengthPrefixed(const std::vector& payload) { + std::vector out; + append_u32_le(out, static_cast(payload.size())); + out.insert(out.end(), payload.begin(), payload.end()); + return out; +} + +} // namespace + +// ----------------------------------------------------------------------------- +// Tests for CalculateLevelBytesLength function. +// ----------------------------------------------------------------------------- + TEST(ParquetUtils, CalculateLevelBytesLength_DATA_PAGE_V2) { std::vector raw = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; AttributesMap attribs = { @@ -158,6 +249,192 @@ TEST(ParquetUtils, CalculateLevelBytesLength_NegativeTotalSize) { EXPECT_THROW(CalculateLevelBytesLength(raw, attribs), InvalidInputException); } +// ----------------------------------------------------------------------------- +// Tests for CountPresentValuesFromDefinitionLevelsV1 function. +// ----------------------------------------------------------------------------- + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RleAllPresent) { + auto def_payload = MakeRleDefPayload(10, 1, 1); + EXPECT_EQ(10u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 10, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RleAllNull) { + auto def_payload = MakeRleDefPayload(10, 0, 1); + EXPECT_EQ(0u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 10, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedMixed) { + std::vector levels = {1, 0, 1, 0, 1, 0, 1, 0}; + auto def_payload = MakeBitPackedDefPayload(levels, 1); + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_MixedRuns) { + auto rle_part = MakeRleDefPayload(4, 1, 1); // 4 present + std::vector levels = {0, 1, 0, 1, 0, 0, 0, 0}; // +2 present + auto bp_part = MakeBitPackedDefPayload(levels, 1); + rle_part.insert(rle_part.end(), bp_part.begin(), bp_part.end()); + EXPECT_EQ(6u, CountPresentValuesFromDefinitionLevelsV1(rle_part, 12, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_InvalidRunLength) { + auto def_payload = MakeRleDefPayload(9, 1, 1); // run_len > num_values + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_InvalidLevelExceedsMax) { + auto def_payload = MakeRleDefPayload(1, 2, 1); // level 2 > max_def_level 1 + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_TruncatedVarint) { + std::vector def_payload = {0x80}; // continuation bit set, no next byte + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_TruncatedRleValue) { + auto def_payload = EncodeUleb128(2); // run_len = 1, but missing repeated value byte + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedCanonical_0to7_88C6FA) { + // Canonical example from Parquet Encodings.md: + // values 0..7 with bit_width=3 are packed as bytes 0x88, 0xC6, 0xFA. + // Bit-packed header for one group of 8 values is varint((1 << 1) | 1) = 0x03. + // + // Important detail: decoder bit_width is derived from max_def_level at runtime. + // So this payload is valid only when max_def_level implies bit_width=3 (e.g. 7). + // Reusing these same bytes with max_def_level=3 (bit_width=2) is a different + // encoding contract and can leave unread trailing bytes by design. + std::vector def_payload = {0x03, 0x88, 0xC6, 0xFA}; + + EXPECT_EQ(1u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 7)); // only value 7 + + // Separate bit_width=2 payload for max_def_level=3. + // This keeps payload bit-width consistent with decoder configuration and + // avoids mixing a 3-bit packed stream with a 2-bit decode expectation. + std::vector levels_bw2 = {0, 1, 2, 3, 0, 1, 2, 0}; // one value at level 3 + auto def_payload_bw2 = MakeBitPackedDefPayload(levels_bw2, 2); + EXPECT_EQ(1u, CountPresentValuesFromDefinitionLevelsV1(def_payload_bw2, 8, 3)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_RleRunLen4_Level1) { + // Manual payload: + // - header 0x08 => RLE run, run_len = 0x08 >> 1 = 4 + // - repeated value byte = 0x01 (bit_width=1, level=1) + std::vector def_payload = {0x08, 0x01}; + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 4, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_BitPackedAlternating_0xAA) { + // Manual payload: + // - header 0x03 => bit-packed, num_groups = 1, run_len = 8 + // - packed byte 0xAA => bits (LSB->MSB): 0,1,0,1,0,1,0,1 + std::vector def_payload = {0x03, 0xAA}; + EXPECT_EQ(4u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_ManualBytes_MixedRleAndBitPacked) { + // Manual payload: + // - 0x06,0x01 => RLE run_len=3, level=1 + // - 0x03,0x0F => bit-packed 8 values, bits: 1,1,1,1,0,0,0,0 + // Total values = 3 + 8 = 11; present count at max_def_level=1 is 3 + 4 = 7. + std::vector def_payload = {0x06, 0x01, 0x03, 0x0F}; + EXPECT_EQ(7u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 11, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitWidth1_ExhaustiveOneGroup) { + // Exhaustive check for all 8-value bit-packed patterns at bit_width=1. + // Payload form: [0x03][packed-byte], where 0x03 means one bit-packed group (8 values). + for (int packed = 0; packed <= 0xFF; ++packed) { + std::vector def_payload = {0x03, static_cast(packed)}; + + size_t ones = 0; + for (int bit = 0; bit < 8; ++bit) { + ones += static_cast((packed >> bit) & 0x01); + } + + EXPECT_EQ(ones, CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1)) + << "packed=0x" << std::hex << packed; + } +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsZeroRleRunLength) { + // Header 0 means RLE with run_len = 0, which is invalid. + std::vector def_payload = {0x00, 0x00}; + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsZeroBitPackedGroups) { + // Header 1 means bit-packed with num_groups = 0, which is invalid. + std::vector def_payload = {0x01}; + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_BitPackedFinalRunAllowsPadding) { + // Corner case: a final bit-packed run is encoded as a full 8-value group, + // while logical num_values ends mid-group. Decode only the logical values + // and ignore padded trailing values in the last group. + // Payload: header=0x03 (1 bit-packed group => 8 values), packed=0x07 (bits 1,1,1,0,0,0,0,0). + std::vector def_payload = {0x03, 0x07}; + EXPECT_EQ(3u, CountPresentValuesFromDefinitionLevelsV1(def_payload, 3, 1)); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsTrailingBytesAfterDecoding) { + // One full bit-packed group (8 values) plus extra trailing byte that must be rejected. + std::vector def_payload = {0x03, 0xAA, 0xFF}; + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 8, 1), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsNonPositiveMaxDefLevel) { + auto def_payload = MakeRleDefPayload(1, 0, 1); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, 1, 0), InvalidInputException); +} + +TEST(ParquetUtils, CountPresentValuesFromDefinitionLevelsV1_RejectsNegativeNumValues) { + auto def_payload = MakeRleDefPayload(1, 1, 1); + EXPECT_THROW(CountPresentValuesFromDefinitionLevelsV1(def_payload, -1, 1), InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for ReadV1RunHeaderUleb128 helper function. +// ----------------------------------------------------------------------------- + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_SingleByteHeader) { + std::vector bytes = {0x06}; // value 6 + size_t offset = 0; + + const uint32_t header = ReadV1RunHeaderUleb128(bytes, offset); + EXPECT_EQ(6u, header); + EXPECT_EQ(1u, offset); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_MultiByteHeaderAndOffsetAdvance) { + std::vector bytes = {0xAA, 0xAC, 0x02}; // second varint = 300 + size_t offset = 1; + + const uint32_t header = ReadV1RunHeaderUleb128(bytes, offset); + EXPECT_EQ(300u, header); + EXPECT_EQ(3u, offset); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_TruncatedVarintThrows) { + std::vector bytes = {0x80}; // continuation bit set, but no following byte + size_t offset = 0; + + EXPECT_THROW(ReadV1RunHeaderUleb128(bytes, offset), InvalidInputException); +} + +TEST(ParquetUtils, ReadV1RunHeaderUleb128_VarintTooLargeThrows) { + std::vector bytes = {0x80, 0x80, 0x80, 0x80, 0x80}; + size_t offset = 0; + + EXPECT_THROW(ReadV1RunHeaderUleb128(bytes, offset), InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for DecompressAndSplit function. +// ----------------------------------------------------------------------------- TEST(ParquetUtils, DecompressAndSplit_DataPageV2_Uncompressed) { AttributesMap attribs_conv = { @@ -229,6 +506,95 @@ TEST(ParquetUtils, DecompressAndSplit_DataPageV2_UnsupportedCompression) { DBPSUnsupportedException); } +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Required_NoLevels) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(4)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(0)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector value_bytes = {0x11, 0x22, 0x33, 0x44}; + auto result = DecompressAndSplit(value_bytes, CompressionCodec::UNCOMPRESSED, attribs); + + EXPECT_TRUE(result.level_bytes.empty()); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(4u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Nullable_RleAllPresent) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(5)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector def_payload = MakeRleDefPayload(5, 1, 1); + std::vector level_bytes = WrapLengthPrefixed(def_payload); + std::vector value_bytes = {0x10, 0x20, 0x30, 0x40, 0x50}; + std::vector plaintext = Join(level_bytes, value_bytes); + + auto result = DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs); + EXPECT_EQ(level_bytes, result.level_bytes); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(5u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_Nullable_BitPackedWithRepLevels) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(8)}, + {"data_page_max_repetition_level", int32_t(1)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + // Rep levels are present but ignored for present-count logic. + std::vector rep_payload = MakeRleDefPayload(8, 0, 1); + std::vector def_payload = MakeBitPackedDefPayload({1, 0, 1, 0, 1, 0, 0, 0}, 1); + std::vector level_bytes = WrapLengthPrefixed(rep_payload); + auto def_wrapped = WrapLengthPrefixed(def_payload); + level_bytes.insert(level_bytes.end(), def_wrapped.begin(), def_wrapped.end()); + + std::vector value_bytes = {0x01, 0x02, 0x03}; + std::vector plaintext = Join(level_bytes, value_bytes); + + auto result = DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs); + EXPECT_EQ(level_bytes, result.level_bytes); + EXPECT_EQ(value_bytes, result.value_bytes); + EXPECT_EQ(3u, result.num_elements); +} + +TEST(ParquetUtils, DecompressAndSplit_DataPageV1_InvalidDefinitionPayload) { + AttributesMap attribs = { + {"page_type", std::string("DATA_PAGE_V1")}, + {"data_page_num_values", int32_t(4)}, + {"data_page_max_repetition_level", int32_t(0)}, + {"data_page_max_definition_level", int32_t(1)}, + {"page_v1_repetition_level_encoding", std::string("RLE")}, + {"page_v1_definition_level_encoding", std::string("RLE")} + }; + + std::vector def_payload = {0x80}; // truncated varint run header + std::vector level_bytes = WrapLengthPrefixed(def_payload); + std::vector value_bytes = {0xAA, 0xBB, 0xCC, 0xDD}; + std::vector plaintext = Join(level_bytes, value_bytes); + + EXPECT_THROW( + DecompressAndSplit(plaintext, CompressionCodec::UNCOMPRESSED, attribs), + InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// Tests for CompressAndJoin function. +// ----------------------------------------------------------------------------- + TEST(ParquetUtils, CompressAndJoin_DataPageV1_Compressed) { AttributesMap attribs = { {"page_type", std::string("DATA_PAGE_V1")}, @@ -241,7 +607,7 @@ TEST(ParquetUtils, CompressAndJoin_DataPageV1_Compressed) { std::vector level_bytes; append_u32_le(level_bytes, 2); // RLE block length - level_bytes.insert(level_bytes.end(), {0x0A, 0x0B}); + level_bytes.insert(level_bytes.end(), {0x04, 0x01}); // RLE run: len=2, level=1 std::vector value_bytes = {0x21, 0x22, 0x23, 0x24}; auto joined = CompressAndJoin(level_bytes, value_bytes, CompressionCodec::SNAPPY, attribs); @@ -385,17 +751,16 @@ TEST(ParquetUtils, Reinterpret_INT32) { reinterpret_cast(values.data()) + values.size() * sizeof(int32_t)); TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + bytes, values.size(), Type::INT32, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(values[i], val); - ++i; } - EXPECT_EQ(values.size(), i); + EXPECT_EQ(values.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_DOUBLE) { @@ -404,17 +769,16 @@ TEST(ParquetUtils, Reinterpret_DOUBLE) { reinterpret_cast(values.data()) + values.size() * sizeof(double)); TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::DOUBLE, std::nullopt, Encoding::PLAIN); + bytes, values.size(), Type::DOUBLE, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_DOUBLE_EQ(values[i], val); - ++i; } - EXPECT_EQ(values.size(), i); + EXPECT_EQ(values.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_INT96) { @@ -432,18 +796,17 @@ TEST(ParquetUtils, Reinterpret_INT96) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT96, std::nullopt, Encoding::PLAIN); + bytes, expected.size(), Type::INT96, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i].lo, val.lo); EXPECT_EQ(expected[i].mid, val.mid); EXPECT_EQ(expected[i].hi, val.hi); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_BYTE_ARRAY) { @@ -464,17 +827,16 @@ TEST(ParquetUtils, Reinterpret_BYTE_ARRAY) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + bytes, expected.size(), Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i], std::vector(val.begin(), val.end())); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { @@ -492,94 +854,103 @@ TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY) { } TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); + bytes, expected.size(), Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); - size_t i = 0; - for (auto val : *buf) { + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + const auto val = buf->GetElement(i); EXPECT_EQ(expected[i], std::vector(val.begin(), val.end())); - ++i; } - EXPECT_EQ(expected.size(), i); + EXPECT_EQ(expected.size(), buf->GetNumElements()); } TEST(ParquetUtils, Reinterpret_UnsupportedEncoding) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::INT32, std::nullopt, Encoding::RLE), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::INT32, std::nullopt, Encoding::RLE), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_RLE_DICTIONARY_Throws) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::INT32, std::nullopt, Encoding::RLE_DICTIONARY), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::INT32, std::nullopt, Encoding::RLE_DICTIONARY), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_BOOLEAN_Throws) { std::vector bytes = {0xB4}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::BOOLEAN, std::nullopt, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::BOOLEAN, std::nullopt, Encoding::PLAIN), DBPSUnsupportedException); } TEST(ParquetUtils, Reinterpret_InvalidDataSize) { std::vector bytes = {0xAA, 0xBB, 0xCC}; auto result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + bytes, 1u, Type::INT32, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); EXPECT_THROW( - { for (auto val : *buf) { (void)val; } }, + { + tcb::span element; + while (buf->ElementsIteratorNext(element)) { (void)element; } + }, InvalidInputException); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_MissingLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, std::nullopt, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer( + bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, std::nullopt, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_ZeroLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, 0, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, 0, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_FIXED_LEN_BYTE_ARRAY_NegativeLength_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; EXPECT_THROW( - ReinterpretValueBytesAsTypedValuesBuffer(bytes, Type::FIXED_LEN_BYTE_ARRAY, -1, Encoding::PLAIN), + ReinterpretValueBytesAsTypedValuesBuffer(bytes, 1u, Type::FIXED_LEN_BYTE_ARRAY, -1, Encoding::PLAIN), InvalidInputException); } TEST(ParquetUtils, Reinterpret_EmptyBytes_FixedSize) { std::vector bytes; TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::DOUBLE, std::nullopt, Encoding::PLAIN); + bytes, 0u, Type::DOUBLE, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); size_t count = 0; - for (auto val : *buf) { (void)val; ++count; } + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + (void)buf->GetElement(i); + ++count; + } EXPECT_EQ(0u, count); } TEST(ParquetUtils, Reinterpret_EmptyBytes_VariableSize) { std::vector bytes; TypedValuesBuffer result = ReinterpretValueBytesAsTypedValuesBuffer( - bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + bytes, 0u, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* buf = std::get_if(&result); ASSERT_NE(nullptr, buf); size_t count = 0; - for (auto val : *buf) { (void)val; ++count; } + for (size_t i = 0; i < buf->GetNumElements(); ++i) { + (void)buf->GetElement(i); + ++count; + } EXPECT_EQ(0u, count); } @@ -594,14 +965,15 @@ TEST(ParquetUtils, RoundTrip_INT32) { reinterpret_cast(values.data()) + values.size() * sizeof(int32_t)); auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::INT32, std::nullopt, Encoding::PLAIN); + input_bytes, values.size(), Type::INT32, std::nullopt, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); TypedBufferI32 write_buf{values.size()}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } @@ -628,14 +1000,15 @@ TEST(ParquetUtils, RoundTrip_BYTE_ARRAY) { } auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); + input_bytes, payloads.size(), Type::BYTE_ARRAY, std::nullopt, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); TypedBufferRawBytesVariableSized write_buf{payloads.size(), input_bytes.size(), true}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } EXPECT_EQ(payloads.size(), pos); @@ -659,7 +1032,7 @@ TEST(ParquetUtils, RoundTrip_FIXED_LEN_BYTE_ARRAY) { } auto read_buf = ReinterpretValueBytesAsTypedValuesBuffer( - input_bytes, Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); + input_bytes, payloads.size(), Type::FIXED_LEN_BYTE_ARRAY, element_len, Encoding::PLAIN); auto* src = std::get_if(&read_buf); ASSERT_NE(nullptr, src); @@ -667,7 +1040,8 @@ TEST(ParquetUtils, RoundTrip_FIXED_LEN_BYTE_ARRAY) { TypedBufferRawBytesFixedSized write_buf{ payloads.size(), 0, RawBytesFixedSizedCodec{static_cast(element_len)}}; size_t pos = 0; - for (auto val : *src) { + for (size_t i = 0; i < src->GetNumElements(); ++i) { + const auto val = src->GetElement(i); write_buf.SetElement(pos++, val); } EXPECT_EQ(payloads.size(), pos); diff --git a/src/processing/typed_buffer.h b/src/processing/typed_buffer.h index b99d810..21f55e0 100644 --- a/src/processing/typed_buffer.h +++ b/src/processing/typed_buffer.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -47,6 +46,7 @@ class ByteBuffer { // Elements are stored contiguously in the span. ByteBuffer( tcb::span elements_span, + size_t num_elements, size_t prefix_size = 0, Codec codec = Codec{}); @@ -64,93 +64,50 @@ class ByteBuffer { size_t prefix_size = 0, Codec codec = Codec{}); + // Getters for immediately available properties. + size_t GetNumElements() const { return num_elements_; } + size_t GetElementSize() const { return element_size_; } + size_t GetRawBufferSize() const { return elements_span_size_; } + // Get and set elements by position with type access from Codec value_type GetElement(size_t position) const; tcb::span GetRawElement(size_t position) const; + tcb::span GetWritableRawElement(size_t position, size_t payload_size); void SetElement(size_t position, const value_type& element); void SetRawElement(size_t position, tcb::span raw); - // Getters for immediately available properties. - size_t GetRawBufferSize() const { return elements_span_.size(); } - size_t GetElementSize() const { return codec_.element_size(); } - - // Get the number of elements in the buffer. - size_t GetNumElements() const; + // Iterator for read-only elements returning raw bytes. + bool ElementsIteratorNext(tcb::span& raw_bytes) const; // Finalizes the write path and transfers the resulting buffer ownership. std::vector FinalizeAndTakeBuffer(); - // Iterator for read-only elements returning a `value_type` - class ConstIterator { - public: - // Iterator traits consumed indirectly by STL iterator machinery. - using iterator_category = std::forward_iterator_tag; - using value_type = typename ByteBuffer::value_type; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - - // Basic forward-iterator operations over encoded elements in elements_span_. - ConstIterator(const ByteBuffer* buffer, size_t cursor_offset); - value_type operator*() const; - ConstIterator& operator++(); - bool operator==(const ConstIterator& other) const; - bool operator!=(const ConstIterator& other) const; - - protected: - size_t ReadAndValidateVariableElementSizeAtCursor() const; - tcb::span RawSpanAtCursor() const; - - const ByteBuffer* buffer_ = nullptr; - size_t cursor_offset_ = 0; - size_t elements_span_size_ = 0; - mutable size_t current_element_size_ = 0; - }; - // Methods used by the STL iterator machinery to iterate over the buffer. - ConstIterator begin() const; - ConstIterator end() const; - - // Iterator for read-only elements returning raw bytes. - // Subclass of ConstIterator that only overrides operator* to return raw spans. - class ConstRawIterator : public ConstIterator { - public: - using iterator_category = std::forward_iterator_tag; - using value_type = tcb::span; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - - using ConstIterator::ConstIterator; - tcb::span operator*() const; - }; - struct RawElementsView { - const ByteBuffer* buffer; - ConstRawIterator begin() const { return ConstRawIterator(buffer, buffer->prefix_size_); } - ConstRawIterator end() const { return ConstRawIterator(buffer, buffer->elements_span_.size()); } - }; - RawElementsView raw_elements() const; - protected: // Helper for reserve heuristics in variable-size parsing. static size_t EstimateOffsetsReserveCountFromSample(tcb::span bytes); - // Helper for calculating the offset of an element by position. - size_t CalculateOffsetOfElement(size_t position) const; - - // Helper to validate the preconditions for reading the buffer with an iterator. - void ValidateIteratorReadPreconditions() const; - // Helper to get a writable span for an element during SetElement calls. tcb::span GetWritableSpanForElement(size_t position, size_t payload_size); // Variables for span elements reading tcb::span elements_span_; - mutable size_t num_elements_; + size_t elements_span_size_; Codec codec_; - + + // Attribute for the number of elements in the buffer (a const) + // - `num_elements_` is a const and passed during construction of both read-only and write buffers. + // - It indicates the expected number of elements in the buffer payload declared upfront. + // - This is treated as an invariant, so if the payload count mismatches, exceptions are thrown. + const size_t num_elements_; + + // Variables for element span iterator. + mutable const uint8_t* element_iterator_current_ptr_; + const uint8_t* element_iterator_end_ptr_; + mutable size_t element_iterator_count_ = 0; + // Variables for determining offset of elements. - size_t prefix_size_ = 0; - size_t element_size_; // for fixed-size elements + const size_t prefix_size_ = 0; + const size_t element_size_; // for fixed-size elements mutable std::vector offsets_; // for variable-size elements // Variables for write buffer. @@ -166,11 +123,12 @@ class ByteBuffer { void InitializeFromSpan() const; void EnsureInitializedFromSpan() const; mutable bool is_initialized_from_span_ = false; + static size_t InitElementSize(const Codec& codec); // Initialization methods and flags for write buffer void InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint); void RebindSpanToWriteBuffer(); - bool is_write_buffer_initialized_ = false; + bool is_write_buffer_enabled_ = false; bool is_write_buffer_finalized_ = false; }; @@ -192,32 +150,54 @@ inline size_t ReadSizeAt(tcb::span bytes, size_t offset) { template ByteBuffer::ByteBuffer( tcb::span elements_span, + size_t num_elements, size_t prefix_size, Codec codec) : elements_span_(elements_span), - num_elements_(kUnsetSize), + elements_span_size_(elements_span.size()), + // `num_elements_` is the expected number of elements in the buffer declared upfront. + // - if the actual payload count mismatches, exceptions are thrown. + num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), + // `element_iterator_current_ptr_` is initialized to the start of the span. + // - it is calculated to point to the start of the span + the prefix size if there is one. + element_iterator_current_ptr_( + elements_span.data() + std::min(prefix_size, elements_span_size_)), + // `element_iterator_end_ptr_` is initialized to the end of the span. + // - it is calculated to point to the start of the span + the size. + element_iterator_end_ptr_(elements_span.data() + elements_span_size_), + element_iterator_count_(0), prefix_size_(prefix_size), - is_initialized_from_span_(false) { + element_size_(InitElementSize(codec_)), + is_write_buffer_enabled_(false), + is_initialized_from_span_(false) {} + +// Initialize element size based on the codec. Wrapper just needed for readability, all resolves in compile-time. +template +inline size_t ByteBuffer::InitElementSize(const Codec& codec) { if constexpr (is_fixed_sized) { - element_size_ = codec_.element_size(); + auto codec_element_size = codec.element_size(); + if (codec_element_size <= 0) { + throw InvalidInputException("Invalid fixed-size buffer: element_size must be greater than zero"); + } + return codec_element_size; } + // For variable-size elements, element size is undefined. + return kUnsetSize; } -// Initializes `num_elements_` and `offsets_` from the span. -// Called in a lazy manner when the buffer is accessed with GetElement or GetNumElements, avoiding unnecessary initialization. +// Initializes `offsets_` from the span. +// Called in a lazy manner when the buffer is accessed with GetElement[i] avoiding unnecessary initialization. template -void ByteBuffer::InitializeFromSpan() const { - if (elements_span_.size() < prefix_size_) { +inline void ByteBuffer::InitializeFromSpan() const { + if (elements_span_size_ < prefix_size_) { throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); } - const size_t readable_size = elements_span_.size() - prefix_size_; + const size_t readable_size = elements_span_size_ - prefix_size_; // No elements to index. Initialize with empty values. if (readable_size == 0) { - num_elements_ = 0; offsets_.clear(); is_initialized_from_span_ = true; return; @@ -232,12 +212,14 @@ void ByteBuffer::InitializeFromSpan() const { if ((readable_size % element_size_) != 0) { throw InvalidInputException("Malformed fixed-size buffer: buffer does not align with element_size"); } - num_elements_ = readable_size / element_size_; - const size_t expected_payload_bytes = num_elements_ * element_size_; - if (expected_payload_bytes != readable_size) { - throw InvalidInputException( - "Malformed fixed-size buffer: computed payload size does not match readable size"); + + // Check if the num_elements passed at contruction time coincides with the calculated from the payload size. + // This is a division of integer values, however it results in a correct integer result because of the modulo guard above. + const size_t num_elements_on_payload = readable_size / element_size_; + if (num_elements_on_payload != num_elements_) { + throw InvalidInputException("Malformed fixed-size buffer: num_elements on payload != num_elements_ expected."); } + offsets_.clear(); is_initialized_from_span_ = true; return; @@ -246,52 +228,54 @@ void ByteBuffer::InitializeFromSpan() const { // Variable-size layout stores [u32 size][element value] back-to-back. // Single pass validates shape and captures per-element prefix offsets. offsets_.clear(); - offsets_.reserve(EstimateOffsetsReserveCountFromSample(elements_span_.subspan(prefix_size_))); + offsets_.reserve(num_elements_); size_t cursor = prefix_size_; - while (cursor < elements_span_.size()) { - if (elements_span_.size() - cursor < kSizePrefixBytes) { + while (cursor < elements_span_size_) { + if (elements_span_size_ - cursor < kSizePrefixBytes) { throw InvalidInputException("Malformed variable-size buffer: truncated length prefix"); } offsets_.push_back(cursor); const size_t current_element_size = ReadSizeAt(elements_span_, cursor); cursor += kSizePrefixBytes; - if (elements_span_.size() - cursor < current_element_size) { + if (elements_span_size_ - cursor < current_element_size) { throw InvalidInputException("Malformed variable-size buffer: truncated element payload"); } cursor += current_element_size; } - num_elements_ = offsets_.size(); + + // Check if the num_elements passed at contruction time coincides with the calculated from the payload size. + const size_t num_elements_on_payload = offsets_.size(); + if (num_elements_on_payload != num_elements_) { + throw InvalidInputException("Malformed variable-size buffer: num_elements on payload != num_elements_ expected."); + } + is_initialized_from_span_ = true; } template -void ByteBuffer::EnsureInitializedFromSpan() const { +inline void ByteBuffer::EnsureInitializedFromSpan() const { // If the span is already initialized, skip it. if (is_initialized_from_span_) { return; } // If the write buffer is initialized, we don't need to initialize from the span. - if (is_write_buffer_initialized_) { + if (is_write_buffer_enabled_) { return; } InitializeFromSpan(); } -// For read-only buffers, gets the number of elements in the buffer and sets num_elements_ if not already set. -// A lighter version to only get num_elements_ and avoid calling InitializeFromSpan that also builds offsets_. -template -size_t ByteBuffer::GetNumElements() const { - if (num_elements_ != kUnsetSize) { - return num_elements_; - } - - // If the buffer is not initialized, initialize it from the span. - EnsureInitializedFromSpan(); - return num_elements_; -} +// ----------------------------------------------------------------------------- +// EstimateOffsetsReserveCountFromSample method +// +// - Reference method for estimating the number of elements in a span. +// - Since num_elements_ is now known and is an invariant, this method is no longer in the active codepath. +// - Nonetheless it captures a useful heuristic for estimating the number of elements in a span from +// the payload for future reference. +// ----------------------------------------------------------------------------- template -size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { +inline size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span bytes) { if (bytes.empty()) return 0; @@ -338,31 +322,32 @@ size_t ByteBuffer::EstimateOffsetsReserveCountFromSample(tcb::span -size_t ByteBuffer::CalculateOffsetOfElement(size_t position) const { - EnsureInitializedFromSpan(); - if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during CalculateOffsetOfElement"); - } - if constexpr (is_fixed_sized) { - return prefix_size_ + (position * element_size_); - } - return offsets_[position]; +inline typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { + return codec_.Decode(GetRawElement(position)); } template -tcb::span ByteBuffer::GetRawElement(size_t position) const { +inline tcb::span ByteBuffer::GetRawElement(size_t position) const { EnsureInitializedFromSpan(); - if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetRawElement"); - } - const size_t offset = CalculateOffsetOfElement(position); // For fixed-size elements are stored contiguously. if constexpr (is_fixed_sized) { + if (position >= num_elements_) { + throw InvalidInputException( + "Element index out of bounds during GetRawElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); + } + const size_t offset = prefix_size_ + (position * element_size_); return elements_span_.subspan(offset, element_size_); } // For variable-size elements, we need to read the size first [u32 size][element]. + if (position >= num_elements_) { + throw InvalidInputException( + "Element index out of bounds during GetRawElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); + } + const size_t offset = offsets_[position]; if (offset == kUnsetSize) { throw InvalidInputException("Element position has not been written yet"); } @@ -370,137 +355,62 @@ tcb::span ByteBuffer::GetRawElement(size_t position) const return elements_span_.subspan(offset + kSizePrefixBytes, element_size); } -template -typename ByteBuffer::value_type ByteBuffer::GetElement(size_t position) const { - return codec_.Decode(GetRawElement(position)); -} - // ----------------------------------------------------------------------------- // Element span iterator -// -// Allows an alternative read of elements_span_ without need for lazy initialization of offsets_, -// so saving execution time when the traversal of the buffer is strictly sequential. -// This is the most common behavior when reading elements in single threaded mode. // ----------------------------------------------------------------------------- template -ByteBuffer::ConstIterator::ConstIterator(const ByteBuffer* buffer, size_t cursor_offset) - : buffer_(buffer), - cursor_offset_(cursor_offset), - elements_span_size_(buffer != nullptr ? buffer->elements_span_.size() : 0u), - current_element_size_(kUnsetSize) {} - -template -inline size_t ByteBuffer::ConstIterator::ReadAndValidateVariableElementSizeAtCursor() const { - // Fixed-sized buffers should not call this method. - if constexpr (is_fixed_sized) { - throw InvalidInputException("ReadAndValidateVariableElementSizeAtCursor is not valid for fixed-size codecs"); - } - - // If the current element size has already been read, return it. - if (current_element_size_ != kUnsetSize) { - return current_element_size_; - } +inline bool ByteBuffer::ElementsIteratorNext(tcb::span& raw_bytes) const { - // Read the current element size and save it to current_element_size_ cached variable. - if ((elements_span_size_ - cursor_offset_) < kSizePrefixBytes) { - throw InvalidInputException("Malformed variable-size buffer: truncated length prefix"); + // Check that this is only used for read-only buffers. + if (is_write_buffer_enabled_) { + throw InvalidInputException("ElementsIteratorNext is only defined for read-only buffers"); } - const size_t current_element_size = ReadSizeAt(buffer_->elements_span_, cursor_offset_); - const size_t payload_offset = cursor_offset_ + kSizePrefixBytes; - if ((elements_span_size_ - payload_offset) < current_element_size) { - throw InvalidInputException("Malformed variable-size buffer: truncated element payload"); + if (elements_span_size_ < prefix_size_) { + throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); } - current_element_size_ = current_element_size; - return current_element_size_; -} -template -tcb::span ByteBuffer::ConstIterator::RawSpanAtCursor() const { - if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { - throw InvalidInputException("Cannot dereference ByteBuffer iterator at end position"); - } - if constexpr (is_fixed_sized) { - return buffer_->elements_span_.subspan(cursor_offset_, buffer_->element_size_); + // If the last element was consumed, check that the number of elements iterated matches the number of elements expected. + if (element_iterator_current_ptr_ == element_iterator_end_ptr_) { + if (element_iterator_count_ != num_elements_) { + throw InvalidInputException( + "Malformed buffer: iterator count mismatch: actual=" + std::to_string(element_iterator_count_) + + " expected=" + std::to_string(num_elements_)); + } + raw_bytes = {}; + return false; } - const size_t current_element_size = ReadAndValidateVariableElementSizeAtCursor(); - const size_t payload_offset = cursor_offset_ + kSizePrefixBytes; - return buffer_->elements_span_.subspan(payload_offset, current_element_size); -} -template -typename ByteBuffer::ConstIterator::value_type ByteBuffer::ConstIterator::operator*() const { - // Decode converts raw bytes into the codec's value_type (e.g. int32_t, float, string_view). - // This keeps the iterator's return type consistent with GetElement across all codecs. - return buffer_->codec_.Decode(RawSpanAtCursor()); -} + const size_t bytes_remaining = + static_cast(element_iterator_end_ptr_ - element_iterator_current_ptr_); -template -typename ByteBuffer::ConstIterator& ByteBuffer::ConstIterator::operator++() { - if (buffer_ == nullptr || cursor_offset_ >= elements_span_size_) { - return *this; - } if constexpr (is_fixed_sized) { - cursor_offset_ += buffer_->element_size_; - return *this; - } - const size_t current_element_size = ReadAndValidateVariableElementSizeAtCursor(); - cursor_offset_ += (kSizePrefixBytes + current_element_size); - current_element_size_ = kUnsetSize; - return *this; -} - -template -bool ByteBuffer::ConstIterator::operator==(const ConstIterator& other) const { - return buffer_ == other.buffer_ && cursor_offset_ == other.cursor_offset_; -} - -template -bool ByteBuffer::ConstIterator::operator!=(const ConstIterator& other) const { - return !(*this == other); -} - -template -tcb::span ByteBuffer::ConstRawIterator::operator*() const { - // Returns the raw bytes for the current element, consistent with GetRawElement. - return this->RawSpanAtCursor(); -} - -template -void ByteBuffer::ValidateIteratorReadPreconditions() const { - if (is_write_buffer_initialized_) { - throw InvalidInputException("Iterator is only available for read buffers"); - } - if (elements_span_.size() < prefix_size_) { - throw InvalidInputException("Malformed buffer: prefix_size exceeds span size"); - } - if constexpr (is_fixed_sized) { - if (element_size_ <= 0) { - throw InvalidInputException("Invalid fixed-size buffer: element_size must be greater than zero"); - } - const size_t readable_size = elements_span_.size() - prefix_size_; - if ((readable_size % element_size_) != 0) { - throw InvalidInputException("Malformed fixed-size buffer: buffer does not align with element_size"); + if (bytes_remaining < element_size_) { + throw InvalidInputException("Malformed fixed-size buffer: truncated element in iterator"); } + raw_bytes = tcb::span(element_iterator_current_ptr_, element_size_); + element_iterator_current_ptr_ += element_size_; + element_iterator_count_++; + return true; } -} -template -typename ByteBuffer::ConstIterator ByteBuffer::begin() const { - ValidateIteratorReadPreconditions(); - return ConstIterator(this, prefix_size_); -} + // Variable-sized elements + if (bytes_remaining < kSizePrefixBytes) { + throw InvalidInputException("Malformed variable-size buffer: truncated length prefix in iterator"); + } + const size_t current_element_size = read_u32_le(element_iterator_current_ptr_); + element_iterator_current_ptr_ += kSizePrefixBytes; -template -typename ByteBuffer::ConstIterator ByteBuffer::end() const { - ValidateIteratorReadPreconditions(); - return ConstIterator(this, elements_span_.size()); -} + const size_t payload_remaining = + static_cast(element_iterator_end_ptr_ - element_iterator_current_ptr_); + if (payload_remaining < current_element_size) { + throw InvalidInputException("Malformed variable-size buffer: truncated element payload in iterator"); + } -template -typename ByteBuffer::RawElementsView ByteBuffer::raw_elements() const { - ValidateIteratorReadPreconditions(); - return RawElementsView{this}; + raw_bytes = tcb::span(element_iterator_current_ptr_, current_element_size); + element_iterator_current_ptr_ += current_element_size; + element_iterator_count_++; + return true; } // ----------------------------------------------------------------------------- @@ -515,10 +425,10 @@ ByteBuffer::ByteBuffer( Codec codec) : num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), - prefix_size_(prefix_size) { + prefix_size_(prefix_size), + element_size_(InitElementSize(codec_)), + is_write_buffer_enabled_(true) { static_assert(is_fixed_sized, "ByteBuffer constructor for fixed-size elements only."); - element_size_ = codec_.element_size(); InitializeForWriteBuffer(0); } @@ -532,15 +442,16 @@ ByteBuffer::ByteBuffer( Codec codec) : num_elements_(num_elements), codec_(std::move(codec)), - element_size_(0), - prefix_size_(prefix_size) { + prefix_size_(prefix_size), + element_size_(kUnsetSize), + is_write_buffer_enabled_(true) { static_assert(!is_fixed_sized, "ByteBuffer constructor for variable-size elements only."); InitializeForWriteBuffer(use_reserve_hint ? reserved_bytes_hint : 0); } // Initializes `write_buffer_`, `offsets_` and `elements_span_` template -void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint) { +inline void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_bytes_hint) { // Fixed-size elements if constexpr (is_fixed_sized) { if (element_size_ <= 0) { @@ -552,7 +463,6 @@ void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_b const size_t fixed_size_total_bytes = prefix_size_ + (num_elements_ * element_size_); write_buffer_.clear(); write_buffer_.resize(fixed_size_total_bytes, static_cast(0)); - is_write_buffer_initialized_ = true; // offsets_ are not used for fixed-size elements. offsets_.clear(); @@ -573,7 +483,6 @@ void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_b write_buffer_.clear(); write_buffer_.resize(prefix_size_, static_cast(0)); write_buffer_.reserve(variable_size_reserved_bytes); - is_write_buffer_initialized_ = true; // offsets_ is initialized so the vector is fully allocated and have random-ish access during writes. offsets_.clear(); @@ -592,8 +501,8 @@ void ByteBuffer::InitializeForWriteBuffer(size_t variable_size_reserved_b template -tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, size_t payload_size) { - if (!is_write_buffer_initialized_) { +inline tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, size_t payload_size) { + if (!is_write_buffer_enabled_) { throw InvalidInputException("Cannot GetWriteSpanForElement: write buffer is not initialized."); } @@ -602,7 +511,9 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, } if (position >= num_elements_) { - throw InvalidInputException("Element position out of range during GetWriteSpanForElement"); + throw InvalidInputException( + "Element index out of bounds during GetWriteSpanForElement: index=" + std::to_string(position) + + " size=" + std::to_string(num_elements_)); } // For fixed-size elements, we write directly at the fixed offset. No need to re-bind the span. @@ -610,7 +521,7 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, if (payload_size != element_size_) { throw InvalidInputException("GetWriteSpanForElement: payload does not match element_size"); } - const size_t offset = CalculateOffsetOfElement(position); + const size_t offset = prefix_size_ + (position * element_size_); auto write_span = tcb::span(write_buffer_.data() + offset, element_size_); return write_span; } @@ -618,7 +529,7 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, // Variable-sized elements - `else` is needed because it's a compile-time check. else { // Defensive check for unlikely extremely large element size that exceeds uint32. - if (payload_size > static_cast(std::numeric_limits::max())) { + if (payload_size > static_cast(std::numeric_limits::max())) [[unlikely]] { throw InvalidInputException("Variable-size element payload exceeds uint32 capacity.. Woohhh!!"); } @@ -630,11 +541,11 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, // This is intentional to allow random writes of elements while the buffer is built. // During FinalizeAndTakeBuffer, the buffer is rebuilt to be sequential and orphaned bytes are removed. const size_t offset = write_buffer_.size(); - offsets_[position] = offset; - append_u32_le(write_buffer_, static_cast(payload_size)); - const size_t payload_offset = write_buffer_.size(); - write_buffer_.resize(payload_offset + payload_size); - auto write_span = tcb::span(write_buffer_.data() + payload_offset, payload_size); + write_buffer_.resize(offset + kSizePrefixBytes + payload_size); + auto offset_ptr = write_buffer_.data() + offset; + + // Write the size prefix + write_u32_le(offset_ptr, static_cast(payload_size)); // Update next_expected_write_position_ for sequential write checking. if (next_expected_write_position_ != kUnsetSize) { @@ -646,12 +557,18 @@ tcb::span ByteBuffer::GetWritableSpanForElement(size_t position, } RebindSpanToWriteBuffer(); - return write_span; + offsets_[position] = offset; + return tcb::span(offset_ptr + kSizePrefixBytes, payload_size);; } } template -void ByteBuffer::SetElement(size_t position, const value_type& element) { +inline tcb::span ByteBuffer::GetWritableRawElement(size_t position, size_t payload_size) { + return GetWritableSpanForElement(position, payload_size); +} + +template +inline void ByteBuffer::SetElement(size_t position, const value_type& element) { if constexpr (is_fixed_sized) { auto write_span = GetWritableSpanForElement(position, element_size_); codec_.Encode(element, write_span); @@ -662,18 +579,18 @@ void ByteBuffer::SetElement(size_t position, const value_type& element) { } template -void ByteBuffer::SetRawElement(size_t position, tcb::span raw) { +inline void ByteBuffer::SetRawElement(size_t position, tcb::span raw) { auto write_span = GetWritableSpanForElement(position, raw.size()); std::memcpy(write_span.data(), raw.data(), raw.size()); } template -std::vector ByteBuffer::FinalizeAndTakeBuffer() { +inline std::vector ByteBuffer::FinalizeAndTakeBuffer() { if (is_write_buffer_finalized_) { throw InvalidInputException("FinalizeAndTakeBuffer: write buffer has already been finalized"); } - if (!is_write_buffer_initialized_) { + if (!is_write_buffer_enabled_) { throw InvalidInputException("FinalizeAndTakeBuffer: write buffer is not initialized"); } @@ -732,15 +649,17 @@ std::vector ByteBuffer::FinalizeAndTakeBuffer() { // Defrag path returns a new buffer; release the original fragmented write buffer. write_buffer_.clear(); write_buffer_.shrink_to_fit(); - is_write_buffer_initialized_ = false; + is_write_buffer_enabled_ = false; is_write_buffer_finalized_ = true; return result; } template -void ByteBuffer::RebindSpanToWriteBuffer() { - elements_span_ = tcb::span(write_buffer_.data(), write_buffer_.size()); +inline void ByteBuffer::RebindSpanToWriteBuffer() { + auto write_buffer_size = write_buffer_.size(); + elements_span_ = tcb::span(write_buffer_.data(), write_buffer_size); + elements_span_size_ = write_buffer_size; } } // namespace dbps::processing diff --git a/src/processing/typed_buffer_codecs.h b/src/processing/typed_buffer_codecs.h index 4794ecb..9e165fd 100644 --- a/src/processing/typed_buffer_codecs.h +++ b/src/processing/typed_buffer_codecs.h @@ -61,75 +61,6 @@ struct PlainValueCodec { } }; -struct StringFixedSizedCodec { - using value_type = std::string_view; - static constexpr bool is_fixed_sized = true; - - explicit StringFixedSizedCodec(size_t element_size_bytes) : element_size_bytes_(element_size_bytes) { - if (element_size_bytes_ <= 0) { - throw InvalidInputException("StringFixedSizedCodec requires element_size_bytes > 0"); - } - } - - static constexpr std::string_view type_name() noexcept { - return "string (fixed-length)"; - } - - size_t element_size() const noexcept { - return element_size_bytes_; - } - - value_type Decode(tcb::span read_span) const { - if (read_span.size() != element_size_bytes_) { - throw InvalidInputException("Decode: read_span size does not match element_size_bytes"); - } - return std::string_view( - reinterpret_cast(read_span.data()), - read_span.size()); - } - - void Encode(const value_type& value, tcb::span write_span) const { - if (write_span.size() != element_size_bytes_) { - throw InvalidInputException("Encode: write_span size does not match element_size_bytes"); - } - if (value.size() != write_span.size()) { - throw InvalidInputException("Encode: value size does not match write_span size"); - } - std::memcpy(write_span.data(), value.data(), write_span.size()); - } - - private: - size_t element_size_bytes_; -}; - -struct StringVariableSizedCodec { - using value_type = std::string_view; - static constexpr bool is_fixed_sized = false; - - static constexpr std::string_view type_name() noexcept { - return "string (variable-length)"; - } - - size_t element_size() const { - throw InvalidInputException("StringVariableSizedCodec does not have a fixed element size"); - } - - value_type Decode(tcb::span read_span) const noexcept { - return std::string_view( - reinterpret_cast(read_span.data()), - read_span.size()); - } - - void Encode(const value_type& value, tcb::span write_span) const { - // Exact match required to prevent short values to leave uninitialized trailing bytes in the buffer - // and prevent longer values from overflowing. - if (value.size() != write_span.size()) { - throw InvalidInputException("Encode: value size does not match write_span size"); - } - std::memcpy(write_span.data(), value.data(), write_span.size()); - } -}; - struct RawBytesFixedSizedCodec { using value_type = tcb::span; static constexpr bool is_fixed_sized = true; diff --git a/src/processing/typed_buffer_test.cpp b/src/processing/typed_buffer_test.cpp index 061299e..7dccff5 100644 --- a/src/processing/typed_buffer_test.cpp +++ b/src/processing/typed_buffer_test.cpp @@ -17,9 +17,11 @@ #include "typed_buffer.h" #include "typed_buffer_codecs.h" +#include "typed_buffer_testing_codecs.h" #include #include +#include #include #include @@ -30,6 +32,8 @@ using dbps::processing::ByteBuffer; using dbps::processing::RawBytesFixedSizedCodec; using dbps::processing::RawBytesVariableSizedCodec; using dbps::processing::kUnsetSize; +using dbps::processing::testing::StringFixedSizedCodec; +using dbps::processing::testing::StringVariableSizedCodec; template class TypedBufferTestProxy : public ByteBuffer { @@ -37,9 +41,10 @@ class TypedBufferTestProxy : public ByteBuffer { public: explicit TypedBufferTestProxy( tcb::span elements_span, + size_t num_elements, size_t prefix_size = 0, Codec codec = Codec{}) - : Base(elements_span, prefix_size, std::move(codec)) {} + : Base(elements_span, num_elements, prefix_size, std::move(codec)) {} TypedBufferTestProxy( size_t num_elements, @@ -95,7 +100,7 @@ std::vector MakePayload(size_t size, uint8_t seed) { TEST(TypedBufferTest, ConstructFixedSize_ValidBuffer_InitializesExpectedState) { std::vector bytes = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); // Trigger lazy span initialization. EXPECT_NO_THROW((void)buffer.GetElement(0)); ExpectCommonState(buffer, 3u, 2u); @@ -104,7 +109,7 @@ TEST(TypedBufferTest, ConstructFixedSize_ValidBuffer_InitializesExpectedState) { TEST(TypedBufferTest, GetElement_FixedSize_ReturnsExpectedSlices) { std::vector bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -126,7 +131,7 @@ TEST(TypedBufferTest, GetElement_FixedSize_WithPrefixSize_SkipsPrefix) { 0xFE, 0xFD, 0xFC, // prefix 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -143,7 +148,7 @@ TEST(TypedBufferTest, GetElement_VariableSize_WithPrefixSize_SkipsPrefix) { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - ByteBuffer buffer(tcb::span(bytes), 2u); + ByteBuffer buffer(tcb::span(bytes), 2u, 2u); const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -158,7 +163,7 @@ TEST(TypedBufferTest, ConstructFixedSize_ZeroElementSize_Throws) { TEST(TypedBufferTest, ConstructFixedSize_NonDivisibleSize_Throws) { std::vector bytes = {0x01, 0x02, 0x03}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } @@ -168,10 +173,10 @@ TEST(TypedBufferTest, ConstructVariableSize_ValidEncodedBuffer_InitializesExpect 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; // Trigger lazy variable-size index parsing. EXPECT_NO_THROW((void)buffer.GetElement(0)); - ExpectCommonState(buffer, 2u, 0u); + ExpectCommonState(buffer, 2u, kUnsetSize); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], 0u); EXPECT_EQ(buffer.GetOffsets()[1], 9u); // 4 bytes length prefix + 5 bytes first payload. @@ -237,7 +242,7 @@ TEST(TypedBufferTest, GetElement_VariableSize_ReturnsExpectedPayload) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; const auto first = buffer.GetElement(0); const auto second = buffer.GetElement(1); @@ -256,10 +261,11 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_WithPrefixSize_SkipsPrefix) { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - ByteBuffer buffer(tcb::span(bytes), 2u); + ByteBuffer buffer(tcb::span(bytes), 2u, 2u); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -273,10 +279,11 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_WithPrefixSize_SkipsPrefix) { 0xFE, 0xFD, 0xFC, // prefix 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -288,10 +295,11 @@ TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_WithPrefixSize_SkipsPrefix) { TEST(TypedBufferTest, Iterate_ReadOnlyFixedSize_TraversesInOrder) { std::vector bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -323,10 +331,11 @@ TEST(TypedBufferTest, Iterate_ReadOnlyVariableSize_TraversesInOrder) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u}; std::vector> collected; - for (const auto element : buffer) { + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { collected.push_back(std::vector(element.begin(), element.end())); } @@ -364,12 +373,14 @@ TEST(TypedBufferTest, TransformFixedSize_ReadIterateWriteFinalize_RoundTrip) { }; // Create a source buffer to read the elements from. - RawBytesFixedSizedBuffer source_buffer(tcb::span(source_bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + RawBytesFixedSizedBuffer source_buffer( + tcb::span(source_bytes), 3u, 0u, RawBytesFixedSizedCodec{kElementSize}); // Create a new buffer to write the transformed elements into. RawBytesFixedSizedBuffer transformed_buffer(3u, 0, RawBytesFixedSizedCodec{kElementSize}); size_t position = 0; - for (const auto element : source_buffer) { + tcb::span element; + while (source_buffer.ElementsIteratorNext(element)) { std::vector source_element(element.begin(), element.end()); std::vector transformed_element = source_element; transformed_element[0] = static_cast(transformed_element[0] + 1u); @@ -395,7 +406,8 @@ TEST(TypedBufferTest, TransformFixedSize_ReadIterateWriteFinalize_RoundTrip) { // Now finalize the transformed buffer and populate a third buffer to read the elements from. std::vector finalized_bytes = transformed_buffer.FinalizeAndTakeBuffer(); - RawBytesFixedSizedBuffer finalized_read_buffer(tcb::span(finalized_bytes), 0, RawBytesFixedSizedCodec{kElementSize}); + RawBytesFixedSizedBuffer finalized_read_buffer( + tcb::span(finalized_bytes), 3u, 0u, RawBytesFixedSizedCodec{kElementSize}); // Compare source and finalized read buffer using the same transformation rule. for (size_t i = 0; i < position; ++i) { @@ -420,11 +432,12 @@ TEST(TypedBufferTest, TransformVariableSize_ReadIterateWriteFinalize_RoundTrip) append_u32_le(source_bytes, 3u); source_bytes.insert(source_bytes.end(), {0x61, 0x62, 0x63}); // "abc" - RawBytesVariableSizedBuffer source_buffer{tcb::span(source_bytes)}; + RawBytesVariableSizedBuffer source_buffer{tcb::span(source_bytes), 3u}; RawBytesVariableSizedBuffer transformed_buffer(3u, source_bytes.size(), true); size_t position = 0; - for (const auto element : source_buffer) { + tcb::span element; + while (source_buffer.ElementsIteratorNext(element)) { std::vector source_element(element.begin(), element.end()); std::vector transformed_element = source_element; transformed_element[0] = static_cast(transformed_element[0] + 1u); @@ -448,7 +461,7 @@ TEST(TypedBufferTest, TransformVariableSize_ReadIterateWriteFinalize_RoundTrip) } std::vector finalized_bytes = transformed_buffer.FinalizeAndTakeBuffer(); - RawBytesVariableSizedBuffer finalized_read_buffer{tcb::span(finalized_bytes)}; + RawBytesVariableSizedBuffer finalized_read_buffer{tcb::span(finalized_bytes), 3u}; // Compare source and finalized read buffer using the same transformation rule. for (size_t i = 0; i < position; ++i) { @@ -468,19 +481,18 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { std::vector empty_bytes; // Fixed-size empty span. - RawBytesFixedSizedBuffer fixed_buffer(tcb::span(empty_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer fixed_buffer(tcb::span(empty_bytes), 0u, 0u, RawBytesFixedSizedCodec{2}); size_t fixed_count = 0; - for (const auto element : fixed_buffer) { - (void)element; + tcb::span element; + while (fixed_buffer.ElementsIteratorNext(element)) { ++fixed_count; } EXPECT_EQ(fixed_count, 0u); // Variable-size empty span. - RawBytesVariableSizedBuffer variable_buffer{tcb::span(empty_bytes)}; + RawBytesVariableSizedBuffer variable_buffer{tcb::span(empty_bytes), 0u}; size_t variable_count = 0; - for (const auto element : variable_buffer) { - (void)element; + while (variable_buffer.ElementsIteratorNext(element)) { ++variable_count; } EXPECT_EQ(variable_count, 0u); @@ -488,7 +500,7 @@ TEST(TypedBufferTest, Iterate_ReadOnlyEmptySpan_VisitsNoElements) { TEST(TypedBufferTest, GetElement_OutOfRange_Throws) { std::vector bytes = {0x01, 0x02, 0x03, 0x04}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_THROW((void)buffer.GetElement(2), InvalidInputException); } @@ -545,7 +557,7 @@ TEST(TypedBufferTest, ConstructWithNumElements_FixedSize_WithPrefixSize_Preserve TEST(TypedBufferTest, ConstructWithNumElements_VariableSize_AllocatesAndSets) { RawBytesVariableSizedBuffer buffer(2u, 8u, true); EXPECT_EQ(buffer.GetNumElements(), 2u); - EXPECT_EQ(buffer.GetElementSize(), 0u); + EXPECT_EQ(buffer.GetElementSize(), kUnsetSize); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], kUnsetSize); EXPECT_EQ(buffer.GetOffsets()[1], kUnsetSize); @@ -756,7 +768,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_PartialWrite_ThrowsAndA std::vector final_buffer = buffer.FinalizeAndTakeBuffer(); EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented after retry). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 2u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); ASSERT_EQ(r0.size(), first.size()); @@ -822,7 +834,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_OutOfOrder_Defragments) EXPECT_NE(final_buffer, raw_before_finalize); // Different byte content. EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented copy). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 3u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); const auto r2 = read_back.GetElement(2); @@ -859,7 +871,7 @@ TEST(TypedBufferTest, FinalizeAndTakeBuffer_VariableSize_Fragmented_Defragments) EXPECT_NE(final_buffer, raw_before_finalize); EXPECT_NE(final_buffer.data(), data_ptr_before); // Different allocation (defragmented copy). - RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer)}; + RawBytesVariableSizedBuffer read_back{tcb::span(final_buffer), 2u}; const auto r0 = read_back.GetElement(0); const auto r1 = read_back.GetElement(1); ASSERT_EQ(r0.size(), first_overwrite.size()); @@ -937,15 +949,91 @@ TEST(TypedBufferTest, SetElement_FixedSize_WrongPayloadSize_Throws) { TEST(TypedBufferTest, SetElement_OnReadOnlyBuffer_Throws) { std::vector bytes = {0x10, 0x11, 0x20, 0x21}; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 2u, 0u, RawBytesFixedSizedCodec{2}); std::vector replacement = {0xAA, 0xBB}; EXPECT_THROW((void)buffer.SetElement(0, tcb::span(replacement)), InvalidInputException); } -TEST(TypedBufferTest, Iterate_OnWriteBuffer_Throws) { +TEST(TypedBufferTest, ElementsIteratorNext_OnWriteBuffer_Throws) { RawBytesFixedSizedBuffer buffer(3u, 0, RawBytesFixedSizedCodec{2u}); - EXPECT_THROW((void)buffer.begin(), InvalidInputException); - EXPECT_THROW((void)buffer.end(), InvalidInputException); + tcb::span element; + EXPECT_THROW((void)buffer.ElementsIteratorNext(element), InvalidInputException); +} + +// ----------------------------------------------------------------------------- +// ElementsIteratorNext tests +// ----------------------------------------------------------------------------- + +TEST(TypedBufferTest, ElementsIteratorNext_FixedSize_TraversesAllElements) { + constexpr size_t kElementSize = 8u; + const std::vector> expected = { + {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17}, + {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27}, + {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37}, + {0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47}, + {0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57}, + }; + + std::vector bytes; + for (const auto& e : expected) { + bytes.insert(bytes.end(), e.begin(), e.end()); + } + + RawBytesFixedSizedBuffer buffer( + tcb::span(bytes), expected.size(), 0u, RawBytesFixedSizedCodec{kElementSize}); + + std::vector> collected; + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { + collected.push_back(std::vector(element.begin(), element.end())); + } + + ASSERT_EQ(collected.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(collected[i], expected[i]); + } + + EXPECT_TRUE(element.empty()); + + tcb::span extra; + EXPECT_FALSE(buffer.ElementsIteratorNext(extra)); + EXPECT_TRUE(extra.empty()); +} + +TEST(TypedBufferTest, ElementsIteratorNext_VariableSize_TraversesAllElements) { + const std::vector> expected = { + MakePayload(11, 0x10), + MakePayload(3, 0x20), + MakePayload(27, 0x30), + MakePayload(1, 0x40), + MakePayload(16, 0x50), + MakePayload(8, 0x60), + }; + + std::vector bytes; + for (const auto& e : expected) { + append_u32_le(bytes, static_cast(e.size())); + bytes.insert(bytes.end(), e.begin(), e.end()); + } + + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), expected.size()}; + + std::vector> collected; + tcb::span element; + while (buffer.ElementsIteratorNext(element)) { + collected.push_back(std::vector(element.begin(), element.end())); + } + + ASSERT_EQ(collected.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(collected[i], expected[i]); + } + + EXPECT_TRUE(element.empty()); + + tcb::span extra; + EXPECT_FALSE(buffer.ElementsIteratorNext(extra)); + EXPECT_TRUE(extra.empty()); } // ----------------------------------------------------------------------------- @@ -954,33 +1042,33 @@ TEST(TypedBufferTest, Iterate_OnWriteBuffer_Throws) { TEST(TypedBufferTest, GetSpanSize_ReturnsUnderlyingSpanByteCount) { std::vector fixed_bytes = {0x10, 0x11, 0x20, 0x21, 0x30, 0x31}; - RawBytesFixedSizedBuffer fixed_buffer(tcb::span(fixed_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer fixed_buffer(tcb::span(fixed_bytes), 3u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_EQ(fixed_buffer.GetRawBufferSize(), 6u); std::vector var_bytes = { 0x02, 0x00, 0x00, 0x00, 0x41, 0x42, 0x03, 0x00, 0x00, 0x00, 0x78, 0x79, 0x7A }; - RawBytesVariableSizedBuffer var_buffer{tcb::span(var_bytes)}; + RawBytesVariableSizedBuffer var_buffer{tcb::span(var_bytes), 2u}; EXPECT_EQ(var_buffer.GetRawBufferSize(), 13u); std::vector empty_bytes; - RawBytesFixedSizedBuffer empty_buffer(tcb::span(empty_bytes), 0, RawBytesFixedSizedCodec{2}); + RawBytesFixedSizedBuffer empty_buffer(tcb::span(empty_bytes), 0u, 0u, RawBytesFixedSizedCodec{2}); EXPECT_EQ(empty_buffer.GetRawBufferSize(), 0u); } // ----------------------------------------------------------------------------- -// ConstRawIterator / raw_elements() tests +// Raw-byte iteration via ElementsIteratorNext tests // ----------------------------------------------------------------------------- TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { - using dbps::processing::StringFixedSizedCodec; std::vector bytes = {'H', 'i', '!', 'B', 'y', 'e'}; ByteBuffer buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{3u}); + tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{3u}); std::vector decoded; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); decoded.push_back(value); } ASSERT_EQ(decoded.size(), 2u); @@ -988,7 +1076,8 @@ TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(decoded[1], "Bye"); std::vector> raw; - for (const auto span : buffer.raw_elements()) { + tcb::span span; + while (buffer.ElementsIteratorNext(span)) { raw.push_back(std::vector(span.begin(), span.end())); } ASSERT_EQ(raw.size(), 2u); @@ -997,17 +1086,17 @@ TEST(TypedBufferTest, RawIterate_FixedSize_ReturnsRawBytesNotDecodedValues) { } TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { - using dbps::processing::StringVariableSizedCodec; std::vector bytes; append_u32_le(bytes, 5u); bytes.insert(bytes.end(), {'H', 'e', 'l', 'l', 'o'}); append_u32_le(bytes, 2u); bytes.insert(bytes.end(), {'O', 'K'}); - ByteBuffer buffer{tcb::span(bytes)}; + ByteBuffer buffer{tcb::span(bytes), 2u}; std::vector decoded; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); decoded.push_back(value); } ASSERT_EQ(decoded.size(), 2u); @@ -1015,7 +1104,8 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(decoded[1], "OK"); std::vector> raw; - for (const auto span : buffer.raw_elements()) { + tcb::span span; + while (buffer.ElementsIteratorNext(span)) { raw.push_back(std::vector(span.begin(), span.end())); } ASSERT_EQ(raw.size(), 2u); @@ -1023,10 +1113,14 @@ TEST(TypedBufferTest, RawIterate_VariableSize_ReturnsRawBytesNotDecodedValues) { EXPECT_EQ(raw[1], (std::vector{'O', 'K'})); } +// ----------------------------------------------------------------------------- +// Tests for border cases +// ----------------------------------------------------------------------------- + TEST(TypedBufferTest, ConstructVariableSize_EmptyBuffer_InitializesEmptyState) { std::vector bytes; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; - ExpectCommonState(buffer, 0u, 0u); + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 0u}; + ExpectCommonState(buffer, 0u, kUnsetSize); EXPECT_TRUE(buffer.GetOffsets().empty()); } @@ -1036,9 +1130,11 @@ TEST(TypedBufferTest, GetNumElements_ReadOnlyVariable_WithPrefix) { 0x05, 0x00, 0x00, 0x00, 0x41, 0x42, 0x43, 0x44, 0x45, // len=5, payload="ABCDE" 0x07, 0x00, 0x00, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37 // len=7, payload="1234567" }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 3u}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 2u, 3u}; EXPECT_EQ(buffer.GetNumElements(), 2u); + // Offsets are populated lazily on first element access. + EXPECT_NO_THROW((void)buffer.GetElement(0)); ASSERT_EQ(buffer.GetOffsets().size(), 2u); EXPECT_EQ(buffer.GetOffsets()[0], 3u); EXPECT_EQ(buffer.GetOffsets()[1], 12u); @@ -1050,7 +1146,7 @@ TEST(TypedBufferTest, GetNumElements_ReadOnlyFixed_WithPrefix) { 0xFE, 0xFD, 0xFC, 0x10, 0x11, 0x20, 0x21, 0x30, 0x31 }; - RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, RawBytesFixedSizedCodec{2u}); + RawBytesFixedSizedBuffer buffer(tcb::span(bytes), 3u, 3u, RawBytesFixedSizedCodec{2u}); EXPECT_EQ(buffer.GetNumElements(), 3u); // Fixed-size buffers never build offsets. @@ -1067,13 +1163,13 @@ TEST(TypedBufferTest, GetNumElements_WriteBuffer_ReturnsConstructorCount) { TEST(TypedBufferTest, GetNumElements_ReadOnly_PrefixExceedsSpan_Throws) { std::vector bytes = {0xAA, 0xBB}; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 3u}; - EXPECT_THROW((void)buffer.GetNumElements(), InvalidInputException); + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u, 3u}; + EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } TEST(TypedBufferTest, ConstructVariableSize_TruncatedLengthPrefix_Throws) { std::vector bytes = {0x01, 0x00, 0x00}; // only 3 bytes - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u}; EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } @@ -1082,7 +1178,7 @@ TEST(TypedBufferTest, ConstructVariableSize_TruncatedPayload_Throws) { std::vector bytes = { 0x05, 0x00, 0x00, 0x00, 0xAA, 0xBB }; - RawBytesVariableSizedBuffer buffer{tcb::span(bytes)}; + RawBytesVariableSizedBuffer buffer{tcb::span(bytes), 1u}; EXPECT_THROW((void)buffer.GetElement(0), InvalidInputException); } diff --git a/src/processing/typed_buffer_testing_codecs.h b/src/processing/typed_buffer_testing_codecs.h new file mode 100644 index 0000000..5f2f985 --- /dev/null +++ b/src/processing/typed_buffer_testing_codecs.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "typed_buffer.h" + + +// ----------------------------------------------------------------------------- +// StringFixedSizedCodec and StringVariableSizedCodec +// +// These codecs are only test only, but help check on border conditions like zero-size and mult-byte values +// +// ----------------------------------------------------------------------------- + +namespace dbps::processing::testing { + +struct StringFixedSizedCodec { + using value_type = std::string_view; + static constexpr bool is_fixed_sized = true; + + explicit StringFixedSizedCodec(size_t element_size_bytes) : element_size_bytes_(element_size_bytes) { + if (element_size_bytes_ == 0) { + throw InvalidInputException("StringFixedSizedCodec requires element_size_bytes > 0"); + } + } + + static constexpr std::string_view type_name() noexcept { + return "string (fixed-length)"; + } + + size_t element_size() const noexcept { + return element_size_bytes_; + } + + value_type Decode(tcb::span read_span) const { + if (read_span.size() != element_size_bytes_) { + throw InvalidInputException("Decode: read_span size does not match element_size_bytes"); + } + return std::string_view( + reinterpret_cast(read_span.data()), + read_span.size()); + } + + void Encode(const value_type& value, tcb::span write_span) const { + if (write_span.size() != element_size_bytes_) { + throw InvalidInputException("Encode: write_span size does not match element_size_bytes"); + } + if (value.size() != write_span.size()) { + throw InvalidInputException("Encode: value size does not match write_span size"); + } + std::memcpy(write_span.data(), value.data(), write_span.size()); + } + +private: + size_t element_size_bytes_; +}; + +struct StringVariableSizedCodec { + using value_type = std::string_view; + static constexpr bool is_fixed_sized = false; + + static constexpr std::string_view type_name() noexcept { + return "string (variable-length)"; + } + + size_t element_size() const { + throw InvalidInputException("StringVariableSizedCodec does not have a fixed element size"); + } + + value_type Decode(tcb::span read_span) const noexcept { + return std::string_view( + reinterpret_cast(read_span.data()), + read_span.size()); + } + + void Encode(const value_type& value, tcb::span write_span) const { + // Exact match required to prevent short values leaving stale trailing bytes, + // and to prevent longer values from overflowing. + if (value.size() != write_span.size()) { + throw InvalidInputException("Encode: value size does not match write_span size"); + } + std::memcpy(write_span.data(), value.data(), write_span.size()); + } +}; + +using TypedBufferStringFixedSized = ByteBuffer; +using TypedBufferStringVariableSized = ByteBuffer; + +} // namespace dbps::processing::testing diff --git a/src/processing/typed_buffer_values.h b/src/processing/typed_buffer_values.h index 3db6547..7a71285 100644 --- a/src/processing/typed_buffer_values.h +++ b/src/processing/typed_buffer_values.h @@ -42,8 +42,6 @@ using TypedBufferI64 = ByteBuffer>; using TypedBufferFloat = ByteBuffer>; using TypedBufferDouble = ByteBuffer>; using TypedBufferInt96 = ByteBuffer>; -using TypedBufferStringFixedSized = ByteBuffer; -using TypedBufferStringVariableSized = ByteBuffer; using TypedBufferRawBytesFixedSized = ByteBuffer; using TypedBufferRawBytesVariableSized = ByteBuffer; @@ -53,8 +51,6 @@ using TypedValuesBuffer = std::variant< TypedBufferFloat, TypedBufferDouble, TypedBufferInt96, - TypedBufferStringFixedSized, - TypedBufferStringVariableSized, TypedBufferRawBytesFixedSized, TypedBufferRawBytesVariableSized >; @@ -70,8 +66,8 @@ inline std::string PrintableTypedValuesBuffer(const TypedValuesBuffer& buffer) { out << BufferType::type_name() << " (" << num_elements << " elements):\n"; - size_t i = 0; - for (const auto element : typed_buffer) { + for (size_t i = 0; i < num_elements; ++i) { + const auto element = typed_buffer.GetElement(i); if constexpr (std::is_same_v) { out << " [" << i << "] [" << element.lo << ", " << element.mid << ", " << element.hi << "]\n"; @@ -83,7 +79,6 @@ inline std::string PrintableTypedValuesBuffer(const TypedValuesBuffer& buffer) { } else { out << " [" << i << "] " << element << "\n"; } - ++i; } return out.str(); diff --git a/src/processing/typed_buffer_values_test.cpp b/src/processing/typed_buffer_values_test.cpp index 38d6843..bcb4394 100644 --- a/src/processing/typed_buffer_values_test.cpp +++ b/src/processing/typed_buffer_values_test.cpp @@ -16,6 +16,7 @@ // under the License. #include "typed_buffer_values.h" +#include "typed_buffer_testing_codecs.h" #include #include @@ -28,6 +29,9 @@ #include "exceptions.h" using namespace dbps::processing; +using dbps::processing::testing::StringFixedSizedCodec; +using dbps::processing::testing::TypedBufferStringFixedSized; +using dbps::processing::testing::TypedBufferStringVariableSized; // ============================================================================= // INT32 @@ -40,7 +44,7 @@ TEST(TypedBufferValuesTest, Int32_ReadBack) { append_i32_le(bytes, 0); append_i32_le(bytes, 2147483647); - TypedBufferI32 buffer{tcb::span(bytes)}; + TypedBufferI32 buffer{tcb::span(bytes), 4u}; EXPECT_EQ(buffer.GetElement(0), 42); EXPECT_EQ(buffer.GetElement(1), -1); @@ -67,7 +71,7 @@ TEST(TypedBufferValuesTest, Int32_WriteRoundTrip) { writer.SetElement(2, 30); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), 10); EXPECT_EQ(reader.GetElement(1), 20); @@ -80,10 +84,11 @@ TEST(TypedBufferValuesTest, Int32_Iterate) { append_i32_le(bytes, 10); append_i32_le(bytes, 15); - TypedBufferI32 buffer{tcb::span(bytes)}; + TypedBufferI32 buffer{tcb::span(bytes), 3u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -105,7 +110,7 @@ TEST(TypedBufferValuesTest, Int32_OutOfOrderWrite_RoundTrip) { writer.SetElement(2, 30); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 4u}; EXPECT_EQ(reader.GetElement(0), 10); EXPECT_EQ(reader.GetElement(1), 20); @@ -125,7 +130,7 @@ TEST(TypedBufferValuesTest, Int32_OverwriteThenRoundTrip) { EXPECT_EQ(writer.GetElement(1), 222); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferI32 reader{tcb::span(finalized)}; + TypedBufferI32 reader{tcb::span(finalized), 2u}; EXPECT_EQ(reader.GetElement(0), 999); EXPECT_EQ(reader.GetElement(1), 222); @@ -159,7 +164,7 @@ TEST(TypedBufferValuesTest, Int96_ReadBack) { AppendInt96LE(bytes, b); AppendInt96LE(bytes, c); - TypedBufferInt96 buffer{tcb::span(bytes)}; + TypedBufferInt96 buffer{tcb::span(bytes), 3u}; ExpectInt96Eq(buffer.GetElement(0), a); ExpectInt96Eq(buffer.GetElement(1), b); @@ -175,7 +180,7 @@ TEST(TypedBufferValuesTest, Int96_WriteRoundTrip) { writer.SetElement(1, b); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferInt96 reader{tcb::span(finalized)}; + TypedBufferInt96 reader{tcb::span(finalized), 2u}; ExpectInt96Eq(reader.GetElement(0), a); ExpectInt96Eq(reader.GetElement(1), b); @@ -189,10 +194,11 @@ TEST(TypedBufferValuesTest, Int96_Iterate) { AppendInt96LE(bytes, a); AppendInt96LE(bytes, b); - TypedBufferInt96 buffer{tcb::span(bytes)}; + TypedBufferInt96 buffer{tcb::span(bytes), 2u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -212,7 +218,7 @@ TEST(TypedBufferValuesTest, Int96_OutOfOrderWrite_RoundTrip) { writer.SetElement(1, b); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferInt96 reader{tcb::span(finalized)}; + TypedBufferInt96 reader{tcb::span(finalized), 3u}; ExpectInt96Eq(reader.GetElement(0), a); ExpectInt96Eq(reader.GetElement(1), b); @@ -229,7 +235,7 @@ TEST(TypedBufferValuesTest, Float_ReadBack) { append_f32_le(bytes, -0.0f); append_f32_le(bytes, 1.0e10f); - TypedBufferFloat buffer{tcb::span(bytes)}; + TypedBufferFloat buffer{tcb::span(bytes), 3u}; EXPECT_FLOAT_EQ(buffer.GetElement(0), 3.14f); EXPECT_FLOAT_EQ(buffer.GetElement(1), -0.0f); @@ -243,7 +249,7 @@ TEST(TypedBufferValuesTest, Float_WriteRoundTrip) { writer.SetElement(2, 0.0f); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferFloat reader{tcb::span(finalized)}; + TypedBufferFloat reader{tcb::span(finalized), 3u}; EXPECT_FLOAT_EQ(reader.GetElement(0), 1.5f); EXPECT_FLOAT_EQ(reader.GetElement(1), -2.25f); @@ -256,10 +262,11 @@ TEST(TypedBufferValuesTest, Float_Iterate) { append_f32_le(bytes, 2.0f); append_f32_le(bytes, 3.0f); - TypedBufferFloat buffer{tcb::span(bytes)}; + TypedBufferFloat buffer{tcb::span(bytes), 3u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.push_back(value); } @@ -284,7 +291,7 @@ TEST(TypedBufferValuesTest, Float_OutOfOrderWrite_InterleavedReads) { EXPECT_FLOAT_EQ(writer.GetElement(2), 99.0f); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferFloat reader{tcb::span(finalized)}; + TypedBufferFloat reader{tcb::span(finalized), 3u}; EXPECT_FLOAT_EQ(reader.GetElement(0), 10.0f); EXPECT_FLOAT_EQ(reader.GetElement(1), 20.0f); @@ -304,7 +311,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_ReadBack) { }; TypedBufferStringFixedSized buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{4}); + tcb::span(bytes), 3u, 0u, StringFixedSizedCodec{4}); EXPECT_EQ(buffer.GetElement(0), "ABCD"); EXPECT_EQ(buffer.GetElement(1), "EFGH"); @@ -320,7 +327,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_WriteRoundTrip) { std::vector finalized = writer.FinalizeAndTakeBuffer(); TypedBufferStringFixedSized reader( - tcb::span(finalized), 0, StringFixedSizedCodec{4}); + tcb::span(finalized), 3u, 0u, StringFixedSizedCodec{4}); EXPECT_EQ(reader.GetElement(0), "AAAA"); EXPECT_EQ(reader.GetElement(1), "BBBB"); @@ -334,10 +341,11 @@ TEST(TypedBufferValuesTest, StringFixedSized_Iterate) { }; TypedBufferStringFixedSized buffer( - tcb::span(bytes), 0, StringFixedSizedCodec{4}); + tcb::span(bytes), 2u, 0u, StringFixedSizedCodec{4}); std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.emplace_back(value); } @@ -361,7 +369,7 @@ TEST(TypedBufferValuesTest, StringFixedSized_OutOfOrderWrite_RoundTrip) { std::vector finalized = writer.FinalizeAndTakeBuffer(); TypedBufferStringFixedSized reader( - tcb::span(finalized), 0, StringFixedSizedCodec{3}); + tcb::span(finalized), 3u, 0u, StringFixedSizedCodec{3}); EXPECT_EQ(reader.GetElement(0), "ZZZ"); EXPECT_EQ(reader.GetElement(1), "BBB"); @@ -384,7 +392,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_ReadBack) { append_u32_le(bytes, 6u); bytes.insert(bytes.end(), {'W', 'o', 'r', 'l', 'd', '!'}); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 2u}; EXPECT_EQ(buffer.GetElement(0), "Hello"); EXPECT_EQ(buffer.GetElement(1), "World!"); @@ -398,7 +406,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_WriteRoundTrip) { writer.SetElement(2, std::string_view("x")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), "short"); EXPECT_EQ(reader.GetElement(1), "a longer string value"); @@ -412,10 +420,11 @@ TEST(TypedBufferValuesTest, StringVariableSized_Iterate) { append_u32_le(bytes, 6u); bytes.insert(bytes.end(), {'b', 'a', 'r', 'b', 'a', 'z'}); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 2u}; std::vector collected; - for (const auto value : buffer) { + for (size_t i = 0; i < buffer.GetNumElements(); ++i) { + const auto value = buffer.GetElement(i); collected.emplace_back(value); } @@ -439,7 +448,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_OutOfOrderWrite_RoundTrip) { EXPECT_EQ(writer.GetElement(0), "replaced"); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 3u}; EXPECT_EQ(reader.GetElement(0), "replaced"); EXPECT_EQ(reader.GetElement(1), "second"); @@ -459,7 +468,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_EmptyStringsMixedWithNonEmpty) { writer.SetElement(4, std::string_view("")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 8u}; EXPECT_EQ(reader.GetElement(0), ""); EXPECT_EQ(reader.GetElement(0).size(), 0u); @@ -477,7 +486,8 @@ TEST(TypedBufferValuesTest, StringVariableSized_EmptyStringsMixedWithNonEmpty) { size_t element_count = 0; std::vector collected; - for (const auto value : reader) { + for (size_t i = 0; i < reader.GetNumElements(); ++i) { + const auto value = reader.GetElement(i); collected.emplace_back(value); ++element_count; } @@ -512,7 +522,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_UTF8_ReadBack) { append_u32_le(bytes, static_cast(emojis.size())); bytes.insert(bytes.end(), emojis.begin(), emojis.end()); - TypedBufferStringVariableSized buffer{tcb::span(bytes)}; + TypedBufferStringVariableSized buffer{tcb::span(bytes), 3u}; EXPECT_EQ(buffer.GetElement(0), "Привет"); EXPECT_EQ(buffer.GetElement(1), "مرحبا"); @@ -528,7 +538,7 @@ TEST(TypedBufferValuesTest, StringVariableSized_UTF8_WriteRoundTrip) { writer.SetElement(3, std::string_view("Ελληνικά")); std::vector finalized = writer.FinalizeAndTakeBuffer(); - TypedBufferStringVariableSized reader{tcb::span(finalized)}; + TypedBufferStringVariableSized reader{tcb::span(finalized), 4u}; EXPECT_EQ(reader.GetElement(0), "Héllo Wörld"); EXPECT_EQ(reader.GetElement(1), "中文测试"); @@ -547,7 +557,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { append_i32_le(i32_bytes, -1); append_i32_le(i32_bytes, 100); - TypedValuesBuffer i32_variant = TypedBufferI32{tcb::span(i32_bytes)}; + TypedValuesBuffer i32_variant = TypedBufferI32{tcb::span(i32_bytes), 3u}; std::string i32_str = PrintableTypedValuesBuffer(i32_variant); EXPECT_NE(i32_str.find("INT32"), std::string::npos); @@ -561,7 +571,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { append_f32_le(float_bytes, 3.14f); append_f32_le(float_bytes, -0.5f); - TypedValuesBuffer float_variant = TypedBufferFloat{tcb::span(float_bytes)}; + TypedValuesBuffer float_variant = TypedBufferFloat{tcb::span(float_bytes), 2u}; std::string float_str = PrintableTypedValuesBuffer(float_variant); EXPECT_NE(float_str.find("FLOAT"), std::string::npos); @@ -577,7 +587,7 @@ TEST(TypedBufferValuesTest, TypedValuesBufferToString_BasicUsage) { std::vector raw_finalized = writer.FinalizeAndTakeBuffer(); TypedValuesBuffer raw_variant = - TypedBufferRawBytesVariableSized{tcb::span(raw_finalized)}; + TypedBufferRawBytesVariableSized{tcb::span(raw_finalized), 2u}; std::string raw_str = PrintableTypedValuesBuffer(raw_variant); EXPECT_NE(raw_str.find("raw bytes (variable-length)"), std::string::npos); diff --git a/src/scripts/dbpa_remote_testapp.cpp b/src/scripts/dbpa_remote_testapp.cpp index f0c06da..f5d8d4e 100644 --- a/src/scripts/dbpa_remote_testapp.cpp +++ b/src/scripts/dbpa_remote_testapp.cpp @@ -247,7 +247,10 @@ class DBPARemoteTestApp { auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); // Encrypt once for the combined payload - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = { + {"page_encoding", "PLAIN"}, + {"page_type", "DICTIONARY_PAGE"}, + {"dict_page_num_values", std::to_string(sample_data.size())}}; auto encrypt_result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { @@ -361,7 +364,7 @@ class DBPARemoteTestApp { auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); // Encrypt - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto encrypt_result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (!encrypt_result || !encrypt_result->success()) { @@ -619,12 +622,17 @@ class DBPARemoteTestApp { std::cout << "Test data: 3 fixed-length strings (8 bytes each)" << std::endl; std::cout << "Original size: " << fixed_length_data.size() << " bytes" << std::endl; - // Build DATA_PAGE_V1 payload: level bytes use RLE blocks with length prefixes + // Build a minimal valid DATA_PAGE_V1 nullable payload (max_def_level=1, max_rep_level=0): + // level bytes layout is [u32 def_payload_len][def_payload]. + // def_payload is hybrid RLE/bit-packed: + // - 0x06 = varint header for an RLE run (LSB=0), run_len = 0x06 >> 1 = 3 + // - 0x01 = repeated definition level value (bit_width=1, value=1 => "present") + // This yields 3 definition levels, all present, matching the 3 fixed-length values below. std::vector level_bytes; - append_u32_le(level_bytes, 3); // repetition level block length - level_bytes.insert(level_bytes.end(), 3, 0xAA); - append_u32_le(level_bytes, 5); // definition level block length - level_bytes.insert(level_bytes.end(), 5, 0xBB); + std::vector def_payload = {0x06, 0x01}; + append_u32_le(level_bytes, static_cast(def_payload.size())); + level_bytes.insert(level_bytes.end(), def_payload.begin(), def_payload.end()); + auto combined_uncompressed = Join(level_bytes, fixed_length_data); auto joined_plaintext = Compress(combined_uncompressed, CompressionCodec::SNAPPY); std::cout << "Compressed size: " << joined_plaintext.size() << " bytes" << std::endl; @@ -632,7 +640,7 @@ class DBPARemoteTestApp { {"page_type", "DATA_PAGE_V1"}, {"data_page_num_values", std::to_string(std::size(test_strings))}, {"data_page_max_definition_level", "1"}, - {"data_page_max_repetition_level", "2"}, + {"data_page_max_repetition_level", "0"}, {"page_v1_repetition_level_encoding", "RLE"}, {"page_v1_definition_level_encoding", "RLE"}, {"page_encoding", "PLAIN"} @@ -690,26 +698,18 @@ class DBPARemoteTestApp { std::vector(decrypted_plaintext.begin(), decrypted_plaintext.end()), CompressionCodec::SNAPPY); size_t offset = 0; - if (offset + 4 > decompressed_combined.size()) { - std::cout << "ERROR: Decompressed payload too small for rep level length" << std::endl; - return false; - } - uint32_t rep_len = read_u32_le(decompressed_combined, static_cast(offset)); - offset += 4; - if (offset + rep_len > decompressed_combined.size()) { - std::cout << "ERROR: Decompressed payload too small for rep level bytes" << std::endl; - return false; - } - auto rep_bytes = std::vector(decompressed_combined.begin() + static_cast(offset), - decompressed_combined.begin() + static_cast(offset + rep_len)); - offset += rep_len; - if (offset + 4 > decompressed_combined.size()) { std::cout << "ERROR: Decompressed payload too small for def level length" << std::endl; return false; } uint32_t def_len = read_u32_le(decompressed_combined, static_cast(offset)); offset += 4; + if (def_len != 2) { + std::cout << "ERROR: Unexpected definition-level payload length" << std::endl; + std::cout << " Expected def_len: 2" << std::endl; + std::cout << " Got def_len: " << def_len << std::endl; + return false; + } if (offset + def_len > decompressed_combined.size()) { std::cout << "ERROR: Decompressed payload too small for def level bytes" << std::endl; return false; @@ -718,8 +718,19 @@ class DBPARemoteTestApp { decompressed_combined.begin() + static_cast(offset + def_len)); offset += def_len; - if (rep_bytes != std::vector{0xAA, 0xAA, 0xAA} || def_bytes != std::vector{0xBB, 0xBB, 0xBB, 0xBB, 0xBB}) { - std::cout << "ERROR: Level bytes content mismatch" << std::endl; + // Expected def-level payload for this demo: + // - 0x06: RLE run header => run_len = 3 + // - 0x01: repeated def level value 1 (present) + if (def_bytes != std::vector{0x06, 0x01}) { + std::cout << "ERROR: Definition-level bytes content mismatch" << std::endl; + return false; + } + + const size_t expected_total_decompressed_size = 4u + 2u + fixed_length_data.size(); + if (decompressed_combined.size() != expected_total_decompressed_size) { + std::cout << "ERROR: Unexpected decompressed payload size" << std::endl; + std::cout << " Expected size: " << expected_total_decompressed_size << std::endl; + std::cout << " Got size: " << decompressed_combined.size() << std::endl; return false; } @@ -729,6 +740,12 @@ class DBPARemoteTestApp { } auto value_bytes = std::vector(decompressed_combined.begin() + static_cast(offset), decompressed_combined.end()); + if (offset != 6u) { + std::cout << "ERROR: Unexpected value-byte offset after level bytes" << std::endl; + std::cout << " Expected offset: 6" << std::endl; + std::cout << " Got offset: " << offset << std::endl; + return false; + } std::cout << "Decompressed value size: " << value_bytes.size() << " bytes" << std::endl; // Verify data integrity @@ -760,7 +777,7 @@ class DBPARemoteTestApp { try { auto empty_data = BuildByteArrayValueBytesForTesting(""); auto compressed_empty = Compress(empty_data, CompressionCodec::SNAPPY); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent_->Encrypt(span(compressed_empty), encoding_attributes); if (result && result->success()) { @@ -780,7 +797,7 @@ class DBPARemoteTestApp { std::string large_data(1000, 'X'); // 1KB of data auto plaintext = BuildByteArrayValueBytesForTesting(large_data); auto compressed_plaintext = Compress(plaintext, CompressionCodec::SNAPPY); - std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}}; + std::map encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}}; auto result = agent_->Encrypt(span(compressed_plaintext), encoding_attributes); if (result && result->success()) { diff --git a/src/scripts/performance_test.cpp b/src/scripts/performance_test.cpp index ef71311..772b4be 100644 --- a/src/scripts/performance_test.cpp +++ b/src/scripts/performance_test.cpp @@ -173,20 +173,26 @@ namespace { size_t num_values, CompressionCodec::type compression_type, const std::string& page_encoding, - int32_t max_definition_level = 1, + int32_t max_definition_level = 0, int32_t max_repetition_level = 2, uint32_t definition_level_block_length = 3, uint32_t repetition_level_block_length = 5) { DataPageBuildResult result; result.level_bytes.clear(); - append_u32_le(result.level_bytes, repetition_level_block_length); // repetition level block length - result.level_bytes.insert(result.level_bytes.end(), - repetition_level_block_length, - 0xAA); - append_u32_le(result.level_bytes, definition_level_block_length); // definition level block length - result.level_bytes.insert(result.level_bytes.end(), - definition_level_block_length, - 0xBB); + + // V1 level bytes are emitted only when the corresponding max level is > 0. + if (max_repetition_level > 0) { + append_u32_le(result.level_bytes, repetition_level_block_length); // repetition level block length + result.level_bytes.insert(result.level_bytes.end(), + repetition_level_block_length, + 0xAA); + } + if (max_definition_level > 0) { + append_u32_le(result.level_bytes, definition_level_block_length); // definition level block length + result.level_bytes.insert(result.level_bytes.end(), + definition_level_block_length, + 0xBB); + } auto combined_uncompressed = Join(result.level_bytes, value_bytes); result.payload = Compress(combined_uncompressed, compression_type); result.attrs = { @@ -204,6 +210,7 @@ namespace { DataPageBuildResult BuildDictionaryPagePayload( const std::vector& value_bytes, + size_t num_values, CompressionCodec::type compression_type, const std::string& page_encoding) { DataPageBuildResult result; @@ -211,7 +218,8 @@ namespace { result.payload = Compress(value_bytes, compression_type); result.attrs = { {"page_type", "DICTIONARY_PAGE"}, - {"page_encoding", page_encoding} + {"page_encoding", page_encoding}, + {"dict_page_num_values", std::to_string(num_values)} }; return result; } @@ -297,6 +305,7 @@ class DBPALocalTestApp { } else if (scenario.page_type == "DICTIONARY_PAGE") { page = BuildDictionaryPagePayload( value_bytes, + num_values, scenario.compression, scenario.page_encoding); } else {