From 08a80b9106e753d83793cf1835afc88f9e25a7ce Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 11 Jun 2026 09:51:26 +0800 Subject: [PATCH 1/5] add readTimeseriesMetadata methods --- .../tsfile/read/TsFileSequenceReader.java | 148 +++++++++++++----- 1 file changed, 108 insertions(+), 40 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index b1fb15b35..5998dd84d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -102,6 +102,7 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.TreeMap; @@ -732,22 +733,23 @@ public TimeseriesMetadata readTimeseriesMetadata( boolean ignoreNotExistDevice, LongConsumer ioSizeConsumer) throws IOException { - readFileMetadata(ioSizeConsumer); - MetadataIndexNode deviceMetadataIndexNode = - tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeConsumer); - if (metadataIndexPair == null) { - if (ignoreNotExistDevice) { - return null; - } - throw new IOException( - Messages.format("error.read.device_not_in_metadata_file", device, file)); - } - ByteBuffer buffer = - readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeConsumer); - MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; - if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + return readTimeseriesMetadata( + device, Optional.empty(), measurement, ignoreNotExistDevice, ioSizeConsumer); + } + + public TimeseriesMetadata readTimeseriesMetadata( + IDeviceID device, + Optional deviceMetadataIndexNodeOffset, + String measurement, + boolean ignoreNotExistDevice, + LongConsumer ioSizeConsumer) + throws IOException { + Pair metadataIndexPair; + MetadataIndexNode metadataIndexNode; + ByteBuffer buffer; + if (deviceMetadataIndexNodeOffset.isPresent()) { + long[] offsetArr = deviceMetadataIndexNodeOffset.get(); + buffer = readData(offsetArr[0], offsetArr[1], ioSizeConsumer); try { metadataIndexNode = deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( @@ -759,6 +761,36 @@ public TimeseriesMetadata readTimeseriesMetadata( metadataIndexPair = getMetadataAndEndOffsetOfMeasurementNode( metadataIndexNode, measurement, false, ioSizeConsumer); + } else { + readFileMetadata(ioSizeConsumer); + MetadataIndexNode deviceMetadataIndexNode = + tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); + metadataIndexPair = + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeConsumer); + if (metadataIndexPair == null) { + if (ignoreNotExistDevice) { + return null; + } + throw new IOException( + Messages.format("error.read.device_not_in_metadata_file", device, file)); + } + buffer = + readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeConsumer); + metadataIndexNode = deviceMetadataIndexNode; + if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + try { + metadataIndexNode = + deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( + buffer, deserializeConfig); + } catch (Exception e) { + logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file); + throw e; + } + metadataIndexPair = + getMetadataAndEndOffsetOfMeasurementNode( + metadataIndexNode, measurement, false, ioSizeConsumer); + } } if (metadataIndexPair == null) { return null; @@ -781,13 +813,15 @@ public TimeseriesMetadata readTimeseriesMetadata( } // when the buffer length is over than Integer.MAX_VALUE, // using tsFileInput to get timeseriesMetadataList - tsFileInput.position(metadataIndexPair.left.getOffset()); - while (tsFileInput.position() < metadataIndexPair.right) { - try { - timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(tsFileInput, true)); - } catch (Exception e1) { - logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); - throw e1; + synchronized (this) { + tsFileInput.position(metadataIndexPair.left.getOffset()); + while (tsFileInput.position() < metadataIndexPair.right) { + try { + timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(tsFileInput, true)); + } catch (Exception e1) { + logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); + throw e1; + } } } } @@ -872,8 +906,21 @@ public List readTimeseriesMetadata( boolean ignoreNotExistDevice, LongConsumer ioSizeRecorder) throws IOException { + return readTimeseriesMetadata( + device, Optional.empty(), measurement, allSensors, ignoreNotExistDevice, ioSizeRecorder); + } + + public List readTimeseriesMetadata( + IDeviceID device, + Optional deviceMetadataIndexNodeOffset, + String measurement, + Set allSensors, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { Pair metadataIndexPair = - getLeafMetadataIndexPair(device, measurement, ioSizeRecorder); + getLeafMetadataIndexPair( + device, deviceMetadataIndexNodeOffset, measurement, ioSizeRecorder); if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return Collections.emptyList(); @@ -927,19 +974,17 @@ public List readTimeseriesMetadata( /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( - IDeviceID device, String measurement, LongConsumer ioSizeRecorder) throws IOException { - readFileMetadata(ioSizeRecorder); - MetadataIndexNode deviceMetadataIndexNode = - tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeRecorder); - if (metadataIndexPair == null) { - return null; - } - ByteBuffer buffer = - readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); - MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; - if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + IDeviceID device, + Optional deviceMetadataIndexNodeOffset, + String measurement, + LongConsumer ioSizeRecorder) + throws IOException { + Pair metadataIndexPair; + MetadataIndexNode metadataIndexNode; + ByteBuffer buffer; + if (deviceMetadataIndexNodeOffset.isPresent()) { + long[] offsetArr = deviceMetadataIndexNodeOffset.get(); + buffer = readData(offsetArr[0], offsetArr[1], ioSizeRecorder); try { metadataIndexNode = deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( @@ -948,10 +993,33 @@ private Pair getLeafMetadataIndexPair( logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file); throw e; } + } else { + readFileMetadata(ioSizeRecorder); + MetadataIndexNode deviceMetadataIndexNode = + tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); metadataIndexPair = - getMetadataAndEndOffsetOfMeasurementNode( - metadataIndexNode, measurement, false, ioSizeRecorder); + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeRecorder); + if (metadataIndexPair == null) { + return null; + } + buffer = + readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); + metadataIndexNode = deviceMetadataIndexNode; + if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + try { + metadataIndexNode = + deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( + buffer, deserializeConfig); + } catch (Exception e) { + logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file); + throw e; + } + } } + metadataIndexPair = + getMetadataAndEndOffsetOfMeasurementNode( + metadataIndexNode, measurement, false, ioSizeRecorder); return metadataIndexPair; } @@ -1626,7 +1694,7 @@ private void generateMetadataIndex( } } - private void generateMetadataIndexUsingTsFileInput( + private synchronized void generateMetadataIndexUsingTsFileInput( IMetadataIndexEntry metadataIndex, long start, long end, From 1e7617d29acf9f7f71b9ecd5a92db3ab7f034b50 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 11 Jun 2026 15:43:55 +0800 Subject: [PATCH 2/5] add ut --- .../tsfile/read/TsFileDeviceIteratorTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java index f75b1ed81..53d12981b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java @@ -24,6 +24,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.TimeRange; @@ -42,6 +43,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; public class TsFileDeviceIteratorTest { private static final String FILE_PATH = @@ -102,6 +104,37 @@ public void test() throws IOException { } } + @Test + public void testReadTimeseriesMetadataWithDeviceMetadataIndexNodeOffset() throws IOException { + try (TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH))) { + registerTableSchema(writer, "table1"); + generateDevice(writer, "table1", 10); + writer.endFile(); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + TsFileDeviceIterator deviceIterator = + reader.getTableDevicesIteratorWithIsAligned("table1", null); + Assert.assertTrue(deviceIterator.hasNext()); + Pair currentDevice = deviceIterator.next(); + long[] deviceMetadataIndexNodeOffset = deviceIterator.getCurrentDeviceMeasurementNodeOffset(); + + TimeseriesMetadata metadataWithoutOffset = + reader.readTimeseriesMetadata(currentDevice.getLeft(), "s1", false); + TimeseriesMetadata metadataWithOffset = + reader.readTimeseriesMetadata( + currentDevice.getLeft(), + Optional.of(deviceMetadataIndexNodeOffset), + "s1", + false, + null); + + Assert.assertEquals("s1", metadataWithoutOffset.getMeasurementId()); + Assert.assertEquals( + metadataWithoutOffset.getMeasurementId(), metadataWithOffset.getMeasurementId()); + } + } + private void registerTableSchema(TsFileIOWriter writer, String tableName) { List schemas = Arrays.asList( From 7926b33345317735df2395d0b39b6cefbe6270ef Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 12 Jun 2026 10:50:25 +0800 Subject: [PATCH 3/5] fix table size accounting --- .../tsfile/write/writer/TsFileIOWriter.java | 5 ++- .../tsfile/write/TsFileWriteApiTest.java | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 0c39f15ec..df6d99b20 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -507,6 +507,10 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { // construct the index tree node for the series currentDevice = currentPath.getIDeviceID(); boolean isTableModel = schema.getTableSchemaMap().containsKey(currentDevice.getTableName()); + String currentTableName = isTableModel ? currentDevice.getTableName() : null; + if (prevDevice == null && currentTableName != null) { + prevTableName = currentTableName; + } if (!currentDevice.equals(prevDevice)) { if (prevDevice != null) { addCurrentIndexNodeToQueue(currentIndexNode, measurementMetadataIndexQueue, out); @@ -515,7 +519,6 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { generateRootNode( measurementMetadataIndexQueue, out, MetadataIndexNodeType.INTERNAL_MEASUREMENT)); currentIndexNode = new MetadataIndexNode(MetadataIndexNodeType.LEAF_MEASUREMENT); - String currentTableName = isTableModel ? currentDevice.getTableName() : null; if (!Objects.equals(currentTableName, prevTableName)) { if (prevTableName != null) { long currentTableSize = out.getPosition() - prevTableMetadataStartOffset; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index feaa27940..fcd761d3f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -1280,6 +1280,47 @@ public void calculateTableSize() throws IOException, WriteProcessException { Assert.assertTrue(tableSizeMap.get("table2") >= 1024 * 1024); } + @Test + public void calculateTableSize2() throws IOException, WriteProcessException { + TableSchema tableSchema1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.BLOB, ColumnCategory.FIELD))); + TableSchema tableSchema2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.BLOB, ColumnCategory.FIELD))); + Tablet tablet1 = + new Tablet( + "table1", + IMeasurementSchema.getMeasurementNameList(tableSchema1.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema1.getColumnSchemas()), + tableSchema1.getColumnTypes()); + tablet1.addTimestamp(0, 0); + tablet1.addValue(0, 0, new byte[1024]); + Tablet tablet2 = + new Tablet( + "table2", + IMeasurementSchema.getMeasurementNameList(tableSchema2.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema2.getColumnSchemas()), + tableSchema2.getColumnTypes()); + tablet2.addTimestamp(0, 0); + tablet2.addValue(0, 0, new byte[1024 * 1024]); + Map tableSizeMap = null; + try (TsFileWriter writer = new TsFileWriter(f)) { + writer.registerTableSchema(tableSchema1); + writer.registerTableSchema(tableSchema2); + writer.writeTable(tablet1); + writer.writeTable(tablet2); + tableSizeMap = writer.getIOWriter().getTableSizeMap(); + } + Assert.assertTrue(tableSizeMap.get("table1") > 1024); + } + @Test public void writeRecord() throws IOException, WriteProcessException, ReadProcessException { setEnv(100 * 1024 * 1024, 10 * 1024); From 296f4992020525178235736abbe8d6d0ef2a7355 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 16 Jun 2026 09:51:24 +0800 Subject: [PATCH 4/5] modify parameter --- .../tsfile/read/TsFileSequenceReader.java | 26 +++++++++---------- .../tsfile/read/TsFileDeviceIteratorTest.java | 7 +---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 8fdad36c2..ac26de239 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -102,7 +102,6 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; -import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.TreeMap; @@ -733,13 +732,12 @@ public TimeseriesMetadata readTimeseriesMetadata( boolean ignoreNotExistDevice, LongConsumer ioSizeConsumer) throws IOException { - return readTimeseriesMetadata( - device, Optional.empty(), measurement, ignoreNotExistDevice, ioSizeConsumer); + return readTimeseriesMetadata(device, null, measurement, ignoreNotExistDevice, ioSizeConsumer); } public TimeseriesMetadata readTimeseriesMetadata( IDeviceID device, - Optional deviceMetadataIndexNodeOffset, + long[] deviceMetadataIndexNodeOffset, String measurement, boolean ignoreNotExistDevice, LongConsumer ioSizeConsumer) @@ -747,9 +745,10 @@ public TimeseriesMetadata readTimeseriesMetadata( Pair metadataIndexPair; MetadataIndexNode metadataIndexNode; ByteBuffer buffer; - if (deviceMetadataIndexNodeOffset.isPresent()) { - long[] offsetArr = deviceMetadataIndexNodeOffset.get(); - buffer = readData(offsetArr[0], offsetArr[1], ioSizeConsumer); + if (deviceMetadataIndexNodeOffset != null) { + buffer = + readData( + deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1], ioSizeConsumer); try { metadataIndexNode = deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( @@ -907,12 +906,12 @@ public List readTimeseriesMetadata( LongConsumer ioSizeRecorder) throws IOException { return readTimeseriesMetadata( - device, Optional.empty(), measurement, allSensors, ignoreNotExistDevice, ioSizeRecorder); + device, null, measurement, allSensors, ignoreNotExistDevice, ioSizeRecorder); } public List readTimeseriesMetadata( IDeviceID device, - Optional deviceMetadataIndexNodeOffset, + long[] deviceMetadataIndexNodeOffset, String measurement, Set allSensors, boolean ignoreNotExistDevice, @@ -975,16 +974,17 @@ public List readTimeseriesMetadata( /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( IDeviceID device, - Optional deviceMetadataIndexNodeOffset, + long[] deviceMetadataIndexNodeOffset, String measurement, LongConsumer ioSizeRecorder) throws IOException { Pair metadataIndexPair; MetadataIndexNode metadataIndexNode; ByteBuffer buffer; - if (deviceMetadataIndexNodeOffset.isPresent()) { - long[] offsetArr = deviceMetadataIndexNodeOffset.get(); - buffer = readData(offsetArr[0], offsetArr[1], ioSizeRecorder); + if (deviceMetadataIndexNodeOffset != null) { + buffer = + readData( + deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1], ioSizeRecorder); try { metadataIndexNode = deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java index 53d12981b..894eb983f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileDeviceIteratorTest.java @@ -43,7 +43,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Optional; public class TsFileDeviceIteratorTest { private static final String FILE_PATH = @@ -123,11 +122,7 @@ public void testReadTimeseriesMetadataWithDeviceMetadataIndexNodeOffset() throws reader.readTimeseriesMetadata(currentDevice.getLeft(), "s1", false); TimeseriesMetadata metadataWithOffset = reader.readTimeseriesMetadata( - currentDevice.getLeft(), - Optional.of(deviceMetadataIndexNodeOffset), - "s1", - false, - null); + currentDevice.getLeft(), deviceMetadataIndexNodeOffset, "s1", false, null); Assert.assertEquals("s1", metadataWithoutOffset.getMeasurementId()); Assert.assertEquals( From 86545295c6df624052b9130e0890ab8cf91a395c Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Tue, 16 Jun 2026 14:49:46 +0800 Subject: [PATCH 5/5] move synchronized --- .../java/org/apache/tsfile/read/TsFileSequenceReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index ac26de239..97dcddc0d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1694,7 +1694,7 @@ private void generateMetadataIndex( } } - private synchronized void generateMetadataIndexUsingTsFileInput( + private void generateMetadataIndexUsingTsFileInput( IMetadataIndexEntry metadataIndex, long start, long end, @@ -1714,7 +1714,7 @@ private synchronized void generateMetadataIndexUsingTsFileInput( needChunkMetadata); } - private void generateMetadataIndexUsingTsFileInput( + private synchronized void generateMetadataIndexUsingTsFileInput( IMetadataIndexEntry metadataIndex, long start, long end,