diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedChunkMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedChunkMetadata.java index 52a1913ca..8b7cfcc4f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedChunkMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedChunkMetadata.java @@ -85,6 +85,18 @@ public void setModified(boolean modified) { } } + @Override + public boolean isDataTypeModifiedAndCannotUseStatistics() { + return timeChunkMetadata.isDataTypeModifiedAndCannotUseStatistics(); + } + + @Override + public void setDataTypeModifiedAndCannotUseStatistics( + boolean dataTypeModifiedAndCannotUseStatistics) { + timeChunkMetadata.setDataTypeModifiedAndCannotUseStatistics( + dataTypeModifiedAndCannotUseStatistics); + } + @Override public boolean isSeq() { return timeChunkMetadata.isSeq(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedTimeSeriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedTimeSeriesMetadata.java index b7a15250c..74ee4db70 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedTimeSeriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/AbstractAlignedTimeSeriesMetadata.java @@ -84,6 +84,18 @@ public void setModified(boolean modified) { } } + @Override + public boolean isDataTypeModifiedAndCannotUseStatistics() { + return timeseriesMetadata.isDataTypeModifiedAndCannotUseStatistics(); + } + + @Override + public void setDataTypeModifiedAndCannotUseStatistics( + boolean dataTypeModifiedAndCannotUseStatistics) { + timeseriesMetadata.setDataTypeModifiedAndCannotUseStatistics( + dataTypeModifiedAndCannotUseStatistics); + } + @Override public boolean isSeq() { return timeseriesMetadata.isSeq(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java index 426486215..09d33989a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ChunkMetadata.java @@ -75,6 +75,10 @@ public class ChunkMetadata implements IChunkMetadata { private boolean modified; + // data type modified + // Unlike the 'modified' property, this property should be passed down to a lower level. + private boolean dataTypeModifiedAndCannotUseStatistics; + /** ChunkLoader of metadata, used to create ChunkReaderWrap. */ private IChunkLoader chunkLoader; @@ -329,7 +333,7 @@ public void setChunkLoader(IChunkLoader chunkLoader) { @Override public boolean isModified() { - return modified; + return modified || dataTypeModifiedAndCannotUseStatistics; } @Override @@ -337,6 +341,17 @@ public void setModified(boolean modified) { this.modified |= modified; } + @Override + public boolean isDataTypeModifiedAndCannotUseStatistics() { + return dataTypeModifiedAndCannotUseStatistics; + } + + @Override + public void setDataTypeModifiedAndCannotUseStatistics( + boolean dataTypeModifiedAndCannotUseStatistics) { + this.dataTypeModifiedAndCannotUseStatistics |= dataTypeModifiedAndCannotUseStatistics; + } + public static long calculateRamSize(String measurementId, TSDataType dataType) { return CHUNK_METADATA_FIXED_RAM_SIZE + RamUsageEstimator.sizeOf(measurementId) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IChunkMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IChunkMetadata.java index f391ed7fe..41d1bbc32 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IChunkMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IChunkMetadata.java @@ -33,6 +33,10 @@ public interface IChunkMetadata extends IMetadata { void setModified(boolean modified); + boolean isDataTypeModifiedAndCannotUseStatistics(); + + void setDataTypeModifiedAndCannotUseStatistics(boolean dataTypeModifiedAndCannotUseStatistics); + boolean isSeq(); void setSeq(boolean seq); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ITimeSeriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ITimeSeriesMetadata.java index c2ddca155..878b8a659 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ITimeSeriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/ITimeSeriesMetadata.java @@ -30,6 +30,10 @@ public interface ITimeSeriesMetadata extends IMetadata { void setModified(boolean modified); + boolean isDataTypeModifiedAndCannotUseStatistics(); + + void setDataTypeModifiedAndCannotUseStatistics(boolean dataTypeModifiedAndCannotUseStatistics); + boolean isSeq(); void setSeq(boolean seq); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java index 1e254e065..790f457d1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java @@ -81,6 +81,10 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata { // modified is true when there are modifications of the series, or from unseq file private boolean modified; + // data type modified + // Unlike the 'modified' property, this property should be passed down to a lower level. + private boolean dataTypeModifiedAndCannotUseStatistics; + private IChunkMetadataLoader chunkMetadataLoader; // used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata @@ -343,14 +347,17 @@ public List getChunkMetadataList() { public List getCopiedChunkMetadataList() { List res = new ArrayList<>(chunkMetadataList.size()); for (IChunkMetadata chunkMetadata : chunkMetadataList) { - res.add(new ChunkMetadata((ChunkMetadata) chunkMetadata)); + ChunkMetadata copiedChunkMetadata = new ChunkMetadata((ChunkMetadata) chunkMetadata); + copiedChunkMetadata.setDataTypeModifiedAndCannotUseStatistics( + dataTypeModifiedAndCannotUseStatistics); + res.add(copiedChunkMetadata); } return res; } @Override public boolean isModified() { - return modified; + return modified || dataTypeModifiedAndCannotUseStatistics; } @Override @@ -358,6 +365,15 @@ public void setModified(boolean modified) { this.modified |= modified; } + public boolean isDataTypeModifiedAndCannotUseStatistics() { + return dataTypeModifiedAndCannotUseStatistics; + } + + public void setDataTypeModifiedAndCannotUseStatistics( + boolean dataTypeModifiedAndCannotUseStatistics) { + this.dataTypeModifiedAndCannotUseStatistics |= dataTypeModifiedAndCannotUseStatistics; + } + @Override public void setSeq(boolean seq) { isSeq = seq; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IChunkReader.java index acda8af6d..e13d32d97 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IChunkReader.java @@ -32,5 +32,7 @@ public interface IChunkReader { void close() throws IOException; + void markDataTypeModifiedAndCannotUseStatistics(); + List loadPageReaderList() throws IOException; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java index 1b870f86e..1227fd012 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/IPageReader.java @@ -41,6 +41,8 @@ default BatchData getAllSatisfiedPageData() throws IOException { void addRecordFilter(Filter filter); + // The 'modified' property is also true when a data type need to be modified in query and + // statistics are no longer available. boolean isModified(); void setModified(boolean modified); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractChunkReader.java index 301134d41..627ace07e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractChunkReader.java @@ -80,4 +80,13 @@ public void close() { public List loadPageReaderList() { return pageReaderList; } + + @Override + public void markDataTypeModifiedAndCannotUseStatistics() { + // In all implementations for now, pageReaderList is completed during construction. + // In other cases, this method should be overridden. + for (IPageReader iPageReader : pageReaderList) { + iPageReader.setModified(true); + } + } }