Skip to content

correct metadata "modified" flags and related logic #17172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shuwenwei wants to merge 5 commits into
base: master
Choose a base branch
from
Open

correct metadata "modified" flags and related logic #17172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
53d7c03
fix setModified
shuwenwei Feb 5, 2026
bdb81cc
add break
shuwenwei Feb 5, 2026
6614f33
fix
shuwenwei Feb 6, 2026
7c13f77
fix
shuwenwei Feb 6, 2026
9ef2522
update pom.xml
shuwenwei Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;

Expand Down Expand Up @@ -87,7 +87,7 @@

@SuppressWarnings("ResultOfMethodCallIgnored")
@RunWith(IoTDBTestRunner.class)
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
@Category({LocalStandaloneIT.class, ClusterIT.class})
public class IoTDBAlterTimeSeriesTypeIT {

private static final Logger log = LoggerFactory.getLogger(IoTDBAlterTimeSeriesTypeIT.class);
Expand Down
80 changes: 19 additions & 61 deletions .../main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.apache.iotdb.db.utils.SchemaUtils;

import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
Expand Down Expand Up @@ -113,13 +112,11 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
context.isDebug(),
context);
if (timeSeriesMetadata != null) {
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
long t2 = System.nanoTime();
List<ModEntry> pathModifications =
context.getPathModifications(
resource, seriesPath.getDeviceId(), seriesPath.getMeasurement());
timeSeriesMetadata.setModified(
timeSeriesMetadata.isModified() || !pathModifications.isEmpty());
timeSeriesMetadata.setModified(!pathModifications.isEmpty());
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications));
int modificationCount = pathModifications.size();
Expand All @@ -141,13 +138,14 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
timeSeriesMetadata =
(TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter);
if (timeSeriesMetadata != null) {
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
timeSeriesMetadata.setChunkMetadataLoader(
new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter));
}
}

if (timeSeriesMetadata != null) {
SchemaUtils.changeTimeseriesMetadataModified(
timeSeriesMetadata, seriesPath.getSeriesType());
if (timeSeriesMetadata.getStatistics().getStartTime()
> timeSeriesMetadata.getStatistics().getEndTime()) {
return null;
Expand Down Expand Up @@ -209,15 +207,12 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
alignedTimeSeriesMetadata =
loadAlignedTimeSeriesMetadataFromDisk(
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows);
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
loadFromMem = true;
alignedTimeSeriesMetadata =
(AbstractAlignedTimeSeriesMetadata)
resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter);
if (alignedTimeSeriesMetadata != null) {
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);

alignedTimeSeriesMetadata.setChunkMetadataLoader(
new MemAlignedChunkMetadataLoader(
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows));
Expand All @@ -226,6 +221,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
}

if (alignedTimeSeriesMetadata != null) {
SchemaUtils.changeAlignedTimeseriesMetadataModified(
alignedTimeSeriesMetadata, targetDataTypeList);
if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
> alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
return null;
Expand Down Expand Up @@ -320,25 +317,18 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
int i = 0;
for (String valueMeasurement : valueMeasurementList) {
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(
resource.getTsFileID(), deviceId, valueMeasurement),
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
context);
exist = (exist || (valueColumn != null));
if (valueColumn != null) {
SchemaUtils.changeAlignedMetadataModified(
valueColumn, alignedPath.getSchemaList().get(i).getType());
}
valueTimeSeriesMetadataList.add(valueColumn);
i++;
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
Expand Down Expand Up @@ -458,55 +448,23 @@ public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata tim
* IOException will be thrown
*/
public static List<IPageReader> loadPageReaderList(
IChunkMetadata chunkMetaData,
Filter globalTimeFilter,
boolean isAligned,
List<TSDataType> targetDataTypeList)
IChunkMetadata chunkMetaData, Filter globalTimeFilter, List<TSDataType> targetDataTypeList)
throws IOException {
checkArgument(chunkMetaData != null, "Can't init null chunkMeta");

IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
IChunkReader chunkReader;
boolean isModified = false;
if (isAligned) {
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) chunkMetaData;
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
if (!SchemaUtils.isUsingSameStatistics(
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
targetDataTypeList.get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) {
isModified = true;
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
}
}
}
IChunkLoader chunkLoader = alignedChunkMetadata.getChunkLoader();
chunkReader = chunkLoader.getChunkReader(alignedChunkMetadata, globalTimeFilter);
} else {
if (!SchemaUtils.isUsingSameStatistics(chunkMetaData.getDataType(), targetDataTypeList.get(0))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(0))) {
isModified = true;
chunkMetaData.setModified(true);
}
IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
try {
chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter);
} catch (ChunkTypeInconsistentException e) {
// if the chunk in tsfile is a value chunk of aligned series, we should skip all data of
// this chunk.
return Collections.emptyList();
}
try {
chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter);
} catch (ChunkTypeInconsistentException e) {
// if the chunk in tsfile is a value chunk of aligned series but registered series is
// non-aligned, we should skip all data of this chunk.
return Collections.emptyList();
}

return isModified
? chunkReader.loadPageReaderList().stream()
.peek(
iPageReader -> {
iPageReader.setModified(true);
})
.collect(Collectors.toList())
: chunkReader.loadPageReaderList();
if (chunkMetaData.isDataTypeModifiedAndCannotUseStatistics()) {
chunkReader.markDataTypeModifiedAndCannotUseStatistics();
}
return chunkReader.loadPageReaderList();
}

/**
Expand Down
47 changes: 2 additions & 45 deletions ...c/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@
import org.apache.tsfile.block.column.Column;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IMetadata;
Expand Down Expand Up @@ -453,15 +451,6 @@ private void initFirstChunkMetadata() throws IOException {
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()));
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false);
if (isAligned) {
SchemaUtils.changeAlignedMetadataModified(
(AbstractAlignedChunkMetadata) firstChunkMetadata,
firstChunkMetadata.getDataType(),
getTsDataTypeList());
} else {
SchemaUtils.changeMetadataModified(
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
}
if (firstChunkMetadata.equals(cachedChunkMetadata.peek())) {
firstChunkMetadata = cachedChunkMetadata.poll();
break;
Expand Down Expand Up @@ -489,45 +478,13 @@ private void unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(

if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) {
firstChunkMetadata = cachedChunkMetadata.poll();
if (isAligned) {
SchemaUtils.changeAlignedMetadataModified(
(AbstractAlignedChunkMetadata) firstChunkMetadata,
firstChunkMetadata.getDataType(),
getTsDataTypeList());
} else {
SchemaUtils.changeMetadataModified(
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
}
}
}

protected void unpackOneTimeSeriesMetadata(ITimeSeriesMetadata timeSeriesMetadata) {
List<IChunkMetadata> chunkMetadataList =
FileLoaderUtils.loadChunkMetadataList(timeSeriesMetadata);
chunkMetadataList.forEach(
chunkMetadata -> {
if (chunkMetadata instanceof AbstractAlignedChunkMetadata) {
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) chunkMetadata;
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
if ((alignedChunkMetadata.getValueChunkMetadataList().get(i) != null)
&& !SchemaUtils.isUsingSameStatistics(
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
getTsDataTypeList().get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(i))) {
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
}
}
chunkMetadata = alignedChunkMetadata;
} else if (chunkMetadata instanceof ChunkMetadata) {
if (!SchemaUtils.isUsingSameStatistics(
chunkMetadata.getDataType(), getTsDataTypeList().get(0))
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(0))) {
chunkMetadata.setModified(true);
}
}
chunkMetadata.setSeq(timeSeriesMetadata.isSeq());
});
chunkMetadataList.forEach(chunkMetadata -> chunkMetadata.setSeq(timeSeriesMetadata.isSeq()));

cachedChunkMetadata.addAll(chunkMetadataList);
}
Expand Down Expand Up @@ -704,7 +661,7 @@ private void unpackOneChunkMetaData(IChunkMetadata chunkMetaData) throws IOExcep
}
List<IPageReader> pageReaderList =
FileLoaderUtils.loadPageReaderList(
chunkMetaData, scanOptions.getGlobalTimeFilter(), isAligned, getTsDataTypeList());
chunkMetaData, scanOptions.getGlobalTimeFilter(), getTsDataTypeList());

// init TsBlockBuilder for each page reader
pageReaderList.forEach(p -> p.initTsBlockBuilder(getTsDataTypeList()));
Expand Down
6 changes: 3 additions & 3 deletions ...rc/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ private static void rewriteStatistics(
TSDataType targetDataType = measurementMap.get(measurement);
if (valueChunkMetadata.getDataType() != targetDataType) {
SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, i, targetDataType);
alignedChunkMetadata.setModified(true);
alignedChunkMetadata.setDataTypeModifiedAndCannotUseStatistics(true);
}
}
}
Expand Down Expand Up @@ -571,7 +571,7 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
return null;
}
chunkMetadataList.set(index, chunkMetadata);
chunkMetadata.setModified(true);
chunkMetadata.setDataTypeModifiedAndCannotUseStatistics(true);
}
if (!useFakeStatistics) {
if (chunkMetadata != null && targetDataType.isCompatible(chunkMetadata.getDataType())) {
Expand All @@ -597,7 +597,7 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
return null;
}
memChunk.setChunkMetadata(rewritedChunkMetadata);
memChunk.getChunkMetaData().setModified(true);
memChunk.getChunkMetaData().setDataTypeModifiedAndCannotUseStatistics(true);
}
if (useFakeStatistics) {
memChunk.initChunkMetaFromTVListsWithFakeStatistics();
Expand Down
5 changes: 5 additions & 0 deletions ...org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,11 @@ public void close() {
// Do nothing because mem chunk reader will not open files
}

@Override
public void markDataTypeModifiedAndCannotUseStatistics() {
// Do nothing because MemPageReader.setModified() is empty
}

@Override
public List<IPageReader> loadPageReaderList() {
return this.pageReaderList;
Expand Down
5 changes: 5 additions & 0 deletions ...n/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemChunkReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,11 @@ public void close() {
// Do nothing because mem chunk reader will not open files
}

@Override
public void markDataTypeModifiedAndCannotUseStatistics() {
// Do nothing because MemPageReader.setModified() is empty
}

@Override
public List<IPageReader> loadPageReaderList() {
return this.pageReaderList;
Expand Down
Loading
Loading