From d084c6051e81fb4074c837b4125ebc182acae254 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Thu, 4 Dec 2025 12:26:31 +0800 Subject: [PATCH] [fix](segmentv3) make it work in cloud (#58681) --- .../org/apache/doris/alter/CloudRollupJobV2.java | 4 ++-- .../doris/alter/CloudSchemaChangeJobV2.java | 2 +- .../doris/cloud/backup/CloudRestoreJob.java | 3 ++- .../cloud/common/util/CloudPropertyAnalyzer.java | 1 - .../cloud/datasource/CloudInternalCatalog.java | 15 +++++++++++++-- .../ext_meta/test_storage_format_v2_1.groovy | 6 +++--- 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 82b137a2c02..2b3a1f8937f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -217,8 +217,8 @@ public class CloudRollupJobV2 extends RollupJobV2 { partitionId, rollupTablet, tabletType, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(), tbl.getBfFpp(), null, rollupSchema, - tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(), - tbl.isInMemory(), true, + tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(), + tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index e0de2b1e3e2..0d02dc1bc48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -239,7 +239,7 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 { tbl.getPartitionInfo().getTabletType(partitionId), shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns, bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(), - tbl.getCompressionType(), + tbl.getCompressionType(), tbl.getStorageFormat(), tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java index 80bd91d9d54..f9b1840f574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java @@ -386,7 +386,8 @@ public class CloudRestoreJob extends RestoreJob { indexMeta.getSchemaHash(), indexMeta.getKeysType(), indexMeta.getShortKeyColumnCount(), localTbl.getCopiedBfColumns(), localTbl.getBfFpp(), indexes, indexMeta.getSchema(), localTbl.getDataSortInfo(), - localTbl.getCompressionType(), localTbl.getStoragePolicy(), + localTbl.getCompressionType(), localTbl.getStorageFormat(), + localTbl.getStoragePolicy(), localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(), localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(), localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java index b41570a25fc..7ac160f4b18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java @@ -30,7 +30,6 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer { forceProperties = ImmutableList.of( RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_INMEMORY), RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM), - RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"), RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY), RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME), RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM), diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index c2359cc59b2..a3ee47960d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -65,6 +65,7 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TTabletType; import com.google.common.base.Preconditions; @@ -171,7 +172,8 @@ public class CloudInternalCatalog extends InternalCatalog { OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId, partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount, bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), - tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(), + tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false, + tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(), tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), @@ -208,7 +210,7 @@ public class CloudInternalCatalog extends InternalCatalog { long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, - String storagePolicy, boolean isInMemory, boolean isShadow, + TStorageFormat storageFormat, String storagePolicy, boolean isInMemory, boolean isShadow, String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, boolean storeRowColumn, int schemaVersion, String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, @@ -319,6 +321,15 @@ public class CloudInternalCatalog extends InternalCatalog { break; } + // Enable external column meta layout when storage_format is V3 (Cloud mode). + switch (storageFormat) { + case V3: + schemaBuilder.setIsExternalSegmentColumnMetaUsed(true); + break; + default: + break; + } + schemaBuilder.setSortColNum(dataSortInfo.getColNum()); for (int i = 0; i < schemaColumns.size(); i++) { Column column = schemaColumns.get(i); diff --git a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy index d7fe1e48e3a..ece39caed92 100644 --- a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy +++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy @@ -19,9 +19,9 @@ suite("test_storage_format_v2_1") { def tableName = "test_storage_format_v2_1_table" // Test 1: Create table with storage_format = V2.1 - sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS test_storage_format_v2_1_table" sql """ - CREATE TABLE ${tableName} ( + CREATE TABLE test_storage_format_v2_1_table ( k bigint, v variant ) @@ -47,7 +47,7 @@ suite("test_storage_format_v2_1") { // Verify table properties def result = sql "SHOW CREATE TABLE ${tableName}" logger.info("Show create table result: ${result}") - assertTrue(result[0][1].contains("V3"), + assertTrue(result[0][1].contains("\"storage_format\" = \"V3\""), "Table should be created with storage_format V3") // Cleanup