nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 07:31:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

[test](be) Migrate variant index storage debugpoint tests (#64348)

Browse Source 
Build a reusable BE index-storage unit-test fixture and
migrate the Variant `variant_p0` debugpoint/index/compaction coverage
that belongs at storage-reader/writer level into focused unit tests.
This PR intentionally keeps the newly-covered product bugs unfixed: the
tests remain as reproducers for Variant sparse stats, Variant
field_pattern index build/drop, patched schema compaction with missing
physical columns, and COUNT_ON_INDEX no-need-read-data behavior.

The migrated coverage includes array inverted-index writer debugpoint
checks, Variant path index lifecycle and compaction coverage,
field-pattern typed index coverage for cumulative/full compaction,
common expression pushdown for `IS NULL` / `IS NOT NULL`, and
`array_contains` common-expression index filtering. Variant and
non-Variant coverage is split into independent test files, with shared
rowset write/read/reload/compact/index-probe helpers kept in
`index_storage_test_util`. The remaining non-test code is limited to
index probe observability needed by the tests.

Follow-up update: refine the probe observability/test framework so unit
tests assert `OlapReaderStatistics::index_probe_events` directly instead
of relying on profile timer snapshots or `last_read_index_id()`. The row
batch model is now index-neutral, synthetic reader-schema injection is
named explicitly, and expr pushdown records structured
source/path/index-id probe events.

The `bloom_filter_must_filter_data` profile/debugpoint portion in
`variant_p0/predefine/test_types_with_indexes_profile.groovy` remains
regression/bloom-filter scope rather than inverted-index storage fixture
scope. Existing BE tests cover bloom-filter reader/writer primitives
separately; this PR migrates the inverted-index storage paths. The Cloud
schema-change transaction-conflict debugpoint remains regression-scope
because it validates cloud schema-change job conflict handling rather
than BE rowset/index storage behavior.
This commit is contained in:
lihangyu authored and GitHub committed 2026-06-25 11:40:28 +08:00
1 parent 1097d51a86
commit 6d4b87692b
24 files changed
+6317 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

be/src/storage/segment/segment.cpp
+1
View File
@@ -321,6 +321,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
// any condition not satisfied, return.
*iter = std::make_unique<EmptySegmentIterator>(*schema);
read_options.stats->filtered_segment_number++;
read_options.stats->rows_stats_filtered += num_rows();
return Status::OK();
}
}
be/test/exprs/function/function_is_null_test.cpp
+6 -5
View File
@@ -174,7 +174,8 @@ TEST_F(FunctionIsNullTest, gc_binlogs_test) {
EXPECT_TRUE(rowset_writer->flush().ok());
EXPECT_TRUE(rowset_writer->build(rowset).ok());
auto check_result = [&](InvertedIndexReader* reader, bool is_null, int expected_result) {
auto check_result = [&](InvertedIndexReader* reader, ColumnId column_id, bool is_null,
int expected_result) {
OlapReaderStatistics stats;
RuntimeState runtime_state;
io::IOContext io_ctx;
@@ -227,8 +228,8 @@ TEST_F(FunctionIsNullTest, gc_binlogs_test) {
auto index_meta = index_metas[0];
auto bkd_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
EXPECT_TRUE(bkd_reader);
check_result(bkd_reader.get(), true, 1);
check_result(bkd_reader.get(), false, 2);
check_result(bkd_reader.get(), 0, true, 1);
check_result(bkd_reader.get(), 0, false, 2);
auto index_metas2 = _tablet_schema->inverted_indexs(1);
EXPECT_FALSE(index_metas2.empty());
@@ -236,8 +237,8 @@ TEST_F(FunctionIsNullTest, gc_binlogs_test) {
auto string_reader =
StringTypeInvertedIndexReader::create_shared(index_meta2, index_file_reader);
EXPECT_TRUE(string_reader);
check_result(string_reader.get(), true, 2);
check_result(string_reader.get(), false, 1);
check_result(string_reader.get(), 1, true, 2);
check_result(string_reader.get(), 1, false, 1);
}
}
be/test/storage/index_storage_lifecycle_test.cpp
+516
View File
@@ -0,0 +1,516 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <algorithm>
#include <map>
#include "core/data_type/data_type_string.h"
#include "storage/predicate/predicate_creator.h"
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
namespace {
std::shared_ptr<ColumnPredicate> title_equals(std::string value) {
return create_comparison_predicate<PredicateType::EQ>(
1, "title", std::make_shared<DataTypeString>(),
Field::create_field<TYPE_STRING>(std::move(value)), false);
}
void expect_applied_title_index(const IndexReadResult& result, int64_t expected_filtered_rows) {
expect_index_filter_stats(result, expected_filtered_rows);
}
} // namespace
class IndexStorageLifecycleTest : public IndexStorageTestFixture {};
TEST_F(IndexStorageLifecycleTest, TextIndexHitAfterCumulativeCompaction) {
const auto index_case =
IndexStorageCaseBuilder("text_index_hit_after_cumulative_compaction")
.tablet_id(110001)
.text_column(TextColumnSpec {.unique_id = 2, .name = "title"})
.inverted_index(IndexSpec::column_index(20001, "idx_title", 2))
.rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0))
.rowset(1, IndexDataSourceSpec::inline_text({"hello", "other"}, 100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets(rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_applied_title_index(read_result.value(), 2);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
IndexReadOptions compacted_read_options;
compacted_read_options.predicates.push_back(title_equals("hello"));
auto compacted_read = read_rowsets(reloaded.value(), compacted_read_options);
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 2);
expect_applied_title_index(compacted_read.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, TextIndexHitAfterFullCompaction) {
IndexTabletOptions options;
options.tablet_id = 110007;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_text({"hello", "other"}, 0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_text({"other", "hello"}, 100));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto compacted = compact_rowsets(IndexCompactionKind::FULL,
{rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({compacted.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_applied_title_index(read_result.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, CountOnIndexPredicateFiltersRows) {
IndexTabletOptions options;
options.tablet_id = 110008;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other", "hello"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
IndexReadOptions read_options;
read_options.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_applied_title_index(read_result.value(), 1);
}
TEST_F(IndexStorageLifecycleTest, DISABLED_CountOnIndexSkipsReadingKeyDataWhenIndexApplied) {
IndexTabletOptions options;
options.tablet_id = 110027;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other", "hello"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
ScopedDebugPoint fail_if_key_column_is_read("segment_iterator._read_columns_by_index",
{{"column_name", "k"}});
IndexReadOptions read_options;
read_options.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
EXPECT_EQ(read_result->stats.raw_rows_read, 2);
expect_applied_title_index(read_result.value(), 1);
}
TEST_F(IndexStorageLifecycleTest, TextPredicateFiltersRowsAfterNullBitmapCheck) {
IndexTabletOptions options;
options.tablet_id = 110010;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(
IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}}));
options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.data_sources.push_back(
IndexDataSourceSpec::inline_text({"hello", "hello world", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
expect_applied_title_index(read_result.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, V1IndexStorageWritesOneFilePerIndex) {
IndexTabletOptions options;
options.tablet_id = 110019;
options.index_storage_format = InvertedIndexStorageFormatPB::V1;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(
IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}}));
options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "hello world", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto probe = probe_rowset(rowset_result.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_EQ(probe->num_segments, 1);
EXPECT_EQ(probe->index_files.expected_files, 2);
expect_index_files(probe.value(), true);
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
expect_applied_title_index(read_result.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, V2IndexStorageWritesCompoundIndexFile) {
IndexTabletOptions options;
options.tablet_id = 110020;
options.index_storage_format = InvertedIndexStorageFormatPB::V2;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(
IndexSpec::column_index(20001, "idx_title_fulltext", 2, {{"parser", "english"}}));
options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_title_string", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "hello world", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto probe = probe_rowset(rowset_result.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_EQ(probe->num_segments, 1);
EXPECT_EQ(probe->index_files.expected_files, 1);
EXPECT_GT(probe->index_files.rowset_meta_entries, 0);
expect_index_files(probe.value(), true);
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
expect_applied_title_index(read_result.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, TextIndexDisabledKeepsFilterStatsAtZero) {
IndexTabletOptions options;
options.tablet_id = 110013;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other", "hello"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
EXPECT_EQ(read_result->stats.rows_inverted_index_filtered, 0);
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageLifecycleTest, TextPredicateWithoutIndexKeepsFilterStatsAtZero) {
IndexTabletOptions options;
options.tablet_id = 110004;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
IndexReadOptions read_options;
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets({rowset_result.value()}, read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
EXPECT_EQ(read_result->stats.rows_inverted_index_filtered, 0);
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageLifecycleTest, DropTextIndexKeepsFilterStatsAtZero) {
const auto index_case =
IndexStorageCaseBuilder("drop_text_index_keeps_filter_stats_at_zero")
.tablet_id(110005)
.text_column(TextColumnSpec {.unique_id = 2, .name = "title"})
.inverted_index(IndexSpec::column_index(20001, "idx_title", 2))
.rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
IndexReadOptions before_drop_options;
before_drop_options.predicates.push_back(title_equals("hello"));
auto before_drop = read_rowsets(rowsets.value(), before_drop_options);
ASSERT_TRUE(before_drop.has_value()) << before_drop.error();
EXPECT_EQ(before_drop->rows_read, 1);
expect_applied_title_index(before_drop.value(), 1);
auto dropped_rowsets = drop_inverted_indexes({IndexSpec::column_index(20001, "idx_title", 2)});
ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error();
ASSERT_EQ(dropped_rowsets->size(), 1);
auto dropped_probe = probe_rowset(dropped_rowsets->front());
ASSERT_TRUE(dropped_probe.has_value()) << dropped_probe.error();
expect_index_files(dropped_probe.value(), false);
auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value());
ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error();
IndexReadOptions after_drop_options;
after_drop_options.predicates.push_back(title_equals("hello"));
auto after_drop = read_rowsets(reloaded_dropped_rowsets.value(), after_drop_options);
ASSERT_TRUE(after_drop.has_value()) << after_drop.error();
EXPECT_EQ(after_drop->rows_read, 1);
EXPECT_EQ(after_drop->stats.rows_inverted_index_filtered, 0);
expect_index_filter_stats(after_drop.value(), 0);
}
TEST_F(IndexStorageLifecycleTest, BuildTextIndexAfterExistingRowsUsesNewIndex) {
const auto index_case =
IndexStorageCaseBuilder("build_text_index_after_existing_rows_uses_new_index")
.tablet_id(110011)
.text_column(TextColumnSpec {.unique_id = 2, .name = "title"})
.rowset(0, IndexDataSourceSpec::inline_text({"hello", "other"}, 0))
.rowset(1, IndexDataSourceSpec::inline_text({"other", "hello"}, 100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
IndexReadOptions before_build_options;
before_build_options.predicates.push_back(title_equals("hello"));
auto before_build = read_rowsets(rowsets.value(), before_build_options);
ASSERT_TRUE(before_build.has_value()) << before_build.error();
EXPECT_EQ(before_build->rows_read, 2);
EXPECT_EQ(before_build->stats.rows_inverted_index_filtered, 0);
expect_index_filter_stats(before_build.value(), 0);
const auto built_index = IndexSpec::column_index(20003, "idx_title_built", 2);
auto built_rowsets = build_inverted_indexes({built_index});
ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error();
ASSERT_EQ(built_rowsets->size(), 2);
for (const auto& rowset : built_rowsets.value()) {
auto probe = probe_rowset(rowset);
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), true);
}
auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value());
ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error();
IndexReadOptions after_build_options;
after_build_options.predicates.push_back(title_equals("hello"));
auto after_build = read_rowsets(reloaded_built_rowsets.value(), after_build_options);
ASSERT_TRUE(after_build.has_value()) << after_build.error();
EXPECT_EQ(after_build->rows_read, 2);
expect_applied_title_index(after_build.value(), 2);
}
TEST_F(IndexStorageLifecycleTest, PatchedSchemaKeepsExistingTextIndexUsable) {
IndexTabletOptions options;
options.tablet_id = 110015;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
IndexSchemaPatch patch;
patch.add_text_columns.push_back(
TextColumnSpec {.unique_id = 3, .name = "body", .nullable = true});
auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_column_unique_id(3));
auto patched_rowsets =
inject_reader_schema_for_rowsets({rowset_result.value()}, std::move(patched_schema));
ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error();
IndexReadOptions read_options;
read_options.return_columns = {0, 1};
read_options.predicates.push_back(title_equals("hello"));
auto read_result = read_rowsets(patched_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
expect_applied_title_index(read_result.value(), 1);
}
TEST_F(IndexStorageLifecycleTest, InjectReaderSchemaUpdatesTabletRowsetMapForIndexDrop) {
IndexTabletOptions options;
options.tablet_id = 110019;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto index = IndexSpec::column_index(20008, "idx_title_patch_drop", 2);
IndexSchemaPatch patch;
patch.add_inverted_indexes.push_back(index);
auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_inverted_index_with_index_id(20008));
auto patched_rowsets =
inject_reader_schema_for_rowsets({rowset_result.value()}, std::move(patched_schema));
ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error();
ASSERT_EQ(patched_rowsets->size(), 1);
ASSERT_TRUE(patched_rowsets->front()->tablet_schema()->has_inverted_index_with_index_id(20008));
auto dropped_rowsets = drop_inverted_indexes({index});
ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error();
ASSERT_EQ(dropped_rowsets->size(), 1);
EXPECT_FALSE(
dropped_rowsets->front()->tablet_schema()->has_inverted_index_with_index_id(20008));
}
TEST_F(IndexStorageLifecycleTest, PatchedSchemaAddDropTextColumnReadsDefaultsAndCompacts) {
IndexTabletOptions options;
options.tablet_id = 110016;
options.text_columns = {
TextColumnSpec {.unique_id = 2, .name = "title"},
TextColumnSpec {.unique_id = 3, .name = "obsolete", .nullable = true},
};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_obsolete", 3));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
IndexBatch batch0;
batch0.keys = {0, 1};
batch0.text_values_by_column = {{"hello", "other"}, {"drop-a", "drop-b"}};
rowset0.batches.push_back(std::move(batch0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
IndexBatch batch1;
batch1.keys = {2, 3};
batch1.text_values_by_column = {{"hello", "later"}, {"drop-c", "drop-d"}};
rowset1.batches.push_back(std::move(batch1));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
IndexSchemaPatch patch;
patch.drop_column_uids.insert(3);
patch.add_text_columns.push_back(
TextColumnSpec {.unique_id = 4, .name = "body", .nullable = true});
auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch);
ASSERT_NE(patched_schema, nullptr);
EXPECT_FALSE(patched_schema->has_column_unique_id(3));
ASSERT_TRUE(patched_schema->has_column_unique_id(4));
auto patched_rowsets = inject_reader_schema_for_rowsets(
{rowset0_result.value(), rowset1_result.value()}, std::move(patched_schema));
ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error();
IndexReadOptions read_options;
read_options.collect_string_values = true;
read_options.need_ordered_result = true;
read_options.return_columns = {1, 2};
auto read_result = read_rowsets(patched_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 4);
ASSERT_TRUE(read_result->string_values_by_uid.contains(2));
ASSERT_TRUE(read_result->string_values_by_uid.contains(4));
EXPECT_EQ(read_result->string_values_by_uid.at(2),
(std::vector<std::optional<std::string>> {"hello", "other", "hello", "later"}));
EXPECT_EQ(read_result->string_values_by_uid.at(4),
(std::vector<std::optional<std::string>> {std::nullopt, std::nullopt, std::nullopt,
std::nullopt}));
IndexReadOptions index_read_options;
index_read_options.predicates.push_back(title_equals("hello"));
auto index_read = read_rowsets(patched_rowsets.value(), index_read_options);
ASSERT_TRUE(index_read.has_value()) << index_read.error();
EXPECT_EQ(index_read->rows_read, 2);
expect_applied_title_index(index_read.value(), 2);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, patched_rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
IndexReadOptions compacted_read_options;
compacted_read_options.collect_string_values = true;
compacted_read_options.need_ordered_result = true;
compacted_read_options.return_columns = {1, 2};
auto compacted_read = read_rowsets({compacted.value()}, compacted_read_options);
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 4);
ASSERT_TRUE(compacted_read->string_values_by_uid.contains(2));
ASSERT_TRUE(compacted_read->string_values_by_uid.contains(4));
EXPECT_EQ(compacted_read->string_values_by_uid.at(2),
(std::vector<std::optional<std::string>> {"hello", "other", "hello", "later"}));
EXPECT_EQ(compacted_read->string_values_by_uid.at(4),
(std::vector<std::optional<std::string>> {std::nullopt, std::nullopt, std::nullopt,
std::nullopt}));
}
} // namespace doris::index_storage_test
be/test/storage/index_storage_schema_patch_test.cpp
+53
View File
@@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
TEST(IndexStorageSchemaPatchTest, AddAndDropTextColumns) {
IndexTabletOptions options;
options.tablet_id = 110014;
options.text_columns = {
TextColumnSpec {.unique_id = 2, .name = "title"},
TextColumnSpec {.unique_id = 3, .name = "obsolete"},
};
options.inverted_indexes.push_back(IndexSpec::column_index(20001, "idx_title", 2));
options.inverted_indexes.push_back(IndexSpec::column_index(20002, "idx_obsolete", 3));
auto base_schema = build_tablet_schema(options);
ASSERT_NE(base_schema, nullptr);
IndexSchemaPatch patch;
patch.drop_column_uids.insert(3);
patch.add_text_columns.push_back(TextColumnSpec {.unique_id = 4, .name = "body"});
auto patched_schema = build_patched_tablet_schema(*base_schema, patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_column_unique_id(2));
EXPECT_FALSE(patched_schema->has_column_unique_id(3));
ASSERT_TRUE(patched_schema->has_column_unique_id(4));
const auto& body = patched_schema->column_by_uid(4);
EXPECT_EQ(body.name(), "body");
EXPECT_EQ(body.type(), FieldType::OLAP_FIELD_TYPE_STRING);
EXPECT_EQ(patched_schema->next_column_unique_id(), 5);
EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(3, "").empty());
}
} // namespace doris::index_storage_test
be/test/storage/test_data/index_storage/variant_path_index.jsonl
+2
View File
@@ -0,0 +1,2 @@
{"b": "one"}
{"b": "other"}
be/test/storage/variant/index_storage_variant_compaction_read_test.cpp
+283
View File
@@ -0,0 +1,283 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "storage/segment/variant/nested_group_provider.h"
#include "storage/variant/index_storage_variant_test_base.h"
namespace doris::index_storage_test {
static bool nested_group_write_path_available() {
auto provider = segment_v2::create_nested_group_read_provider();
return provider != nullptr && provider->should_enable_nested_group_read_path();
}
class IndexStorageVariantCompactionReadTest : public IndexStorageTestFixture {
protected:
void run_deep_sparse_variant_lifecycle(bool external_segment_meta, int64_t tablet_id);
void run_nested_group_variant_lifecycle(bool external_segment_meta, int64_t tablet_id);
};
void IndexStorageVariantCompactionReadTest::run_deep_sparse_variant_lifecycle(
bool external_segment_meta, int64_t tablet_id) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.sparse_hash_shard_count = 2;
IndexTabletOptions options;
options.tablet_id = tablet_id;
options.external_segment_meta = external_segment_meta;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h0", "deep": {"rare_a": "a0"}, "cold0": "c0"})",
R"({"hot": "h1", "deep": {"rare_b": "b0"}, "cold1": "c1"})"},
0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
auto rowset0_probe = probe_rowset(rowset0_result.value());
ASSERT_TRUE(rowset0_probe.has_value()) << rowset0_probe.error();
EXPECT_TRUE(has_variant_layout(rowset0_probe.value(), 2, "hot"));
EXPECT_TRUE(has_sparse_path_stat(rowset0_probe.value(), "deep.rare_a"));
EXPECT_TRUE(has_sparse_path_stat(rowset0_probe.value(), "deep.rare_b"));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h2", "deep": {"rare_a": "a1"}, "cold2": "c2"})",
R"({"hot": "h3", "deep": {"rare_c": "c0"}, "cold3": "c3"})"},
100));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto read_result = read_rowsets({rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 4);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE,
{rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "hot"));
EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_a"));
EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_b"));
EXPECT_TRUE(has_sparse_path_stat(compacted_probe.value(), "deep.rare_c"));
auto compacted_read = read_rowsets({compacted.value()});
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 4);
}
void IndexStorageVariantCompactionReadTest::run_nested_group_variant_lifecycle(
bool external_segment_meta, int64_t tablet_id) {
if (!nested_group_write_path_available()) {
GTEST_SKIP() << "NestedGroup write path is not available in this build";
}
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.sparse_hash_shard_count = 2;
variant.enable_nested_group = true;
IndexTabletOptions options;
options.tablet_id = tablet_id;
options.external_segment_meta = external_segment_meta;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_variant(
{R"({"owner": "alice", "profile": {"region": "us"}, "items": [{"sku": "a", "qty": 1}]})",
R"({"owner": "bob", "profile": {"region": "eu"}, "items": [{"sku": "b", "qty": 2}]})"},
0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"owner": "carol", "profile": {"region": "apac"}, "items": [{"sku": "c", "qty": 3}]})",
R"({"owner": "dave", "profile": {"region": "us"}, "items": [{"sku": "d", "qty": 4}]})"},
100));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE,
{rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
ASSERT_TRUE(compacted.value()->tablet_schema()->has_column_unique_id(2));
EXPECT_TRUE(compacted.value()->tablet_schema()->column_by_uid(2).variant_enable_nested_group());
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
EXPECT_TRUE(has_variant_parent(compacted_probe.value(), 2));
EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "owner"));
EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 2, "profile.region"));
auto compacted_read = read_rowsets({compacted.value()});
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 4);
}
TEST_F(IndexStorageVariantCompactionReadTest, WriteReadProbeAndCumulativeCompact) {
IndexTabletOptions options;
options.tablet_id = 110002;
options.external_segment_meta = true;
options.variant_columns = {VariantColumnSpec {}};
options.variant_columns[0].unique_id = 2;
options.variant_columns[0].name = "v";
options.variant_columns[0].max_subcolumns_count = 4;
options.variant_columns[0].sparse_hash_shard_count = 2;
options.variant_columns[0].predefined_paths = {
VariantPathSpec {.path = "a",
.type = FieldType::OLAP_FIELD_TYPE_INT,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(
IndexBatch::single_variant({R"({"a": 1, "b": "one"})", R"({"a": 2, "c": 20})"}, 0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(
IndexBatch::single_variant({R"({"a": 3, "b": "three"})", R"({"a": 4, "d": 40})"}, 100));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto read_result = read_rowsets({rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 4);
auto probe_result = probe_rowset(rowset0_result.value());
ASSERT_TRUE(probe_result.has_value()) << probe_result.error();
EXPECT_EQ(probe_result->num_rows, 2);
EXPECT_EQ(probe_result->num_segments, 1);
EXPECT_TRUE(probe_result->contains_relative_path("a"));
EXPECT_TRUE(probe_result->contains_relative_path("b"));
expect_index_files(probe_result.value(), false);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE,
{rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
EXPECT_TRUE(compacted_probe->contains_relative_path("a"));
EXPECT_TRUE(compacted_probe->contains_relative_path("b"));
}
TEST_F(IndexStorageVariantCompactionReadTest, DeepSparseVariantCompactsWithExternalSegmentMeta) {
run_deep_sparse_variant_lifecycle(true, 110023);
}
TEST_F(IndexStorageVariantCompactionReadTest, DeepSparseVariantCompactsWithoutExternalSegmentMeta) {
run_deep_sparse_variant_lifecycle(false, 110024);
}
TEST_F(IndexStorageVariantCompactionReadTest, NestedGroupVariantCompactsWithExternalSegmentMeta) {
run_nested_group_variant_lifecycle(true, 110025);
}
TEST_F(IndexStorageVariantCompactionReadTest,
NestedGroupVariantCompactsWithoutExternalSegmentMeta) {
run_nested_group_variant_lifecycle(false, 110026);
}
TEST_F(IndexStorageVariantCompactionReadTest, VariantDocModeWritesDocValueColumnsAfterCompaction) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 4;
variant.enable_doc_mode = true;
variant.doc_materialization_min_rows = 100000;
variant.doc_hash_shard_count = 2;
IndexTabletOptions options;
options.tablet_id = 110012;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(
IndexBatch::single_variant({R"({"a": "one", "b": 1})", R"({"a": "two", "c": 2})"}, 0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"a": "three", "d": 3})", R"({"a": "four", "e": 4})"}, 100));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto before_compaction_read = read_rowsets({rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(before_compaction_read.has_value()) << before_compaction_read.error();
EXPECT_EQ(before_compaction_read->rows_read, 4);
auto rowset0_probe = probe_rowset(rowset0_result.value());
ASSERT_TRUE(rowset0_probe.has_value()) << rowset0_probe.error();
EXPECT_TRUE(has_doc_value_column(rowset0_probe.value()));
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE,
{rowset0_result.value(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
EXPECT_TRUE(has_doc_value_column(compacted_probe.value()));
auto compacted_read = read_rowsets({compacted.value()});
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 4);
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_compaction_schema_test.cpp
+208
View File
@@ -0,0 +1,208 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "exec/common/variant_util.h"
#include "storage/variant/index_storage_variant_test_base.h"
namespace doris::index_storage_test {
class IndexStorageVariantCompactionSchemaTest : public IndexStorageTestFixture {
protected:
};
TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsHonorsZeroLimitAndTieOrdering) {
variant_util::PathToNoneNullValues stats {
{"alpha", 3},
{"beta", 3},
{"gamma", 1},
};
TabletSchema::PathsSetInfo unlimited;
variant_util::VariantCompactionUtil::get_subpaths(0, stats, unlimited);
EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("beta")));
EXPECT_TRUE(unlimited.sub_path_set.contains(StringRef("gamma")));
EXPECT_TRUE(unlimited.sparse_path_set.empty());
TabletSchema::PathsSetInfo top_one;
variant_util::VariantCompactionUtil::get_subpaths(1, stats, top_one);
EXPECT_TRUE(top_one.sub_path_set.contains(StringRef("beta")));
EXPECT_FALSE(top_one.sub_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(top_one.sparse_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(top_one.sparse_path_set.contains(StringRef("gamma")));
}
TEST(IndexStorageVariantCompactionUtilTest, GetSubpathsKeepsAllPathsAtLimitAndHandlesEmptyStats) {
variant_util::PathToNoneNullValues exact_limit {
{"alpha", 2},
{"beta", 1},
};
TabletSchema::PathsSetInfo at_limit;
variant_util::VariantCompactionUtil::get_subpaths(2, exact_limit, at_limit);
EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(at_limit.sub_path_set.contains(StringRef("beta")));
EXPECT_TRUE(at_limit.sparse_path_set.empty());
TabletSchema::PathsSetInfo empty;
variant_util::VariantCompactionUtil::get_subpaths(1, {}, empty);
EXPECT_TRUE(empty.sub_path_set.empty());
EXPECT_TRUE(empty.sparse_path_set.empty());
}
TEST(IndexStorageVariantCompactionUtilTest, EmptyInputsKeepVariantSchemaWithoutPathStats) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
IndexTabletOptions options;
options.tablet_id = 110054;
options.variant_columns = {std::move(variant)};
auto base_schema = build_tablet_schema(options);
ASSERT_NE(base_schema, nullptr);
EXPECT_EQ(
variant_util::VariantCompactionUtil::calculate_variant_extended_schema({}, base_schema),
nullptr);
auto compaction_schema = std::make_shared<TabletSchema>(*base_schema);
auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(
{}, compaction_schema);
ASSERT_TRUE(status.ok()) << status.to_string();
ASSERT_TRUE(compaction_schema->has_column_unique_id(2));
const auto* path_set_info = compaction_schema->try_path_set_info(2);
ASSERT_NE(path_set_info, nullptr);
EXPECT_TRUE(path_set_info->typed_path_set.empty());
EXPECT_TRUE(path_set_info->sub_path_set.empty());
EXPECT_TRUE(path_set_info->sparse_path_set.empty());
}
TEST_F(IndexStorageVariantCompactionSchemaTest,
VariantMaxSubcolumnsCountZeroMaterializesAllObservedPaths) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 0;
IndexTabletOptions options;
options.tablet_id = 110049;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_variant(
{R"({"alpha": "a0", "beta": "b0"})", R"({"alpha": "a1", "gamma": "g1"})"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto compaction_schema = std::make_shared<TabletSchema>(*tablet_schema());
auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(
{rowset_result.value()}, compaction_schema);
ASSERT_TRUE(status.ok()) << status.to_string();
const auto* path_set_info = compaction_schema->try_path_set_info(2);
ASSERT_NE(path_set_info, nullptr);
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta")));
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("gamma")));
EXPECT_TRUE(path_set_info->sparse_path_set.empty());
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "alpha"));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "beta"));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "gamma"));
}
TEST_F(IndexStorageVariantCompactionSchemaTest,
VariantPathCountAtLimitMaterializesAllObservedPaths) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 2;
IndexTabletOptions options;
options.tablet_id = 110050;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_variant(
{R"({"alpha": "a0", "beta": "b0"})", R"({"alpha": "a1", "beta": "b1"})"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto compaction_schema = std::make_shared<TabletSchema>(*tablet_schema());
auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(
{rowset_result.value()}, compaction_schema);
ASSERT_TRUE(status.ok()) << status.to_string();
const auto* path_set_info = compaction_schema->try_path_set_info(2);
ASSERT_NE(path_set_info, nullptr);
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("alpha")));
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("beta")));
EXPECT_TRUE(path_set_info->sparse_path_set.empty());
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "alpha"));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "beta"));
}
TEST_F(IndexStorageVariantCompactionSchemaTest,
VariantTypedPathAndDynamicSubpathStaySeparatedInCompactionSchema) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 2;
variant.predefined_paths = {
VariantPathSpec {.path = "typed_i",
.type = FieldType::OLAP_FIELD_TYPE_INT,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
IndexTabletOptions options;
options.tablet_id = 110052;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_variant(
{R"({"typed_i": 1, "hot": "h0"})", R"({"typed_i": 2, "cold": "c1"})"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto compaction_schema = std::make_shared<TabletSchema>(*tablet_schema());
auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(
{rowset_result.value()}, compaction_schema);
ASSERT_TRUE(status.ok()) << status.to_string();
const auto* path_set_info = compaction_schema->try_path_set_info(2);
ASSERT_NE(path_set_info, nullptr);
EXPECT_TRUE(path_set_info->typed_path_set.contains("typed_i"));
EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("typed_i")));
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("hot")));
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("cold")));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "typed_i"));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "hot"));
EXPECT_TRUE(schema_has_extracted_variant_path(*compaction_schema, 2, "cold"));
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_debug_point_test.cpp
+210
View File
@@ -0,0 +1,210 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <algorithm>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include "core/column/column_array.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/column/column_variant.h"
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_string.h"
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
namespace {
constexpr int32_t kVariantUid = 2;
constexpr int64_t kArrayPathIndexId = 210001;
constexpr std::string_view kArrayPath = "c_arr";
VariantColumnSpec array_text_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.nullable = true;
variant.max_subcolumns_count = 10;
variant.predefined_paths = {
VariantPathSpec {.path = std::string(kArrayPath),
.type = FieldType::OLAP_FIELD_TYPE_ARRAY,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = FieldType::OLAP_FIELD_TYPE_STRING,
.array_item_nullable = true},
};
return variant;
}
ColumnPtr single_array_variant_column(const std::vector<std::optional<std::string>>& elements,
bool array_is_null = false) {
auto values = ColumnString::create();
auto item_null_map = ColumnUInt8::create();
if (!array_is_null) {
for (const auto& element : elements) {
if (element.has_value()) {
values->insert_data(element->data(), element->size());
item_null_map->insert_value(0);
} else {
values->insert_default();
item_null_map->insert_value(1);
}
}
}
auto nullable_items = ColumnNullable::create(std::move(values), std::move(item_null_map));
auto offsets = ColumnArray::ColumnOffsets::create();
offsets->insert_value(array_is_null ? 0 : elements.size());
auto array = ColumnArray::create(std::move(nullable_items), std::move(offsets));
auto array_null_map = ColumnUInt8::create();
array_null_map->insert_value(array_is_null ? 1 : 0);
auto nullable_array = ColumnNullable::create(std::move(array), std::move(array_null_map));
auto variant = ColumnVariant::create(10, false);
variant->insert_default();
auto array_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeArray>(
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())));
CHECK(variant->add_sub_column(PathInData(std::string(kArrayPath), true),
std::move(nullable_array), array_type));
return variant;
}
ColumnPtr single_null_variant_column() {
auto variant = ColumnVariant::create(10, false);
variant->insert_default();
auto null_map = ColumnUInt8::create();
null_map->insert_value(1);
return ColumnNullable::create(std::move(variant), std::move(null_map));
}
IndexRowsetSpec single_array_variant_rowset(int64_t version,
std::vector<std::optional<std::string>> elements,
bool array_is_null = false) {
IndexRowsetSpec rowset;
rowset.version = version;
rowset.batches.push_back(IndexBatch::single_variant_column(
single_array_variant_column(elements, array_is_null), static_cast<int32_t>(version)));
return rowset;
}
IndexRowsetSpec single_null_variant_rowset(int64_t version) {
IndexRowsetSpec rowset;
rowset.version = version;
rowset.batches.push_back(IndexBatch::single_variant_column(single_null_variant_column(),
static_cast<int32_t>(version)));
return rowset;
}
} // namespace
class IndexStorageVariantDebugPointTest : public IndexStorageTestFixture {
protected:
RowsetSharedPtr write_array_rowset_with_expected_field_count(
int64_t version, std::vector<std::optional<std::string>> elements,
int32_t expected_field_count, bool array_is_null = false) {
ScopedDebugPoint debug_point(
"array_inverted_index.write_index",
{{"single_array_field_count", std::to_string(expected_field_count)}});
auto rowset = write_rowset(
single_array_variant_rowset(version, std::move(elements), array_is_null));
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_TRUE(rowset.has_value()) << rowset.error();
if (!rowset.has_value()) {
return nullptr;
}
auto probe = probe_rowset(rowset.value());
EXPECT_TRUE(probe.has_value()) << probe.error();
if (probe.has_value()) {
expect_index_files(probe.value(), true);
}
return rowset.value();
}
RowsetSharedPtr write_null_variant_rowset_with_array_debug_point(int64_t version) {
ScopedDebugPoint debug_point("array_inverted_index.write_index",
{{"single_array_field_count", "0"}});
auto rowset = write_rowset(single_null_variant_rowset(version));
EXPECT_TRUE(rowset.has_value()) << rowset.error();
if (!rowset.has_value()) {
return nullptr;
}
return rowset.value();
}
};
TEST_F(IndexStorageVariantDebugPointTest, ArrayPathIndexWriteCountsNonNullElements) {
IndexTabletOptions options;
options.tablet_id = 110029;
options.variant_columns = {array_text_variant_column()};
options.inverted_indexes.push_back(IndexSpec::field_pattern_index(
kArrayPathIndexId, "index_inverted_c_arr", kVariantUid, std::string(kArrayPath)));
ASSERT_TRUE(create_tablet(options).ok());
std::vector<RowsetSharedPtr> rowsets;
rowsets.push_back(
write_array_rowset_with_expected_field_count(0, {"amory", "is", "committer"}, 3));
rowsets.push_back(write_array_rowset_with_expected_field_count(1, {"amory", "better"}, 2));
rowsets.push_back(write_array_rowset_with_expected_field_count(2, {"amory", std::nullopt}, 1));
rowsets.push_back(
write_array_rowset_with_expected_field_count(3, {std::nullopt, std::nullopt}, 0));
rowsets.push_back(write_array_rowset_with_expected_field_count(4, {}, 0));
rowsets.push_back(write_array_rowset_with_expected_field_count(5, {}, 0, true));
rowsets.push_back(write_null_variant_rowset_with_array_debug_point(6));
ASSERT_TRUE(std::all_of(rowsets.begin(), rowsets.end(),
[](const auto& rowset) { return rowset != nullptr; }));
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets);
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 7);
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
expect_index_files(compacted_probe.value(), true);
}
TEST_F(IndexStorageVariantDebugPointTest, ArrayPathIndexAcceptsMixedTypedElements) {
IndexTabletOptions options;
options.tablet_id = 110030;
options.variant_columns = {array_text_variant_column()};
options.inverted_indexes.push_back(IndexSpec::field_pattern_index(
kArrayPathIndexId, "index_inverted_c_arr", kVariantUid, std::string(kArrayPath)));
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_variant(
{R"({"c_arr": ["text"]})", R"({"c_arr": [1.1]})", R"({"c_arr": [1.0]})",
R"({"c_arr": [90]})", R"({"c_arr": [90999999999999]})"},
0));
rowset.batches.back().parse_to = ParseConfig::ParseTo::OnlyDocValueColumn;
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
auto probe = probe_rowset(rowset_result.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), true);
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_dynamic_path_pruning_test.cpp
+333
View File
@@ -0,0 +1,333 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "common/config.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "storage/predicate/predicate_creator.h"
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
namespace {
constexpr int32_t kVariantUid = 2;
constexpr int64_t kPagePruneLowValueBound = 900000000000LL;
int64_t page_prune_offset(size_t row) {
uint64_t mixed = static_cast<uint64_t>(row + 1) * 11400714819323198485ull;
mixed ^= mixed >> 33;
return static_cast<int64_t>(mixed % static_cast<uint64_t>(kPagePruneLowValueBound));
}
std::shared_ptr<ColumnPredicate> bigint_equals(int32_t column_id, std::string column_name,
int64_t value) {
return create_comparison_predicate<PredicateType::EQ>(
column_id, std::move(column_name), std::make_shared<DataTypeInt64>(),
Field::create_field<TYPE_BIGINT>(value), false);
}
std::shared_ptr<ColumnPredicate> bigint_greater(int32_t column_id, std::string column_name,
int64_t value) {
return create_comparison_predicate<PredicateType::GT>(
column_id, std::move(column_name), std::make_shared<DataTypeInt64>(),
Field::create_field<TYPE_BIGINT>(value), false);
}
void expect_optional_page_zone_map_filter_stats(const IndexReadResult& result,
int64_t selected_rows, int64_t total_rows,
int64_t max_filtered_rows) {
EXPECT_GE(result.stats.raw_rows_read, selected_rows);
EXPECT_LE(result.stats.raw_rows_read, total_rows);
EXPECT_GE(result.stats.rows_stats_filtered, 0);
EXPECT_LE(result.stats.rows_stats_filtered, max_filtered_rows);
if (result.stats.raw_rows_read < total_rows) {
EXPECT_GT(result.stats.rows_stats_filtered, 0);
}
}
DataTypePtr nullable_int64_target_type() {
return make_nullable(std::make_shared<DataTypeInt64>());
}
VariantColumnSpec dynamic_variant_column(bool is_bf_column = false) {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.is_bf_column = is_bf_column;
variant.max_subcolumns_count = 4;
return variant;
}
std::vector<std::string> split_dynamic_variant_rows(size_t low_rows, int64_t low_value,
size_t high_rows, int64_t high_value) {
// Keep both ranges in one data source. The fixture flushes the rowset writer after
// each data source, so splitting low/high ranges into separate sources creates two
// segments and lets segment ZoneMap prune before page ZoneMap or BloomFilter can run.
std::vector<std::string> rows;
rows.reserve(low_rows + high_rows);
for (size_t i = 0; i < low_rows; ++i) {
rows.push_back(R"({"dynamic_i": )" + std::to_string(low_value + page_prune_offset(i)) +
"}");
}
for (size_t i = 0; i < high_rows; ++i) {
rows.push_back(R"({"dynamic_i": )" +
std::to_string(high_value + page_prune_offset(low_rows + i)) + "}");
}
return rows;
}
std::vector<std::string> interleaved_dynamic_variant_rows(size_t pairs, int64_t low_value,
int64_t high_value) {
// BF assertions must keep both segment and page ZoneMaps matched. Alternating values keeps each
// page range wide enough that ZoneMap cannot prune before the BloomFilter reader is exercised.
std::vector<std::string> rows;
rows.reserve(pairs * 2);
for (size_t i = 0; i < pairs; ++i) {
rows.push_back(R"({"dynamic_i": )" + std::to_string(low_value) + "}");
rows.push_back(R"({"dynamic_i": )" + std::to_string(high_value) + "}");
}
return rows;
}
class IndexStorageVariantDynamicPathPruningTest : public IndexStorageTestFixture {
protected:
void SetUp() override {
IndexStorageTestFixture::SetUp();
_old_zone_map_row_num_threshold = config::zone_map_row_num_threshold;
// Keep zone maps available on small test pages when the compacted layout exposes them.
config::zone_map_row_num_threshold = 20;
}
void TearDown() override {
config::zone_map_row_num_threshold = _old_zone_map_row_num_threshold;
IndexStorageTestFixture::TearDown();
}
void verify_dynamic_path_filter(const std::vector<RowsetSharedPtr>& rowsets,
const std::function<void(const IndexReadResult&)>& verify,
int64_t greater_than = 50) {
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets);
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.dynamic_i");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] =
nullable_int64_target_type();
read_options.predicates.push_back(
bigint_greater(path_column_id, path_column.name(), greater_than));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
verify(read_result.value());
expect_index_filter_stats(read_result.value(), 0);
}
private:
int32_t _old_zone_map_row_num_threshold = 20;
};
TEST_F(IndexStorageVariantDynamicPathPruningTest,
DynamicPathSegmentZoneMapPrunesBeforeCompactionWithoutPredefinedTypedPath) {
const auto index_case =
IndexStorageCaseBuilder("variant_dynamic_path_segment_zone_map_prune")
.tablet_id(110060)
.variant_column(dynamic_variant_column())
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"dynamic_i": 1})", R"({"dynamic_i": 2})",
R"({"dynamic_i": 100})", R"({"dynamic_i": 101})"},
0),
2)
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
verify_dynamic_path_filter(rowsets.value(), [=](const IndexReadResult& result) {
EXPECT_EQ(result.rows_read, 2);
expect_raw_rows_read(result, 2);
expect_segment_pruned(result, 1);
});
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
verify_dynamic_path_filter(reloaded.value(), [=](const IndexReadResult& result) {
EXPECT_EQ(result.rows_read, 2);
EXPECT_GT(result.stats.raw_rows_read, 0);
});
}
// Compaction may merge low-value and high-value input segments into one output segment. In that
// layout the segment ZoneMap spans both values and cannot be expected to prune the low rows.
TEST_F(IndexStorageVariantDynamicPathPruningTest,
DynamicPathSegmentZoneMapAfterCompactionKeepsPathReadableWithoutPredefinedTypedPath) {
const auto index_case =
IndexStorageCaseBuilder("variant_dynamic_path_segment_zone_map_after_compaction")
.tablet_id(110063)
.variant_column(dynamic_variant_column())
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"dynamic_i": 1})", R"({"dynamic_i": 2})",
R"({"dynamic_i": 100})", R"({"dynamic_i": 101})"},
0),
2)
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
verify_dynamic_path_filter(reloaded.value(), [=](const IndexReadResult& result) {
EXPECT_EQ(result.rows_read, 2);
EXPECT_GT(result.stats.raw_rows_read, 0);
});
}
TEST_F(IndexStorageVariantDynamicPathPruningTest,
DynamicPathPageZoneMapPrunesWithoutPredefinedTypedPath) {
constexpr size_t kLowRows = 2048;
constexpr size_t kHighRows = 2048;
constexpr int64_t kLowValueBase = 1;
constexpr int64_t kHighValueBase = 2000000000000LL;
constexpr int64_t kPagePruneThreshold = 1000000000000LL;
IndexTabletOptions options;
options.tablet_id = 110061;
options.storage_page_size = 4096;
options.variant_columns.push_back(dynamic_variant_column());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.max_rows_per_segment = static_cast<int64_t>(kLowRows + kHighRows);
rowset.data_sources.push_back(IndexDataSourceSpec::inline_variant(
split_dynamic_variant_rows(kLowRows, kLowValueBase, kHighRows, kHighValueBase), 0));
ASSERT_TRUE(create_tablet(options).ok());
auto rowsets = write_rowsets({rowset});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
const auto total_rows = static_cast<int64_t>(kLowRows + kHighRows);
const auto high_rows = static_cast<int64_t>(kHighRows);
verify_dynamic_path_filter(
rowsets.value(),
[=](const IndexReadResult& result) {
EXPECT_EQ(result.rows_read, kHighRows);
expect_segment_pruned(result, 0);
EXPECT_GE(result.stats.raw_rows_read, high_rows);
EXPECT_LE(result.stats.raw_rows_read, total_rows);
},
kPagePruneThreshold);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
verify_dynamic_path_filter(
reloaded.value(),
[=](const IndexReadResult& result) {
EXPECT_EQ(result.rows_read, kHighRows);
expect_segment_pruned(result, 0);
// Compaction can produce a layout where page ZoneMap pruning is not triggered.
// Keep the result contract strict and validate stats only when raw_rows_read
// proves page pruning actually happened.
expect_optional_page_zone_map_filter_stats(result, high_rows, total_rows,
static_cast<int64_t>(kLowRows));
},
kPagePruneThreshold);
}
TEST_F(IndexStorageVariantDynamicPathPruningTest,
DynamicPathBloomFilterPrunesWithoutPredefinedTypedPath) {
constexpr size_t kLowRows = 2048;
constexpr size_t kHighRows = 2048;
constexpr int64_t kMissingValueInsideSegmentZoneMap = 50;
IndexTabletOptions options;
options.tablet_id = 110062;
options.storage_page_size = 4096;
options.variant_columns.push_back(dynamic_variant_column(true));
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.max_rows_per_segment = static_cast<int64_t>(kLowRows + kHighRows);
rowset.data_sources.push_back(IndexDataSourceSpec::inline_variant(
interleaved_dynamic_variant_rows(kLowRows, 1, 100), 0));
ASSERT_TRUE(create_tablet(options).ok());
auto rowsets = write_rowsets({rowset});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto verify_bloom_filter = [&](const std::vector<RowsetSharedPtr>& target_rowsets) {
auto readable_rowsets = rowsets_with_variant_extended_schema(target_rowsets);
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.dynamic_i");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] =
nullable_int64_target_type();
read_options.predicates.push_back(bigint_equals(path_column_id, path_column.name(),
kMissingValueInsideSegmentZoneMap));
ScopedDebugPoint must_filter("bloom_filter_must_filter_data");
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 0);
expect_bloom_filter_filtered(read_result.value(), kLowRows + kHighRows);
expect_index_filter_stats(read_result.value(), 0);
};
verify_bloom_filter(rowsets.value());
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
verify_bloom_filter(reloaded.value());
}
} // namespace
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_expr_pushdown_test.cpp
+471
View File
@@ -0,0 +1,471 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/primitive_type.h"
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr_context.h"
#include "exprs/vliteral.h"
#include "testutil/index_storage_test_util.h"
#include "testutil/mock/mock_slot_ref.h"
namespace doris::index_storage_test {
namespace {
constexpr int32_t kVariantUid = 2;
constexpr int64_t kIntPatternIndexId = 220201;
constexpr int64_t kArrayPatternIndexId = 220202;
constexpr int64_t kSparseIntPatternIndexId = 220203;
constexpr std::string_view kArrayPath = "inventors";
VariantColumnSpec typed_pattern_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.max_subcolumns_count = 2;
variant.predefined_paths = {
VariantPathSpec {.path = "int_*",
.type = FieldType::OLAP_FIELD_TYPE_INT,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
};
return variant;
}
VariantColumnSpec array_text_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.nullable = true;
variant.max_subcolumns_count = 10;
variant.predefined_paths = {
VariantPathSpec {.path = std::string(kArrayPath),
.type = FieldType::OLAP_FIELD_TYPE_ARRAY,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = FieldType::OLAP_FIELD_TYPE_STRING,
.array_item_nullable = true},
};
return variant;
}
DataTypePtr nullable_array_text_type() {
return make_nullable(
std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeString>())));
}
VExprSPtr make_string_literal(std::string value) {
TExprNode node;
node.__set_node_type(TExprNodeType::STRING_LITERAL);
node.__set_type(create_type_desc(PrimitiveType::TYPE_STRING));
node.__set_num_children(0);
node.__set_is_nullable(false);
TStringLiteral literal;
literal.__set_value(std::move(value));
node.__set_string_literal(literal);
return VLiteral::create_shared(node);
}
VExprContextSPtr make_is_null_context(int32_t block_column_id, bool is_null) {
TFunctionName fn_name;
fn_name.__set_db_name("");
fn_name.__set_function_name(is_null ? "is_null_pred" : "is_not_null_pred");
TFunction fn;
fn.__set_name(fn_name);
fn.__set_binary_type(TFunctionBinaryType::BUILTIN);
fn.__set_arg_types({create_type_desc(PrimitiveType::TYPE_INT)});
fn.__set_ret_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
fn.__set_has_var_args(false);
TExprNode node;
node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
node.__set_node_type(TExprNodeType::IS_NULL_PRED);
node.__set_fn(fn);
node.__set_num_children(1);
node.__set_is_nullable(false);
auto root = VectorizedFnCall::create_shared(node);
root->add_child(std::make_shared<MockSlotRef>(
block_column_id, make_nullable(std::make_shared<DataTypeInt32>())));
return VExprContext::create_shared(root);
}
VExprContextSPtr make_array_contains_context(int32_t block_column_id, std::string value) {
TFunctionName fn_name;
fn_name.__set_db_name("");
fn_name.__set_function_name("array_contains");
TFunction fn;
fn.__set_name(fn_name);
fn.__set_binary_type(TFunctionBinaryType::BUILTIN);
fn.__set_arg_types({create_type_desc(PrimitiveType::TYPE_ARRAY),
create_type_desc(PrimitiveType::TYPE_STRING)});
fn.__set_ret_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
fn.__set_has_var_args(false);
TExprNode node;
node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
node.__set_node_type(TExprNodeType::FUNCTION_CALL);
node.__set_fn(fn);
node.__set_num_children(2);
node.__set_is_nullable(true);
auto root = VectorizedFnCall::create_shared(node);
root->add_child(std::make_shared<MockSlotRef>(block_column_id, nullable_array_text_type()));
root->add_child(make_string_literal(std::move(value)));
return VExprContext::create_shared(root);
}
IndexReadOptions make_is_null_read_options(const TabletSchema& schema, int32_t path_column_id,
bool is_null) {
const auto& path_column = schema.column(path_column_id);
IndexReadOptions options;
options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
options.target_cast_type_for_variants[path_column.name()] =
make_nullable(std::make_shared<DataTypeInt32>());
options.common_expr_ctxs_push_down.push_back(make_is_null_context(1, is_null));
return options;
}
IndexReadOptions make_array_contains_read_options(const TabletSchema& schema,
int32_t path_column_id, std::string value) {
const auto& path_column = schema.column(path_column_id);
IndexReadOptions options;
options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
options.target_cast_type_for_variants[path_column.name()] = nullable_array_text_type();
options.common_expr_ctxs_push_down.push_back(make_array_contains_context(1, std::move(value)));
return options;
}
} // namespace
class IndexStorageVariantExprPushdownTest : public IndexStorageTestFixture {};
TEST_F(IndexStorageVariantExprPushdownTest, IsNullExprUsesVariantFieldPatternIndex) {
const auto index_case =
IndexStorageCaseBuilder("variant_is_null_expr_field_pattern_index")
.tablet_id(110034)
.variant_column(typed_pattern_variant_column())
.inverted_index(IndexSpec::field_pattern_index(
kIntPatternIndexId, "idx_v_int_glob", kVariantUid, "int_*"))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"int_1": 42})", R"({"int_2": 7})"}, 0))
.rowset(1, IndexDataSourceSpec::inline_variant(
{R"({"int_1": null})", R"({"int_1": 8})"}, 100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
auto not_null_read =
read_rowsets(readable_rowsets.value(),
make_is_null_read_options(*tablet_schema(), path_column_id, false));
ASSERT_TRUE(not_null_read.has_value()) << not_null_read.error();
EXPECT_EQ(not_null_read->rows_read, 2);
expect_index_filter_stats(not_null_read.value(), 2);
auto null_read =
read_rowsets(readable_rowsets.value(),
make_is_null_read_options(*tablet_schema(), path_column_id, true));
ASSERT_TRUE(null_read.has_value()) << null_read.error();
EXPECT_EQ(null_read->rows_read, 2);
expect_index_filter_stats(null_read.value(), 2);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.int_1");
ASSERT_EQ(compacted_path_column_id, path_column_id);
auto compacted_not_null = read_rowsets(
readable_compacted.value(),
make_is_null_read_options(*tablet_schema(), compacted_path_column_id, false));
ASSERT_TRUE(compacted_not_null.has_value()) << compacted_not_null.error();
EXPECT_EQ(compacted_not_null->rows_read, 2);
expect_index_filter_stats(compacted_not_null.value(), 2);
auto compacted_null = read_rowsets(
readable_compacted.value(),
make_is_null_read_options(*tablet_schema(), compacted_path_column_id, true));
ASSERT_TRUE(compacted_null.has_value()) << compacted_null.error();
EXPECT_EQ(compacted_null->rows_read, 2);
expect_index_filter_stats(compacted_null.value(), 2);
}
TEST_F(IndexStorageVariantExprPushdownTest, IsNullExprSkipsSparseVariantFieldPatternIndex) {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.sparse_hash_shard_count = 2;
// "rare" is kept in the sparse map instead of a materialized typed subcolumn. The
// field-pattern index metadata should not make sparse map entries look indexable unless the
// writer creates a real sparse-path inverted index.
const auto index_case =
IndexStorageCaseBuilder("variant_sparse_is_null_expr_field_pattern_index")
.tablet_id(110038)
.variant_column(std::move(variant))
.inverted_index(IndexSpec::field_pattern_index(
kSparseIntPatternIndexId, "idx_v_sparse_rare", kVariantUid, "rare"))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"hot": 1, "rare": 10})", R"({"hot": 2})",
R"({"hot": 3, "rare": null})"},
0))
.rowset(1, IndexDataSourceSpec::inline_variant(
{R"({"hot": 4, "rare": 20})", R"({"hot": 5})"}, 100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto read_schema = build_schema_with_variant_path_column(*tablet_schema(), kVariantUid, "rare",
FieldType::OLAP_FIELD_TYPE_INT);
ASSERT_NE(read_schema, nullptr);
auto readable_rowsets =
inject_reader_schema_for_rowsets(rowsets.value(), std::move(read_schema));
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.rare");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
auto not_null_read =
read_rowsets(readable_rowsets.value(),
make_is_null_read_options(*tablet_schema(), path_column_id, false));
ASSERT_TRUE(not_null_read.has_value()) << not_null_read.error();
EXPECT_EQ(not_null_read->rows_read, 2);
expect_index_filter_stats(not_null_read.value(), 0);
auto null_read =
read_rowsets(readable_rowsets.value(),
make_is_null_read_options(*tablet_schema(), path_column_id, true));
ASSERT_TRUE(null_read.has_value()) << null_read.error();
EXPECT_EQ(null_read->rows_read, 3);
expect_index_filter_stats(null_read.value(), 0);
}
TEST_F(IndexStorageVariantExprPushdownTest, ArrayContainsUsesVariantArrayFieldPatternIndex) {
const auto index_case =
IndexStorageCaseBuilder("variant_array_contains_field_pattern_index")
.tablet_id(110035)
.variant_column(array_text_variant_column())
.inverted_index(IndexSpec::field_pattern_index(kArrayPatternIndexId,
"idx_v_inventors", kVariantUid,
std::string(kArrayPath)))
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"inventors":["a", "b", "c"]})", R"({"inventors":[]})",
R"({"inventors":["d", "e", "f", "g", "h", "i", "j", "k", "l"]})",
R"({"inventors":null})",
R"({"inventors":["m", "n", "o", "p", "q", "r", "s", "t", "u"]})",
R"({"inventors":[null, null, null]})",
R"({"inventors":["v", "w", "x", "y", "z"]})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.inventors");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
ScopedDebugPoint debug_point("array_func.array_contains", {{"result_bitmap", "1"}});
auto read =
read_rowsets(readable_rowsets.value(),
make_array_contains_read_options(*tablet_schema(), path_column_id, "w"));
ASSERT_TRUE(read.has_value()) << read.error();
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_EQ(read->rows_read, 1);
expect_index_filter_stats(read.value(), 6);
}
TEST_F(IndexStorageVariantExprPushdownTest,
ArrayContainsFiltersMissesDisabledIndexAndCompactedReload) {
const auto index_case =
IndexStorageCaseBuilder("variant_array_contains_filter_stats_matrix")
.tablet_id(110036)
.variant_column(array_text_variant_column())
.inverted_index(IndexSpec::field_pattern_index(kArrayPatternIndexId,
"idx_v_inventors", kVariantUid,
std::string(kArrayPath)))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"inventors":["w", "amory"]})", R"({"inventors":[]})",
R"({"inventors":null})"},
0))
.rowset(1, IndexDataSourceSpec::inline_variant(
{R"({"inventors":["w", "x"]})", R"({"inventors":[null]})",
R"({"inventors":["m"]})", R"({"inventors":["n", null]})"},
100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.inventors");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
{
ScopedDebugPoint debug_point("array_func.array_contains", {{"result_bitmap", "1"}});
auto hit_read = read_rowsets(
readable_rowsets.value(),
make_array_contains_read_options(*tablet_schema(), path_column_id, "w"));
ASSERT_TRUE(hit_read.has_value()) << hit_read.error();
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_EQ(hit_read->rows_read, 2);
expect_index_filter_stats(hit_read.value(), 5);
}
{
ScopedDebugPoint debug_point("array_func.array_contains", {{"result_bitmap", "0"}});
auto miss_read = read_rowsets(
readable_rowsets.value(),
make_array_contains_read_options(*tablet_schema(), path_column_id, "missing"));
ASSERT_TRUE(miss_read.has_value()) << miss_read.error();
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_EQ(miss_read->rows_read, 0);
expect_index_filter_stats(miss_read.value(), 7);
}
auto disabled_options = make_array_contains_read_options(*tablet_schema(), path_column_id, "w");
disabled_options.enable_inverted_index_query = false;
auto disabled_read = read_rowsets(readable_rowsets.value(), std::move(disabled_options));
ASSERT_TRUE(disabled_read.has_value()) << disabled_read.error();
EXPECT_EQ(disabled_read->rows_read, 2);
expect_index_filter_stats(disabled_read.value(), 0);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 7);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.inventors");
ASSERT_EQ(compacted_path_column_id, path_column_id);
ScopedDebugPoint debug_point("array_func.array_contains", {{"result_bitmap", "2"}});
auto compacted_read = read_rowsets(
readable_compacted.value(),
make_array_contains_read_options(*tablet_schema(), compacted_path_column_id, "w"));
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_EQ(compacted_read->rows_read, 2);
expect_index_filter_stats(compacted_read.value(), 5);
}
TEST_F(IndexStorageVariantExprPushdownTest, ArrayContainsAllHitKeepsFilterStatsAtZeroFilteredRows) {
const auto index_case =
IndexStorageCaseBuilder("variant_array_contains_all_hit_filter_stats")
.tablet_id(110039)
.variant_column(array_text_variant_column())
.inverted_index(IndexSpec::field_pattern_index(kArrayPatternIndexId,
"idx_v_inventors", kVariantUid,
std::string(kArrayPath)))
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"inventors":["w"]})", R"({"inventors":["w", "x"]})",
R"({"inventors":["a", "w"]})", R"({"inventors":["w", null]})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.inventors");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
ScopedDebugPoint debug_point("array_func.array_contains", {{"result_bitmap", "4"}});
auto read =
read_rowsets(readable_rowsets.value(),
make_array_contains_read_options(*tablet_schema(), path_column_id, "w"));
ASSERT_TRUE(read.has_value()) << read.error();
EXPECT_GT(debug_point.execute_num(), 0);
EXPECT_EQ(read->rows_read, 4);
expect_index_filter_stats(read.value(), 0);
}
// Mixed scalar/nested/unsupported JSON shapes keep semantic array_contains results while the index
// filter still accounts for skipped rows.
TEST_F(IndexStorageVariantExprPushdownTest,
ArrayContainsMixedNestedAndUnsupportedInputsFilterRows) {
const auto index_case =
IndexStorageCaseBuilder("variant_array_contains_mixed_nested_unsupported_filter")
.tablet_id(110043)
.variant_column(array_text_variant_column())
.inverted_index(IndexSpec::field_pattern_index(kArrayPatternIndexId,
"idx_v_inventors", kVariantUid,
std::string(kArrayPath)))
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"inventors":["w", 1, true]})",
R"({"inventors":[["w"], "x"]})",
R"({"inventors":[{"name":"w"}, "y"]})", R"({"inventors":"w"})",
R"({"inventors":{"name":"w"}})", R"({"inventors":["z"]})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.inventors");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
auto read =
read_rowsets(readable_rowsets.value(),
make_array_contains_read_options(*tablet_schema(), path_column_id, "w"));
ASSERT_TRUE(read.has_value()) << read.error();
EXPECT_EQ(read->rows_read, 1);
expect_index_filter_stats(read.value(), 5);
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_field_pattern_index_test.cpp
+796
View File
@@ -0,0 +1,796 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <cstdint>
#include <memory>
#include <string>
#include <string_view>
#include <vector>
#include "common/config.h"
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/value/vdatetime_value.h"
#include "storage/predicate/predicate_creator.h"
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
namespace {
constexpr int32_t kVariantUid = 2;
constexpr int64_t kIntPatternIndexId = 210201;
constexpr int64_t kStringPatternIndexId = 210202;
constexpr int64_t kBigIntPatternIndexId = 210203;
constexpr int64_t kDoublePatternIndexId = 210204;
constexpr int64_t kBoolPatternIndexId = 210205;
constexpr int64_t kDatePatternIndexId = 210206;
constexpr int64_t kDateTimePatternIndexId = 210207;
constexpr int32_t kPagePruneLowValueBound = 900000;
int32_t page_prune_offset(size_t row) {
uint64_t mixed = static_cast<uint64_t>(row + 1) * 11400714819323198485ull;
mixed ^= mixed >> 33;
return static_cast<int32_t>(mixed % static_cast<uint64_t>(kPagePruneLowValueBound));
}
std::shared_ptr<ColumnPredicate> typed_equals(int32_t column_id, std::string column_name,
DataTypePtr data_type, Field value) {
return create_comparison_predicate<PredicateType::EQ>(column_id, std::move(column_name),
std::move(data_type), value, false);
}
std::shared_ptr<ColumnPredicate> int_equals(int32_t column_id, std::string column_name,
int32_t value) {
return create_comparison_predicate<PredicateType::EQ>(
column_id, std::move(column_name), std::make_shared<DataTypeInt32>(),
Field::create_field<TYPE_INT>(value), false);
}
std::shared_ptr<ColumnPredicate> int_greater(int32_t column_id, std::string column_name,
int32_t value) {
return create_comparison_predicate<PredicateType::GT>(
column_id, std::move(column_name), std::make_shared<DataTypeInt32>(),
Field::create_field<TYPE_INT>(value), false);
}
void expect_optional_page_zone_map_filter_stats(const IndexReadResult& result,
int64_t selected_rows, int64_t total_rows,
int64_t max_filtered_rows) {
EXPECT_GE(result.stats.raw_rows_read, selected_rows);
EXPECT_LE(result.stats.raw_rows_read, total_rows);
EXPECT_GE(result.stats.rows_stats_filtered, 0);
EXPECT_LE(result.stats.rows_stats_filtered, max_filtered_rows);
if (result.stats.raw_rows_read < total_rows) {
EXPECT_GT(result.stats.rows_stats_filtered, 0);
}
}
DataTypePtr nullable_target_type(const DataTypePtr& type) {
return make_nullable(type);
}
DataTypePtr nullable_int32_target_type() {
return make_nullable(std::make_shared<DataTypeInt32>());
}
DataTypePtr nullable_int64_target_type() {
return make_nullable(std::make_shared<DataTypeInt64>());
}
void disable_bkd_skip_for_filter_stats_assertions(IndexReadOptions* read_options) {
// Tiny three-row segments can hit the default 50% BKD skip threshold before the field-pattern
// index contributes rows_inverted_index_filtered.
read_options->inverted_index_skip_threshold = 100;
}
VariantColumnSpec typed_pattern_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.max_subcolumns_count = 2;
variant.predefined_paths = {
VariantPathSpec {.path = "int_*",
.type = FieldType::OLAP_FIELD_TYPE_INT,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "string_*",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
};
return variant;
}
VariantColumnSpec multi_typed_pattern_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.max_subcolumns_count = 6;
variant.predefined_paths = {
VariantPathSpec {.path = "big_*",
.type = FieldType::OLAP_FIELD_TYPE_BIGINT,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "double_*",
.type = FieldType::OLAP_FIELD_TYPE_DOUBLE,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "bool_*",
.type = FieldType::OLAP_FIELD_TYPE_BOOL,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
};
return variant;
}
VariantColumnSpec date_time_pattern_variant_column() {
VariantColumnSpec variant;
variant.unique_id = kVariantUid;
variant.name = "v";
variant.max_subcolumns_count = 4;
variant.predefined_paths = {
VariantPathSpec {.path = "date_*",
.type = FieldType::OLAP_FIELD_TYPE_DATEV2,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "datetime_*",
.type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME_GLOB,
.array_item_type = {},
.array_item_nullable = true},
};
return variant;
}
Field date_v2_field(int64_t yyyymmdd) {
DateV2Value<DateV2ValueType> value;
value.from_date_int64(yyyymmdd);
return Field::create_field<TYPE_DATEV2>(value);
}
Field datetime_v2_field(int64_t yyyymmddhhmmss) {
DateV2Value<DateTimeV2ValueType> value;
value.from_date_int64(yyyymmddhhmmss);
return Field::create_field<TYPE_DATETIMEV2>(value);
}
std::vector<std::string> split_int_variant_rows(size_t low_rows, int32_t low_value,
size_t high_rows, int32_t high_value) {
// Keep both ranges in one data source. The fixture flushes the rowset writer after
// each data source, so splitting low/high ranges into separate sources creates two
// segments and lets segment ZoneMap prune before page ZoneMap or BloomFilter can run.
std::vector<std::string> rows;
rows.reserve(low_rows + high_rows);
for (size_t i = 0; i < low_rows; ++i) {
rows.push_back(R"({"int_1": )" + std::to_string(low_value + page_prune_offset(i)) + "}");
}
for (size_t i = 0; i < high_rows; ++i) {
rows.push_back(R"({"int_1": )" +
std::to_string(high_value + page_prune_offset(low_rows + i)) + "}");
}
return rows;
}
std::vector<std::string> interleaved_int_variant_rows(size_t pairs, int32_t low_value,
int32_t high_value) {
// BF assertions must keep both segment and page ZoneMaps matched. Alternating values keeps each
// page range wide enough that ZoneMap cannot prune before the BloomFilter reader is exercised.
std::vector<std::string> rows;
rows.reserve(pairs * 2);
for (size_t i = 0; i < pairs; ++i) {
rows.push_back(R"({"int_1": )" + std::to_string(low_value) + "}");
rows.push_back(R"({"int_1": )" + std::to_string(high_value) + "}");
}
return rows;
}
} // namespace
class IndexStorageVariantFieldPatternIndexTest : public IndexStorageTestFixture {
protected:
void SetUp() override {
IndexStorageTestFixture::SetUp();
_old_zone_map_row_num_threshold = config::zone_map_row_num_threshold;
// Keep zone maps available on small test pages when the compacted layout exposes them.
config::zone_map_row_num_threshold = 20;
}
void TearDown() override {
config::zone_map_row_num_threshold = _old_zone_map_row_num_threshold;
IndexStorageTestFixture::TearDown();
}
void run_typed_int_field_pattern_index_lifecycle(IndexCompactionKind compaction_kind,
int64_t tablet_id);
private:
int32_t _old_zone_map_row_num_threshold = 20;
};
void IndexStorageVariantFieldPatternIndexTest::run_typed_int_field_pattern_index_lifecycle(
IndexCompactionKind compaction_kind, int64_t tablet_id) {
const auto int_index = IndexSpec::field_pattern_index(kIntPatternIndexId, "idx_v_int_glob",
kVariantUid, "int_*");
const auto string_index = IndexSpec::field_pattern_index(
kStringPatternIndexId, "idx_v_string_glob", kVariantUid, "string_*");
const auto index_case = IndexStorageCaseBuilder("typed_int_field_pattern_index_lifecycle")
.tablet_id(tablet_id)
.variant_column(typed_pattern_variant_column())
.inverted_index(int_index)
.inverted_index(string_index)
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"int_1": 42, "string_1": "sample"})",
R"({"int_1": 7, "string_1": "other"})"},
0))
.rowset(1, IndexDataSourceSpec::inline_variant(
{R"({"int_1": 42, "string_1": "sample"})",
R"({"int_1": 8, "string_1": "other"})"},
100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(int_equals(path_column_id, path_column.name(), 42));
auto before_compaction = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(before_compaction.has_value()) << before_compaction.error();
EXPECT_EQ(before_compaction->rows_read, 2);
expect_index_filter_stats(before_compaction.value(), 2);
IndexReadOptions range_read_options;
range_read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
range_read_options.target_cast_type_for_variants[path_column.name()] =
nullable_int32_target_type();
range_read_options.predicates.push_back(int_greater(path_column_id, path_column.name(), 10));
auto range_before_compaction = read_rowsets(readable_rowsets.value(), range_read_options);
ASSERT_TRUE(range_before_compaction.has_value()) << range_before_compaction.error();
EXPECT_EQ(range_before_compaction->rows_read, 2);
expect_index_filter_stats(range_before_compaction.value(), 2);
auto compacted = compact_rowsets(compaction_kind, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.int_1");
ASSERT_EQ(compacted_path_column_id, path_column_id);
auto after_compaction = read_rowsets(readable_compacted.value(), read_options);
ASSERT_TRUE(after_compaction.has_value()) << after_compaction.error();
EXPECT_EQ(after_compaction->rows_read, 2);
expect_index_filter_stats(after_compaction.value(), 2);
auto range_after_compaction = read_rowsets(readable_compacted.value(), range_read_options);
ASSERT_TRUE(range_after_compaction.has_value()) << range_after_compaction.error();
EXPECT_EQ(range_after_compaction->rows_read, 2);
expect_index_filter_stats(range_after_compaction.value(), 2);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest, TypedIntIndexAfterCumulativeCompaction) {
run_typed_int_field_pattern_index_lifecycle(IndexCompactionKind::CUMULATIVE, 110032);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest, TypedIntIndexAfterFullCompaction) {
run_typed_int_field_pattern_index_lifecycle(IndexCompactionKind::FULL, 110033);
}
// Non-INT typed Variant field-pattern indexes should filter rows for each physical storage type.
TEST_F(IndexStorageVariantFieldPatternIndexTest, BigIntDoubleAndBoolFieldPatternIndexesFilterRows) {
const auto index_case =
IndexStorageCaseBuilder("variant_multi_typed_field_pattern_index_matrix")
.tablet_id(110040)
.variant_column(multi_typed_pattern_variant_column())
.inverted_index(IndexSpec::field_pattern_index(
kBigIntPatternIndexId, "idx_v_big_glob", kVariantUid, "big_*"))
.inverted_index(IndexSpec::field_pattern_index(
kDoublePatternIndexId, "idx_v_double_glob", kVariantUid, "double_*"))
.inverted_index(IndexSpec::field_pattern_index(
kBoolPatternIndexId, "idx_v_bool_glob", kVariantUid, "bool_*"))
// Use distinct hit counts per path so filter stats validate the selected
// field-pattern index rather than only proving that some index was applied.
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"big_1": 9000000000, "double_1": 3.5, "bool_1": true})",
R"({"big_1": 7, "double_1": 1.25, "bool_1": false})",
R"({"big_1": 9000000000, "double_1": 7.5, "bool_1": false})",
R"({"big_1": 8, "double_1": 8.5, "bool_1": true})",
R"({"big_1": 9, "double_1": 9.5, "bool_1": true})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
auto read_and_verify = [&](std::string_view path, FieldType expected_storage_type,
DataTypePtr data_type, Field value, int64_t expected_rows,
int64_t expected_filtered_rows) {
const int32_t path_column_id = column_id_by_path("v." + std::string(path));
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
EXPECT_EQ(path_column.type(), expected_storage_type)
<< "unexpected storage type for " << path_column.name();
EXPECT_EQ(data_type->get_storage_field_type(), expected_storage_type)
<< "unexpected predicate type for " << path_column.name();
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] =
nullable_target_type(data_type);
disable_bkd_skip_for_filter_stats_assertions(&read_options);
read_options.predicates.push_back(
typed_equals(path_column_id, path_column.name(), data_type, value));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, expected_rows);
expect_index_filter_stats(read_result.value(), expected_filtered_rows);
};
read_and_verify("big_1", FieldType::OLAP_FIELD_TYPE_BIGINT, std::make_shared<DataTypeInt64>(),
Field::create_field<TYPE_BIGINT>(Int64(9000000000LL)), 2, 3);
read_and_verify("double_1", FieldType::OLAP_FIELD_TYPE_DOUBLE,
std::make_shared<DataTypeFloat64>(),
Field::create_field<TYPE_DOUBLE>(Float64(3.5)), 1, 4);
read_and_verify("bool_1", FieldType::OLAP_FIELD_TYPE_BOOL, std::make_shared<DataTypeBool>(),
Field::create_field<TYPE_BOOLEAN>(UInt8(1)), 3, 2);
}
// DATEV2/DATETIMEV2 Variant field-pattern indexes should filter rows for matching paths.
TEST_F(IndexStorageVariantFieldPatternIndexTest, DateAndDateTimeFieldPatternIndexesFilterRows) {
const auto index_case =
IndexStorageCaseBuilder("variant_date_time_field_pattern_index_matrix")
.tablet_id(110044)
.variant_column(date_time_pattern_variant_column())
.inverted_index(IndexSpec::field_pattern_index(
kDatePatternIndexId, "idx_v_date_glob", kVariantUid, "date_*"))
.inverted_index(IndexSpec::field_pattern_index(kDateTimePatternIndexId,
"idx_v_datetime_glob",
kVariantUid, "datetime_*"))
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"date_1": "2024-01-02", "datetime_1": "2024-01-02 03:04:05"})",
R"({"date_1": "2024-01-03", "datetime_1": "2024-01-03 03:04:05"})",
R"({"date_1": "2024-01-02", "datetime_1": "2024-01-04 03:04:05"})",
R"({"date_1": "2024-01-04", "datetime_1": "2024-01-05 03:04:05"})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
auto read_and_verify = [&](std::string_view path, DataTypePtr data_type, Field value,
int64_t expected_rows, int64_t expected_filtered_rows) {
const int32_t path_column_id = column_id_by_path("v." + std::string(path));
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] =
nullable_target_type(data_type);
disable_bkd_skip_for_filter_stats_assertions(&read_options);
read_options.predicates.push_back(
typed_equals(path_column_id, path_column.name(), data_type, value));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, expected_rows);
expect_index_filter_stats(read_result.value(), expected_filtered_rows);
};
read_and_verify("date_1", std::make_shared<DataTypeDateV2>(), date_v2_field(20240102), 2, 2);
read_and_verify("datetime_1", std::make_shared<DataTypeDateTimeV2>(),
datetime_v2_field(20240102030405), 1, 3);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest, TypedVariantPathSegmentZoneMapPrunesWholeSegment) {
const auto index_case = IndexStorageCaseBuilder("variant_typed_path_segment_zone_map_prune")
.tablet_id(110041)
.variant_column(typed_pattern_variant_column())
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"int_1": 1})", R"({"int_1": 2})",
R"({"int_1": 100})", R"({"int_1": 101})"},
0),
2)
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(int_greater(path_column_id, path_column.name(), 50));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_raw_rows_read(read_result.value(), 2);
expect_segment_pruned(read_result.value(), 1);
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest, TypedVariantPathPageZoneMapPrunesWithinSegment) {
constexpr size_t kLowRows = 2048;
constexpr size_t kHighRows = 2048;
constexpr int32_t kLowValueBase = 1;
constexpr int32_t kHighValueBase = 10000000;
constexpr int32_t kPagePruneThreshold = 1000000;
auto variant = typed_pattern_variant_column();
IndexTabletOptions options;
options.tablet_id = 110045;
options.storage_page_size = 4096;
options.variant_columns.push_back(std::move(variant));
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.max_rows_per_segment = static_cast<int64_t>(kLowRows + kHighRows);
rowset.data_sources.push_back(IndexDataSourceSpec::inline_variant(
split_int_variant_rows(kLowRows, kLowValueBase, kHighRows, kHighValueBase), 0));
ASSERT_TRUE(create_tablet(options).ok());
auto rowsets = write_rowsets({rowset});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
// Use dispersed low/high ranges so page-level pruning does not depend on one exact page
// boundary or compression layout.
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(
int_greater(path_column_id, path_column.name(), kPagePruneThreshold));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, kHighRows);
expect_segment_pruned(read_result.value(), 0);
const auto total_rows = static_cast<int64_t>(kLowRows + kHighRows);
const auto high_rows = static_cast<int64_t>(kHighRows);
// Compaction can produce a layout where page ZoneMap pruning is not triggered. Keep the result
// contract strict and validate stats only when raw_rows_read proves page pruning actually
// happened.
expect_optional_page_zone_map_filter_stats(read_result.value(), high_rows, total_rows,
static_cast<int64_t>(kLowRows));
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest,
DisabledInvertedIndexQueryDoesNotFilterVariantFieldPatternIndex) {
const auto int_index = IndexSpec::field_pattern_index(kIntPatternIndexId, "idx_v_int_glob",
kVariantUid, "int_*");
const auto index_case =
IndexStorageCaseBuilder("variant_disabled_inverted_index_query_reverse_case")
.tablet_id(110046)
.variant_column(typed_pattern_variant_column())
.inverted_index(int_index)
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"int_1": 42})", R"({"int_1": 7})", R"({"int_1": 42})",
R"({"int_1": 8})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(int_equals(path_column_id, path_column.name(), 42));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_index_filter_stats(read_result.value(), 0);
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest,
TargetCastTypeMismatchDoesNotDisableExtractedPathZoneMapPruning) {
const auto index_case =
IndexStorageCaseBuilder("variant_unsafe_target_cast_zone_map_reverse_case")
.tablet_id(110047)
.variant_column(typed_pattern_variant_column())
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"int_1": 1})", R"({"int_1": 2})", R"({"int_1": 100})",
R"({"int_1": 101})"},
0),
2)
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
// Keep nullability aligned with Variant typed paths so this case checks the stored
// physical type mismatch, not a nullable/non-nullable wrapper mismatch.
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int64_target_type();
read_options.predicates.push_back(int_greater(path_column_id, path_column.name(), 50));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
// The production guard only uses target-cast mismatches to reject storage predicates for
// Variant-typed columns. This test reads an already extracted typed path, so the synthetic
// target-cast mismatch does not disable segment ZoneMap pruning.
EXPECT_EQ(read_result->rows_read, 2);
expect_raw_rows_read(read_result.value(), 2);
expect_segment_pruned(read_result.value(), 1);
expect_zone_map_filtered(read_result.value(), 2);
expect_index_filter_stats(read_result.value(), 0);
}
// Expected-red: this is separate from the DORIS-26471 BF case fix. Segment-level typed-path ZoneMap
// pruning works, but rows_stats_filtered is still not updated on this path.
TEST_F(IndexStorageVariantFieldPatternIndexTest,
TypedVariantPathZoneMapRowsStatsFilteredCountsPrunedRows) {
const auto index_case =
IndexStorageCaseBuilder("variant_typed_path_segment_zone_map_rows_stats")
.tablet_id(110042)
.variant_column(typed_pattern_variant_column())
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"int_1": 1})", R"({"int_1": 2})", R"({"int_1": 100})",
R"({"int_1": 101})"},
0),
2)
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(int_greater(path_column_id, path_column.name(), 50));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_raw_rows_read(read_result.value(), 2);
expect_segment_pruned(read_result.value(), 1);
expect_zone_map_filtered(read_result.value(), 2);
expect_index_filter_stats(read_result.value(), 0);
}
TEST_F(IndexStorageVariantFieldPatternIndexTest,
TypedVariantPathPageZoneMapRowsStatsFilteredCountsPrunedRows) {
constexpr size_t kLowRows = 2048;
constexpr size_t kHighRows = 2048;
constexpr int32_t kLowValueBase = 1;
constexpr int32_t kHighValueBase = 10000000;
constexpr int32_t kPagePruneThreshold = 1000000;
IndexTabletOptions options;
options.tablet_id = 110048;
options.storage_page_size = 4096;
options.variant_columns.push_back(typed_pattern_variant_column());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.max_rows_per_segment = static_cast<int64_t>(kLowRows + kHighRows);
rowset.data_sources.push_back(IndexDataSourceSpec::inline_variant(
split_int_variant_rows(kLowRows, kLowValueBase, kHighRows, kHighValueBase), 0));
ASSERT_TRUE(create_tablet(options).ok());
auto rowsets = write_rowsets({rowset});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
// Use dispersed low/high ranges so page ZoneMap pruning is observable even if the exact
// compacted page boundary differs by build configuration.
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(
int_greater(path_column_id, path_column.name(), kPagePruneThreshold));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, kHighRows);
expect_segment_pruned(read_result.value(), 0);
const auto total_rows = static_cast<int64_t>(kLowRows + kHighRows);
const auto high_rows = static_cast<int64_t>(kHighRows);
// Compaction can produce a layout where page ZoneMap pruning is not triggered. Keep the result
// contract strict and validate stats only when raw_rows_read proves page pruning actually
// happened.
expect_optional_page_zone_map_filter_stats(read_result.value(), high_rows, total_rows,
static_cast<int64_t>(kLowRows));
expect_index_filter_stats(read_result.value(), 0);
}
// Variant typed-path BloomFilter is inherited from the parent Variant column and filters equality
// misses before ZoneMap pruning.
TEST_F(IndexStorageVariantFieldPatternIndexTest, VariantTypedPathBloomFilterFiltersMissingValue) {
constexpr size_t kLowRows = 2048;
constexpr size_t kHighRows = 2048;
constexpr int32_t kMissingValueInsideSegmentZoneMap = 50;
auto variant = typed_pattern_variant_column();
variant.is_bf_column = true;
IndexTabletOptions options;
options.tablet_id = 110049;
options.storage_page_size = 4096;
options.variant_columns.push_back(std::move(variant));
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.max_rows_per_segment = static_cast<int64_t>(kLowRows + kHighRows);
rowset.data_sources.push_back(
IndexDataSourceSpec::inline_variant(interleaved_int_variant_rows(kLowRows, 1, 100), 0));
ASSERT_TRUE(create_tablet(options).ok());
auto rowsets = write_rowsets({rowset});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_1");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
// Variant extracted paths are nullable in the extended schema. Keep the target cast nullable so
// can_apply_predicate_safely allows this case to exercise the BloomFilter reader.
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(
int_equals(path_column_id, path_column.name(), kMissingValueInsideSegmentZoneMap));
ScopedDebugPoint must_filter("bloom_filter_must_filter_data");
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 0);
expect_bloom_filter_filtered(read_result.value(), kLowRows + kHighRows);
expect_index_filter_stats(read_result.value(), 0);
}
// Expected-red: once a sparse Variant child has sparse statistics but no materialized subcolumn, an
// exact typed-path predicate should still be able to use those stats for pushdown observability.
TEST_F(IndexStorageVariantFieldPatternIndexTest,
DISABLED_SparseChildPathStatsCanDriveTypedPathPushdown) {
auto variant = typed_pattern_variant_column();
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = 1;
const auto index_case =
IndexStorageCaseBuilder("variant_sparse_child_path_stats_pushdown")
.tablet_id(110050)
.variant_column(std::move(variant))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"int_1": 1, "int_rare": 999})", R"({"int_1": 2})",
R"({"int_1": 3})", R"({"int_1": 4})"},
0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.int_rare");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.enable_inverted_index_query = false;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_int32_target_type();
read_options.predicates.push_back(int_equals(path_column_id, path_column.name(), 999));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 1);
EXPECT_GT(read_result->stats.rows_stats_filtered, 0);
expect_index_filter_stats(read_result.value(), 0);
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_path_index_lifecycle_test.cpp
+527
View File
@@ -0,0 +1,527 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "storage/variant/index_storage_variant_test_base.h"
namespace doris::index_storage_test {
class IndexStorageVariantPathIndexLifecycleTest : public IndexStorageTestFixture {
protected:
void run_variant_path_count_on_index_filters_rows(bool verify_key_column_not_read);
};
void IndexStorageVariantPathIndexLifecycleTest::run_variant_path_count_on_index_filters_rows(
bool verify_key_column_not_read) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.predefined_paths = {
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
const auto index_case =
IndexStorageCaseBuilder("variant_path_count_on_index_skips_data")
.tablet_id(110037)
.variant_column(std::move(variant))
.inverted_index(IndexSpec::field_pattern_index(220401, "idx_v_b_count", 2, "b"))
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"b": "one"})", R"({"b": "other"})", R"({"b": "one"})"}, 0))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.b");
ASSERT_GE(path_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.push_down_agg_type_opt = TPushAggOp::COUNT_ON_INDEX;
read_options.target_cast_type_for_variants[path_column.name()] = nullable_string_target_type();
read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "one"));
if (verify_key_column_not_read) {
read_options.return_columns = {static_cast<uint32_t>(path_column_id)};
}
auto verify_read_result = [&](const Result<IndexReadResult>& read_result) {
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
EXPECT_EQ(read_result->stats.raw_rows_read, 2);
expect_index_filter_stats(read_result.value(), 1);
};
auto read_and_verify = [&](const std::vector<RowsetSharedPtr>& readable) {
if (verify_key_column_not_read) {
ScopedDebugPoint fail_if_key_column_is_read("segment_iterator._read_columns_by_index",
{{"column_name", "k"}});
auto read_result = read_rowsets(readable, read_options);
verify_read_result(read_result);
return;
}
auto read_result = read_rowsets(readable, read_options);
verify_read_result(read_result);
};
read_and_verify(readable_rowsets.value());
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 3);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.b");
ASSERT_EQ(compacted_path_column_id, path_column_id);
read_and_verify(readable_compacted.value());
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest,
DISABLED_BuildAndDropVariantPathIndexAfterExistingRows) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.predefined_paths = {
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
const auto index_case =
IndexStorageCaseBuilder("build_and_drop_variant_path_index_after_existing_rows")
.tablet_id(110018)
.variant_column(std::move(variant))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"b": "one"})", R"({"b": "other"})"}, 0))
.rowset(1, IndexDataSourceSpec::inline_variant(
{R"({"b": "other"})", R"({"b": "one"})"}, 100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.b");
ASSERT_GE(path_column_id, 0);
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_string_target_type();
read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "one"));
auto before_build = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(before_build.has_value()) << before_build.error();
EXPECT_EQ(before_build->rows_read, 2);
expect_index_filter_stats(before_build.value(), 0);
const auto path_index = IndexSpec::field_pattern_index(20004, "idx_v_b_built", 2, "b");
auto built_rowsets = build_inverted_indexes({path_index});
ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error();
ASSERT_EQ(built_rowsets->size(), 2);
for (const auto& rowset : built_rowsets.value()) {
auto probe = probe_rowset(rowset);
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), true);
}
auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value());
ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error();
auto after_build = read_rowsets(reloaded_built_rowsets.value(), read_options);
ASSERT_TRUE(after_build.has_value()) << after_build.error();
EXPECT_EQ(after_build->rows_read, 2);
expect_index_filter_stats(after_build.value(), 2);
auto dropped_rowsets = drop_inverted_indexes({path_index});
ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error();
ASSERT_EQ(dropped_rowsets->size(), 2);
for (const auto& rowset : dropped_rowsets.value()) {
auto probe = probe_rowset(rowset);
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), false);
}
auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value());
ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error();
auto after_drop = read_rowsets(reloaded_dropped_rowsets.value(), read_options);
ASSERT_TRUE(after_drop.has_value()) << after_drop.error();
EXPECT_EQ(after_drop->rows_read, 2);
expect_index_filter_stats(after_drop.value(), 0);
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest,
DISABLED_DropVariantPathIndexWithoutMaterializedPathRemovesMetadata) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
IndexTabletOptions options;
options.tablet_id = 110021;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(
IndexBatch::single_variant({R"({"b": "one"})", R"({"b": "other"})"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
const auto missing_path_index =
IndexSpec::field_pattern_index(20005, "idx_v_missing", 2, "missing");
auto built_rowsets = build_inverted_indexes({missing_path_index});
ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error();
ASSERT_EQ(built_rowsets->size(), 1);
EXPECT_FALSE(built_rowsets->front()
->tablet_schema()
->inverted_index_by_field_pattern(2, "missing")
.empty());
auto dropped_rowsets = drop_inverted_indexes({missing_path_index});
ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error();
ASSERT_EQ(dropped_rowsets->size(), 1);
EXPECT_TRUE(dropped_rowsets->front()
->tablet_schema()
->inverted_index_by_field_pattern(2, "missing")
.empty());
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest,
DISABLED_DropOneVariantPathIndexKeepsSiblingPathIndexUsable) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.predefined_paths = {
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "c",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
const auto index_case =
IndexStorageCaseBuilder("drop_one_variant_path_index_keeps_sibling_path_index_usable")
.tablet_id(110022)
.variant_column(std::move(variant))
.rowset(0,
IndexDataSourceSpec::inline_variant({R"({"b": "drop", "c": "keep"})",
R"({"b": "other", "c": "other"})"},
0))
.rowset(1,
IndexDataSourceSpec::inline_variant({R"({"b": "other", "c": "other"})",
R"({"b": "drop", "c": "keep"})"},
100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t b_column_id = column_id_by_path("v.b");
const int32_t c_column_id = column_id_by_path("v.c");
ASSERT_GE(b_column_id, 0);
ASSERT_GE(c_column_id, 0);
const auto& b_column = tablet_schema()->column(b_column_id);
const auto& c_column = tablet_schema()->column(c_column_id);
IndexReadOptions b_read_options;
b_read_options.return_columns = {0, static_cast<uint32_t>(b_column_id)};
b_read_options.target_cast_type_for_variants[b_column.name()] = nullable_string_target_type();
b_read_options.predicates.push_back(string_equals(b_column_id, b_column.name(), "drop"));
IndexReadOptions c_read_options;
c_read_options.return_columns = {0, static_cast<uint32_t>(c_column_id)};
c_read_options.target_cast_type_for_variants[c_column.name()] = nullable_string_target_type();
c_read_options.predicates.push_back(string_equals(c_column_id, c_column.name(), "keep"));
const auto b_index = IndexSpec::field_pattern_index(20006, "idx_v_b_drop", 2, "b");
const auto c_index = IndexSpec::field_pattern_index(20007, "idx_v_c_keep", 2, "c");
auto built_rowsets = build_inverted_indexes({b_index, c_index});
ASSERT_TRUE(built_rowsets.has_value()) << built_rowsets.error();
ASSERT_EQ(built_rowsets->size(), 2);
for (const auto& rowset : built_rowsets.value()) {
auto probe = probe_rowset(rowset);
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), true);
}
auto reloaded_built_rowsets = reload_rowsets(built_rowsets.value());
ASSERT_TRUE(reloaded_built_rowsets.has_value()) << reloaded_built_rowsets.error();
auto b_after_build = read_rowsets(reloaded_built_rowsets.value(), b_read_options);
ASSERT_TRUE(b_after_build.has_value()) << b_after_build.error();
EXPECT_EQ(b_after_build->rows_read, 2);
expect_index_filter_stats(b_after_build.value(), 2);
auto c_after_build = read_rowsets(reloaded_built_rowsets.value(), c_read_options);
ASSERT_TRUE(c_after_build.has_value()) << c_after_build.error();
EXPECT_EQ(c_after_build->rows_read, 2);
expect_index_filter_stats(c_after_build.value(), 2);
auto dropped_rowsets = drop_inverted_indexes({b_index});
ASSERT_TRUE(dropped_rowsets.has_value()) << dropped_rowsets.error();
ASSERT_EQ(dropped_rowsets->size(), 2);
for (const auto& rowset : dropped_rowsets.value()) {
EXPECT_TRUE(rowset->tablet_schema()->inverted_index_by_field_pattern(2, "b").empty());
EXPECT_FALSE(rowset->tablet_schema()->inverted_index_by_field_pattern(2, "c").empty());
auto probe = probe_rowset(rowset);
ASSERT_TRUE(probe.has_value()) << probe.error();
expect_index_files(probe.value(), true);
}
auto reloaded_dropped_rowsets = reload_rowsets(dropped_rowsets.value());
ASSERT_TRUE(reloaded_dropped_rowsets.has_value()) << reloaded_dropped_rowsets.error();
auto b_after_drop = read_rowsets(reloaded_dropped_rowsets.value(), b_read_options);
ASSERT_TRUE(b_after_drop.has_value()) << b_after_drop.error();
EXPECT_EQ(b_after_drop->rows_read, 2);
expect_index_filter_stats(b_after_drop.value(), 0);
auto c_after_drop = read_rowsets(reloaded_dropped_rowsets.value(), c_read_options);
ASSERT_TRUE(c_after_drop.has_value()) << c_after_drop.error();
EXPECT_EQ(c_after_drop->rows_read, 2);
expect_index_filter_stats(c_after_drop.value(), 2);
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest, VariantPathIndexHitAfterCumulativeCompaction) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.predefined_paths = {
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
const auto index_case =
IndexStorageCaseBuilder("variant_path_index_hit_after_cumulative_compaction")
.tablet_id(110006)
.variant_column(std::move(variant))
.inverted_index(IndexSpec::field_pattern_index(20002, "idx_v_b", 2, "b"))
.rowset(0, IndexDataSourceSpec::inline_variant(
{R"({"b": "one"})", R"({"b": "other"})"}, 0))
.rowset(1, IndexDataSourceSpec::variant_jsonl(
"be/test/storage/test_data/index_storage/"
"variant_path_index.jsonl",
100, 1))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.b");
ASSERT_GE(path_column_id, 0);
const auto& path_column = tablet_schema()->column(path_column_id);
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_string_target_type();
read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "one"));
auto read_result = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 2);
expect_index_filter_stats(read_result.value(), 1);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.b");
ASSERT_EQ(compacted_path_column_id, path_column_id);
auto compacted_read = read_rowsets(readable_compacted.value(), read_options);
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 2);
expect_index_filter_stats(compacted_read.value(), 2);
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest, VariantPathCountOnIndexFiltersRows) {
run_variant_path_count_on_index_filters_rows(false);
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest, VariantPathCountOnIndexSkipsReadingKeyData) {
run_variant_path_count_on_index_filters_rows(true);
}
TEST_F(IndexStorageVariantPathIndexLifecycleTest,
VariantPathEqualityChoosesStringIndexAfterCompaction) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 10;
variant.predefined_paths = {
VariantPathSpec {.path = "string1",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "string2",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
VariantPathSpec {.path = "array_string",
.type = FieldType::OLAP_FIELD_TYPE_ARRAY,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = FieldType::OLAP_FIELD_TYPE_STRING,
.array_item_nullable = true},
};
IndexSpec phrase_index =
IndexSpec::field_pattern_index(210101, "idx_v_string1_phrase", 2, "string1");
phrase_index.properties["parser"] = "unicode";
phrase_index.properties["support_phrase"] = "true";
const auto string_index =
IndexSpec::field_pattern_index(210102, "idx_v_string1_string", 2, "string1");
IndexSpec sibling_phrase_index =
IndexSpec::field_pattern_index(210103, "idx_v_string2_phrase", 2, "string2");
sibling_phrase_index.properties["parser"] = "unicode";
sibling_phrase_index.properties["support_phrase"] = "false";
const auto sibling_string_index =
IndexSpec::field_pattern_index(210104, "idx_v_string2_string", 2, "string2");
const auto array_string_index =
IndexSpec::field_pattern_index(210105, "idx_v_array_string", 2, "array_string");
const auto index_case =
IndexStorageCaseBuilder("variant_path_equality_chooses_string_index_after_compaction")
.tablet_id(110031)
.variant_column(std::move(variant))
.inverted_index(IndexSpec::column_index(210100, "idx_v_parent", 2))
.inverted_index(std::move(phrase_index))
.inverted_index(string_index)
.inverted_index(std::move(sibling_phrase_index))
.inverted_index(sibling_string_index)
.inverted_index(array_string_index)
// Keep hit sets asymmetric so a phrase, sibling, or array index cannot satisfy
// the same row/filter-count assertions as the exact string path index.
.rowset(0,
IndexDataSourceSpec::inline_variant(
{R"({"string1": "hello", "string2": "side", "array_string": ["array-only"]})",
R"({"string1": "hello world", "string2": "side", "array_string": ["hello"]})",
R"({"string1": "world", "string2": "world", "array_string": ["world"]})",
R"({"string1": "other", "string2": "world", "array_string": ["hello"]})",
R"({"string1": "hello", "string2": "side", "array_string": ["other"]})"},
0))
.rowset(1,
IndexDataSourceSpec::inline_variant(
{R"({"string1": "hello", "string2": "side", "array_string": ["hello"]})",
R"({"string1": "other", "string2": "world", "array_string": ["array-only"]})",
R"({"string1": "world", "string2": "side", "array_string": ["hello"]})",
R"({"string1": "other", "string2": "side", "array_string": ["world"]})",
R"({"string1": "hello", "string2": "side", "array_string": ["other"]})"},
100))
.build();
ASSERT_TRUE(create_tablet(index_case.tablet_options).ok());
auto rowsets = write_rowsets(index_case.rowsets);
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto readable_rowsets = rowsets_with_variant_extended_schema(rowsets.value());
ASSERT_TRUE(readable_rowsets.has_value()) << readable_rowsets.error();
const int32_t path_column_id = column_id_by_path("v.string1");
ASSERT_GE(path_column_id, 0);
const auto& path_column = tablet_schema()->column(path_column_id);
const int32_t sibling_path_column_id = column_id_by_path("v.string2");
ASSERT_GE(sibling_path_column_id, 0);
const auto& sibling_path_column = tablet_schema()->column(sibling_path_column_id);
auto expect_path_index_filtered_rows = [](const IndexReadResult& result,
int64_t expected_filtered_rows) {
expect_index_filter_stats(result, expected_filtered_rows);
};
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(path_column_id)};
read_options.target_cast_type_for_variants[path_column.name()] = nullable_string_target_type();
read_options.predicates.push_back(string_equals(path_column_id, path_column.name(), "hello"));
IndexReadOptions sibling_read_options;
sibling_read_options.return_columns = {0, static_cast<uint32_t>(sibling_path_column_id)};
sibling_read_options.target_cast_type_for_variants[sibling_path_column.name()] =
nullable_string_target_type();
sibling_read_options.predicates.push_back(
string_equals(sibling_path_column_id, sibling_path_column.name(), "world"));
auto before_compaction = read_rowsets(readable_rowsets.value(), read_options);
ASSERT_TRUE(before_compaction.has_value()) << before_compaction.error();
EXPECT_EQ(before_compaction->rows_read, 4);
expect_path_index_filtered_rows(before_compaction.value(), 6);
auto sibling_before_compaction = read_rowsets(readable_rowsets.value(), sibling_read_options);
ASSERT_TRUE(sibling_before_compaction.has_value()) << sibling_before_compaction.error();
EXPECT_EQ(sibling_before_compaction->rows_read, 3);
expect_path_index_filtered_rows(sibling_before_compaction.value(), 7);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 10);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto readable_compacted = rowsets_with_variant_extended_schema(reloaded.value());
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
const int32_t compacted_path_column_id = column_id_by_path("v.string1");
ASSERT_EQ(compacted_path_column_id, path_column_id);
const int32_t compacted_sibling_path_column_id = column_id_by_path("v.string2");
ASSERT_EQ(compacted_sibling_path_column_id, sibling_path_column_id);
auto after_compaction = read_rowsets(readable_compacted.value(), read_options);
ASSERT_TRUE(after_compaction.has_value()) << after_compaction.error();
EXPECT_EQ(after_compaction->rows_read, 4);
expect_path_index_filtered_rows(after_compaction.value(), 6);
auto sibling_after_compaction = read_rowsets(readable_compacted.value(), sibling_read_options);
ASSERT_TRUE(sibling_after_compaction.has_value()) << sibling_after_compaction.error();
EXPECT_EQ(sibling_after_compaction->rows_read, 3);
expect_path_index_filtered_rows(sibling_after_compaction.value(), 7);
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_schema_evolution_test.cpp
+308
View File
@@ -0,0 +1,308 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "exec/common/variant_util.h"
#include "storage/variant/index_storage_variant_test_base.h"
namespace doris::index_storage_test {
class IndexStorageVariantSchemaEvolutionTest : public IndexStorageTestFixture {
protected:
void run_multi_variant_columns_keep_shared_path_isolated(bool external_segment_meta,
int64_t tablet_id);
};
// DORIS-26466 is intentionally kept out for now; the product path is not being fixed yet.
#if 0
TEST_F(IndexStorageVariantSchemaEvolutionTest, PatchedSchemaAddDropVariantColumnCompactsNewRows) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 4;
VariantColumnSpec obsolete;
obsolete.unique_id = 3;
obsolete.name = "old_v";
obsolete.max_subcolumns_count = 4;
IndexTabletOptions options;
options.tablet_id = 110017;
options.variant_columns = {std::move(variant), std::move(obsolete)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
IndexBatch batch0;
batch0.keys = {0, 1};
batch0.variant_jsons_by_column = {
{R"({"a": "old-one"})", R"({"a": "old-two"})"},
{R"({"gone": "drop-one"})", R"({"gone": "drop-two"})"},
};
rowset0.batches.push_back(std::move(batch0));
auto rowset0_result = write_rowset(rowset0);
ASSERT_TRUE(rowset0_result.has_value()) << rowset0_result.error();
VariantColumnSpec added;
added.unique_id = 4;
added.name = "v2";
added.nullable = true;
added.max_subcolumns_count = 4;
added.predefined_paths = {
VariantPathSpec {.path = "x",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
IndexSchemaPatch patch;
patch.drop_column_uids.insert(3);
patch.add_variant_columns.push_back(std::move(added));
auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_column_unique_id(2));
EXPECT_FALSE(patched_schema->has_column_unique_id(3));
ASSERT_TRUE(patched_schema->has_column_unique_id(4));
auto patched_old_rowsets =
inject_reader_schema_for_rowsets({rowset0_result.value()}, std::move(patched_schema));
ASSERT_TRUE(patched_old_rowsets.has_value()) << patched_old_rowsets.error();
IndexRowsetSpec rowset1;
rowset1.version = 1;
IndexBatch batch1;
batch1.keys = {2, 3};
batch1.variant_jsons_by_column = {
{R"({"a": "new-one"})", R"({"a": "new-two"})"},
{R"({"x": "added-one"})", R"({"x": "added-two"})"},
};
rowset1.batches.push_back(std::move(batch1));
auto rowset1_result = write_rowset(rowset1);
ASSERT_TRUE(rowset1_result.has_value()) << rowset1_result.error();
auto read_result = read_rowsets({patched_old_rowsets->front(), rowset1_result.value()});
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 4);
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE,
{patched_old_rowsets->front(), rowset1_result.value()});
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
EXPECT_EQ(compacted.value()->num_rows(), 4);
auto compacted_probe = probe_rowset(compacted.value());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
EXPECT_TRUE(has_variant_layout(compacted_probe.value(), 4, "x"));
EXPECT_FALSE(has_variant_parent(compacted_probe.value(), 3));
auto compacted_read = read_rowsets({compacted.value()});
ASSERT_TRUE(compacted_read.has_value()) << compacted_read.error();
EXPECT_EQ(compacted_read->rows_read, 4);
}
#endif
TEST_F(IndexStorageVariantSchemaEvolutionTest,
MissingRequiredVariantColumnFailsExtendedInfoAggregation) {
IndexTabletOptions options;
options.tablet_id = 110018;
options.text_columns = {TextColumnSpec {.unique_id = 2, .name = "title"}};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset;
rowset.version = 0;
rowset.batches.push_back(IndexBatch::single_text({"hello", "other"}, 0));
auto rowset_result = write_rowset(rowset);
ASSERT_TRUE(rowset_result.has_value()) << rowset_result.error();
VariantColumnSpec missing_required_variant;
missing_required_variant.unique_id = 3;
missing_required_variant.name = "required_v";
missing_required_variant.nullable = false;
IndexSchemaPatch patch;
patch.add_variant_columns.push_back(std::move(missing_required_variant));
auto patched_schema = build_patched_tablet_schema(*tablet_schema(), patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_column_unique_id(3));
auto patched_rowsets =
inject_reader_schema_for_rowsets({rowset_result.value()}, std::move(patched_schema));
ASSERT_TRUE(patched_rowsets.has_value()) << patched_rowsets.error();
std::unordered_map<int32_t, variant_util::VariantExtendedInfo> variant_extended_info;
auto status = variant_util::VariantCompactionUtil::aggregate_variant_extended_info(
patched_rowsets->front(), &variant_extended_info);
EXPECT_FALSE(status.ok());
EXPECT_NE(status.to_string().find("column not found in segment"), std::string::npos)
<< status.to_string();
}
void IndexStorageVariantSchemaEvolutionTest::run_multi_variant_columns_keep_shared_path_isolated(
bool external_segment_meta, int64_t tablet_id) {
VariantColumnSpec left;
left.unique_id = 2;
left.name = "v_left";
left.max_subcolumns_count = 2;
VariantColumnSpec right;
right.unique_id = 3;
right.name = "v_right";
right.max_subcolumns_count = 2;
IndexTabletOptions options;
options.tablet_id = tablet_id;
options.external_segment_meta = external_segment_meta;
options.variant_columns = {std::move(left), std::move(right)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
IndexBatch batch0;
batch0.keys = {0, 1};
batch0.variant_jsons_by_column = {
{R"({"shared": "left-0", "left_only": "l0"})",
R"({"shared": "left-1", "left_only": "l1"})"},
{R"({"shared": "right-0", "right_only": "r0"})",
R"({"shared": "right-1", "right_only": "r1"})"},
};
rowset0.batches.push_back(std::move(batch0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
IndexBatch batch1;
batch1.keys = {2, 3};
batch1.variant_jsons_by_column = {
{R"({"shared": "left-2", "left_only": "l2"})",
R"({"shared": "left-3", "left_only": "l3"})"},
{R"({"shared": "right-2", "right_only": "r2"})",
R"({"shared": "right-3", "right_only": "r3"})"},
};
rowset1.batches.push_back(std::move(batch1));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
std::unordered_map<int32_t, variant_util::VariantExtendedInfo> extended_info;
for (const auto& rowset : rowsets.value()) {
auto status = variant_util::VariantCompactionUtil::aggregate_variant_extended_info(
rowset, &extended_info);
ASSERT_TRUE(status.ok()) << status.to_string();
}
ASSERT_TRUE(extended_info.contains(2));
ASSERT_TRUE(extended_info.contains(3));
EXPECT_EQ(extended_info.at(2).path_to_none_null_values.at("shared"), 4);
EXPECT_EQ(extended_info.at(3).path_to_none_null_values.at("shared"), 4);
EXPECT_TRUE(extended_info.at(2).path_to_none_null_values.contains("left_only"));
EXPECT_TRUE(extended_info.at(3).path_to_none_null_values.contains("right_only"));
EXPECT_FALSE(extended_info.at(2).path_to_none_null_values.contains("right_only"));
EXPECT_FALSE(extended_info.at(3).path_to_none_null_values.contains("left_only"));
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
ASSERT_EQ(compacted.value()->num_rows(), 4);
auto probe = probe_rowset(compacted.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_TRUE(has_variant_layout(probe.value(), 2, "shared"));
EXPECT_TRUE(has_variant_layout(probe.value(), 2, "left_only"));
EXPECT_TRUE(has_variant_layout(probe.value(), 3, "shared"));
EXPECT_TRUE(has_variant_layout(probe.value(), 3, "right_only"));
EXPECT_FALSE(has_variant_layout(probe.value(), 2, "right_only"));
EXPECT_FALSE(has_variant_layout(probe.value(), 3, "left_only"));
auto read_result = read_rowsets({compacted.value()});
ASSERT_TRUE(read_result.has_value()) << read_result.error();
EXPECT_EQ(read_result->rows_read, 4);
}
TEST_F(IndexStorageVariantSchemaEvolutionTest,
MultiVariantColumnsKeepSharedPathIsolatedWithExternalSegmentMeta) {
run_multi_variant_columns_keep_shared_path_isolated(true, 110047);
}
TEST_F(IndexStorageVariantSchemaEvolutionTest,
MultiVariantColumnsKeepSharedPathIsolatedWithoutExternalSegmentMeta) {
run_multi_variant_columns_keep_shared_path_isolated(false, 110053);
}
TEST_F(IndexStorageVariantSchemaEvolutionTest, SchemaPatchHelperKeepsVariantPathMetadataIsolated) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.predefined_paths = {
VariantPathSpec {.path = "a",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
IndexTabletOptions options;
options.tablet_id = 110048;
options.variant_columns = {std::move(variant)};
options.inverted_indexes = {IndexSpec::field_pattern_index(230001, "idx_v_a", 2, "a")};
auto base_schema = build_tablet_schema(options);
ASSERT_NE(base_schema, nullptr);
VariantColumnSpec modified;
modified.unique_id = 2;
modified.name = "v";
modified.predefined_paths = {
VariantPathSpec {.path = "b",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
VariantColumnSpec added;
added.unique_id = 3;
added.name = "v_added";
added.predefined_paths = {
VariantPathSpec {.path = "a",
.type = FieldType::OLAP_FIELD_TYPE_STRING,
.nullable = true,
.pattern_type = PatternTypePB::MATCH_NAME,
.array_item_type = {},
.array_item_nullable = true},
};
IndexSchemaPatch patch;
patch.modify_variant_columns.emplace(2, std::move(modified));
patch.add_variant_columns.push_back(std::move(added));
patch.add_inverted_indexes.push_back(
IndexSpec::field_pattern_index(230002, "idx_v_added_a", 3, "a"));
auto patched_schema = build_patched_tablet_schema(*base_schema, patch);
ASSERT_NE(patched_schema, nullptr);
EXPECT_TRUE(schema_has_variant_path(*patched_schema, 2, "a"));
EXPECT_TRUE(schema_has_variant_path(*patched_schema, 2, "b"));
EXPECT_TRUE(schema_has_variant_path(*patched_schema, 3, "a"));
EXPECT_FALSE(schema_has_variant_path(*patched_schema, 3, "b"));
EXPECT_FALSE(patched_schema->inverted_index_by_field_pattern(2, "a").empty());
EXPECT_FALSE(patched_schema->inverted_index_by_field_pattern(3, "a").empty());
EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(2, "missing").empty());
EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(3, "b").empty());
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_schema_patch_test.cpp
+86
View File
@@ -0,0 +1,86 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
TEST(IndexStorageVariantSchemaPatchTest, AddDropIndexAndModifyVariantProperties) {
IndexTabletOptions options;
options.tablet_id = 110003;
options.variant_columns = {VariantColumnSpec {}};
options.variant_columns[0].unique_id = 2;
options.variant_columns[0].name = "v";
options.inverted_indexes.push_back(
IndexSpec::field_pattern_index(20001, "idx_v_old", 2, "old"));
auto base_schema = build_tablet_schema(options);
VariantColumnSpec modified = options.variant_columns[0];
modified.max_subcolumns_count = 16;
modified.enable_doc_mode = true;
modified.doc_materialization_min_rows = 128;
VariantColumnSpec added;
added.unique_id = 3;
added.name = "v2";
added.max_subcolumns_count = 2;
IndexSchemaPatch patch;
patch.modify_variant_columns.emplace(2, modified);
patch.add_variant_columns.push_back(added);
patch.drop_index_names.insert("idx_v_old");
patch.add_inverted_indexes.push_back(
IndexSpec::field_pattern_index(20002, "idx_v_new", 2, "a"));
auto patched_schema = build_patched_tablet_schema(*base_schema, patch);
ASSERT_NE(patched_schema, nullptr);
ASSERT_TRUE(patched_schema->has_column_unique_id(2));
ASSERT_TRUE(patched_schema->has_column_unique_id(3));
const auto& variant = patched_schema->column_by_uid(2);
EXPECT_EQ(variant.variant_max_subcolumns_count(), 16);
EXPECT_TRUE(variant.variant_enable_doc_mode());
EXPECT_TRUE(patched_schema->inverted_index_by_field_pattern(2, "old").empty());
auto new_indexes = patched_schema->inverted_index_by_field_pattern(2, "a");
ASSERT_EQ(new_indexes.size(), 1);
EXPECT_EQ(new_indexes[0]->index_name(), "idx_v_new");
}
TEST(IndexStorageVariantSchemaPatchTest, VariantDocAndNestedGroupOptionsArePreserved) {
IndexTabletOptions options;
options.tablet_id = 110009;
options.variant_columns = {VariantColumnSpec {}};
options.variant_columns[0].unique_id = 2;
options.variant_columns[0].name = "v";
options.variant_columns[0].enable_doc_mode = true;
options.variant_columns[0].doc_materialization_min_rows = 64;
options.variant_columns[0].doc_hash_shard_count = 8;
options.variant_columns[0].enable_nested_group = true;
auto schema = build_tablet_schema(options);
ASSERT_NE(schema, nullptr);
ASSERT_TRUE(schema->has_column_unique_id(2));
const auto& variant = schema->column_by_uid(2);
EXPECT_TRUE(variant.variant_enable_doc_mode());
EXPECT_EQ(variant.variant_doc_materialization_min_rows(), 64);
EXPECT_EQ(variant.variant_doc_hash_shard_count(), 8);
EXPECT_TRUE(variant.variant_enable_nested_group());
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_sparse_stats_test.cpp
+353
View File
@@ -0,0 +1,353 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "exec/common/variant_util.h"
#include "storage/variant/index_storage_variant_test_base.h"
namespace doris::index_storage_test {
class IndexStorageVariantSparseStatsTest : public IndexStorageTestFixture {
protected:
void run_sparse_stats_limit_boundary_case(int32_t sparse_stats_limit,
size_t expected_sparse_stat_entries,
bool expect_b_c_sparse_stat, int64_t tablet_id);
};
TEST_F(IndexStorageVariantSparseStatsTest, VariantCompactionSchemaTopNRecordsSparseOverflowPaths) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = 2;
IndexTabletOptions options;
options.tablet_id = 110045;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h0", "warm": "w0", "rare0": "r0"})",
R"({"hot": "h1", "warm": "w1", "rare1": "r1"})", R"({"hot": "h2", "rare2": "r2"})"},
0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h3", "warm": "w3", "rare3": "r3"})", R"({"hot": "h4", "rare4": "r4"})"},
100));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compaction_schema = std::make_shared<TabletSchema>(*tablet_schema());
auto status = variant_util::VariantCompactionUtil::get_extended_compaction_schema(
rowsets.value(), compaction_schema);
ASSERT_TRUE(status.ok()) << status.to_string();
const auto* path_set_info = compaction_schema->try_path_set_info(2);
ASSERT_NE(path_set_info, nullptr);
EXPECT_TRUE(path_set_info->sub_path_set.contains(StringRef("hot")));
EXPECT_FALSE(path_set_info->sub_path_set.contains(StringRef("warm")));
EXPECT_TRUE(path_set_info->sparse_path_set.contains(StringRef("warm")));
EXPECT_TRUE(path_set_info->sparse_path_set.contains(StringRef("rare0")));
}
TEST_F(IndexStorageVariantSparseStatsTest, VariantSparseStatsLimitIsPreservedAfterCompaction) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = 2;
IndexTabletOptions options;
options.tablet_id = 110046;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h0", "warm": "w0", "rare0": "r0"})",
R"({"hot": "h1", "warm": "w1", "rare1": "r1"})", R"({"hot": "h2", "rare2": "r2"})"},
0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h3", "warm": "w3", "rare3": "r3"})", R"({"hot": "h4", "rare4": "r4"})"},
100));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
ASSERT_EQ(compacted.value()->num_rows(), 5);
auto probe = probe_rowset(compacted.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_TRUE(has_variant_layout(probe.value(), 2, "hot"));
EXPECT_FALSE(has_variant_layout(probe.value(), 2, "warm"));
EXPECT_TRUE(has_sparse_path_stat(probe.value(), "warm"));
EXPECT_LE(sparse_stat_entry_count(probe.value()), 2);
}
TEST_F(IndexStorageVariantSparseStatsTest, VariantSparseStatsLimitOneIsPreservedAfterCompaction) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = 1;
IndexTabletOptions options;
options.tablet_id = 110051;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h0", "warm": "w0", "rare0": "r0"})",
R"({"hot": "h1", "warm": "w1", "rare1": "r1"})", R"({"hot": "h2", "rare2": "r2"})"},
0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(IndexBatch::single_variant(
{R"({"hot": "h3", "warm": "w3", "rare3": "r3"})", R"({"hot": "h4", "rare4": "r4"})"},
100));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
ASSERT_EQ(compacted.value()->num_rows(), 5);
auto probe = probe_rowset(compacted.value());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_TRUE(has_variant_layout(probe.value(), 2, "hot"));
EXPECT_LE(sparse_stat_entry_count(probe.value()), 1);
}
void IndexStorageVariantSparseStatsTest::run_sparse_stats_limit_boundary_case(
int32_t sparse_stats_limit, size_t expected_sparse_stat_entries,
bool expect_b_c_sparse_stat, int64_t tablet_id) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = sparse_stats_limit;
IndexTabletOptions options;
options.tablet_id = tablet_id;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(
IndexBatch::single_variant({R"({"a": "hot-0", "b": "scalar-0", "d": "dense-0"})",
R"({"a": "hot-1", "b": "scalar-1", "d": "dense-1"})",
R"({"a": "hot-2", "b": "scalar-2", "d": "dense-2"})"},
0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(
IndexBatch::single_variant({R"({"a": "hot-3", "b": "scalar-3", "d": "dense-3"})",
R"({"a": "hot-4", "b": {"c": "child-0"}})"},
100));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
ASSERT_EQ(compacted.value()->num_rows(), 5);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto child_schema = build_schema_with_variant_path_column(*tablet_schema(), 2, "v.b.c",
FieldType::OLAP_FIELD_TYPE_STRING);
ASSERT_NE(child_schema, nullptr);
auto child_readable =
inject_reader_schema_for_rowsets(reloaded.value(), std::move(child_schema));
ASSERT_TRUE(child_readable.has_value()) << child_readable.error();
auto probe = probe_rowset(child_readable->front());
ASSERT_TRUE(probe.has_value()) << probe.error();
EXPECT_TRUE(has_variant_layout(probe.value(), 2, "a"));
EXPECT_TRUE(has_sparse_path_stat(probe.value(), "b"));
EXPECT_EQ(has_sparse_path_stat(probe.value(), "b.c"), expect_b_c_sparse_stat);
EXPECT_EQ(sparse_stat_entry_count(probe.value()), expected_sparse_stat_entries);
const int32_t child_column_id = column_id_by_path("v.b.c");
ASSERT_GE(child_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& child_column = tablet_schema()->column(child_column_id);
IndexReadOptions child_read_options;
child_read_options.need_ordered_result = true;
child_read_options.return_columns = {0, static_cast<uint32_t>(child_column_id)};
child_read_options.collect_string_values = true;
auto child_read_result = read_rowsets(child_readable.value(), child_read_options);
ASSERT_TRUE(child_read_result.has_value()) << child_read_result.error();
ASSERT_EQ(child_read_result->rows_read, 5);
ASSERT_TRUE(child_read_result->string_values_by_uid.contains(child_column.unique_id()));
const auto& child_values = child_read_result->string_values_by_uid.at(child_column.unique_id());
ASSERT_EQ(child_values.size(), 5);
EXPECT_EQ(std::count(child_values.begin(), child_values.end(),
std::optional<std::string> {"child-0"}),
1)
<< describe_optional_string_values(child_values);
EXPECT_EQ(std::count(child_values.begin(), child_values.end(), std::nullopt), 4)
<< describe_optional_string_values(child_values);
auto missing_schema = build_schema_with_variant_path_column(*tablet_schema(), 2, "v.b.missing",
FieldType::OLAP_FIELD_TYPE_STRING);
ASSERT_NE(missing_schema, nullptr);
auto missing_readable =
inject_reader_schema_for_rowsets(child_readable.value(), std::move(missing_schema));
ASSERT_TRUE(missing_readable.has_value()) << missing_readable.error();
const int32_t missing_column_id = column_id_by_path("v.b.missing");
ASSERT_GE(missing_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& missing_column = tablet_schema()->column(missing_column_id);
IndexReadOptions missing_read_options;
missing_read_options.need_ordered_result = true;
missing_read_options.return_columns = {0, static_cast<uint32_t>(missing_column_id)};
missing_read_options.collect_string_values = true;
auto missing_read_result = read_rowsets(missing_readable.value(), missing_read_options);
ASSERT_TRUE(missing_read_result.has_value()) << missing_read_result.error();
ASSERT_EQ(missing_read_result->rows_read, 5);
ASSERT_TRUE(missing_read_result->string_values_by_uid.contains(missing_column.unique_id()));
const auto& missing_values =
missing_read_result->string_values_by_uid.at(missing_column.unique_id());
ASSERT_EQ(missing_values.size(), 5);
EXPECT_TRUE(std::all_of(missing_values.begin(), missing_values.end(), [](const auto& value) {
return !value.has_value();
})) << describe_optional_string_values(missing_values);
}
TEST_F(IndexStorageVariantSparseStatsTest,
SparseStatsBeforeLimitReadsRecordedExactChildAndMissingChildNull) {
run_sparse_stats_limit_boundary_case(/*sparse_stats_limit=*/4,
/*expected_sparse_stat_entries=*/3,
/*expect_b_c_sparse_stat=*/true, 110055);
}
TEST_F(IndexStorageVariantSparseStatsTest,
SparseStatsAtLimitReadsRecordedExactChildAndMissingChildNull) {
run_sparse_stats_limit_boundary_case(/*sparse_stats_limit=*/3,
/*expected_sparse_stat_entries=*/3,
/*expect_b_c_sparse_stat=*/true, 110056);
}
TEST_F(IndexStorageVariantSparseStatsTest,
SparseStatsAfterLimitTruncationReadsUnrecordedExactChildAndMissingChildNull) {
run_sparse_stats_limit_boundary_case(/*sparse_stats_limit=*/2,
/*expected_sparse_stat_entries=*/2,
/*expect_b_c_sparse_stat=*/false, 110057);
}
TEST_F(IndexStorageVariantSparseStatsTest,
ExactSparsePathReadsHiddenChildAfterSparseStatsLimitTruncated) {
VariantColumnSpec variant;
variant.unique_id = 2;
variant.name = "v";
variant.max_subcolumns_count = 1;
variant.max_sparse_column_statistics_size = 2;
IndexTabletOptions options;
options.tablet_id = 110028;
options.variant_columns = {std::move(variant)};
ASSERT_TRUE(create_tablet(options).ok());
IndexRowsetSpec rowset0;
rowset0.version = 0;
rowset0.batches.push_back(
IndexBatch::single_variant({R"({"a": "hot-0", "b": "scalar-0", "d": "dense-0"})",
R"({"a": "hot-1", "b": "scalar-1", "d": "dense-1"})",
R"({"a": "hot-2", "b": "scalar-2", "d": "dense-2"})"},
0));
IndexRowsetSpec rowset1;
rowset1.version = 1;
rowset1.batches.push_back(
IndexBatch::single_variant({R"({"a": "hot-3", "b": "scalar-3", "d": "dense-3"})",
R"({"a": "hot-4", "b": {"c": "child-0"}})"},
100));
auto rowsets = write_rowsets({rowset0, rowset1});
ASSERT_TRUE(rowsets.has_value()) << rowsets.error();
auto compacted = compact_rowsets(IndexCompactionKind::CUMULATIVE, rowsets.value());
ASSERT_TRUE(compacted.has_value()) << compacted.error();
ASSERT_NE(compacted.value(), nullptr);
ASSERT_EQ(compacted.value()->num_rows(), 5);
auto reloaded = reload_rowsets({compacted.value()});
ASSERT_TRUE(reloaded.has_value()) << reloaded.error();
auto read_schema = build_schema_with_variant_path_column(*tablet_schema(), 2, "b",
FieldType::OLAP_FIELD_TYPE_VARIANT);
ASSERT_NE(read_schema, nullptr);
auto readable_compacted =
inject_reader_schema_for_rowsets(reloaded.value(), std::move(read_schema));
ASSERT_TRUE(readable_compacted.has_value()) << readable_compacted.error();
auto compacted_probe = probe_rowset(readable_compacted->front());
ASSERT_TRUE(compacted_probe.has_value()) << compacted_probe.error();
ASSERT_TRUE(has_variant_layout(compacted_probe.value(), 2, "a"));
ASSERT_TRUE(has_sparse_path_stat(compacted_probe.value(), "b"));
ASSERT_FALSE(has_sparse_path_stat(compacted_probe.value(), "b.c"));
const int32_t b_column_id = column_id_by_path("v.b");
ASSERT_GE(b_column_id, 0) << dump_schema_paths(*tablet_schema());
const auto& b_column = tablet_schema()->column(b_column_id);
ASSERT_TRUE(b_column.is_variant_type());
IndexReadOptions read_options;
read_options.return_columns = {0, static_cast<uint32_t>(b_column_id)};
read_options.collect_variant_values = true;
auto read_result = read_rowsets(readable_compacted.value(), read_options);
ASSERT_TRUE(read_result.has_value()) << read_result.error();
ASSERT_EQ(read_result->rows_read, 5);
ASSERT_TRUE(read_result->variant_values_by_uid.contains(b_column.unique_id()));
const auto& b_values = read_result->variant_values_by_uid.at(b_column.unique_id());
ASSERT_EQ(b_values.size(), 5);
const auto has_hidden_child = std::any_of(b_values.begin(), b_values.end(), [](const auto& v) {
return v.has_value() && v->find("child-0") != std::string::npos;
});
std::ostringstream serialized_values;
for (const auto& value : b_values) {
serialized_values << (value.has_value() ? value.value() : "NULL") << '\n';
}
EXPECT_TRUE(has_hidden_child) << serialized_values.str();
}
} // namespace doris::index_storage_test
be/test/storage/variant/index_storage_variant_test_base.h
+136
View File
@@ -0,0 +1,136 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <algorithm>
#include <optional>
#include <sstream>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_string.h"
#include "storage/predicate/predicate_creator.h"
#include "testutil/index_storage_test_util.h"
namespace doris::index_storage_test {
inline std::shared_ptr<ColumnPredicate> string_equals(int32_t column_id, std::string column_name,
std::string value) {
return create_comparison_predicate<PredicateType::EQ>(
column_id, std::move(column_name), std::make_shared<DataTypeString>(),
Field::create_field<TYPE_STRING>(std::move(value)), false);
}
inline DataTypePtr nullable_string_target_type() {
return make_nullable(std::make_shared<DataTypeString>());
}
inline bool has_doc_value_column(const IndexRowsetProbe& probe) {
return std::any_of(probe.segments.begin(), probe.segments.end(), [](const auto& segment) {
return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(),
[](const auto& column) { return column.is_doc_value_column; });
});
}
inline bool has_variant_layout(const IndexRowsetProbe& probe, int32_t parent_unique_id,
std::string_view relative_path) {
return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) {
return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(),
[&](const auto& column) {
return column.parent_unique_id == parent_unique_id &&
column.relative_path == relative_path;
});
});
}
inline bool has_variant_parent(const IndexRowsetProbe& probe, int32_t parent_unique_id) {
return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) {
return std::any_of(
segment.variant_columns.begin(), segment.variant_columns.end(),
[&](const auto& column) { return column.parent_unique_id == parent_unique_id; });
});
}
inline bool has_sparse_path_stat(const IndexRowsetProbe& probe, std::string_view relative_path) {
return std::any_of(probe.segments.begin(), probe.segments.end(), [&](const auto& segment) {
return std::any_of(segment.variant_columns.begin(), segment.variant_columns.end(),
[&](const auto& column) {
auto it =
column.sparse_non_null_size.find(std::string(relative_path));
return it != column.sparse_non_null_size.end() && it->second > 0;
});
});
}
inline bool schema_has_variant_path(const TabletSchema& schema, int32_t parent_unique_id,
std::string_view relative_path) {
if (!schema.has_column_unique_id(parent_unique_id)) {
return false;
}
const auto& parent = schema.column_by_uid(parent_unique_id);
for (const auto& subcolumn : parent.get_sub_columns()) {
if (subcolumn == nullptr) {
continue;
}
if (subcolumn->name() == relative_path ||
(subcolumn->has_path_info() &&
subcolumn->path_info_ptr()->copy_pop_front().get_path() == relative_path)) {
return true;
}
}
return false;
}
inline bool schema_has_extracted_variant_path(const TabletSchema& schema, int32_t parent_unique_id,
std::string_view relative_path) {
for (int32_t column_id = 0; column_id < schema.num_columns(); ++column_id) {
const auto& column = schema.column(column_id);
if (!column.is_extracted_column() || column.parent_unique_id() != parent_unique_id ||
!column.has_path_info()) {
continue;
}
if (column.path_info_ptr()->copy_pop_front().get_path() == relative_path) {
return true;
}
}
return false;
}
inline size_t sparse_stat_entry_count(const IndexRowsetProbe& probe) {
size_t count = 0;
for (const auto& segment : probe.segments) {
for (const auto& column : segment.variant_columns) {
count += column.sparse_non_null_size.size();
}
}
return count;
}
inline std::string describe_optional_string_values(
const std::vector<std::optional<std::string>>& values) {
std::ostringstream out;
for (const auto& value : values) {
out << (value.has_value() ? value.value() : "NULL") << '\n';
}
return out.str();
}
} // namespace doris::index_storage_test
be/test/storage/segment/variant_column_writer_reader_test.cpp → be/test/storage/variant/variant_column_writer_reader_test.cpp
View File
File renamed without changes.
be/test/storage/compaction/variant_doc_mode_compaction_test.cpp → be/test/storage/variant/variant_doc_mode_compaction_test.cpp
View File
File renamed without changes.
be/test/storage/segment/variant_stats_calculator_test.cpp → be/test/storage/variant/variant_stats_calculator_test.cpp
+45 -1
View File
@@ -31,6 +31,7 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "exec/common/variant_util.h"
#include "storage/tablet/tablet_schema.h"
#include "util/json/path_in_data.h"
@@ -119,6 +120,28 @@ protected:
return ColumnMap::create(std::move(keys), std::move(values), std::move(offsets));
}
ColumnPtr create_sparse_map_column_for_limit_test() {
auto keys = ColumnString::create();
auto values = ColumnString::create();
auto offsets = ColumnArray::ColumnOffsets::create();
offsets->insert_value(0);
keys->insert_data("hot", 3);
values->insert_data("h0", 2);
keys->insert_data("warm", 4);
values->insert_data("w0", 2);
offsets->insert_value(2);
keys->insert_data("hot", 3);
values->insert_data("h1", 2);
keys->insert_data("cold", 4);
values->insert_data("c1", 2);
offsets->insert_value(4);
return ColumnMap::create(std::move(keys), std::move(values), std::move(offsets));
}
TabletSchemaSPtr _tablet_schema;
std::unique_ptr<SegmentFooterPB> _footer;
};
@@ -233,6 +256,27 @@ TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithSparseColumn) {
EXPECT_TRUE(column_meta.has_variant_statistics());
}
TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsStopsAddingKeysAtSparseStatsLimit) {
auto map_column = create_sparse_map_column_for_limit_test();
VariantStatisticsPB stats;
variant_util::VariantCompactionUtil::calculate_variant_stats(*map_column, &stats, 2, 1, 2);
ASSERT_EQ(stats.sparse_column_non_null_size_size(), 2);
EXPECT_EQ(stats.sparse_column_non_null_size().at("hot"), 2);
EXPECT_EQ(stats.sparse_column_non_null_size().at("warm"), 1);
EXPECT_FALSE(stats.sparse_column_non_null_size().contains("cold"));
}
TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithZeroSparseStatsLimitDropsAllNewKeys) {
auto map_column = create_sparse_map_column_for_limit_test();
VariantStatisticsPB stats;
variant_util::VariantCompactionUtil::calculate_variant_stats(*map_column, &stats, 0, 1, 2);
EXPECT_TRUE(stats.sparse_column_non_null_size().empty());
}
TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithMissingFooterEntry) {
// Create variant sub column but don't add corresponding footer entry
TabletColumn sub_column = create_variant_column(2, "variant_col.missing_sub", 1, "missing_sub");
@@ -482,4 +526,4 @@ TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithFooterOffset) {
EXPECT_EQ(_footer->columns(footer_offset).none_null_size(), 2);
}
} // namespace doris::segment_v2
} // namespace doris::segment_v2
be/test/storage/segment/variant_util_test.cpp → be/test/storage/variant/variant_util_test.cpp
View File
File renamed without changes.
be/test/testutil/index_storage_test_util.cpp
+1578
View File
File diff suppressed because it is too large. Load diff
be/test/testutil/index_storage_test_util.h
+346
View File
@@ -0,0 +1,346 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <gtest/gtest.h>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
#include "common/status.h"
#include "core/data_type/data_type.h"
#include "exprs/vexpr_fwd.h"
#include "runtime/descriptors.h"
#include "storage/olap_common.h"
#include "storage/olap_define.h"
#include "storage/predicate/column_predicate.h"
#include "storage/rowset/rowset.h"
#include "storage/tablet/tablet.h"
#include "storage/tablet/tablet_schema.h"
#include "util/json/json_parser.h"
namespace doris {
class DataDir;
struct DebugPoint;
class StorageEngine;
namespace segment_v2 {
class InvertedIndexQueryCache;
class InvertedIndexSearcherCache;
} // namespace segment_v2
namespace index_storage_test {
struct TextColumnSpec {
int32_t unique_id = 2;
std::string name = "title";
FieldType type = FieldType::OLAP_FIELD_TYPE_STRING;
bool nullable = false;
};
struct VariantPathSpec {
std::string path;
FieldType type = FieldType::OLAP_FIELD_TYPE_STRING;
bool nullable = true;
PatternTypePB pattern_type = PatternTypePB::MATCH_NAME;
std::optional<FieldType> array_item_type;
bool array_item_nullable = true;
};
struct VariantColumnSpec {
int32_t unique_id = 2;
std::string name = "v";
bool nullable = false;
bool is_bf_column = false;
int32_t max_subcolumns_count = 8;
int32_t max_sparse_column_statistics_size = 10000;
int32_t sparse_hash_shard_count = 0;
bool enable_doc_mode = false;
int64_t doc_materialization_min_rows = 0;
int32_t doc_hash_shard_count = 0;
bool enable_nested_group = false;
std::vector<VariantPathSpec> predefined_paths;
};
struct IndexSpec {
int64_t index_id = 10001;
std::string name = "idx_v";
int32_t column_uid = 2;
std::string suffix_path;
std::map<std::string, std::string> properties;
static IndexSpec column_index(int64_t index_id, std::string name, int32_t column_uid,
std::map<std::string, std::string> properties = {});
static IndexSpec field_pattern_index(int64_t index_id, std::string name, int32_t column_uid,
std::string field_pattern);
};
struct IndexTabletOptions {
int64_t tablet_id = 100000;
bool include_key_column = true;
bool external_segment_meta = true;
int64_t storage_page_size = 0;
InvertedIndexStorageFormatPB index_storage_format = InvertedIndexStorageFormatPB::V2;
std::vector<TextColumnSpec> text_columns;
std::vector<VariantColumnSpec> variant_columns;
std::vector<IndexSpec> inverted_indexes;
};
struct IndexBatch {
std::vector<int32_t> keys;
std::vector<std::vector<std::string>> text_values_by_column;
std::vector<std::vector<std::string>> variant_jsons_by_column;
std::vector<ColumnPtr> variant_columns_by_column;
bool deprecated_enable_flatten_nested = false;
bool check_duplicate_json_path = false;
ParseConfig::ParseTo parse_to = ParseConfig::ParseTo::OnlySubcolumns;
static IndexBatch single_text(std::vector<std::string> values, int32_t first_key = 0);
static IndexBatch single_variant(std::vector<std::string> jsons, int32_t first_key = 0);
static IndexBatch single_variant_column(ColumnPtr column, int32_t first_key = 0);
size_t num_rows() const;
};
enum class IndexDataSourceKind {
INLINE_TEXT_ROWS,
INLINE_VARIANT_ROWS,
VARIANT_JSONL_FILE,
};
struct IndexDataSourceSpec {
IndexDataSourceKind kind = IndexDataSourceKind::INLINE_TEXT_ROWS;
std::vector<std::string> rows;
std::string file_path;
int32_t first_key = 0;
size_t batch_size = 4096;
static IndexDataSourceSpec inline_text(std::vector<std::string> values, int32_t first_key = 0);
static IndexDataSourceSpec inline_variant(std::vector<std::string> jsons,
int32_t first_key = 0);
static IndexDataSourceSpec variant_jsonl(std::string file_path, int32_t first_key = 0,
size_t batch_size = 4096);
};
struct IndexRowsetSpec {
int64_t version = 0;
int64_t max_rows_per_segment = 200;
DataWriteType write_type = DataWriteType::TYPE_DIRECT;
bool add_to_tablet = true;
std::vector<IndexDataSourceSpec> data_sources;
std::vector<IndexBatch> batches;
};
struct IndexReadOptions {
ReaderType reader_type = ReaderType::READER_QUERY;
bool need_ordered_result = false;
std::vector<uint32_t> return_columns;
std::vector<std::shared_ptr<ColumnPredicate>> predicates;
std::map<std::string, DataTypePtr> target_cast_type_for_variants;
std::map<int32_t, TColumnAccessPaths> all_access_paths;
std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
VExprContextSPtrs common_expr_ctxs_push_down;
bool collect_string_values = false;
bool collect_variant_values = false;
bool enable_inverted_index_query = true;
bool enable_fallback_on_missing_inverted_index = true;
bool enable_no_need_read_data_opt = true;
bool enable_common_expr_pushdown = true;
bool enable_inverted_index_query_cache = false;
std::optional<int32_t> inverted_index_skip_threshold = std::nullopt;
TPushAggOp::type push_down_agg_type_opt = TPushAggOp::NONE;
};
struct IndexReadResult {
int64_t rows_read = 0;
OlapReaderStatistics stats;
std::map<int32_t, std::vector<std::optional<std::string>>> string_values_by_uid;
std::map<int32_t, std::vector<std::optional<std::string>>> variant_values_by_uid;
bool inverted_index_used() const;
};
struct IndexColumnLayout {
int32_t parent_unique_id = -1;
std::string full_path;
std::string relative_path;
int64_t none_null_size = 0;
bool is_sparse_column = false;
bool is_doc_value_column = false;
std::map<std::string, int64_t> sparse_non_null_size;
};
struct IndexSegmentLayout {
int64_t segment_id = -1;
int64_t num_rows = 0;
std::vector<IndexColumnLayout> variant_columns;
bool contains_relative_path(const std::string& path) const;
};
struct IndexFileProbe {
int64_t expected_files = 0;
int64_t existing_files = 0;
int64_t rowset_meta_entries = 0;
int64_t rowset_meta_index_size = 0;
std::vector<std::string> missing_files;
bool all_expected_files_exist() const { return expected_files == existing_files; }
};
struct IndexRowsetProbe {
int64_t num_rows = 0;
int64_t num_segments = 0;
std::vector<IndexSegmentLayout> segments;
IndexFileProbe index_files;
bool contains_relative_path(const std::string& path) const;
};
enum class IndexCompactionKind {
CUMULATIVE,
FULL,
};
struct IndexSchemaPatch {
std::vector<TextColumnSpec> add_text_columns;
std::vector<VariantColumnSpec> add_variant_columns;
std::set<int32_t> drop_column_uids;
std::map<int32_t, VariantColumnSpec> modify_variant_columns;
std::vector<IndexSpec> add_inverted_indexes;
std::set<int64_t> drop_index_ids;
std::set<std::string> drop_index_names;
};
struct IndexStorageCase {
std::string name;
IndexTabletOptions tablet_options;
std::vector<IndexRowsetSpec> rowsets;
};
class IndexStorageCaseBuilder {
public:
explicit IndexStorageCaseBuilder(std::string name);
IndexStorageCaseBuilder& tablet_id(int64_t tablet_id);
IndexStorageCaseBuilder& text_column(TextColumnSpec column);
IndexStorageCaseBuilder& variant_column(VariantColumnSpec column);
IndexStorageCaseBuilder& inverted_index(IndexSpec index);
IndexStorageCaseBuilder& rowset(int64_t version, IndexDataSourceSpec data_source,
int64_t max_rows_per_segment = 200);
IndexStorageCase build() const;
private:
IndexStorageCase _case;
};
TabletSchemaPB build_tablet_schema_pb(const IndexTabletOptions& options);
TabletSchemaSPtr build_tablet_schema(const IndexTabletOptions& options);
TabletSchemaPB apply_schema_patch(const TabletSchema& base_schema, const IndexSchemaPatch& patch);
TabletSchemaSPtr build_patched_tablet_schema(const TabletSchema& base_schema,
const IndexSchemaPatch& patch);
TabletSchemaSPtr build_schema_with_variant_path_column(const TabletSchema& base_schema,
int32_t parent_unique_id,
std::string relative_path, FieldType type);
void expect_index_filter_stats(const IndexReadResult& result, int64_t expected_filtered_rows);
void expect_raw_rows_read(const IndexReadResult& result, int64_t expected_raw_rows_read);
void expect_segment_pruned(const IndexReadResult& result, int64_t expected_filtered_segments);
void expect_zone_map_filtered(const IndexReadResult& result, int64_t expected_filtered_rows);
void expect_bloom_filter_filtered(const IndexReadResult& result, int64_t expected_filtered_rows);
void expect_inverted_index_used(const IndexReadResult& result);
void expect_index_files(const IndexRowsetProbe& probe, bool expected_present);
std::string dump_schema_paths(const TabletSchema& schema);
class ScopedDebugPoint {
public:
explicit ScopedDebugPoint(std::string name, std::map<std::string, std::string> params = {});
~ScopedDebugPoint();
int64_t execute_num() const;
private:
std::string _name;
bool _enable_debug_points = false;
std::shared_ptr<DebugPoint> _debug_point;
};
class IndexStorageTestFixture : public testing::Test {
public:
~IndexStorageTestFixture() override;
protected:
void SetUp() override;
void TearDown() override;
Status create_tablet(const IndexTabletOptions& options);
Result<RowsetSharedPtr> write_rowset(const IndexRowsetSpec& spec);
Result<std::vector<RowsetSharedPtr>> write_rowsets(const std::vector<IndexRowsetSpec>& specs);
Result<IndexReadResult> read_rowsets(const std::vector<RowsetSharedPtr>& rowsets,
IndexReadOptions options = {});
Result<RowsetSharedPtr> compact_rowsets(IndexCompactionKind kind,
const std::vector<RowsetSharedPtr>& rowsets);
Result<RowsetSharedPtr> compact_rowsets_and_reload(IndexCompactionKind kind,
const std::vector<RowsetSharedPtr>& rowsets);
Result<RowsetSharedPtr> reload_rowset(const RowsetSharedPtr& rowset);
Result<std::vector<RowsetSharedPtr>> reload_rowsets(
const std::vector<RowsetSharedPtr>& rowsets);
// Synthetic reader-schema injection for storage UTs. This mutates fixture tablet metadata and
// reloads rowsets with the supplied schema; it does not model a real schema-change job.
Result<std::vector<RowsetSharedPtr>> inject_reader_schema_for_rowsets(
const std::vector<RowsetSharedPtr>& rowsets, TabletSchemaSPtr schema);
Result<std::vector<RowsetSharedPtr>> build_inverted_indexes(
const std::vector<IndexSpec>& indexes);
Result<std::vector<RowsetSharedPtr>> build_inverted_indexes_and_reload(
const std::vector<IndexSpec>& indexes);
Result<std::vector<RowsetSharedPtr>> drop_inverted_indexes(
const std::vector<IndexSpec>& indexes);
Result<std::vector<RowsetSharedPtr>> drop_inverted_indexes_and_reload(
const std::vector<IndexSpec>& indexes);
Result<IndexRowsetProbe> probe_rowset(const RowsetSharedPtr& rowset);
void use_rowset_schema(const RowsetSharedPtr& rowset);
Result<std::vector<RowsetSharedPtr>> rowsets_with_variant_extended_schema(
const std::vector<RowsetSharedPtr>& rowsets);
int32_t column_id_by_path(const std::string& path) const;
const TabletSchemaSPtr& tablet_schema() const { return _tablet_schema; }
const TabletSharedPtr& tablet() const { return _tablet; }
StorageEngine* storage_engine() const { return _engine_ref; }
DataDir* data_dir() const { return _data_dir.get(); }
private:
std::string _test_dir;
std::string _tmp_dir;
TabletSchemaSPtr _tablet_schema;
TabletSharedPtr _tablet;
StorageEngine* _engine_ref = nullptr;
std::unique_ptr<DataDir> _data_dir;
std::unique_ptr<segment_v2::InvertedIndexSearcherCache> _inverted_index_searcher_cache;
std::unique_ptr<segment_v2::InvertedIndexQueryCache> _inverted_index_query_cache;
};
} // namespace index_storage_test
} // namespace doris
regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy
+49
View File
@@ -51,6 +51,55 @@ suite("test_variant_bloom_filter", "nonConcurrent") {
}
}
sql """DROP TABLE IF EXISTS test_variant_typed_int_bloom_filter"""
sql "set default_variant_enable_doc_mode = false"
sql "set enable_common_expr_pushdown = true"
// Keep doc mode controlled by the session variable above. The column-level
// variant_enable_doc_mode property is mutually exclusive with typed-path
// properties such as variant_max_subcolumns_count.
sql """
CREATE TABLE test_variant_typed_int_bloom_filter (
k bigint,
v variant<'int_1' : int, properties(
"variant_max_subcolumns_count" = "2",
"variant_enable_typed_paths_to_sparse" = "false"
)>
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 1
properties(
"replication_num" = "1",
"disable_auto_compaction" = "true",
"bloom_filter_columns" = "v"
);
"""
sql """
INSERT INTO test_variant_typed_int_bloom_filter VALUES
(1, '{"int_1": 1}'),
(2, '{"int_1": 2}'),
(3, '{"int_1": 100}'),
(4, '{"int_1": 101}');
"""
sql """sync"""
def matched = sql """
select k, cast(v['int_1'] as int) from test_variant_typed_int_bloom_filter
where cast(v['int_1'] as int) in (1, 101)
order by k
"""
assertEquals("[[1, 1], [4, 101]]", matched.toString())
try {
GetDebugPoint().enableDebugPointForAllBEs("bloom_filter_must_filter_data")
def missing = sql """
select k from test_variant_typed_int_bloom_filter
where cast(v['int_1'] as int) = 50
"""
assertEquals(0, missing.size())
} finally {
GetDebugPoint().disableDebugPointForAllBEs("bloom_filter_must_filter_data")
}
sql """DROP TABLE IF EXISTS ${index_table}"""
int seed = Math.floor(Math.random() * 7)
def var_def = "variant"
regression-test/suites/variant_p0/column_name.groovy
+10 -6
View File
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("regression_test_variant_column_name", "variant_type"){
suite("regression_test_variant_column_name", "p0, nonConcurrent"){
def table_name = "var_column_name"
sql "DROP TABLE IF EXISTS ${table_name}"
sql """
@@ -64,10 +64,14 @@ suite("regression_test_variant_column_name", "variant_type"){
qt_sql "select cast(Tags[''] as text) from var_column_name order by cast(Tags[''] as string)"
// name with `.`
sql "truncate table var_column_name"
test {
sql """insert into var_column_name values (7, '{"a.b": "UPPER CASE", "a.c": "lower case", "a" : {"b" : 123}, "a" : {"c" : 456}}')"""
exception "may contains duplicated entry"
// When parser-side duplicate path deduplication is disabled, dotted keys and nested paths
// collide during Variant materialization and should report duplicated entry.
setBeConfigTemporary([variant_enable_duplicate_json_path_check: false]) {
sql "truncate table var_column_name"
test {
sql """insert into var_column_name values (7, '{"a.b": "UPPER CASE", "a.c": "lower case", "a" : {"b" : 123}, "a" : {"c" : 456}}')"""
exception "may contains duplicated entry"
}
}
for (int i = 0; i < 7; i++) {
sql """insert into var_column_name select * from var_column_name"""
@@ -93,4 +97,4 @@ suite("regression_test_variant_column_name", "variant_type"){
sql """insert into var_column_name values (8, '{"${key}": "test"}')"""
exception "Key length exceeds maximum allowed size of 255 bytes."
}
}
}