From a35707da74f8df0d6502df95feaf734dd9d20d9d Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 3 Nov 2026 16:37:26 +0800 Subject: [PATCH] [fix](variant) Write Variant V2 sparse keys in byte order (#70001) --- .../segment/variant/v2/variant_shredder.cpp | 5 + .../variant_column_writer_reader_test.cpp | 87 ++++++++++++ .../data/variant_p0/sparse_key_byte_order.out | 129 ++++++++++++++++++ .../variant_p0/sparse_key_byte_order.groovy | 69 ++++++++++ 4 files changed, 290 insertions(+) create mode 100644 regression-test/data/variant_p0/sparse_key_byte_order.out create mode 100644 regression-test/suites/variant_p0/sparse_key_byte_order.groovy diff --git a/be/src/storage/segment/variant/v2/variant_shredder.cpp b/be/src/storage/segment/variant/v2/variant_shredder.cpp index e9164a513d7..6175cbaef4c 100644 --- a/be/src/storage/segment/variant/v2/variant_shredder.cpp +++ b/be/src/storage/segment/variant/v2/variant_shredder.cpp @@ -509,6 +509,11 @@ struct VariantShredder::Impl { DorisVector sparse_plan = build_sparse_plan(selection, candidate_builders); select_sparse_statistics(&sparse_plan); + // Readers search full path bytes: a-b must precede a.c, unlike component order. + // Select statistics first to retain the existing encounter-order budget tie-break. + std::ranges::sort(sparse_plan, [](const SparsePlan& left, const SparsePlan& right) { + return *left.path < *right.path; + }); RETURN_IF_ERROR(append_binary_rows(sparse_plan, sparse_maps)); DorisVector bucket_statistics(options.sparse_bucket_count); publish_sparse_statistics(sparse_plan, &bucket_statistics, result); diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 0e8ebdaf46e..4b63f624480 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -74,6 +74,7 @@ #include "storage/segment/variant/nested_group_provider.h" #include "storage/segment/variant/nested_group_streaming_write_plan.h" #include "storage/segment/variant/sparse_column_merge_iterator.h" +#include "storage/segment/variant/v2/variant_assembler.h" #include "storage/segment/variant/v2/variant_column_writer.h" #include "storage/segment/variant/v2/variant_path_builder.h" #include "storage/segment/variant/v2/variant_shredder.h" @@ -968,6 +969,92 @@ TEST(VariantPathBuilderTest, ShredderReusesCanonicalPathsAcrossAppends) { EXPECT_EQ(selected.type->to_string(*selected.column, 1), "2"); } +TEST(VariantShredderTest, SparseRowsUseByteOrderWithoutChangingStatisticsSelection) { + ColumnPtr source; + DataTypePtr source_type; + ASSERT_TRUE(create_variant_writer_source( + VariantWriterInput::V2, + {R"({"z":1,"a-b":1,"a b":3,"a$b":4,"a%2Eb":5,"a":{"c":2}})", R"({"z":2})", + R"({"z":3})"}, + 1, false, {}, &source, &source_type) + .ok()); + const auto& values = assert_cast(*source); + for (const auto layout : {segment_v2::VariantShredderPhysicalLayout::ORDINARY, + segment_v2::VariantShredderPhysicalLayout::DOC}) { + for (const uint32_t buckets : {1, 3}) { + SCOPED_TRACE(testing::Message() + << "layout=" << static_cast(layout) << " buckets=" << buckets); + segment_v2::VariantShredderOptions options; + options.physical_layout = layout; + options.max_subcolumns_count = 1; + options.sparse_bucket_count = buckets; + options.max_sparse_column_statistics_size = 1; + options.doc_bucket_count = buckets; + options.doc_materialization_min_rows = 4; + segment_v2::VariantShredder shredder(options); + segment_v2::VariantShredder::TestAccess::set_binary_cells_per_chunk(shredder, 2); + ASSERT_TRUE(shredder.append(values.read_view(), 0, values.size()).ok()); + segment_v2::VariantShreddedColumns output; + ASSERT_TRUE(shredder.finish(&output).ok()); + ASSERT_EQ(output.binary_buckets.size(), buckets); + for (const auto& bucket : output.binary_buckets) { + const auto& map = assert_cast(*bucket.column); + const auto& keys = assert_cast(map.get_keys()); + for (size_t row = 0; row < map.size(); ++row) { + std::vector actual; + for (size_t cell = map.offset_at(row); + cell < map.offset_at(row) + map.size_at(row); ++cell) { + actual.push_back(keys.get_data_at(cell).to_string()); + } + auto expected = actual; + std::ranges::sort(expected); + EXPECT_EQ(actual, expected) << "row=" << row; + } + } + if (layout == segment_v2::VariantShredderPhysicalLayout::ORDINARY) { + ASSERT_EQ(output.materialized.size(), 1); + EXPECT_EQ(output.materialized.front().path.get_path(), "z"); + const std::vector> cases { + {"a.c", "2"}, {"a-b", "1"}, {"a b", "3"}, {"a$b", "4"}, {"a%2Eb", "5"}}; + for (const auto& [path, expected] : cases) { + SCOPED_TRACE(path); + const uint32_t bucket = variant_util::variant_binary_shard_of( + {path.data(), path.size()}, buckets); + const auto& map = + assert_cast(*output.binary_buckets[bucket].column); + segment_v2::variant_v2::VariantAssemblerOptions read_options; + read_options.requested_path = PathInData(path); + read_options.storage_map_kind = segment_v2::variant_v2::StorageMapKind::SPARSE; + auto assembler = segment_v2::variant_v2::VariantAssembler::create( + std::move(read_options)); + ASSERT_TRUE(assembler.has_value()) << assembler.error(); + ColumnNullable::MutablePtr result; + ASSERT_TRUE((*assembler) + ->assemble({.num_rows = values.size(), + .materialized_columns = {}, + .storage_map = &map}, + &result) + .ok()); + const auto& read_values = + assert_cast(result->get_nested_column()); + EXPECT_EQ(variant_v2_json_at(read_values, 0), expected); + EXPECT_FALSE(result->is_null_at(0)); + EXPECT_TRUE(result->is_null_at(1)); + EXPECT_TRUE(result->is_null_at(2)); + } + if (buckets == 1) { + // Component order encounters a.c first; physical byte order must not + // change which path wins the one-entry statistics budget. + EXPECT_EQ(output.statistics.sparse_column_non_null_size.size(), 1); + EXPECT_EQ(output.statistics.sparse_column_non_null_size.at("a.c"), 1); + EXPECT_EQ(output.binary_buckets.front().statistics.sparse_column_non_null_size, + output.statistics.sparse_column_non_null_size); + } + } + } + } +} + TEST(VariantShredderTest, ReusesRootAndNestedPathsForSharedMetadata) { DataTypeVariantV2SerDe serde; DataTypeSerDe::FormatOptions format_options; diff --git a/regression-test/data/variant_p0/sparse_key_byte_order.out b/regression-test/data/variant_p0/sparse_key_byte_order.out new file mode 100644 index 00000000000..785f7cbcb32 --- /dev/null +++ b/regression-test/data/variant_p0/sparse_key_byte_order.out @@ -0,0 +1,129 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 11 12 13 14 15 {"c":15} {"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !before_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + +-- !after_compaction -- +1 31 32 33 34 35 {"c":35} {"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1} +2 \N \N \N \N \N \N {"z":2} +3 \N \N \N \N \N \N {"z":3} +4 21 22 23 24 25 {"c":25} {"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4} +5 \N \N \N \N \N \N {"z":5} +6 \N \N \N \N \N \N {"z":6} + diff --git a/regression-test/suites/variant_p0/sparse_key_byte_order.groovy b/regression-test/suites/variant_p0/sparse_key_byte_order.groovy new file mode 100644 index 00000000000..654d80ca3ce --- /dev/null +++ b/regression-test/suites/variant_p0/sparse_key_byte_order.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("sparse_key_byte_order", "p0") { + // The parser rejects an explicit doc-mode property beside sparse properties. + sql "SET default_variant_enable_doc_mode = false" + for (def keyModel : ["DUPLICATE", "UNIQUE"]) { + for (def shardCount : [1, 3]) { + for (def docMode : [false, true]) { + def tableName = "sparse_key_byte_order_${keyModel.toLowerCase()}_${shardCount}_${docMode}" + def mowProperty = keyModel == "UNIQUE" ? ', "enable_unique_key_merge_on_write"="true"' : '' + def variantProperties = docMode ? + '"variant_enable_doc_mode"="true", "variant_enable_nested_group"="false", ' + + '"variant_doc_materialization_min_rows"="1000000", ' + + "\"variant_doc_hash_shard_count\"=\"${shardCount}\"" : + '"variant_max_subcolumns_count"="1", "variant_enable_typed_paths_to_sparse"="false", ' + + '"variant_enable_nested_group"="false", ' + + "\"variant_sparse_hash_shard_count\"=\"${shardCount}\"" + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + id INT, + v VARIANT + ) ${keyModel} KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1", "disable_auto_compaction"="true"${mowProperty}) + """ + // z is hotter than every other leaf, leaving the punctuation keys sparse. + sql """INSERT INTO ${tableName} VALUES + (1, parse_to_variant('{"z":1,"a-b":11,"a b":12,"a\$b":13,"a%2Eb":14,"a":{"c":15}}')), + (2, parse_to_variant('{"z":2}')), + (3, parse_to_variant('{"z":3}')) + """ + sql """INSERT INTO ${tableName} VALUES + (4, parse_to_variant('{"z":4,"a-b":21,"a b":22,"a\$b":23,"a%2Eb":24,"a":{"c":25}}')), + (5, parse_to_variant('{"z":5}')), + (6, parse_to_variant('{"z":6}')) + """ + if (keyModel == "UNIQUE") { + sql """INSERT INTO ${tableName} VALUES + (1, parse_to_variant('{"z":1,"a-b":31,"a b":32,"a\$b":33,"a%2Eb":34,"a":{"c":35}}')), + (2, parse_to_variant('{"z":2}')), + (3, parse_to_variant('{"z":3}')) + """ + } + def projection = """SELECT id, v['a-b'], v['a b'], v['a\$b'], v['a%2Eb'], + v['a']['c'], CAST(v['a'] AS STRING), CAST(v AS STRING) + FROM ${tableName} ORDER BY id""" + qt_before_compaction projection + trigger_and_wait_compaction(tableName, "full", 1800) + qt_after_compaction projection + } + } + } +}