[fix](variant) Write Variant V2 sparse keys in byte order (#70001)

be/src/storage/segment/variant/v2/variant_shredder.cpp

@@ -509,6 +509,11 @@ struct VariantShredder::Impl {
 
         DorisVector<SparsePlan> sparse_plan = build_sparse_plan(selection, candidate_builders);
         select_sparse_statistics(&sparse_plan);
+        // Readers search full path bytes: a-b must precede a.c, unlike component order.
+        // Select statistics first to retain the existing encounter-order budget tie-break.
+        std::ranges::sort(sparse_plan, [](const SparsePlan& left, const SparsePlan& right) {
+            return *left.path < *right.path;
+        });
         RETURN_IF_ERROR(append_binary_rows(sparse_plan, sparse_maps));
         DorisVector<VariantStatistics> bucket_statistics(options.sparse_bucket_count);
         publish_sparse_statistics(sparse_plan, &bucket_statistics, result);

be/test/storage/variant/variant_column_writer_reader_test.cpp

@@ -74,6 +74,7 @@
 #include "storage/segment/variant/nested_group_provider.h"
 #include "storage/segment/variant/nested_group_streaming_write_plan.h"
 #include "storage/segment/variant/sparse_column_merge_iterator.h"
+#include "storage/segment/variant/v2/variant_assembler.h"
 #include "storage/segment/variant/v2/variant_column_writer.h"
 #include "storage/segment/variant/v2/variant_path_builder.h"
 #include "storage/segment/variant/v2/variant_shredder.h"
@@ -968,6 +969,92 @@ TEST(VariantPathBuilderTest, ShredderReusesCanonicalPathsAcrossAppends) {
     EXPECT_EQ(selected.type->to_string(*selected.column, 1), "2");
 }
 
+TEST(VariantShredderTest, SparseRowsUseByteOrderWithoutChangingStatisticsSelection) {
+    ColumnPtr source;
+    DataTypePtr source_type;
+    ASSERT_TRUE(create_variant_writer_source(
+                        VariantWriterInput::V2,
+                        {R"({"z":1,"a-b":1,"a b":3,"a$b":4,"a%2Eb":5,"a":{"c":2}})", R"({"z":2})",
+                         R"({"z":3})"},
+                        1, false, {}, &source, &source_type)
+                        .ok());
+    const auto& values = assert_cast<const ColumnVariantV2&>(*source);
+    for (const auto layout : {segment_v2::VariantShredderPhysicalLayout::ORDINARY,
+                              segment_v2::VariantShredderPhysicalLayout::DOC}) {
+        for (const uint32_t buckets : {1, 3}) {
+            SCOPED_TRACE(testing::Message()
+                         << "layout=" << static_cast<int>(layout) << " buckets=" << buckets);
+            segment_v2::VariantShredderOptions options;
+            options.physical_layout = layout;
+            options.max_subcolumns_count = 1;
+            options.sparse_bucket_count = buckets;
+            options.max_sparse_column_statistics_size = 1;
+            options.doc_bucket_count = buckets;
+            options.doc_materialization_min_rows = 4;
+            segment_v2::VariantShredder shredder(options);
+            segment_v2::VariantShredder::TestAccess::set_binary_cells_per_chunk(shredder, 2);
+            ASSERT_TRUE(shredder.append(values.read_view(), 0, values.size()).ok());
+            segment_v2::VariantShreddedColumns output;
+            ASSERT_TRUE(shredder.finish(&output).ok());
+            ASSERT_EQ(output.binary_buckets.size(), buckets);
+            for (const auto& bucket : output.binary_buckets) {
+                const auto& map = assert_cast<const ColumnMap&>(*bucket.column);
+                const auto& keys = assert_cast<const ColumnString&>(map.get_keys());
+                for (size_t row = 0; row < map.size(); ++row) {
+                    std::vector<std::string> actual;
+                    for (size_t cell = map.offset_at(row);
+                         cell < map.offset_at(row) + map.size_at(row); ++cell) {
+                        actual.push_back(keys.get_data_at(cell).to_string());
+                    }
+                    auto expected = actual;
+                    std::ranges::sort(expected);
+                    EXPECT_EQ(actual, expected) << "row=" << row;
+                }
+            }
+            if (layout == segment_v2::VariantShredderPhysicalLayout::ORDINARY) {
+                ASSERT_EQ(output.materialized.size(), 1);
+                EXPECT_EQ(output.materialized.front().path.get_path(), "z");
+                const std::vector<std::pair<std::string, std::string>> cases {
+                        {"a.c", "2"}, {"a-b", "1"}, {"a b", "3"}, {"a$b", "4"}, {"a%2Eb", "5"}};
+                for (const auto& [path, expected] : cases) {
+                    SCOPED_TRACE(path);
+                    const uint32_t bucket = variant_util::variant_binary_shard_of(
+                            {path.data(), path.size()}, buckets);
+                    const auto& map =
+                            assert_cast<const ColumnMap&>(*output.binary_buckets[bucket].column);
+                    segment_v2::variant_v2::VariantAssemblerOptions read_options;
+                    read_options.requested_path = PathInData(path);
+                    read_options.storage_map_kind = segment_v2::variant_v2::StorageMapKind::SPARSE;
+                    auto assembler = segment_v2::variant_v2::VariantAssembler::create(
+                            std::move(read_options));
+                    ASSERT_TRUE(assembler.has_value()) << assembler.error();
+                    ColumnNullable::MutablePtr result;
+                    ASSERT_TRUE((*assembler)
+                                        ->assemble({.num_rows = values.size(),
+                                                    .materialized_columns = {},
+                                                    .storage_map = &map},
+                                                   &result)
+                                        .ok());
+                    const auto& read_values =
+                            assert_cast<const ColumnVariantV2&>(result->get_nested_column());
+                    EXPECT_EQ(variant_v2_json_at(read_values, 0), expected);
+                    EXPECT_FALSE(result->is_null_at(0));
+                    EXPECT_TRUE(result->is_null_at(1));
+                    EXPECT_TRUE(result->is_null_at(2));
+                }
+                if (buckets == 1) {
+                    // Component order encounters a.c first; physical byte order must not
+                    // change which path wins the one-entry statistics budget.
+                    EXPECT_EQ(output.statistics.sparse_column_non_null_size.size(), 1);
+                    EXPECT_EQ(output.statistics.sparse_column_non_null_size.at("a.c"), 1);
+                    EXPECT_EQ(output.binary_buckets.front().statistics.sparse_column_non_null_size,
+                              output.statistics.sparse_column_non_null_size);
+                }
+            }
+        }
+    }
+}
+
 TEST(VariantShredderTest, ReusesRootAndNestedPathsForSharedMetadata) {
     DataTypeVariantV2SerDe serde;
     DataTypeSerDe::FormatOptions format_options;

regression-test/data/variant_p0/sparse_key_byte_order.out

@@ -0,0 +1,129 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !before_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	11	12	13	14	15	{"c":15}	{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !before_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+
+-- !after_compaction --
+1	31	32	33	34	35	{"c":35}	{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2	\N	\N	\N	\N	\N	\N	{"z":2}
+3	\N	\N	\N	\N	\N	\N	{"z":3}
+4	21	22	23	24	25	{"c":25}	{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5	\N	\N	\N	\N	\N	\N	{"z":5}
+6	\N	\N	\N	\N	\N	\N	{"z":6}
+

regression-test/suites/variant_p0/sparse_key_byte_order.groovy

@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("sparse_key_byte_order", "p0") {
+    // The parser rejects an explicit doc-mode property beside sparse properties.
+    sql "SET default_variant_enable_doc_mode = false"
+    for (def keyModel : ["DUPLICATE", "UNIQUE"]) {
+        for (def shardCount : [1, 3]) {
+            for (def docMode : [false, true]) {
+                def tableName = "sparse_key_byte_order_${keyModel.toLowerCase()}_${shardCount}_${docMode}"
+                def mowProperty = keyModel == "UNIQUE" ? ', "enable_unique_key_merge_on_write"="true"' : ''
+                def variantProperties = docMode ?
+                    '"variant_enable_doc_mode"="true", "variant_enable_nested_group"="false", ' +
+                    '"variant_doc_materialization_min_rows"="1000000", ' +
+                    "\"variant_doc_hash_shard_count\"=\"${shardCount}\"" :
+                    '"variant_max_subcolumns_count"="1", "variant_enable_typed_paths_to_sparse"="false", ' +
+                    '"variant_enable_nested_group"="false", ' +
+                    "\"variant_sparse_hash_shard_count\"=\"${shardCount}\""
+                sql "DROP TABLE IF EXISTS ${tableName}"
+                sql """
+                    CREATE TABLE ${tableName} (
+                        id INT,
+                        v VARIANT<PROPERTIES(${variantProperties})>
+                    ) ${keyModel} KEY(id)
+                    DISTRIBUTED BY HASH(id) BUCKETS 1
+                    PROPERTIES("replication_num"="1", "disable_auto_compaction"="true"${mowProperty})
+                """
+                // z is hotter than every other leaf, leaving the punctuation keys sparse.
+                sql """INSERT INTO ${tableName} VALUES
+                    (1, parse_to_variant('{"z":1,"a-b":11,"a b":12,"a\$b":13,"a%2Eb":14,"a":{"c":15}}')),
+                    (2, parse_to_variant('{"z":2}')),
+                    (3, parse_to_variant('{"z":3}'))
+                """
+                sql """INSERT INTO ${tableName} VALUES
+                    (4, parse_to_variant('{"z":4,"a-b":21,"a b":22,"a\$b":23,"a%2Eb":24,"a":{"c":25}}')),
+                    (5, parse_to_variant('{"z":5}')),
+                    (6, parse_to_variant('{"z":6}'))
+                """
+                if (keyModel == "UNIQUE") {
+                    sql """INSERT INTO ${tableName} VALUES
+                        (1, parse_to_variant('{"z":1,"a-b":31,"a b":32,"a\$b":33,"a%2Eb":34,"a":{"c":35}}')),
+                        (2, parse_to_variant('{"z":2}')),
+                        (3, parse_to_variant('{"z":3}'))
+                    """
+                }
+                def projection = """SELECT id, v['a-b'], v['a b'], v['a\$b'], v['a%2Eb'],
+                    v['a']['c'], CAST(v['a'] AS STRING), CAST(v AS STRING)
+                    FROM ${tableName} ORDER BY id"""
+                qt_before_compaction projection
+                trigger_and_wait_compaction(tableName, "full", 1800)
+                qt_after_compaction projection
+            }
+        }
+    }
+}