From 211a79a011554bb561192e83d28cbd4fb5e25021 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 19 Oct 2026 10:52:21 +0800 Subject: [PATCH] [fix](be) Count variant data in information_schema.column_data_sizes (#68444) --- .../schema_column_data_sizes_scanner.cpp | 53 +++++--- be/src/storage/merger.cpp | 5 +- be/src/storage/segment/column_writer.cpp | 13 ++ be/src/storage/segment/column_writer.h | 22 ++-- .../variant/v2/variant_column_writer.cpp | 13 ++ .../variant/v2/variant_column_writer.h | 3 + .../variant/variant_column_writer_impl.cpp | 28 ++++ .../variant/variant_column_writer_impl.h | 20 ++- .../segment_0.dat | Bin 5586 -> 5588 bytes .../segment_1.dat | Bin 5588 -> 5590 bytes .../segment_0.dat | Bin 5402 -> 5404 bytes .../segment_1.dat | Bin 5404 -> 5406 bytes .../segment_0.dat | Bin 5446 -> 5448 bytes .../segment_1.dat | Bin 5258 -> 5260 bytes .../segment_0.dat | Bin 5262 -> 5264 bytes .../segment_1.dat | Bin 5074 -> 5076 bytes .../segment_0.dat | Bin 5582 -> 5584 bytes .../segment_1.dat | Bin 5582 -> 5584 bytes .../segment_0.dat | Bin 5398 -> 5400 bytes .../segment_1.dat | Bin 5398 -> 5400 bytes .../segment_0.dat | Bin 5384 -> 5386 bytes .../segment_1.dat | Bin 5161 -> 5163 bytes .../segment_0.dat | Bin 5200 -> 5202 bytes .../segment_1.dat | Bin 4977 -> 4979 bytes .../segment_0.dat | Bin 12635 -> 12639 bytes .../segment_1.dat | Bin 12645 -> 12649 bytes .../segment_0.dat | Bin 3513 -> 3515 bytes .../segment_1.dat | Bin 3525 -> 3527 bytes .../segment_0.dat | Bin 3515 -> 3517 bytes .../segment_1.dat | Bin 3527 -> 3529 bytes .../segment_0.dat | Bin 3513 -> 3515 bytes .../segment_1.dat | Bin 3525 -> 3527 bytes .../segment_0.dat | Bin 3513 -> 3515 bytes .../segment_1.dat | Bin 3525 -> 3527 bytes .../segment_0.dat | Bin 2854 -> 2856 bytes .../segment_1.dat | Bin 3056 -> 3058 bytes .../segment_2.dat | Bin 2448 -> 2450 bytes .../variant_column_writer_reader_test.cpp | 14 ++ ...st_query_sys_column_data_sizes_variant.out | 33 +++++ ...query_sys_column_data_sizes_variant.groovy | 120 ++++++++++++++++++ 40 files changed, 287 insertions(+), 37 deletions(-) create mode 100644 regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out create mode 100644 regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy diff --git a/be/src/information_schema/schema_column_data_sizes_scanner.cpp b/be/src/information_schema/schema_column_data_sizes_scanner.cpp index 52df991c710..2ffba6a1563 100644 --- a/be/src/information_schema/schema_column_data_sizes_scanner.cpp +++ b/be/src/information_schema/schema_column_data_sizes_scanner.cpp @@ -111,40 +111,57 @@ void SchemaColumnDataSizesScanner::_collect_column_data_sizes_from_rowsets( // Get column data page stats from each segment footer and aggregate by column_unique_id for (const auto& segment : segments) { - auto collector = [&](const segment_v2::ColumnMetaPB& column_meta) { + auto collector = [&](const segment_v2::ColumnMetaPB& column_meta, + uint32_t column_unique_id) { if (column_meta.has_compressed_data_bytes() && column_meta.has_uncompressed_data_bytes()) { - auto cid = schema->field_index(column_meta.unique_id()); + auto cid = schema->field_index(column_unique_id); if (cid == -1) { return; } - // Aggregate stats by column_unique_id - if (aggregated_stats.contains(column_meta.unique_id())) { - auto& existing_stats = aggregated_stats[column_meta.unique_id()]; - existing_stats.compressed_data_bytes += column_meta.compressed_data_bytes(); - existing_stats.uncompressed_data_bytes += - column_meta.uncompressed_data_bytes(); - existing_stats.raw_data_bytes += column_meta.raw_data_bytes(); - } else { - aggregated_stats[column_meta.unique_id()] = ColumnDataSizeInfo { + auto [it, inserted] = aggregated_stats.try_emplace(column_unique_id); + auto& stats = it->second; + if (inserted) { + stats = ColumnDataSizeInfo { .backend_id = backend_id_, .table_id = table_id, .index_id = index_id, .partition_id = partition_id, .tablet_id = tablet_id, .rowset_id = rowset_id, - .column_unique_id = column_meta.unique_id(), + .column_unique_id = column_unique_id, .column_name = schema->column(cid).name(), .column_type = TabletColumn::get_string_by_field_type( - static_cast(column_meta.type())), - .compressed_data_bytes = column_meta.compressed_data_bytes(), - .uncompressed_data_bytes = column_meta.uncompressed_data_bytes(), - .raw_data_bytes = column_meta.raw_data_bytes(), + schema->column(cid).type()), + .compressed_data_bytes = 0, + .uncompressed_data_bytes = 0, + .raw_data_bytes = 0, }; } + stats.compressed_data_bytes += column_meta.compressed_data_bytes(); + stats.uncompressed_data_bytes += column_meta.uncompressed_data_bytes(); + stats.raw_data_bytes += column_meta.raw_data_bytes(); + } + }; + auto footer_collector = [&](const segment_v2::ColumnMetaPB& column_meta) { + // A variant meta counts the data of all its inner writers, whose own metas carry no + // data bytes. Compaction writes variant subcolumns as columns of their own, with + // data bytes but no unique id; count them to their root variant column. + uint32_t column_unique_id = column_meta.unique_id(); + if (column_meta.unique_id() == -1 && column_meta.has_compressed_data_bytes()) { + DORIS_CHECK(column_meta.has_column_path_info()); + column_unique_id = column_meta.column_path_info().parrent_column_unique_id(); + } + collector(column_meta, column_unique_id); + if (column_meta.type() == static_cast(FieldType::OLAP_FIELD_TYPE_VARIANT)) { + // External column meta (V3) embeds the sparse and doc value columns of a + // variant into its root column meta. + for (const auto& child : column_meta.children_columns()) { + collector(child, column_unique_id); + } } }; - st = segment->traverse_column_meta_pbs(collector); + st = segment->traverse_column_meta_pbs(footer_collector); if (!st.ok()) { continue; } @@ -354,4 +371,4 @@ Status SchemaColumnDataSizesScanner::_fill_block_impl(Block* block) { return Status::OK(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp index 566adea31ac..a743e90467e 100644 --- a/be/src/storage/merger.cpp +++ b/be/src/storage/merger.cpp @@ -706,8 +706,9 @@ Status Merger::vertical_merge_rowsets( const auto& col = tablet_schema.column(col_ordinal); int32_t uid = col.unique_id(); - // Variant columns (root or subcolumn): raw_data_bytes is 0 (TODO in writer), - // cannot estimate from footer, fallback to default for the entire group. + // Variant columns (root or subcolumn): subcolumns written as columns of their own + // (e.g. by compaction) are not counted in the root's raw_data_bytes, cannot estimate + // from footer, fallback to default for the entire group. if (uid < 0 || col.is_variant_type()) { need_fallback = true; break; diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 685076a9402..9b94884c8be 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -1342,6 +1342,19 @@ uint64_t VariantColumnWriter::estimate_buffer_size() { return _impl->estimate_buffer_size(); } +uint64_t VariantColumnWriter::get_raw_data_bytes() const { + return _impl->get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes); +} + +uint64_t VariantColumnWriter::get_total_uncompressed_data_pages_bytes() const { + return _impl->get_total_data_pages_bytes( + &ColumnWriter::get_total_uncompressed_data_pages_bytes); +} + +uint64_t VariantColumnWriter::get_total_compressed_data_pages_bytes() const { + return _impl->get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes); +} + Status VariantColumnWriter::finish() { return _impl->finish(); } diff --git a/be/src/storage/segment/column_writer.h b/be/src/storage/segment/column_writer.h index fc8c7db59ba..d79b35d1647 100644 --- a/be/src/storage/segment/column_writer.h +++ b/be/src/storage/segment/column_writer.h @@ -210,6 +210,8 @@ public: virtual uint64_t get_raw_data_bytes() const = 0; virtual uint64_t get_total_uncompressed_data_pages_bytes() const = 0; virtual uint64_t get_total_compressed_data_pages_bytes() const = 0; + // Points to one of the three data bytes getters above. + using DataBytesGetter = uint64_t (ColumnWriter::*)() const; // used for append not null data. virtual Status append_data(const uint8_t** ptr, size_t num_rows) = 0; @@ -620,16 +622,14 @@ public: Status write_bloom_filter_index() override; ordinal_t get_next_rowid() const override { return _next_rowid; } - uint64_t get_raw_data_bytes() const override { - return 0; // TODO - } + uint64_t get_raw_data_bytes() const override { return _writer->get_raw_data_bytes(); } uint64_t get_total_uncompressed_data_pages_bytes() const override { - return 0; // TODO + return _writer->get_total_uncompressed_data_pages_bytes(); } uint64_t get_total_compressed_data_pages_bytes() const override { - return 0; // TODO + return _writer->get_total_compressed_data_pages_bytes(); } Status append_nulls(size_t num_rows) override { @@ -688,17 +688,11 @@ public: Status write_bloom_filter_index() override; ordinal_t get_next_rowid() const override { return _next_rowid; } - uint64_t get_raw_data_bytes() const override { - return 0; // TODO - } + uint64_t get_raw_data_bytes() const override; - uint64_t get_total_uncompressed_data_pages_bytes() const override { - return 0; // TODO - } + uint64_t get_total_uncompressed_data_pages_bytes() const override; - uint64_t get_total_compressed_data_pages_bytes() const override { - return 0; // TODO - } + uint64_t get_total_compressed_data_pages_bytes() const override; Status append_nulls(size_t num_rows) override { return Status::NotSupported("variant writer can not append_nulls"); diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.cpp b/be/src/storage/segment/variant/v2/variant_column_writer.cpp index 12a06a84ab2..0790f1c8db0 100644 --- a/be/src/storage/segment/variant/v2/variant_column_writer.cpp +++ b/be/src/storage/segment/variant/v2/variant_column_writer.cpp @@ -359,4 +359,17 @@ uint64_t VariantV2ColumnWriter::estimate_buffer_size() { return size; } +uint64_t VariantV2ColumnWriter::get_total_data_pages_bytes( + ColumnWriter::DataBytesGetter getter) const { + DORIS_CHECK(_is_finalized); + uint64_t size = std::invoke(getter, _root_writer.get()); + for (const auto& writer : _subcolumn_writers) { + size += std::invoke(getter, writer.get()); + } + if (_binary_writer != nullptr) { + size += _binary_writer->get_total_data_pages_bytes(getter); + } + return size; +} + } // namespace doris::segment_v2 diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.h b/be/src/storage/segment/variant/v2/variant_column_writer.h index 6a41bf6e821..ea982f58a87 100644 --- a/be/src/storage/segment/variant/v2/variant_column_writer.h +++ b/be/src/storage/segment/variant/v2/variant_column_writer.h @@ -59,6 +59,9 @@ public: Status write_inverted_index(); Status write_bloom_filter_index(); uint64_t estimate_buffer_size(); + // Sums `getter`, one of the ColumnWriter data bytes getters, over every column writer of this + // variant: the root, the materialized subcolumns and the sparse or doc value columns. + uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const; private: Status _write_root(const IColumn* root_jsonb, int& column_id); diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp b/be/src/storage/segment/variant/variant_column_writer_impl.cpp index bb4559884d8..a443482a9e6 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp +++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp @@ -211,6 +211,18 @@ uint64_t UnifiedSparseColumnWriter::estimate_buffer_size() const { return size; } +uint64_t UnifiedSparseColumnWriter::get_total_data_pages_bytes( + ColumnWriter::DataBytesGetter getter) const { + uint64_t size = 0; + if (_single_writer) { + size += std::invoke(getter, _single_writer.get()); + } + for (const auto& w : _bucket_writers) { + size += std::invoke(getter, w.get()); + } + return size; +} + Status UnifiedSparseColumnWriter::finish() { if (_single_writer) { RETURN_IF_ERROR(_single_writer->finish()); @@ -350,6 +362,17 @@ uint64_t VariantDocWriter::estimate_buffer_size() const { return size; } +uint64_t VariantDocWriter::get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const { + uint64_t size = 0; + for (const auto& writer : _subcolumn_writers) { + size += std::invoke(getter, writer.get()); + } + for (const auto& writer : _doc_value_column_writers) { + size += std::invoke(getter, writer.get()); + } + return size; +} + Status VariantDocWriter::finish() { for (auto& writer : _subcolumn_writers) { RETURN_IF_ERROR(writer->finish()); @@ -520,6 +543,11 @@ uint64_t VariantColumnWriterImpl::estimate_buffer_size() { return _v2_writer ? _v2_writer->estimate_buffer_size() : 0; } +uint64_t VariantColumnWriterImpl::get_total_data_pages_bytes( + ColumnWriter::DataBytesGetter getter) const { + return _v2_writer->get_total_data_pages_bytes(getter); +} + VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts, TabletColumnPtr column) : ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) { diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.h b/be/src/storage/segment/variant/variant_column_writer_impl.h index 6ba064c3927..4cd882eaed2 100644 --- a/be/src/storage/segment/variant/variant_column_writer_impl.h +++ b/be/src/storage/segment/variant/variant_column_writer_impl.h @@ -62,6 +62,8 @@ public: virtual Status write_inverted_index() = 0; virtual Status write_bloom_filter_index() = 0; virtual uint64_t estimate_buffer_size() const = 0; + // Sums `getter`, one of the ColumnWriter data bytes getters, over the owned column writers. + virtual uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const = 0; virtual void merge_stats_to(VariantStatistics* stats) const = 0; }; @@ -80,6 +82,7 @@ public: Status write_inverted_index() override; Status write_bloom_filter_index() override; uint64_t estimate_buffer_size() const override; + uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const override; void merge_stats_to(VariantStatistics* stats) const override; private: @@ -127,6 +130,7 @@ public: Status write_zone_map() override; Status write_inverted_index() override; Status write_bloom_filter_index() override; + uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const override; void merge_stats_to(VariantStatistics* stats) const override; private: @@ -171,6 +175,7 @@ public: Status write_inverted_index(); Status write_bloom_filter_index(); uint64_t estimate_buffer_size(); + uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const; private: Status _ensure_writer(); @@ -205,15 +210,15 @@ public: ordinal_t get_next_rowid() const override { return _next_rowid; } uint64_t get_raw_data_bytes() const override { - return 0; // TODO + return _get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes); } uint64_t get_total_uncompressed_data_pages_bytes() const override { - return 0; // TODO + return _get_total_data_pages_bytes(&ColumnWriter::get_total_uncompressed_data_pages_bytes); } uint64_t get_total_compressed_data_pages_bytes() const override { - return 0; // TODO + return _get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes); } Status append_nulls(size_t num_rows) override { @@ -228,6 +233,15 @@ public: Status finalize(); private: + template + uint64_t _get_total_data_pages_bytes(Func func) const { + uint64_t size = std::invoke(func, _doc_value_column_writer.get()); + for (const auto& writer : _subcolumn_writers) { + size += std::invoke(func, writer.get()); + } + return size; + } + Status _append(const uint8_t* null_map, const uint8_t** ptr, size_t num_rows); Status _ensure_input_format(const VariantColumnData& column); Status _initialize_v2_shredder(); diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat index 7e5fce17d26514c2fe5d9f15371471059d72de56..f4c19d63ff3b91560e4832efae7f29f8b008e27e 100644 GIT binary patch delta 41 xcmcbleMNhN8!uz)WOv>w&Mhn_7`L!oU`*M3g7+Ug&jB6=h7a<(<}L<7h5$kQ4lMux delta 39 vcmcbjeMx(R8!uz?WOv>wc7_v-3>O#~HlO1C$Ii2lhk-$-(p=BQAjl8^2mlN0 diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat index b6d9603d7490a09e5e963515313c4c2237ec7934..dcdd42cbf186aa18bf644d5e5e0998d05a4b73da 100644 GIT binary patch delta 41 zcmV+^0M`H1E7mKpNDTpnlSvJF3A6;r0kj0r0c5kv4gLoV!VCZa(zid~L@-h@N=OfF delta 39 vcmcbneMNhN2QOpuWKZ5Ic7_v-3>O#~HlN}B$If$rhk?O+6JiZ1 diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_not_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat index d5bfc49f8e79c32907ef7f234fd08ed19462bdce..027959a37a197a1f0fe4886fdf73081539d55a21 100644 GIT binary patch delta 41 xcmbQEHBW29IUdH=$>({hIJdBzVBErTfiY!s0Pg{Io)#VkhIQo^`dtiy3;|1>4(tE` delta 39 vcmbQIHAid1IUdI5$>({h*cnbRGF)I}*c`-rfSsp_hk;?gweB1jgCIiy4^IqJ diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_0.dat index 2aac77babc1367b9ac1d818e4109e376599072a6..776cac5ba8ffdb79cd98ab61994da752a8d39a93 100644 GIT binary patch delta 41 xcmX@6bwX>y3SP$6$t!uQIA<`QV4T5xfzfcY7M}n+k1Y=a!~UD%jxGj4h5$0#4A}qx delta 39 vcmX@1bxdo+3SP$M$t!uQ*cnbRGF)I}*sQ}Rz|LdM!@#gpUhj*GL69K;16m7s diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_with_variant_bloom_vertical_uncompressed/segment_1.dat index 4ca8478218c799d92737548f988b60dae7c3bfdb..83ec1ff3d36bdc16ed183e5509f57ee1be2f6de0 100644 GIT binary patch delta 41 xcmeCu?9tqi$;;R}Ig7W7^BVIB#%s(M7{fNd=AF#ObBLRP;cKGaSr>yKLjXEz4ygbD delta 39 vcmeCt?9$wj$;;S0Ig7W7o#6x{!v#i$&2M=pv+*3@W?*>69lX!QAjl8^{Ot@* diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_0.dat index a2bcbf9c6f62ada3404453b5769cf6ed26f8532c..6a6a987fd58dd959324b5b1d67f9f756d97794e1 100644 GIT binary patch delta 41 xcmeCvoS?ZukC(A^vOaGW=M3f(j5C-oFdA;&z delta 39 vcmbQB*{8WdkC(A|vOaGWJHrV^h6{`gn>X=ZV&nPC&A`x@!?WMTAjl8^@DvNz diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v2_nullable_without_variant_bloom_vertical_compressed/segment_1.dat index 60a3d754fdaf2683edb0121f34e5cd5d3f6d2083..c525473cbd29a6e32a7c31223ea322cf82e59294 100644 GIT binary patch delta 41 xcmcblenox5dmhHt$sc&CIIl6EV7$hBfiY}zA#W%fPYX8#gOXT}lZ!!+Apl!b4gLTC delta 51 zcmcbjeo1}9dmhH-$sc&C_!&+xGF)I}xWULEG?$Tsak3$w^yX>2Mr=Gy+zbq&YM*ml H41x>+zfTVh diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat index cf870d36a45e28ee6085379c2099320ff3800892..274293b55a639a3d092c5a5b7132eea53efebcdd 100644 GIT binary patch delta 46 zcmV+}0MY->E6^*jE)EX!1IPjM1JD6v*Z}|t0h0_58?#jo`vw8KlTj6Q2r~6MlSD95 EF@z5fxc~qF delta 44 zcmV+{0Mq}_E6yvhE)EO;$N>P*0RY$m00;q-4i6i%R}T9I0lAY?6?F*sl%1(WFj6rx C(+)@g diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat index 0c70bedef5590137a640650cb0efe6955216eb2c..50b2ccd920846f36f02bd7329986d799805cbfc7 100644 GIT binary patch delta 46 zcmX@7eL;JJ9v|Ol<`aycnJ+M=++bwjV4TduFTFXG?>8Ic?#Y3or5v`ri_={Uf(!wF CqYoGW delta 44 zcmcbheNKCW9v=_G2}Xtsj0`sz88{dx^YKe>4(I#L#<+8GuxKgA_Z-$(7lR-}04%2s AI{*Lx diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat index c92c2d9d7cbc6d4d0f9c0c25d00de8491bd7311b..1605671fedad2d3458f5b51dc8f864faea7051ec 100644 GIT binary patch delta 37 tcmbQHHA8E|7GBQJ%qJK>Ghbj#*}R+gE*s77G8FS6O0TO7#TM2<-N77G8FS6O0TO7#TM2<-N+-CGO+ diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat index 38b7998ace2e0938766ab79d51066d2b46736525..30ac1ee33edc97c159d0e375e01727a36cc16669 100644 GIT binary patch delta 37 tcmeCs>eAYf%*Xke=>+3vrVEUQo3r=~*cf+Do-LBfv1Uqysf$67Aprjv4AuYu delta 35 rcmeCu>d@Md%*W1ff|20@Bg5t#J_9z!os;K^q;g#S(DvKKAjl8^$|?*r diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_1.dat index bd262d787f1778ae816979f88ddfc0a27ca8a4e0..1b055d7e1fd00936084a0921fc2c0ba602219812 100644 GIT binary patch delta 46 zcmV+}0MY-cD61&24h{~m1IPie1JD6h*Z}|t0h8l!q-2eap delta 44 zcmZ3ju~K6LA0H3H2}Xtsj0`sz88{dxzvGkMY|b~0m2v0fbHbq}aL0?+|5v+xb_1p&L0;}c~FeFdk3L@-h@JAw~W delta 35 rcmcblaY19lOI~({6O0TO7#TKy;{C+RxO4JT;ZzQm#7bWmgCIiy_&*G{ diff --git a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_compressed/segment_1.dat index ffe02c1030f21883929c93515f1006c8a9af5d36..156d248d7ad2ee82d237327b695751a5cbc499f8 100644 GIT binary patch delta 37 vcmV+=0NVfYCi5n+lnn{61IPie1JD6hvzrZg1p&L06cbel)d0chL@-h@8!iq3 delta 35 rcmeyY_EBxa6kc|Q6O0TO7#TLt<*i_4+&NiHIFv(nv%QjwL69K;-k%Ei diff --git a/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/fixed_partial_sequence_variant_row_store_vertical/segment_0.dat index 2194947a02311c12ccdc92e9909c47b2149c2f4b..8cff17ddb9cb842150f96f391e442e16f35a5462 100644 GIT binary patch delta 62 zcmcbebU$gs8x_W>li#Xzac*Hb!MKIx0%OYNI@NGipx`yFF3tv~6O0W^7Z|lRr)$4p S<>8cJU^pT7`ntH=3MPJtUQ7;3=B?E)!HrwL52V{6d45o delta 58 zcmaEv^fYP1Cl$uYlRv9;u``@tWVpb{u(?e&oE0c^PYWSbsQreOhgXJyA@gGLHWz~+ FLja`;6qNt~ diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_0.dat index fe23b80c28d1df4f8c137d08031941a40fa412a7..5273fd05748c23bbad55f29d27f8b8984eebb433 100644 GIT binary patch delta 41 xcmdlfy<2+2K32x&$@^I=IJdBzVBErTfiY#X9a}y-PdWz!LshBmMi+x1LjXNV4Ws}7 delta 39 vcmdljy;FL_K32xY$@^I=*cnbRGF)I}*zCZT&(4#|!NBm~+`|GFgCIiy4h9VF diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_0_horizontal/segment_1.dat index 04ce67e4f776afd3c0912098fde742e3e54b1a58..ddd81eb8f4dd03d8a345b7d28d2017c9feef7ad5 100644 GIT binary patch delta 41 xcmX>qeO!9ODOSej$){N>IJdBzVBErTfiY#X7h66%Pc{bwLzdy^OD+aMh5$#l4rc%W delta 39 vcmX>ueN=kGDOSeD$){N>*cnbRGF)I}*zCiW&(4#{!N8y#=$7DO5M&4d2tf-; diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_0.dat index fa7e17a77bc6ea5383414d7b9560eaa6168b2bc2..0d221a14cf9e4d398d49876be9bfb9065bc4d3f7 100644 GIT binary patch delta 41 xcmdljy;pjJ1v_KwWJ~rc&Mhn_7`L!oU`*M(i~TDrPbLQg!%V|9uUrg*3;{064k`cu delta 39 vcmdlhy<2*N1v_K&WJ~rcc7_v-3>O#~Ht%8o%F2_@!NB0n#A5AY5M&4d?lKB~ diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_1_vertical/segment_1.dat index 8edad123858bde8bfc4eff38f28df0b97b8fe5ed..4eb8de195a4371c3ab885ed9f7ba3b46612a41b8 100644 GIT binary patch delta 41 xcmX>ueNuXZ3p-=$WLNeo&Mhn_7`L!oU`*M3jQuMsPc8=oL;a@>FI^0R3;{bd4-^0Z delta 39 vcmX>peO!8j3p-=;WLNeoc7_v-3>O#~HlJYs%F2_?!N6d9W!6j=gCIiy0BH<% diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_0.dat index fe23b80c28d1df4f8c137d08031941a40fa412a7..5273fd05748c23bbad55f29d27f8b8984eebb433 100644 GIT binary patch delta 41 xcmdlfy<2+2K32x&$@^I=IJdBzVBErTfiY#X9a}y-PdWz!LshBmMi+x1LjXNV4Ws}7 delta 39 vcmdljy;FL_K32xY$@^I=*cnbRGF)I}*zCZT&(4#|!NBm~+`|GFgCIiy4h9VF diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_2_horizontal/segment_1.dat index 04ce67e4f776afd3c0912098fde742e3e54b1a58..ddd81eb8f4dd03d8a345b7d28d2017c9feef7ad5 100644 GIT binary patch delta 41 xcmX>qeO!9ODOSej$){N>IJdBzVBErTfiY#X7h66%Pc{bwLzdy^OD+aMh5$#l4rc%W delta 39 vcmX>ueN=kGDOSeD$){N>*cnbRGF)I}*zCiW&(4#{!N8y#=$7DO5M&4d2tf-; diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_3_vertical/segment_0.dat index 10a3e9baea0cf6358ce18edff40730090fb408fc..0d08884ec60299801951de402e934662f1315c7b 100644 GIT binary patch delta 41 xcmdlfy<2+2K32x&$@^I=IJdBzVBErTfiY#X9a}y-PdWz!!_<_`?_CUn3;{u;4q^ZR delta 39 vcmdljy;FL_K32xY$@^I=*cnbRGF)I}*zCZT&(4#|!N9qeO!9ODOSej$){N>IJdBzVBErTfiY#X7h66%Pc{bw!&(PJT^EBOLjXc@4N?FA delta 39 vcmX>ueN=kGDOSeD$){N>*cnbRGF)I}*zCiW&(4#{!NBlDp=G{{L69K;51|ZJ diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_0.dat index 9a119a5b17dffca517a4f2c5f40d53e704c2b84f..f9625d6766c0109210dcab5b13d272b01d6e99c0 100644 GIT binary patch delta 41 zcmV+^0M`Ge7N{1mtONmrldS}J3D5(`0nh``0Y$Sc1?~k5<^})&r$HaZL@-h@KXeW| delta 39 vcmZ1>woGip8WzUJ$!l3E*cnbRGF)I}*sRC;j+N&L8v}#ZoiJk;gCIiy1a%A# diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_1.dat index b4c7f6a370517dd36076ac2db664433f69c09393..74a3a39f727ce622fc328f106f610d694a792fe4 100644 GIT binary patch delta 41 xcmew${z-hpT^7dX$@f?)IHxe5V4T8yfl+gFENeR(Pdz&WgRPEpy^BGRAplAJ4PyWR delta 39 vcmew){y}`hT^7d1$@f?)*cnbRGF)I}*c{K=&c;*A&cMKJ_2QL_L69K;AlwY} diff --git a/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat b/be/test/storage/test_data/segment_flusher_format/variant_row_store_segment_creator/segment_2.dat index 540713ba36381b9f3a414c587afaa83d6d2b696e..af4d09fc4be3c72942ec9f7efa1f740eb99dc4f0 100644 GIT binary patch delta 41 xcmbOrJV|)NQ)b5I$Afn5~Ajl8^{5cD3 diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp b/be/test/storage/variant/variant_column_writer_reader_test.cpp index 423152955ed..0e907d08924 100644 --- a/be/test/storage/variant/variant_column_writer_reader_test.cpp +++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp @@ -1704,6 +1704,8 @@ protected: RETURN_IF_ERROR(writer->finish()); RETURN_IF_ERROR(writer->write_data()); + // The file holds only the data pages of this variant so far. + EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(), file_writer->bytes_appended()); RETURN_IF_ERROR(writer->write_ordinal_index()); RETURN_IF_ERROR(writer->write_zone_map()); if (index_write_policy == VariantIndexWritePolicy::BLOOM_AND_INVERTED) { @@ -1775,6 +1777,8 @@ protected: RETURN_IF_ERROR(writer->finish()); RETURN_IF_ERROR(writer->write_data()); + // The file holds only the data pages of this subcolumn so far. + EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(), file_writer->bytes_appended()); RETURN_IF_ERROR(writer->write_ordinal_index()); RETURN_IF_ERROR(writer->write_zone_map()); RETURN_IF_ERROR(file_writer->close()); @@ -3951,6 +3955,10 @@ TEST_F(VariantColumnWriterReaderTest, test_write_data_normal) { EXPECT_TRUE(st.ok()) << st.msg(); st = writer->write_data(); EXPECT_TRUE(st.ok()) << st.msg(); + // The root, the subcolumns and the sparse columns all count to the variant. + EXPECT_EQ(writer->get_total_compressed_data_pages_bytes(), file_writer->bytes_appended()); + EXPECT_GT(writer->get_total_uncompressed_data_pages_bytes(), 0); + EXPECT_GT(writer->get_raw_data_bytes(), 0); st = writer->write_ordinal_index(); EXPECT_TRUE(st.ok()) << st.msg(); st = writer->write_zone_map(); @@ -5241,6 +5249,12 @@ TEST_P(VariantSpecializedWriterCompatibilityTest, doc_compact_writer_round_trip) EXPECT_TRUE(doc_compact_writer->finish().ok()); EXPECT_TRUE(root_writer->write_data().ok()); EXPECT_TRUE(doc_compact_writer->write_data().ok()); + // The file holds only data pages so far: the root of the root-only writer, plus the doc value + // column and the materialized subcolumns of the doc compact writer. + EXPECT_GT(doc_compact_writer->get_total_compressed_data_pages_bytes(), 0); + EXPECT_EQ(root_writer->get_total_compressed_data_pages_bytes() + + doc_compact_writer->get_total_compressed_data_pages_bytes(), + file_writer->bytes_appended()); EXPECT_TRUE(root_writer->write_ordinal_index().ok()); EXPECT_TRUE(doc_compact_writer->write_ordinal_index().ok()); EXPECT_TRUE(file_writer->close().ok()); diff --git a/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out b/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out new file mode 100644 index 00000000000..9233a27e436 --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_column_data_sizes_variant.out @@ -0,0 +1,33 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + +-- !sizes -- +id BIGINT true true true +v VARIANT true true true + diff --git a/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy b/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy new file mode 100644 index 00000000000..71463b2c933 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_column_data_sizes_variant.groovy @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_column_data_sizes_variant", "query,p0") { + // A variant column keeps its data in the root, the materialized subcolumns and the sparse or + // doc value columns. Compaction writes them as columns of their own, and external column meta + // (V3) embeds the sparse and doc value columns into the root meta. All of them count to the + // variant column. + def checkVariantDataSizes = { String tableName -> + for (int i = 0; i < 3; i++) { + sql """ + INSERT INTO ${tableName} + SELECT number, parse_to_variant(concat('{"a":', number, ',"b":"', repeat('x', number % 32), + '","c":{"d":', number * 2, '},"k', number % 16, '":"', repeat('y', number % 64), '"}')) + FROM numbers("number" = "10000") + """ + } + def tabletId = sql("SHOW TABLETS FROM ${tableName}")[0][0] + def check = { + qt_sizes """ + SELECT COLUMN_NAME, COLUMN_TYPE, SUM(COMPRESSED_DATA_BYTES) > 0, + SUM(UNCOMPRESSED_DATA_BYTES) > 0, SUM(RAW_DATA_BYTES) > 0 + FROM information_schema.column_data_sizes + WHERE TABLET_ID = ${tabletId} + GROUP BY COLUMN_NAME, COLUMN_TYPE + ORDER BY COLUMN_NAME + """ + // The data pages of all columns lie in the segment files and make up most of them, so + // a part of the variant that is not counted, or counted twice, breaks the bounds. + def bytes = sql """ + SELECT SUM(c.data_page_bytes), SUM(r.DATA_DISK_SIZE) + FROM ( + SELECT BACKEND_ID, ROWSET_ID, SUM(COMPRESSED_DATA_BYTES) AS data_page_bytes + FROM information_schema.column_data_sizes + WHERE TABLET_ID = ${tabletId} + GROUP BY BACKEND_ID, ROWSET_ID + ) c + JOIN information_schema.rowsets r + ON r.BACKEND_ID = c.BACKEND_ID AND r.ROWSET_ID = c.ROWSET_ID + """ + long dataPageBytes = bytes[0][0] as long + long segmentBytes = bytes[0][1] as long + logger.info("${tableName}: ${dataPageBytes} data page bytes in ${segmentBytes} segment bytes") + assertTrue(dataPageBytes <= segmentBytes && dataPageBytes * 2 > segmentBytes, + "${tableName}: ${dataPageBytes} data page bytes in ${segmentBytes} segment bytes") + } + + check() + trigger_and_wait_compaction(tableName, "full") + check() + } + + // Materialized subcolumns and bucketized sparse columns, with the column metas in the footer + // (V2) and in the external column meta region (V3). + sql "set default_variant_enable_doc_mode = false" + sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_v2" + sql """ + CREATE TABLE test_column_data_sizes_variant_v2 ( + id BIGINT, + v VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V2") + """ + checkVariantDataSizes("test_column_data_sizes_variant_v2") + + sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_v3" + sql """ + CREATE TABLE test_column_data_sizes_variant_v3 ( + id BIGINT, + v VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3") + """ + checkVariantDataSizes("test_column_data_sizes_variant_v3") + + // Doc value columns and materialized subcolumns in doc mode. + sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_doc_v2" + sql """ + CREATE TABLE test_column_data_sizes_variant_doc_v2 ( + id BIGINT, + v VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V2") + """ + checkVariantDataSizes("test_column_data_sizes_variant_doc_v2") + + sql "DROP TABLE IF EXISTS test_column_data_sizes_variant_doc_v3" + sql """ + CREATE TABLE test_column_data_sizes_variant_doc_v3 ( + id BIGINT, + v VARIANT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1", "disable_auto_compaction" = "true", "storage_format" = "V3") + """ + checkVariantDataSizes("test_column_data_sizes_variant_doc_v3") +}