[feature](be) Add cache block hole-fill planning

### What problem does this PR solve?

Issue Number: None

Related PR: #67292

Problem Summary: Partial query ranges can cover unordered and overlapping fragments inside one File Cache block. Add a block-local planner that unions existing coverage, takes its complement to produce sorted holes, and applies the dedicated three-parameter coalescing policy without crossing the block boundary. The coalescing size limit controls combining holes; one mandatory hole may be larger and remains one remote read.

### Release note

None

### Check List (For Author)

- Test: Unit Test
    - ./run-be-ut.sh -j100 --run '--filter=HoleFillPlannerTest.*'
- Behavior changed: No; this commit adds an internal hole-fill planner.
- Does this need documentation: No
This commit is contained in:
bobhan1 committed 2026-10-29 16:28:49 +08:00
1 parent a547700289
commit 7cfe76cbbd
3 files changed
+262

No files matched your search

+91
View File
@@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "io/cache/hole_fill_planner.h"
#include <algorithm>
#include <limits>
#include <utility>
#include "common/logging.h"
namespace doris::io {
Status HoleFillPlanner::plan(const FileRange& block,
const std::vector<FileRange>& covered_intervals,
const FileRangeCoalesceOptions& options,
std::vector<FileRange>* output_ranges) {
DORIS_CHECK(output_ranges != nullptr);
RETURN_IF_ERROR(options.validate());
if (block.size == 0) {
return Status::InvalidArgument("hole-fill block at offset {} is empty", block.offset);
}
if (block.offset > std::numeric_limits<size_t>::max() - block.size) {
return Status::InvalidArgument("hole-fill block at offset {} with size {} overflows",
block.offset, block.size);
}
for (const auto& covered : covered_intervals) {
if (covered.size == 0) {
return Status::InvalidArgument("covered interval at offset {} is empty",
covered.offset);
}
if (covered.offset < block.offset || covered.offset > block.end() ||
covered.size > block.end() - covered.offset) {
return Status::InvalidArgument(
"covered interval at offset {} with size {} exceeds block [{}, {})",
covered.offset, covered.size, block.offset, block.end());
}
}
auto sorted_coverage = covered_intervals;
std::ranges::sort(sorted_coverage, [](const FileRange& left, const FileRange& right) {
return left.offset < right.offset;
});
std::vector<FileRange> covered_union;
covered_union.reserve(sorted_coverage.size());
for (const auto& covered : sorted_coverage) {
if (covered_union.empty() || covered_union.back().end() < covered.offset) {
covered_union.push_back(covered);
} else {
covered_union.back().size = std::max(covered_union.back().end(), covered.end()) -
covered_union.back().offset;
}
}
std::vector<FileRange> holes;
size_t cursor = block.offset;
for (const auto& covered : covered_union) {
DORIS_CHECK(cursor <= covered.offset);
if (cursor < covered.offset) {
holes.push_back({.offset = cursor, .size = covered.offset - cursor});
}
cursor = covered.end();
}
if (cursor < block.end()) {
holes.push_back({.offset = cursor, .size = block.end() - cursor});
}
auto coalesced_holes = FileRangeCoalescer::coalesce(holes, options);
for (const auto& hole : coalesced_holes) {
DORIS_CHECK(block.offset <= hole.offset);
DORIS_CHECK(hole.end() <= block.end());
}
*output_ranges = std::move(coalesced_holes);
return Status::OK();
}
} // namespace doris::io
+39
View File
@@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <vector>
#include "common/status.h"
#include "io/fs/file_range_coalescer.h"
namespace doris::io {
/// Computes the remote reads needed to complete one File Cache block. Covered intervals are first
/// unioned, their complement inside `block` becomes the hole list, and the supplied coalescing
/// policy may combine nearby holes without ever crossing the block boundary.
class HoleFillPlanner {
public:
/// `covered_intervals` may overlap and arrive in any order, but each interval must be non-empty
/// and contained by `block`. The output is sorted, disjoint, and contained by `block`.
static Status plan(const FileRange& block, const std::vector<FileRange>& covered_intervals,
const FileRangeCoalesceOptions& options,
std::vector<FileRange>* output_ranges);
};
} // namespace doris::io
+132
View File
@@ -0,0 +1,132 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "io/cache/hole_fill_planner.h"
#include <gtest/gtest.h>
#include <vector>
namespace doris::io {
namespace {
constexpr size_t KiB(size_t value) {
return value * 1024;
}
const FileRangeCoalesceOptions kOptions {
.max_gap_bytes = KiB(32),
.max_range_bytes = KiB(1024),
.max_read_amplification_ratio = 2.0,
};
TEST(HoleFillPlannerTest, MergesNearbyHolesWithinAmplificationLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const std::vector<FileRange> covered {{.offset = KiB(544), .size = KiB(480)},
{.offset = 0, .size = KiB(256)},
{.offset = KiB(384), .size = KiB(32)}};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(256), .size = KiB(288)}}));
}
TEST(HoleFillPlannerTest, KeepsHolesSeparateWhenGapExceedsLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const std::vector<FileRange> covered {{.offset = KiB(480), .size = KiB(64)}};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(480)},
{.offset = KiB(544), .size = KiB(480)}}));
}
TEST(HoleFillPlannerTest, AppliesAmplificationLimitAfterGapCheck) {
const FileRange block {.offset = 0, .size = KiB(1024)};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block,
{{.offset = KiB(8), .size = KiB(16)},
{.offset = KiB(32), .size = KiB(992)}},
kOptions, &ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(32)}}));
ASSERT_TRUE(HoleFillPlanner::plan(block,
{{.offset = KiB(4), .size = KiB(16)},
{.offset = KiB(24), .size = KiB(1000)}},
kOptions, &ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(4)},
{.offset = KiB(20), .size = KiB(4)}}));
}
TEST(HoleFillPlannerTest, UsesOnlyTheCurrentPhysicalEofBlock) {
const FileRange block {.offset = KiB(1024), .size = KiB(300)};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block, {{.offset = KiB(1124), .size = KiB(100)}}, kOptions,
&ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(1024), .size = KiB(100)},
{.offset = KiB(1224), .size = KiB(100)}}));
for (const auto& range : ranges) {
EXPECT_GE(range.offset, block.offset);
EXPECT_LE(range.end(), block.end());
}
}
TEST(HoleFillPlannerTest, HandlesEmptyAndCompleteCoverage) {
const FileRange block {.offset = 0, .size = KiB(1024)};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block, {}, kOptions, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));
ASSERT_TRUE(HoleFillPlanner::plan(block, {block}, kOptions, &ranges).ok());
EXPECT_TRUE(ranges.empty());
}
TEST(HoleFillPlannerTest, KeepsSingleHoleLargerThanCoalesceLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const FileRangeCoalesceOptions options {
.max_gap_bytes = KiB(32),
.max_range_bytes = KiB(128),
.max_read_amplification_ratio = 2.0,
};
std::vector<FileRange> ranges;
ASSERT_TRUE(HoleFillPlanner::plan(block, {}, options, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));
}
TEST(HoleFillPlannerTest, RejectsCoverageOutsideBlockWithoutChangingOutput) {
const FileRange block {.offset = KiB(1024), .size = KiB(1024)};
std::vector<FileRange> ranges {{.offset = 7, .size = 11}};
EXPECT_FALSE(HoleFillPlanner::plan(block, {{.offset = KiB(512), .size = KiB(1024)}}, kOptions,
&ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 7, .size = 11}}));
}
} // namespace
} // namespace doris::io