mirror of
https://github.com/apache/doris.git
synced 2026-11-04 15:39:47 +00:00
[feature](be) Add cache block hole-fill planning
### What problem does this PR solve? Issue Number: None Related PR: #67292 Problem Summary: Partial query ranges can cover unordered and overlapping fragments inside one File Cache block. Add a block-local planner that unions existing coverage, takes its complement to produce sorted holes, and applies the dedicated three-parameter coalescing policy without crossing the block boundary. The coalescing size limit controls combining holes; one mandatory hole may be larger and remains one remote read. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh -j100 --run '--filter=HoleFillPlannerTest.*' - Behavior changed: No; this commit adds an internal hole-fill planner. - Does this need documentation: No
This commit is contained in:
1 parent
a547700289
commit
7cfe76cbbd
3 files changed
+262
No files matched your search
+91
@@ -0,0 +1,91 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "io/cache/hole_fill_planner.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
#include "common/logging.h"
|
||||
|
||||
namespace doris::io {
|
||||
|
||||
Status HoleFillPlanner::plan(const FileRange& block,
|
||||
const std::vector<FileRange>& covered_intervals,
|
||||
const FileRangeCoalesceOptions& options,
|
||||
std::vector<FileRange>* output_ranges) {
|
||||
DORIS_CHECK(output_ranges != nullptr);
|
||||
RETURN_IF_ERROR(options.validate());
|
||||
if (block.size == 0) {
|
||||
return Status::InvalidArgument("hole-fill block at offset {} is empty", block.offset);
|
||||
}
|
||||
if (block.offset > std::numeric_limits<size_t>::max() - block.size) {
|
||||
return Status::InvalidArgument("hole-fill block at offset {} with size {} overflows",
|
||||
block.offset, block.size);
|
||||
}
|
||||
for (const auto& covered : covered_intervals) {
|
||||
if (covered.size == 0) {
|
||||
return Status::InvalidArgument("covered interval at offset {} is empty",
|
||||
covered.offset);
|
||||
}
|
||||
if (covered.offset < block.offset || covered.offset > block.end() ||
|
||||
covered.size > block.end() - covered.offset) {
|
||||
return Status::InvalidArgument(
|
||||
"covered interval at offset {} with size {} exceeds block [{}, {})",
|
||||
covered.offset, covered.size, block.offset, block.end());
|
||||
}
|
||||
}
|
||||
|
||||
auto sorted_coverage = covered_intervals;
|
||||
std::ranges::sort(sorted_coverage, [](const FileRange& left, const FileRange& right) {
|
||||
return left.offset < right.offset;
|
||||
});
|
||||
std::vector<FileRange> covered_union;
|
||||
covered_union.reserve(sorted_coverage.size());
|
||||
for (const auto& covered : sorted_coverage) {
|
||||
if (covered_union.empty() || covered_union.back().end() < covered.offset) {
|
||||
covered_union.push_back(covered);
|
||||
} else {
|
||||
covered_union.back().size = std::max(covered_union.back().end(), covered.end()) -
|
||||
covered_union.back().offset;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<FileRange> holes;
|
||||
size_t cursor = block.offset;
|
||||
for (const auto& covered : covered_union) {
|
||||
DORIS_CHECK(cursor <= covered.offset);
|
||||
if (cursor < covered.offset) {
|
||||
holes.push_back({.offset = cursor, .size = covered.offset - cursor});
|
||||
}
|
||||
cursor = covered.end();
|
||||
}
|
||||
if (cursor < block.end()) {
|
||||
holes.push_back({.offset = cursor, .size = block.end() - cursor});
|
||||
}
|
||||
|
||||
auto coalesced_holes = FileRangeCoalescer::coalesce(holes, options);
|
||||
for (const auto& hole : coalesced_holes) {
|
||||
DORIS_CHECK(block.offset <= hole.offset);
|
||||
DORIS_CHECK(hole.end() <= block.end());
|
||||
}
|
||||
*output_ranges = std::move(coalesced_holes);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace doris::io
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "common/status.h"
|
||||
#include "io/fs/file_range_coalescer.h"
|
||||
|
||||
namespace doris::io {
|
||||
|
||||
/// Computes the remote reads needed to complete one File Cache block. Covered intervals are first
|
||||
/// unioned, their complement inside `block` becomes the hole list, and the supplied coalescing
|
||||
/// policy may combine nearby holes without ever crossing the block boundary.
|
||||
class HoleFillPlanner {
|
||||
public:
|
||||
/// `covered_intervals` may overlap and arrive in any order, but each interval must be non-empty
|
||||
/// and contained by `block`. The output is sorted, disjoint, and contained by `block`.
|
||||
static Status plan(const FileRange& block, const std::vector<FileRange>& covered_intervals,
|
||||
const FileRangeCoalesceOptions& options,
|
||||
std::vector<FileRange>* output_ranges);
|
||||
};
|
||||
|
||||
} // namespace doris::io
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "io/cache/hole_fill_planner.h"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace doris::io {
|
||||
namespace {
|
||||
|
||||
constexpr size_t KiB(size_t value) {
|
||||
return value * 1024;
|
||||
}
|
||||
|
||||
const FileRangeCoalesceOptions kOptions {
|
||||
.max_gap_bytes = KiB(32),
|
||||
.max_range_bytes = KiB(1024),
|
||||
.max_read_amplification_ratio = 2.0,
|
||||
};
|
||||
|
||||
TEST(HoleFillPlannerTest, MergesNearbyHolesWithinAmplificationLimit) {
|
||||
const FileRange block {.offset = 0, .size = KiB(1024)};
|
||||
const std::vector<FileRange> covered {{.offset = KiB(544), .size = KiB(480)},
|
||||
{.offset = 0, .size = KiB(256)},
|
||||
{.offset = KiB(384), .size = KiB(32)}};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());
|
||||
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(256), .size = KiB(288)}}));
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, KeepsHolesSeparateWhenGapExceedsLimit) {
|
||||
const FileRange block {.offset = 0, .size = KiB(1024)};
|
||||
const std::vector<FileRange> covered {{.offset = KiB(480), .size = KiB(64)}};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());
|
||||
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(480)},
|
||||
{.offset = KiB(544), .size = KiB(480)}}));
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, AppliesAmplificationLimitAfterGapCheck) {
|
||||
const FileRange block {.offset = 0, .size = KiB(1024)};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block,
|
||||
{{.offset = KiB(8), .size = KiB(16)},
|
||||
{.offset = KiB(32), .size = KiB(992)}},
|
||||
kOptions, &ranges)
|
||||
.ok());
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(32)}}));
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block,
|
||||
{{.offset = KiB(4), .size = KiB(16)},
|
||||
{.offset = KiB(24), .size = KiB(1000)}},
|
||||
kOptions, &ranges)
|
||||
.ok());
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(4)},
|
||||
{.offset = KiB(20), .size = KiB(4)}}));
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, UsesOnlyTheCurrentPhysicalEofBlock) {
|
||||
const FileRange block {.offset = KiB(1024), .size = KiB(300)};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, {{.offset = KiB(1124), .size = KiB(100)}}, kOptions,
|
||||
&ranges)
|
||||
.ok());
|
||||
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(1024), .size = KiB(100)},
|
||||
{.offset = KiB(1224), .size = KiB(100)}}));
|
||||
for (const auto& range : ranges) {
|
||||
EXPECT_GE(range.offset, block.offset);
|
||||
EXPECT_LE(range.end(), block.end());
|
||||
}
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, HandlesEmptyAndCompleteCoverage) {
|
||||
const FileRange block {.offset = 0, .size = KiB(1024)};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, {}, kOptions, &ranges).ok());
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, {block}, kOptions, &ranges).ok());
|
||||
EXPECT_TRUE(ranges.empty());
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, KeepsSingleHoleLargerThanCoalesceLimit) {
|
||||
const FileRange block {.offset = 0, .size = KiB(1024)};
|
||||
const FileRangeCoalesceOptions options {
|
||||
.max_gap_bytes = KiB(32),
|
||||
.max_range_bytes = KiB(128),
|
||||
.max_read_amplification_ratio = 2.0,
|
||||
};
|
||||
std::vector<FileRange> ranges;
|
||||
|
||||
ASSERT_TRUE(HoleFillPlanner::plan(block, {}, options, &ranges).ok());
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));
|
||||
}
|
||||
|
||||
TEST(HoleFillPlannerTest, RejectsCoverageOutsideBlockWithoutChangingOutput) {
|
||||
const FileRange block {.offset = KiB(1024), .size = KiB(1024)};
|
||||
std::vector<FileRange> ranges {{.offset = 7, .size = 11}};
|
||||
|
||||
EXPECT_FALSE(HoleFillPlanner::plan(block, {{.offset = KiB(512), .size = KiB(1024)}}, kOptions,
|
||||
&ranges)
|
||||
.ok());
|
||||
|
||||
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 7, .size = 11}}));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace doris::io
|
||||
Reference in new issue
Block a user