[fix](be) Validate S3 upload buffer alignment on config updates

### What problem does this PR solve?

Issue Number: None

Related PR: #67292, #67579

Problem Summary: File Cache initialization rejects an S3 upload buffer size that is smaller than a cache block or not a multiple of the block size. The buffer size is mutable, but runtime updates did not register the same validation, so a later update could violate the layout assumed by cached reads and hole-fill deduplication. Register the existing alignment constraint as a config validator and cover accepted updates, rejected updates with value rollback, invalid block sizes, and File Cache disabled behavior. Cache probing, read-ahead, and hole-fill algorithms remain unchanged.

### Release note

Reject S3 upload buffer configuration updates that violate File Cache block alignment while File Cache is enabled.

### Check List (For Author)

- Test: Unit Test — 66 tests from 4 suites pass with ./run-be-ut.sh --run -j100 (ASAN/PCH), including configuration validation and partial-block/async cache writeback. Direct Release/PCH BE build passes with ./build.sh --be -j100. Clang-format 16, build hygiene, and diff checks pass. No cluster or clang-tidy run.
- Behavior changed: Yes — invalid runtime S3 upload buffer updates return INVALID_ARGUMENT and retain the previous value; defaults and IO mechanisms are unchanged.
- Does this need documentation: No — applies the existing File Cache startup constraint to runtime configuration updates.
This commit is contained in:
bobhan1 committed 2026-10-29 16:28:49 +08:00
1 parent fd6df2c335
commit a881473fa1
2 files changed
+57

No files matched your search

+7
View File
@@ -1515,6 +1515,13 @@ DEFINE_mInt32(tablet_path_check_batch_size, "1000");
// it must be larger than or equal to 5MB
DEFINE_mInt64(s3_write_buffer_size, "5242880");
// Startup checks this alignment too. Reject runtime updates such as 5.5 MiB with 1 MiB
// cache blocks, which would let upload writers create non-grid cache blocks.
DEFINE_Validator(s3_write_buffer_size, [](int64_t value) {
return !enable_file_cache ||
(file_cache_each_block_size > 0 && value >= file_cache_each_block_size &&
value % file_cache_each_block_size == 0);
});
// Log interval when doing s3 upload task
DEFINE_mInt32(s3_file_writer_log_interval_second, "60");
DEFINE_mInt64(file_cache_max_file_reader_cache_size, "1000000");
+50
View File
@@ -20,6 +20,7 @@
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
#include <cstdint>
#include <map>
#include <ostream>
#include <string>
@@ -27,6 +28,7 @@
#include "common/status.h"
#include "gtest/gtest_pred_impl.h"
#include "util/defer_op.h"
namespace doris {
using namespace config;
@@ -149,4 +151,52 @@ TEST_F(ConfigTest, UpdateConfigs) {
EXPECT_EQ(cfg_std_string, "doris_config_test_string");
}
TEST(S3WriteBufferConfigTest, ValidateRuntimeUpdates) {
const bool saved_enable_file_cache = config::enable_file_cache;
const int64_t saved_block_size = config::file_cache_each_block_size;
const int64_t saved_buffer_size = config::s3_write_buffer_size;
auto* saved_config_map = config::full_conf_map;
Defer restore {[&]() {
config::enable_file_cache = saved_enable_file_cache;
config::file_cache_each_block_size = saved_block_size;
config::s3_write_buffer_size = saved_buffer_size;
config::full_conf_map = saved_config_map;
}};
// Exercise the production registry without changing the configuration dump used by other tests.
config::full_conf_map = nullptr;
constexpr int64_t kMiB = 1024 * 1024;
config::enable_file_cache = true;
config::file_cache_each_block_size = kMiB;
ASSERT_TRUE(config::set_config("s3_write_buffer_size", "8388608").ok());
EXPECT_EQ(config::s3_write_buffer_size, 8 * kMiB);
const int64_t invalid_sizes[] = {-1, 0, kMiB / 2, 5 * kMiB + kMiB / 2};
for (const auto size : invalid_sizes) {
const auto status = config::set_config("s3_write_buffer_size", std::to_string(size));
EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
EXPECT_EQ(config::s3_write_buffer_size, 8 * kMiB);
}
// Reject upload buffers smaller than one cache block.
config::file_cache_each_block_size = 8 * kMiB;
EXPECT_TRUE(config::set_config("s3_write_buffer_size", "4194304")
.is<ErrorCode::INVALID_ARGUMENT>());
EXPECT_EQ(config::s3_write_buffer_size, 8 * kMiB);
ASSERT_TRUE(config::set_config("s3_write_buffer_size", "16777216").ok());
EXPECT_EQ(config::s3_write_buffer_size, 16 * kMiB);
// Reject an invalid block size before evaluating the modulo operation.
config::file_cache_each_block_size = 0;
EXPECT_TRUE(config::set_config("s3_write_buffer_size", "8388608")
.is<ErrorCode::INVALID_ARGUMENT>());
EXPECT_EQ(config::s3_write_buffer_size, 16 * kMiB);
// Preserve the existing behavior when File Cache is disabled.
config::enable_file_cache = false;
config::file_cache_each_block_size = kMiB;
ASSERT_TRUE(config::set_config("s3_write_buffer_size", "5767168").ok());
EXPECT_EQ(config::s3_write_buffer_size, 5 * kMiB + kMiB / 2);
}
} // namespace doris