branch-4.1: [fix](be) Stop background workers before startup exit #68652 (#68653)

### What problem does this PR solve?

Issue Number: None

Related PR: #68652

Problem Summary:

This PR backports the startup-failure lifecycle fix from #68652 to
`branch-4.1`.

When a BE service fails to start with `enable_graceful_exit_check=true`,
the startup failure path calls `exit(1)` while StorageEngine background
workers are still running. Process-wide static destruction can then race
with a path GC worker that is using the function-local static RE2
matcher in `TabletManager::get_tablet_id_and_schema_hash_from_path()`,
causing a null-address crash in `re2::Prog::SearchOnePass`.

The fix stops and joins created server starters, releases the backend
service, and explicitly destroys `ExecEnv` before calling `exit(1)`. The
immediate `_exit(1)` path remains unchanged when graceful exit checking
is disabled.

The backport has the same stable patch-id as #68652.

### Release note

None

### Check List (For Author)

- Test:
  - [x] Manual test
- Targeted clang-format 16 check passed for
`be/src/service/doris_main.cpp` on `branch-4.1`.
- A local BE build could not start because the macOS environment does
not have Homebrew; `run buildall` is requested below for full build
validation.
- Behavior changed:
  - [ ] Yes
  - [x] No. Only abnormal BE startup cleanup ordering changes.
- Does this need documentation?
  - [x] No
  - [ ] Yes
This commit is contained in:
Gavin Chou authored and GitHub committed 2026-10-08 10:13:24 +08:00
1 parent b9dc93d45e
commit 6217a2c618
1 file changed
+54 -15
+54 -15
View File
@@ -326,10 +326,18 @@ struct Checker {
// running the leak check on this abnormal-exit path reports them as false-positive
// leaks. enable_graceful_exit_check is honored so memleak-check mode still runs LSAN.
[[noreturn]] static void exit_on_startup_failure() {
google::FlushLogFiles(google::GLOG_INFO);
if (!doris::config::enable_graceful_exit_check) {
google::FlushLogFiles(google::GLOG_INFO);
_exit(1);
}
// exit() starts destroying function-local statics while background threads are still
// running. Tear down ExecEnv first so StorageEngine workers are stopped and joined before
// they can race with those destructors.
if (doris::ExecEnv::ready()) {
doris::ExecEnv::GetInstance()->destroy();
}
google::FlushLogFiles(google::GLOG_INFO);
exit(1);
}
@@ -635,11 +643,38 @@ int main(int argc, char** argv) {
doris::ThriftRpcHelper::setup(exec_env);
// 1. thrift server with be_port
std::shared_ptr<doris::BaseBackendService> service;
std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
std::unique_ptr<doris::server::IServerStarter> brpc_starter;
std::unique_ptr<doris::server::IServerStarter> http_starter;
std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
std::unique_ptr<doris::server::IServerStarter> flight_starter;
bool backend_thrift_started = false;
bool brpc_started = false;
bool http_started = false;
bool heartbeat_thrift_started = false;
bool flight_started = false;
auto stop_and_join_server = [](std::unique_ptr<doris::server::IServerStarter>& starter,
bool started) {
if (starter != nullptr) {
if (started) {
starter->stop();
}
starter->join();
}
};
std::function<void(Status&, std::string_view)> stop_work_if_error = [&](Status& status,
std::string_view msg) {
if (!status.ok()) {
std::cerr << msg << '\n';
service->stop_works();
if (doris::config::enable_graceful_exit_check) {
stop_and_join_server(flight_starter, flight_started);
stop_and_join_server(heartbeat_thrift_starter, heartbeat_thrift_started);
stop_and_join_server(http_starter, http_started);
stop_and_join_server(backend_thrift_starter, backend_thrift_started);
stop_and_join_server(brpc_starter, brpc_started);
service.reset();
}
exit_on_startup_failure();
}
};
@@ -652,44 +687,48 @@ int main(int argc, char** argv) {
exec_env);
}
std::unique_ptr<doris::server::IServerStarter> backend_thrift_starter;
EXIT_IF_ERROR(doris::server::create_backend_thrift_starter(exec_env, doris::config::be_port,
service, &backend_thrift_starter));
status = doris::server::create_backend_thrift_starter(exec_env, doris::config::be_port, service,
&backend_thrift_starter);
stop_work_if_error(status, "Failed to create BE server, exiting");
status = backend_thrift_starter->start();
backend_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE server did not start correctly, exiting");
// 2. brpc service
std::unique_ptr<doris::server::IServerStarter> brpc_starter;
EXIT_IF_ERROR(doris::server::create_brpc_starter(
exec_env, doris::config::brpc_port, doris::config::brpc_num_threads, &brpc_starter));
status = doris::server::create_brpc_starter(exec_env, doris::config::brpc_port,
doris::config::brpc_num_threads, &brpc_starter);
stop_work_if_error(status, "Failed to create BRPC service, exiting");
status = brpc_starter->start();
brpc_started = status.ok();
stop_work_if_error(status, "BRPC service did not start correctly, exiting");
// 3. http service
std::unique_ptr<doris::server::IServerStarter> http_starter;
EXIT_IF_ERROR(doris::server::create_http_starter(exec_env, doris::config::webserver_port,
doris::config::webserver_num_workers,
&http_starter));
status =
doris::server::create_http_starter(exec_env, doris::config::webserver_port,
doris::config::webserver_num_workers, &http_starter);
stop_work_if_error(status, "Failed to create BE HTTP service, exiting");
status = http_starter->start();
http_started = status.ok();
stop_work_if_error(status, "Doris Be http service did not start correctly, exiting");
// 4. heart beat server
doris::ClusterInfo* cluster_info = exec_env->cluster_info();
std::unique_ptr<doris::server::IServerStarter> heartbeat_thrift_starter;
status = doris::server::create_heartbeat_thrift_starter(
exec_env, doris::config::heartbeat_service_port,
doris::config::heartbeat_service_thread_count, cluster_info, &heartbeat_thrift_starter);
stop_work_if_error(status, "Heartbeat services did not start correctly, exiting");
status = heartbeat_thrift_starter->start();
heartbeat_thrift_started = status.ok();
stop_work_if_error(status, "Doris BE HeartBeat Service did not start correctly, exiting: " +
status.to_string());
// 5. arrow flight service
std::unique_ptr<doris::server::IServerStarter> flight_starter;
EXIT_IF_ERROR(doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
&flight_starter));
status = doris::server::create_flight_starter(doris::config::arrow_flight_sql_port,
&flight_starter);
stop_work_if_error(status, "Failed to create Arrow Flight service, exiting");
status = flight_starter->start();
flight_started = status.ok();
stop_work_if_error(
status, "Arrow Flight Service did not start correctly, exiting, " + status.to_string());