nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 07:31:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Fix](fragment) avoid query-ctx map clear self-deadlock when stop FragmentMgr (#62954)

Browse Source 
Problem Summary:

Previously `ConcurrentContextMap::clear()` destroyed map values while
holding the shard_mutex. For `_query_ctx_map_delay_delete`, releasing
the last QueryContext reference can run `QueryContext::~QueryContext()`,
which calls `FragmentMgr::remove_query_context()` and re-enters the same
delay-delete map
through `erase()`. This can throw std::system_error with "Resource
deadlock avoided" during BE shutdown.
```bash
terminate called after throwing an instance of 'std::system_error'
  what():  Resource deadlock avoided
*** Query id: 0-0 ***
*** is nereids: 0 ***
*** tablet id: 0 ***
*** Aborted at 1777288253 (unix time) try "date -d @1777288253" if you are using GNU date ***
*** Current BE git commitID: f060f176c9 ***
*** SIGABRT unknown detail explain (@0x7a53) received by PID 31315 (TID 31315 OR 0x7f6d057be200) from PID 31315; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at ../src/common/signal_handler.h:420
 1# 0x00007F6D05BD7420 in /lib/x86_64-linux-gnu/libpthread.so.0
 2# raise at ../sysdeps/unix/sysv/linux/raise.c:51
 3# abort at /build/glibc-SzIz7B/glibc-2.31/stdlib/abort.c:81
 4# __gnu_cxx::__verbose_terminate_handler() [clone .cold] in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
 5# __cxxabiv1::__terminate(void (*)()) in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
 6# 0x00005608D3BC2AB9 in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
 7# 0x00005608A68DBCFE in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
 8# 0x00005608AAF679AE in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
 9# std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
10# decltype (std::allocator_traits<std::allocator<std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > > >::destroy({parm#2}, {parm#3})) phmap::allocator_traits<std::allocator<std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > > >::destroy_impl<std::allocator<std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > >, std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > >(int, std::allocator<std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > >&, std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> >*) at /var/local/thirdparty/installed/include/parallel_hashmap/phmap_base.h:1542
11# phmap::priv::raw_hash_set<phmap::priv::FlatHashMapPolicy<doris::TUniqueId, std::shared_ptr<doris::QueryContext> >, phmap::Hash<doris::TUniqueId>, phmap::EqualTo<doris::TUniqueId>, std::allocator<std::pair<doris::TUniqueId const, std::shared_ptr<doris::QueryContext> > > >::clear() at /var/local/thirdparty/installed/include/parallel_hashmap/phmap.h:1279
12# doris::ConcurrentContextMap<doris::TUniqueId, std::shared_ptr<doris::QueryContext>, doris::QueryContext>::clear() at /root/doris/be/build_ASAN/../src/runtime/fragment_mgr.cpp:311
13# doris::FragmentMgr::stop() at /root/doris/be/build_ASAN/../src/runtime/fragment_mgr.cpp:348
14# doris::ExecEnv::destroy() at /root/doris/be/build_ASAN/../src/runtime/exec_env_init.cpp:819
15# main in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be
16# __libc_start_main at ../csu/libc-start.c:342
17# _start in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be

```

```text
FragmentMgr::stop()
  -> _query_ctx_map_delay_delete.clear()
      -> unique_lock(unique_query_id_lock)
      -> map.clear()
          -> QueryContext::~QueryContext()
              -> FragmentMgr::remove_query_context(query_id)
                  -> _query_ctx_map_delay_delete.erase(query_id)
                      -> unique_lock(unique_query_id_lock)        <- dead-lock happened
                      -> map.erase(query_id)
```
This commit is contained in:
linrrarity authored and GitHub committed 2026-05-08 11:54:58 +08:00
1 parent 6e6729cd83
commit b3af03dafc
2 files changed
+45 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

be/src/runtime/fragment_mgr.cpp
+15 -2
View File
@@ -298,9 +298,22 @@ void ConcurrentContextMap<Key, Value, ValueType>::insert(const Key& query_id,
template <typename Key, typename Value, typename ValueType>
void ConcurrentContextMap<Key, Value, ValueType>::clear() {
// Avoid self-deadlock from releasing the last QueryContext
// in _query_ctx_map_delay_delete:
// FragmentMgr::stop()
// -> _query_ctx_map_delay_delete.clear()
// -> unique_lock(query_id_lock)
// -> map.clear()
// -> QueryContext::~QueryContext()
// -> FragmentMgr::remove_query_context(query_id)
// -> _query_ctx_map_delay_delete.erase(query_id)
// -> unique_lock(query_id_lock) <- deadlock
for (auto& pair : _internal_map) {
std::unique_lock lock(*pair.first);
auto& map = pair.second;
phmap::flat_hash_map<Key, Value> map;
{
std::unique_lock lock(*pair.first);
map.swap(pair.second);
}
map.clear();
}
}
be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp
+30
View File
@@ -153,4 +153,34 @@ TEST_F(FragmentMgrCrossClusterCancelTest, CancelWorkerInvalidQueryDetectionSkips
EXPECT_TRUE(queries_pipeline_task_leak.empty());
}
TEST(FragmentMgrDelayDeleteMapTest, ClearShouldNotAbortWhenReleasingLastQueryContextRef) {
auto* exec_env = ExecEnv::GetInstance();
auto* previous_fragment_mgr = exec_env->_fragment_mgr;
exec_env->_fragment_mgr = new FragmentMgr(exec_env);
TUniqueId query_id;
query_id.__set_hi(101);
query_id.__set_lo(202);
TQueryOptions query_options;
query_options.__set_query_type(TQueryType::SELECT);
query_options.__set_execution_timeout(60);
query_options.__set_mem_limit(64L * 1024 * 1024);
TNetworkAddress fe_addr;
fe_addr.hostname = "127.0.0.1";
fe_addr.port = 9030;
auto query_ctx =
QueryContext::create(query_id, exec_env, query_options, fe_addr,
/*is_nereids*/ true, fe_addr, QuerySource::INTERNAL_FRONTEND);
exec_env->_fragment_mgr->_query_ctx_map_delay_delete.insert(query_id, query_ctx);
query_ctx.reset();
exec_env->_fragment_mgr->_query_ctx_map_delay_delete.clear();
exec_env->_fragment_mgr->stop();
delete exec_env->_fragment_mgr;
exec_env->_fragment_mgr = previous_fragment_mgr;
}
} // namespace doris