[Fix](func) Keep multi-match regex cache owner alive (#66909)

### What problem does this PR solve?

Problem Summary:

`multi_match_any` compiles each pattern array and stores the result in a
process-wide, direct-mapped cache.

The previous implementation retained the cache entry's shared_ptr only
while preparing the regex database and scratch space. The subsequent
scan used a **raw** `Regexps*`. If another concurrent request inserted a
different pattern array into the same cache bucket, the old cache entry
could be replaced and destroyed while it was still being scanned,
resulting in a use-after-free.

This change keeps the cache entry's shared_ptr alive for the entire
hs_scan operation. Cache eviction can still replace the global entry,
but the compiled database is not destroyed until all active scans have
completed.

### Release note

Fix a potential use-after-free in multi_match_any when concurrent
pattern arrays collide in the compiled-regex cache.

be/src/exprs/function/functions_multi_string_search.cpp

@@ -18,17 +18,12 @@
 // https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsMultiStringSearch.h
 // and modified by Doris
 
-#include <hs/hs_common.h>
-#include <hs/hs_runtime.h>
+#include "exprs/function/functions_multi_string_search.h"
 
 #include <algorithm>
-#include <boost/iterator/iterator_facade.hpp>
 #include <cstddef>
-#include <limits>
 #include <memory>
-#include <optional>
 #include <utility>
-#include <vector>
 
 #include "common/status.h"
 #include "core/block/block.h"
@@ -41,15 +36,10 @@
 #include "core/column/column_string.h"
 #include "core/column/column_vector.h"
 #include "core/data_type/data_type.h"
-#include "core/data_type/data_type_number.h" // IWYU pragma: keep
-#include "core/field.h"
 #include "core/pod_array_fwd.h"
-#include "core/string_ref.h"
 #include "core/types.h"
-#include "exprs/aggregate/aggregate_function.h"
 #include "exprs/function/function.h"
 #include "exprs/function/function_helpers.h"
-#include "exprs/function/regexps.h"
 #include "exprs/function/simple_function_factory.h"
 
 namespace doris {
@@ -165,190 +155,6 @@ private:
     }
 };
 
-/// For more readable instantiations of MultiMatchAnyImpl<>
-struct MultiMatchTraits {
-    enum class Find { Any, AnyIndex };
-};
-
-template <PrimitiveType PType, MultiMatchTraits::Find Find, bool WithEditDistance>
-struct FunctionMultiMatchAnyImpl {
-    using ResultType = typename PrimitiveTypeTraits<PType>::CppType;
-    static constexpr PrimitiveType ResultPType = PType;
-
-    static constexpr bool FindAny = (Find == MultiMatchTraits::Find::Any);
-    static constexpr bool FindAnyIndex = (Find == MultiMatchTraits::Find::AnyIndex);
-
-    static constexpr auto name = "multi_match_any";
-
-    static auto get_return_type() {
-        return std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>();
-    }
-
-    /**
-     * Prepares the regular expressions and scratch space for Hyperscan.
-     *
-     * This function takes a vector of needles (substrings to search for) and initializes
-     * the regular expressions and scratch space required for Hyperscan, a high-performance
-     * regular expression matching library.
-     *
-     */
-    static Status prepare_regexps_and_scratch(const std::vector<StringRef>& needles,
-                                              multiregexps::Regexps*& regexps,
-                                              multiregexps::ScratchPtr& smart_scratch) {
-        multiregexps::DeferredConstructedRegexpsPtr deferred_constructed_regexps =
-                multiregexps::getOrSet</*SaveIndices*/
-                                       FindAnyIndex, WithEditDistance>(needles, std::nullopt);
-        regexps = deferred_constructed_regexps->get();
-
-        hs_scratch_t* scratch = nullptr;
-        hs_error_t err = hs_clone_scratch(regexps->getScratch(), &scratch);
-
-        if (err != HS_SUCCESS) {
-            return Status::InternalError("could not clone scratch space for vectorscan");
-        }
-
-        smart_scratch.reset(scratch);
-        return Status::OK();
-    }
-
-    /**
-     * Static callback function to handle the match results of the hs_scan function.
-     *
-     * This function is called when a matching substring is found while scanning with
-     * Hyperscan. It updates the result based on the match information.
-     *
-     */
-    static int on_match([[maybe_unused]] unsigned int id, unsigned long long /* from */, // NOLINT
-                        unsigned long long /* to */,                                     // NOLINT
-                        unsigned int /* flags */, void* context) {
-        if constexpr (FindAnyIndex) {
-            *reinterpret_cast<ResultType*>(context) = id;
-        } else if constexpr (FindAny) {
-            *reinterpret_cast<ResultType*>(context) = 1;
-        }
-        /// Once we hit the callback, there is no need to search for others.
-        return 1;
-    }
-
-    static Status vector_constant(const ColumnString::Chars& haystack_data,
-                                  const ColumnString::Offsets& haystack_offsets,
-                                  const Array& needles_arr, PaddedPODArray<ResultType>& res,
-                                  PaddedPODArray<UInt64>& offsets, bool allow_hyperscan,
-                                  size_t max_hyperscan_regexp_length,
-                                  size_t max_hyperscan_regexp_total_length) {
-        if (!allow_hyperscan) {
-            return Status::InvalidArgument("Hyperscan functions are disabled");
-        }
-
-        std::vector<StringRef> needles;
-        needles.reserve(needles_arr.size());
-        for (const auto& needle : needles_arr) {
-            const auto& tmp = needle.get<TYPE_STRING>();
-            needles.emplace_back(StringRef {tmp.data(), tmp.size()});
-        }
-
-        res.resize(haystack_offsets.size());
-
-        if (needles_arr.empty()) {
-            std::fill(res.begin(), res.end(), 0);
-            return Status::OK();
-        }
-
-        multiregexps::Regexps* regexps = nullptr;
-        multiregexps::ScratchPtr smart_scratch;
-        RETURN_IF_ERROR(prepare_regexps_and_scratch(needles, regexps, smart_scratch));
-
-        const size_t haystack_offsets_size = haystack_offsets.size();
-        UInt64 offset = 0;
-        for (size_t i = 0; i < haystack_offsets_size; ++i) {
-            UInt64 length = haystack_offsets[i] - offset;
-            /// vectorscan restriction.
-            if (length > std::numeric_limits<UInt32>::max()) {
-                return Status::InternalError("too long string to search");
-            }
-            /// zero the result, scan, check, update the offset.
-            res[i] = 0;
-            hs_error_t err = hs_scan(
-                    regexps->getDB(), reinterpret_cast<const char*>(haystack_data.data()) + offset,
-                    static_cast<unsigned>(length), 0, smart_scratch.get(), on_match, &res[i]);
-            if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
-                return Status::InternalError("failed to scan with vectorscan");
-            }
-            offset = haystack_offsets[i];
-        }
-
-        return Status::OK();
-    }
-
-    static Status vector_vector(const ColumnString::Chars& haystack_data,
-                                const ColumnString::Offsets& haystack_offsets,
-                                const IColumn& needles_data,
-                                const ColumnArray::Offsets64& needles_offsets,
-                                PaddedPODArray<ResultType>& res, PaddedPODArray<UInt64>& offsets,
-                                bool allow_hyperscan, size_t max_hyperscan_regexp_length,
-                                size_t max_hyperscan_regexp_total_length) {
-        if (!allow_hyperscan) {
-            return Status::InvalidArgument("Hyperscan functions are disabled");
-        }
-
-        res.resize(haystack_offsets.size());
-
-        size_t prev_haystack_offset = 0;
-        size_t prev_needles_offset = 0;
-
-        const auto& nested_column =
-                assert_cast<const ColumnNullable&>(needles_data).get_nested_column();
-        const auto* needles_data_string = check_and_get_column<ColumnString>(nested_column);
-
-        if (!needles_data_string) {
-            return Status::InvalidArgument("needles should be string column");
-        }
-
-        std::vector<StringRef> needles;
-        for (size_t i = 0; i < haystack_offsets.size(); ++i) {
-            needles.reserve(needles_offsets[i] - prev_needles_offset);
-
-            for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j) {
-                needles.emplace_back(needles_data_string->get_data_at(j));
-            }
-            if (needles.empty()) {
-                res[i] = 0;
-                prev_haystack_offset = haystack_offsets[i];
-                prev_needles_offset = needles_offsets[i];
-                continue;
-            }
-
-            multiregexps::Regexps* regexps = nullptr;
-            multiregexps::ScratchPtr smart_scratch;
-            RETURN_IF_ERROR(prepare_regexps_and_scratch(needles, regexps, smart_scratch));
-
-            const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset;
-
-            /// vectorscan restriction.
-            if (cur_haystack_length > std::numeric_limits<UInt32>::max()) {
-                return Status::InternalError("too long string to search");
-            }
-
-            /// zero the result, scan, check, update the offset.
-            res[i] = 0;
-            hs_error_t err = hs_scan(
-                    regexps->getDB(),
-                    reinterpret_cast<const char*>(haystack_data.data()) + prev_haystack_offset,
-                    static_cast<unsigned>(cur_haystack_length), 0, smart_scratch.get(), on_match,
-                    &res[i]);
-            if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
-                return Status::InternalError("failed to scan with vectorscan");
-            }
-
-            prev_haystack_offset = haystack_offsets[i];
-            prev_needles_offset = needles_offsets[i];
-            needles.clear();
-        }
-
-        return Status::OK();
-    }
-};
-
 using FunctionMultiMatchAny = FunctionsMultiStringSearch<FunctionMultiMatchAnyImpl<
         TYPE_TINYINT, MultiMatchTraits::Find::Any, /*WithEditDistance*/ false>>;
 

be/src/exprs/function/functions_multi_string_search.h

@@ -0,0 +1,230 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Functions/FunctionsMultiStringSearch.h
+// and modified by Doris
+
+#pragma once
+
+#include <hs/hs_runtime.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "common/status.h"
+#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/data_type/data_type_number.h"
+#include "core/field.h"
+#include "core/pod_array_fwd.h"
+#include "core/string_ref.h"
+#include "core/types.h"
+#include "exprs/function/function_helpers.h"
+#include "exprs/function/regexps.h"
+
+namespace doris {
+
+/// For more readable instantiations of MultiMatchAnyImpl<>
+struct MultiMatchTraits {
+    enum class Find { Any, AnyIndex };
+};
+
+template <PrimitiveType PType, MultiMatchTraits::Find Find, bool WithEditDistance>
+struct FunctionMultiMatchAnyImpl {
+    using ResultType = typename PrimitiveTypeTraits<PType>::CppType;
+    static constexpr PrimitiveType ResultPType = PType;
+
+    static constexpr bool FindAny = (Find == MultiMatchTraits::Find::Any);
+    static constexpr bool FindAnyIndex = (Find == MultiMatchTraits::Find::AnyIndex);
+
+    static constexpr auto name = "multi_match_any";
+
+    static auto get_return_type() {
+        return std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>();
+    }
+
+    /**
+     * Prepares the regular expressions and scratch space for Hyperscan.
+     *
+     * This function takes a vector of needles (substrings to search for) and initializes
+     * the regular expressions and scratch space required for Hyperscan, a high-performance
+     * regular expression matching library.
+     *
+     */
+    static Status prepare_regexps_and_scratch(const std::vector<StringRef>& needles,
+                                              multiregexps::RegexpsPtr& regexps,
+                                              multiregexps::ScratchPtr& smart_scratch) {
+        multiregexps::DeferredConstructedRegexpsPtr deferred_constructed_regexps =
+                multiregexps::getOrSet</*SaveIndices*/
+                                       FindAnyIndex, WithEditDistance>(needles, std::nullopt);
+        regexps = deferred_constructed_regexps->get();
+
+        hs_scratch_t* scratch = nullptr;
+        hs_error_t err = hs_clone_scratch(regexps->getScratch(), &scratch);
+
+        if (err != HS_SUCCESS) {
+            return Status::InternalError("could not clone scratch space for vectorscan");
+        }
+
+        smart_scratch.reset(scratch);
+        return Status::OK();
+    }
+
+    /**
+     * Static callback function to handle the match results of the hs_scan function.
+     *
+     * This function is called when a matching substring is found while scanning with
+     * Hyperscan. It updates the result based on the match information.
+     *
+     */
+    static int on_match([[maybe_unused]] unsigned int id, unsigned long long /* from */, // NOLINT
+                        unsigned long long /* to */,                                     // NOLINT
+                        unsigned int /* flags */, void* context) {
+        if constexpr (FindAnyIndex) {
+            *reinterpret_cast<ResultType*>(context) = id;
+        } else if constexpr (FindAny) {
+            *reinterpret_cast<ResultType*>(context) = 1;
+        }
+        /// Once we hit the callback, there is no need to search for others.
+        return 1;
+    }
+
+    static Status vector_constant(const ColumnString::Chars& haystack_data,
+                                  const ColumnString::Offsets& haystack_offsets,
+                                  const Array& needles_arr, PaddedPODArray<ResultType>& res,
+                                  PaddedPODArray<UInt64>& offsets, bool allow_hyperscan,
+                                  size_t max_hyperscan_regexp_length,
+                                  size_t max_hyperscan_regexp_total_length) {
+        if (!allow_hyperscan) {
+            return Status::InvalidArgument("Hyperscan functions are disabled");
+        }
+
+        std::vector<StringRef> needles;
+        needles.reserve(needles_arr.size());
+        for (const auto& needle : needles_arr) {
+            const auto& tmp = needle.get<TYPE_STRING>();
+            needles.emplace_back(StringRef {tmp.data(), tmp.size()});
+        }
+
+        res.resize(haystack_offsets.size());
+
+        if (needles_arr.empty()) {
+            std::fill(res.begin(), res.end(), 0);
+            return Status::OK();
+        }
+
+        multiregexps::RegexpsPtr regexps;
+        multiregexps::ScratchPtr smart_scratch;
+        RETURN_IF_ERROR(prepare_regexps_and_scratch(needles, regexps, smart_scratch));
+
+        const size_t haystack_offsets_size = haystack_offsets.size();
+        UInt64 offset = 0;
+        for (size_t i = 0; i < haystack_offsets_size; ++i) {
+            UInt64 length = haystack_offsets[i] - offset;
+            /// vectorscan restriction.
+            if (length > std::numeric_limits<UInt32>::max()) {
+                return Status::InternalError("too long string to search");
+            }
+            /// zero the result, scan, check, update the offset.
+            res[i] = 0;
+            hs_error_t err = hs_scan(
+                    regexps->getDB(), reinterpret_cast<const char*>(haystack_data.data()) + offset,
+                    static_cast<unsigned>(length), 0, smart_scratch.get(), on_match, &res[i]);
+            if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
+                return Status::InternalError("failed to scan with vectorscan");
+            }
+            offset = haystack_offsets[i];
+        }
+
+        return Status::OK();
+    }
+
+    static Status vector_vector(const ColumnString::Chars& haystack_data,
+                                const ColumnString::Offsets& haystack_offsets,
+                                const IColumn& needles_data,
+                                const ColumnArray::Offsets64& needles_offsets,
+                                PaddedPODArray<ResultType>& res, PaddedPODArray<UInt64>& offsets,
+                                bool allow_hyperscan, size_t max_hyperscan_regexp_length,
+                                size_t max_hyperscan_regexp_total_length) {
+        if (!allow_hyperscan) {
+            return Status::InvalidArgument("Hyperscan functions are disabled");
+        }
+
+        res.resize(haystack_offsets.size());
+
+        size_t prev_haystack_offset = 0;
+        size_t prev_needles_offset = 0;
+
+        const auto& nested_column =
+                assert_cast<const ColumnNullable&>(needles_data).get_nested_column();
+        const auto* needles_data_string = check_and_get_column<ColumnString>(nested_column);
+
+        if (!needles_data_string) {
+            return Status::InvalidArgument("needles should be string column");
+        }
+
+        std::vector<StringRef> needles;
+        for (size_t i = 0; i < haystack_offsets.size(); ++i) {
+            needles.reserve(needles_offsets[i] - prev_needles_offset);
+
+            for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j) {
+                needles.emplace_back(needles_data_string->get_data_at(j));
+            }
+            if (needles.empty()) {
+                res[i] = 0;
+                prev_haystack_offset = haystack_offsets[i];
+                prev_needles_offset = needles_offsets[i];
+                continue;
+            }
+
+            multiregexps::RegexpsPtr regexps;
+            multiregexps::ScratchPtr smart_scratch;
+            RETURN_IF_ERROR(prepare_regexps_and_scratch(needles, regexps, smart_scratch));
+
+            const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset;
+
+            /// vectorscan restriction.
+            if (cur_haystack_length > std::numeric_limits<UInt32>::max()) {
+                return Status::InternalError("too long string to search");
+            }
+
+            /// zero the result, scan, check, update the offset.
+            res[i] = 0;
+            hs_error_t err = hs_scan(
+                    regexps->getDB(),
+                    reinterpret_cast<const char*>(haystack_data.data()) + prev_haystack_offset,
+                    static_cast<unsigned>(cur_haystack_length), 0, smart_scratch.get(), on_match,
+                    &res[i]);
+            if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
+                return Status::InternalError("failed to scan with vectorscan");
+            }
+
+            prev_haystack_offset = haystack_offsets[i];
+            prev_needles_offset = needles_offsets[i];
+            needles.clear();
+        }
+
+        return Status::OK();
+    }
+};
+
+} // namespace doris

be/src/exprs/function/regexps.h

@@ -68,24 +68,26 @@ private:
     ScratchPtr scratch;
 };
 
+using RegexpsPtr = std::shared_ptr<Regexps>;
+
 class DeferredConstructedRegexps {
 public:
     explicit DeferredConstructedRegexps(std::function<Regexps()> constructor_)
             : constructor(std::move(constructor_)) {}
 
-    Regexps* get() {
+    RegexpsPtr get() {
         std::lock_guard lock(mutex);
         if (regexps) {
-            return &*regexps;
+            return regexps;
         }
-        regexps = constructor();
-        return &*regexps;
+        regexps = std::make_shared<Regexps>(constructor());
+        return regexps;
     }
 
 private:
     std::mutex mutex;
     std::function<Regexps()> constructor;
-    std::optional<Regexps> regexps;
+    RegexpsPtr regexps;
 };
 
 using DeferredConstructedRegexpsPtr = std::shared_ptr<DeferredConstructedRegexps>;

be/test/exprs/function/functions_multi_string_search_test.cpp

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/function/functions_multi_string_search.h"
+
+#include <gtest/gtest.h>
+
+#include <string>
+#include <vector>
+
+namespace doris {
+
+TEST(FunctionsMultiStringSearchTest, KeepRegexpsAliveAfterCacheEviction) {
+    using Impl = FunctionMultiMatchAnyImpl<TYPE_TINYINT, MultiMatchTraits::Find::Any, false>;
+
+    std::vector<String> target_patterns = {"needle"};
+    std::vector<StringRef> target_refs = {{target_patterns[0].data(), target_patterns[0].size()}};
+    const size_t target_bucket =
+            multiregexps::GlobalCacheTable::getBucketIndexFor(target_patterns, std::nullopt);
+
+    std::vector<String> collision_patterns;
+    for (size_t i = 0;; ++i) {
+        collision_patterns = {"collision-" + std::to_string(i)};
+        if (multiregexps::GlobalCacheTable::getBucketIndexFor(collision_patterns, std::nullopt) ==
+            target_bucket) {
+            break;
+        }
+    }
+    std::vector<StringRef> collision_refs = {
+            {collision_patterns[0].data(), collision_patterns[0].size()}};
+
+    multiregexps::RegexpsPtr target_regexps;
+    multiregexps::ScratchPtr target_scratch;
+    ASSERT_TRUE(
+            Impl::prepare_regexps_and_scratch(target_refs, target_regexps, target_scratch).ok());
+
+    auto collision_owner = multiregexps::getOrSet<false, false>(collision_refs, std::nullopt);
+    ASSERT_NE(nullptr, collision_owner->get());
+
+    Impl::ResultType result = 0;
+    const std::string haystack = "find the needle";
+    const hs_error_t err = hs_scan(target_regexps->getDB(), haystack.data(),
+                                   static_cast<unsigned>(haystack.size()), 0, target_scratch.get(),
+                                   Impl::on_match, &result);
+
+    EXPECT_EQ(HS_SCAN_TERMINATED, err);
+    EXPECT_EQ(1, result);
+}
+
+} // namespace doris