diff --git a/be/src/exprs/function/match.cpp b/be/src/exprs/function/match.cpp index 4d747df0e56..03258b60206 100644 --- a/be/src/exprs/function/match.cpp +++ b/be/src/exprs/function/match.cpp @@ -19,6 +19,12 @@ #include +#include +#include +#include +#include +#include + #include "core/field.h" #include "runtime/query_context.h" #include "runtime/runtime_state.h" @@ -44,6 +50,253 @@ const InvertedIndexAnalyzerCtx* get_match_analyzer_ctx(FunctionContext* context) return analyzer_ctx; } +enum class PhraseMode { EXACT, PREFIX, EDGE }; + +class StreamingPhraseMatcher { +public: + StreamingPhraseMatcher(const std::vector& query_tokens, PhraseMode mode) + : _mode(mode), + _query_size(query_tokens.size()), + _last_word((_query_size - 1) / 64), + _last_bit(uint64_t {1} << ((_query_size - 1) % 64)), + _state(_last_word + 1, 0), + _pending_masks(_last_word + 1, 0), + _active_marks(_last_word + 1, 0), + _first_term(query_tokens.front().get_single_term()), + _last_term(query_tokens.back().get_single_term()) { + for (size_t pos = 0; pos < _query_size; ++pos) { + if ((_mode == PhraseMode::PREFIX && pos == _query_size - 1) || + (_mode == PhraseMode::EDGE && (pos == 0 || pos == _query_size - 1))) { + continue; + } + auto& words = _exact_masks[query_tokens[pos].get_single_term()]; + const size_t word = pos / 64; + const uint64_t bit = uint64_t {1} << (pos % 64); + if (!words.empty() && words.back().first == word) { + words.back().second |= bit; + } else { + words.emplace_back(word, bit); + } + } + } + + void reset_row() { + for (size_t word : _pending_touched) { + _pending_masks[word] = 0; + } + _pending_touched.clear(); + _pending_sorted = true; + _pending_position = -1; + _previous_position = -1; + _active_words.clear(); + ++_generation; + } + + bool feed(const std::string& term, int64_t position) { + if (_pending_position >= 0 && position != _pending_position) { + if (advance()) { + return true; + } + } + if (_pending_position < 0) { + start_position(position); + } + const auto mask_it = _exact_masks.find(term); + if (mask_it != _exact_masks.end()) { + const auto& mask_words = mask_it->second; + if (mask_words.size() / 4 > _candidate_words.size()) { + for (size_t word : _candidate_words) { + auto it = std::lower_bound( + mask_words.begin(), mask_words.end(), word, + [](const auto& entry, size_t value) { return entry.first < value; }); + if (it != mask_words.end() && it->first == word) { + add_pending_mask(word, it->second); + } + } + } else { + size_t candidate = 0; + for (const auto& [word, mask] : mask_words) { + while (candidate < _candidate_words.size() && + _candidate_words[candidate] < word) { + ++candidate; + } + if (candidate == _candidate_words.size()) { + break; + } + if (_candidate_words[candidate] == word) { + add_pending_mask(word, mask); + } + } + } + } + const bool first_matches = _mode == PhraseMode::EDGE && + (_query_size == 1 ? term.find(_first_term) != std::string::npos + : term.ends_with(_first_term)); + const bool last_matches = + (_mode == PhraseMode::PREFIX || (_mode == PhraseMode::EDGE && _query_size > 1)) && + term.starts_with(_last_term); + + if (first_matches) { + add_pending_mask(0, 1); + } + if (last_matches && + std::binary_search(_candidate_words.begin(), _candidate_words.end(), _last_word)) { + add_pending_mask(_last_word, _last_bit); + } + return false; + } + + bool finish() { return _pending_position >= 0 && advance(); } + +private: + void start_position(int64_t position) { + if (_previous_position >= 0 && position != _previous_position + 1) { + _active_words.clear(); + ++_generation; + } + _pending_position = position; + _candidate_words.clear(); + _candidate_words.push_back(0); + for (size_t word : _active_words) { + if (_candidate_words.back() != word) { + _candidate_words.push_back(word); + } + if (word < _last_word && (_state[word] >> 63) != 0) { + _candidate_words.push_back(word + 1); + } + } + } + + void add_pending_mask(size_t word, uint64_t mask) { + if (_pending_masks[word] == 0) { + if (!_pending_touched.empty() && word < _pending_touched.back()) { + _pending_sorted = false; + } + _pending_touched.push_back(word); + } + _pending_masks[word] |= mask; + } + + bool advance() { + const int64_t position = _pending_position; + _pending_position = -1; + _previous_position = position; + + const uint64_t old_generation = _generation++; + _next_active_words.clear(); + if (!_pending_touched.empty()) { + if (!_pending_sorted) { + std::sort(_pending_touched.begin(), _pending_touched.end()); + } + for (auto it = _pending_touched.rbegin(); it != _pending_touched.rend(); ++it) { + const size_t word = *it; + const uint64_t previous = _active_marks[word] == old_generation ? _state[word] : 0; + const uint64_t carry = word > 0 && _active_marks[word - 1] == old_generation + ? _state[word - 1] >> 63 + : 0; + const uint64_t next = + ((previous << 1) | carry | uint64_t {word == 0}) & _pending_masks[word]; + if (next != 0) { + _state[word] = next; + _active_marks[word] = _generation; + _next_active_words.push_back(word); + } + } + } + for (size_t word : _pending_touched) { + _pending_masks[word] = 0; + } + _pending_touched.clear(); + _pending_sorted = true; + std::reverse(_next_active_words.begin(), _next_active_words.end()); + _active_words.swap(_next_active_words); + return !_active_words.empty() && _active_words.back() == _last_word && + (_state[_last_word] & _last_bit) != 0; + } + + PhraseMode _mode; + size_t _query_size; + size_t _last_word; + uint64_t _last_bit; + std::vector _state; + std::vector _pending_masks; + std::vector _active_marks; + std::vector _pending_touched; + std::vector _candidate_words; + std::vector _active_words; + std::vector _next_active_words; + std::string _first_term; + std::string _last_term; + std::unordered_map>> _exact_masks; + int64_t _pending_position = -1; + int64_t _previous_position = -1; + uint64_t _generation = 1; + bool _pending_sorted = true; +}; + +template +bool for_each_data_element_tokens(const FunctionMatchBase& function, const std::string& column_name, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnString* string_col, size_t row, + const ColumnArray::Offsets64* array_offsets, + const ColumnUInt8::Container* array_element_null_map, + Callback&& callback) { + const size_t begin = array_offsets ? (row == 0 ? 0 : (*array_offsets)[row - 1]) : row; + const size_t end = array_offsets ? (*array_offsets)[row] : row + 1; + const bool keyword = analyzer_ctx && + (!analyzer_ctx->requires_analysis() || analyzer_ctx->analyzer == nullptr); + segment_v2::TermInfo keyword_token; + int32_t unused_array_offset = 0; + for (size_t element = begin; element < end; ++element) { + if (array_element_null_map && (*array_element_null_map)[element]) { + continue; + } + if (keyword) { + keyword_token.term = string_col->get_data_at(element).to_string(); + if (callback(std::span(&keyword_token, 1))) { + return true; + } + continue; + } + auto tokens = function.analyse_data_token(column_name, analyzer_ctx, string_col, element, + nullptr, unused_array_offset); + if (tokens.empty()) { + continue; + } + if (callback(std::span(tokens))) { + return true; + } + } + return false; +} + +bool match_phrase_data_tokens(const FunctionMatchBase& function, const std::string& column_name, + const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnString* string_col, size_t row, + const ColumnArray::Offsets64* array_offsets, + const ColumnUInt8::Container* array_element_null_map, + StreamingPhraseMatcher& matcher) { + matcher.reset_row(); + int64_t position_base = 0; + const bool matched = for_each_data_element_tokens( + function, column_name, analyzer_ctx, string_col, row, array_offsets, + array_element_null_map, [&](std::span tokens) { + const bool analyzed = analyzer_ctx && analyzer_ctx->requires_analysis() && + analyzer_ctx->analyzer != nullptr; + int32_t last_position = 0; + for (const auto& token : tokens) { + const int32_t position = analyzed ? token.position : 1; + if (matcher.feed(token.get_single_term(), position_base + position)) { + return true; + } + last_position = position; + } + position_base += last_position; + return false; + }); + return matched || matcher.finish(); +} + } // namespace Status FunctionMatchBase::evaluate_inverted_index( @@ -140,6 +393,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); const auto* values = check_and_get_column(source_col.get()); const ColumnArray* array_col = nullptr; + const ColumnUInt8::Container* array_element_null_map = nullptr; if (is_column(source_col.get())) { array_col = check_and_get_column(source_col.get()); if (array_col && !array_col->get_data().is_column_string()) { @@ -153,6 +407,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, if (is_column_nullable(array_col->get_data())) { const auto& array_nested_null_column = reinterpret_cast(array_col->get_data()); + array_element_null_map = &array_nested_null_column.get_null_map_column().get_data(); values = check_and_get_column( *(array_nested_null_column.get_nested_column_ptr())); } else { @@ -174,7 +429,7 @@ Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, vec_res.resize_fill(input_rows_count); RETURN_IF_ERROR(execute_match(context, column_name, match_query_str, input_rows_count, values, analyzer_ctx, (array_col ? &(array_col->get_offsets()) : nullptr), - vec_res)); + vec_res, array_element_null_map)); block.replace_by_position(result, std::move(res)); return Status::OK(); @@ -237,8 +492,9 @@ std::vector FunctionMatchBase::analyse_query_str_token( inline std::vector FunctionMatchBase::analyse_data_token( const std::string& column_name, const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnString* string_col, int32_t current_block_row_idx, - const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset) const { + const ColumnString* string_col, size_t current_block_row_idx, + const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset, + const ColumnUInt8::Container* array_element_null_map) const { std::vector data_tokens; if (analyzer_ctx == nullptr) { return data_tokens; @@ -250,6 +506,9 @@ inline std::vector FunctionMatchBase::analyse_data_token( if (array_offsets) { for (auto next_src_array_offset = (*array_offsets)[current_block_row_idx]; current_src_array_offset < next_src_array_offset; ++current_src_array_offset) { + if (array_element_null_map && (*array_element_null_map)[current_src_array_offset]) { + continue; + } const auto& str_ref = string_col->get_data_at(current_src_array_offset); if (!requires_analysis) { data_tokens.emplace_back(str_ref.to_string()); @@ -258,9 +517,12 @@ inline std::vector FunctionMatchBase::analyse_data_token( auto reader = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::create_reader( analyzer_ctx->char_filter_map); reader->init(str_ref.data, (int)str_ref.size, true); - data_tokens = + auto element_tokens = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result( reader, analyzer_ctx->analyzer.get()); + for (auto& token : element_tokens) { + data_tokens.emplace_back(std::move(token)); + } } } else { const auto& str_ref = string_col->get_data_at(current_block_row_idx); @@ -297,7 +559,8 @@ Status FunctionMatchAny::execute_match(FunctionContext* context, const std::stri const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str, column_name); @@ -311,21 +574,23 @@ Status FunctionMatchAny::execute_match(FunctionContext* context, const std::stri return Status::OK(); } - auto current_src_array_offset = 0; - for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); + std::unordered_set query_terms; + for (const auto& token : query_tokens) { + query_terms.emplace(token.get_single_term()); + } - // TODO: more efficient impl - for (auto& term_info : query_tokens) { - auto it = std::find_if(data_tokens.begin(), data_tokens.end(), - [&](const segment_v2::TermInfo& info) { - return info.get_single_term() == term_info.get_single_term(); - }); - if (it != data_tokens.end()) { - result[i] = true; - break; - } + for (int i = 0; i < input_rows_count; i++) { + if (for_each_data_element_tokens(*this, column_name, analyzer_ctx, string_col, i, + array_offsets, array_element_null_map, + [&](std::span data_tokens) { + for (const auto& info : data_tokens) { + if (query_terms.contains(info.get_single_term())) { + return true; + } + } + return false; + })) { + result[i] = true; } } @@ -337,7 +602,8 @@ Status FunctionMatchAll::execute_match(FunctionContext* context, const std::stri const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str, column_name); @@ -351,26 +617,32 @@ Status FunctionMatchAll::execute_match(FunctionContext* context, const std::stri return Status::OK(); } - auto current_src_array_offset = 0; + std::unordered_map query_term_ids; + for (const auto& token : query_tokens) { + query_term_ids.emplace(token.get_single_term(), query_term_ids.size()); + } + std::vector last_seen_row; + for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); - - // TODO: more efficient impl - auto find_count = 0; - for (auto& term_info : query_tokens) { - auto it = std::find_if(data_tokens.begin(), data_tokens.end(), - [&](const segment_v2::TermInfo& info) { - return info.get_single_term() == term_info.get_single_term(); - }); - if (it != data_tokens.end()) { - ++find_count; - } else { - break; - } - } - - if (find_count == query_tokens.size()) { + size_t remaining = query_term_ids.size(); + if (for_each_data_element_tokens( + *this, column_name, analyzer_ctx, string_col, i, array_offsets, + array_element_null_map, [&](std::span data_tokens) { + if (last_seen_row.empty()) { + last_seen_row.resize(query_term_ids.size(), 0); + } + for (const auto& info : data_tokens) { + auto it = query_term_ids.find(info.get_single_term()); + if (it != query_term_ids.end() && + last_seen_row[it->second] != static_cast(i) + 1) { + last_seen_row[it->second] = static_cast(i) + 1; + if (--remaining == 0) { + return true; + } + } + } + return false; + })) { result[i] = true; } } @@ -378,12 +650,12 @@ Status FunctionMatchAll::execute_match(FunctionContext* context, const std::stri return Status::OK(); } -Status FunctionMatchPhrase::execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, - size_t input_rows_count, const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { +Status FunctionMatchPhrase::execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, const ColumnString* string_col, + const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str, column_name); @@ -397,43 +669,10 @@ Status FunctionMatchPhrase::execute_match(FunctionContext* context, const std::s return Status::OK(); } - auto current_src_array_offset = 0; + StreamingPhraseMatcher matcher(query_tokens, PhraseMode::EXACT); for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); - - // TODO: more efficient impl - bool matched = false; - auto data_it = data_tokens.begin(); - while (data_it != data_tokens.end()) { - // find position of first token - data_it = - std::find_if(data_it, data_tokens.end(), [&](const segment_v2::TermInfo& info) { - return info.get_single_term() == query_tokens[0].get_single_term(); - }); - if (data_it != data_tokens.end()) { - matched = true; - auto data_it_next = ++data_it; - auto query_it = query_tokens.begin() + 1; - // compare query_tokens after the first to data_tokens one by one - while (query_it != query_tokens.end()) { - if (data_it_next == data_tokens.end() || - data_it_next->get_single_term() != query_it->get_single_term()) { - matched = false; - break; - } - query_it++; - data_it_next++; - } - - if (matched) { - break; - } - } - } - - // check matched - if (matched) { + if (match_phrase_data_tokens(*this, column_name, analyzer_ctx, string_col, i, array_offsets, + array_element_null_map, matcher)) { result[i] = true; } } @@ -445,7 +684,8 @@ Status FunctionMatchPhrasePrefix::execute_match( FunctionContext* context, const std::string& column_name, const std::string& match_query_str, size_t input_rows_count, const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str, column_name); @@ -459,52 +699,23 @@ Status FunctionMatchPhrasePrefix::execute_match( return Status::OK(); } - int32_t current_src_array_offset = 0; + StreamingPhraseMatcher matcher(query_tokens, PhraseMode::PREFIX); for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); - - int64_t dis_count = data_tokens.size() - query_tokens.size(); - if (dis_count < 0) { - continue; - } - - for (size_t j = 0; j < dis_count + 1; j++) { - if (data_tokens[j].get_single_term() == query_tokens[0].get_single_term() || - query_tokens.size() == 1) { - bool match = true; - for (size_t k = 0; k < query_tokens.size(); k++) { - const std::string& data_token = data_tokens[j + k].get_single_term(); - const std::string& query_token = query_tokens[k].get_single_term(); - if (k == query_tokens.size() - 1) { - if (data_token.compare(0, query_token.size(), query_token) != 0) { - match = false; - break; - } - } else { - if (data_token != query_token) { - match = false; - break; - } - } - } - if (match) { - result[i] = true; - break; - } - } + if (match_phrase_data_tokens(*this, column_name, analyzer_ctx, string_col, i, array_offsets, + array_element_null_map, matcher)) { + result[i] = true; } } return Status::OK(); } -Status FunctionMatchRegexp::execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, - size_t input_rows_count, const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { +Status FunctionMatchRegexp::execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, const ColumnString* string_col, + const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); VLOG_DEBUG << "begin to run FunctionMatchRegexp::execute_match, parser_type: " @@ -544,25 +755,25 @@ Status FunctionMatchRegexp::execute_match(FunctionContext* context, const std::s }; try { - auto current_src_array_offset = 0; for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); - - for (auto& input : data_tokens) { - bool is_match = false; - const auto& input_str = input.get_single_term(); - if (hs_scan(database, input_str.data(), (uint32_t)input_str.size(), 0, scratch, - on_match, (void*)&is_match) != HS_SUCCESS) { - LOG(ERROR) << "hyperscan match failed: " << input_str; - break; - } - - if (is_match) { - result[i] = true; - break; - } - } + for_each_data_element_tokens( + *this, column_name, analyzer_ctx, string_col, i, array_offsets, + array_element_null_map, [&](std::span data_tokens) { + for (const auto& input : data_tokens) { + bool is_match = false; + const auto& input_str = input.get_single_term(); + if (hs_scan(database, input_str.data(), (uint32_t)input_str.size(), 0, + scratch, on_match, (void*)&is_match) != HS_SUCCESS) { + LOG(ERROR) << "hyperscan match failed: " << input_str; + return true; + } + if (is_match) { + result[i] = true; + return true; + } + } + return false; + }); } } _CLFINALLY({ @@ -577,7 +788,8 @@ Status FunctionMatchPhraseEdge::execute_match( FunctionContext* context, const std::string& column_name, const std::string& match_query_str, size_t input_rows_count, const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const { + ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map) const { RETURN_IF_ERROR(check(context, name)); auto query_tokens = analyse_query_str_token(analyzer_ctx, match_query_str, column_name); @@ -591,49 +803,11 @@ Status FunctionMatchPhraseEdge::execute_match( return Status::OK(); } - int32_t current_src_array_offset = 0; + StreamingPhraseMatcher matcher(query_tokens, PhraseMode::EDGE); for (int i = 0; i < input_rows_count; i++) { - auto data_tokens = analyse_data_token(column_name, analyzer_ctx, string_col, i, - array_offsets, current_src_array_offset); - - int64_t dis_count = data_tokens.size() - query_tokens.size(); - if (dis_count < 0) { - continue; - } - - for (size_t j = 0; j < dis_count + 1; j++) { - bool match = true; - if (query_tokens.size() == 1) { - if (data_tokens[j].get_single_term().find(query_tokens[0].get_single_term()) == - std::string::npos) { - match = false; - } - } else { - for (size_t k = 0; k < query_tokens.size(); k++) { - const std::string& data_token = data_tokens[j + k].get_single_term(); - const std::string& query_token = query_tokens[k].get_single_term(); - if (k == 0) { - if (!data_token.ends_with(query_token)) { - match = false; - break; - } - } else if (k == query_tokens.size() - 1) { - if (!data_token.starts_with(query_token)) { - match = false; - break; - } - } else { - if (data_token != query_token) { - match = false; - break; - } - } - } - } - if (match) { - result[i] = true; - break; - } + if (match_phrase_data_tokens(*this, column_name, analyzer_ctx, string_col, i, array_offsets, + array_element_null_map, matcher)) { + result[i] = true; } } diff --git a/be/src/exprs/function/match.h b/be/src/exprs/function/match.h index 84d0e4374bf..a4818360ddb 100644 --- a/be/src/exprs/function/match.h +++ b/be/src/exprs/function/match.h @@ -72,12 +72,12 @@ public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override; - virtual Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const = 0; + virtual Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const = 0; doris::segment_v2::InvertedIndexQueryType get_query_type_from_fn_name() const; @@ -87,8 +87,9 @@ public: std::vector analyse_data_token( const std::string& column_name, const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnString* string_col, int32_t current_block_row_idx, - const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset) const; + const ColumnString* string_col, size_t current_block_row_idx, + const ColumnArray::Offsets64* array_offsets, int32_t& current_src_array_offset, + const ColumnUInt8::Container* array_element_null_map = nullptr) const; Status check(FunctionContext* context, const std::string& function_name) const; @@ -107,12 +108,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; class FunctionMatchAll : public FunctionMatchBase { @@ -122,12 +123,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; class FunctionMatchPhrase : public FunctionMatchBase { @@ -137,12 +138,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; class FunctionMatchPhrasePrefix : public FunctionMatchBase { @@ -152,12 +153,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; class FunctionMatchRegexp : public FunctionMatchBase { @@ -167,12 +168,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; class FunctionMatchPhraseEdge : public FunctionMatchBase { @@ -182,12 +183,12 @@ public: String get_name() const override { return name; } - Status execute_match(FunctionContext* context, const std::string& column_name, - const std::string& match_query_str, size_t input_rows_count, - const ColumnString* string_col, - const InvertedIndexAnalyzerCtx* analyzer_ctx, - const ColumnArray::Offsets64* array_offsets, - ColumnUInt8::Container& result) const override; + Status execute_match( + FunctionContext* context, const std::string& column_name, + const std::string& match_query_str, size_t input_rows_count, + const ColumnString* string_col, const InvertedIndexAnalyzerCtx* analyzer_ctx, + const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result, + const ColumnUInt8::Container* array_element_null_map = nullptr) const override; }; } // namespace doris diff --git a/be/test/exprs/function/function_match_test.cpp b/be/test/exprs/function/function_match_test.cpp index 0f06b29e2a3..4758e82f05e 100644 --- a/be/test/exprs/function/function_match_test.cpp +++ b/be/test/exprs/function/function_match_test.cpp @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -25,6 +26,8 @@ #include #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_string.h" @@ -470,6 +473,403 @@ TEST(FunctionMatchTest, array_offset_handling) { } } +TEST(FunctionMatchTest, array_match_uses_tokens_from_all_elements) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + auto string_col = ColumnString::create(); + for (const std::string value : {"alpha token", "tail only", "head only", "omega token", + "unrelated head", "unrelated tail", "alpha token", ""}) { + string_col->insert_data(value.data(), value.size()); + } + ColumnArray::Offsets64 array_offsets = {2, 4, 6, 8, 8}; + + FunctionMatchAny match_any; + FunctionMatchAll match_all; + FunctionMatchRegexp match_regexp; + FunctionMatchPhrase match_phrase; + FunctionMatchPhrasePrefix match_phrase_prefix; + FunctionMatchPhraseEdge match_phrase_edge; + auto check_match = [&](const FunctionMatchBase& function, const std::string& query, + const std::vector& expected) { + SCOPED_TRACE(function.get_name() + ": " + query); + ColumnUInt8::Container result(array_offsets.size(), 0); + ASSERT_TRUE(function.execute_match(context.get(), "tags", query, array_offsets.size(), + string_col.get(), ctx.ctx.get(), &array_offsets, result) + .ok()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(result[row], expected[row]) << "row: " << row; + } + }; + check_match(match_any, "alpha", {1, 0, 0, 1, 0}); + check_match(match_any, "omega", {0, 1, 0, 0, 0}); + check_match(match_all, "alpha tail", {1, 0, 0, 0, 0}); + check_match(match_all, "alpha alpha", {1, 0, 0, 1, 0}); + check_match(match_regexp, "^alpha$", {1, 0, 0, 1, 0}); + check_match(match_phrase, "alpha token", {1, 0, 0, 1, 0}); + check_match(match_phrase, "token tail", {1, 0, 0, 0, 0}); + check_match(match_phrase_prefix, "alpha tok", {1, 0, 0, 1, 0}); + check_match(match_phrase_prefix, "token tai", {1, 0, 0, 0, 0}); + check_match(match_phrase_edge, "pha tok", {1, 0, 0, 1, 0}); + check_match(match_phrase_edge, "ken tai", {1, 0, 0, 0, 0}); + + segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("keyword", {}); + builder.add_token_filter_config("lowercase", {}); + auto provider = + std::make_shared(builder.build()); + ctx.ctx->parser_type = InvertedIndexParserType::PARSER_NONE; + ctx.ctx->analyzer_name = "custom_keyword_lowercase"; + ctx.ctx->analyzer_provider = provider; + ctx.ctx->analyzer = provider->get_analyzer(); + check_match(match_any, "ALPHA TOKEN", {1, 0, 0, 1, 0}); + check_match(match_phrase, "ALPHA TOKEN", {1, 0, 0, 1, 0}); + + ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_NONE); + check_match(match_any, "alpha token", {1, 0, 0, 1, 0}); + check_match(match_all, "alpha token", {1, 0, 0, 1, 0}); + check_match(match_phrase, "alpha token", {1, 0, 0, 1, 0}); + check_match(match_phrase_prefix, "alpha", {1, 0, 0, 1, 0}); + check_match(match_phrase_edge, "pha", {1, 0, 0, 1, 0}); + check_match(match_any, "", {0, 0, 0, 1, 0}); +} + +TEST(FunctionMatchTest, array_phrase_spans_non_null_elements) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + auto string_col = ColumnString::create(); + for (const std::string value : + {"hello", "world", "hello", "hidden", "world", "hello", "there", "world"}) { + string_col->insert_data(value.data(), value.size()); + } + ColumnArray::Offsets64 offsets = {2, 5, 8}; + ColumnUInt8::Container null_map = {0, 0, 0, 1, 0, 0, 0, 0}; + + auto check_match = [&](const FunctionMatchBase& function, const std::string& query, + const std::vector& expected) { + SCOPED_TRACE(function.get_name() + ": " + query); + ColumnUInt8::Container result(expected.size(), 0); + ASSERT_TRUE(function.execute_match(context.get(), "tags", query, expected.size(), + string_col.get(), ctx.ctx.get(), &offsets, result, + &null_map) + .ok()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(result[row], expected[row]) << "row: " << row; + } + }; + check_match(FunctionMatchPhrase {}, "hello world", {1, 1, 0}); + check_match(FunctionMatchPhrasePrefix {}, "hello wor", {1, 1, 0}); + check_match(FunctionMatchPhraseEdge {}, "llo wor", {1, 1, 0}); + check_match(FunctionMatchPhrase {}, "hello hidden world", {0, 0, 0}); +} + +TEST(FunctionMatchTest, array_phrase_respects_analyzer_positions) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::Settings delimiter_settings; + delimiter_settings.set("preserve_original", "true"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", tokenizer_settings); + builder.add_token_filter_config("word_delimiter", delimiter_settings); + auto provider = + std::make_shared(builder.build()); + auto ctx = std::make_unique(); + ctx->analyzer_name = "word_delimiter_with_original"; + ctx->analyzer_provider = provider; + ctx->analyzer = provider->get_analyzer(); + + auto string_col = ColumnString::create(); + string_col->insert_data("foo-bar", 7); + string_col->insert_data("baz", 3); + ColumnArray::Offsets64 offsets = {2}; + + FunctionMatchPhrase match_phrase; + int32_t unused_offset = 0; + auto data_tokens = match_phrase.analyse_data_token("tags", ctx.get(), string_col.get(), 0, + nullptr, unused_offset); + ASSERT_EQ(data_tokens.size(), 3); + EXPECT_EQ(data_tokens[0].position, data_tokens[1].position); + auto query_tokens = match_phrase.analyse_query_str_token(ctx.get(), "foo-bar baz", "tags"); + ASSERT_EQ(query_tokens.size(), 4); + + ColumnUInt8::Container result(1, 0); + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", "foo-bar baz", 1, string_col.get(), + ctx.get(), &offsets, result) + .ok()); + EXPECT_EQ(result[0], 0); + result[0] = 0; + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", "foo bar baz", 1, string_col.get(), + ctx.get(), &offsets, result) + .ok()); + EXPECT_EQ(result[0], 1); +} + +TEST(FunctionMatchTest, array_phrase_preserves_leading_analyzer_gap) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + + segment_v2::inverted_index::Settings tokenizer_settings; + tokenizer_settings.set("tokenize_on_chars", "[whitespace]"); + segment_v2::inverted_index::CustomAnalyzerConfig::Builder builder; + builder.with_tokenizer_config("char_group", tokenizer_settings); + builder.add_token_filter_config("word_delimiter", {}); + auto provider = + std::make_shared(builder.build()); + auto ctx = std::make_unique(); + ctx->analyzer_name = "word_delimiter"; + ctx->analyzer_provider = provider; + ctx->analyzer = provider->get_analyzer(); + + auto string_col = ColumnString::create(); + string_col->insert_data("hello", 5); + string_col->insert_data("--- world", 9); + ColumnArray::Offsets64 offsets = {2}; + + FunctionMatchPhrase match_phrase; + int32_t unused_offset = 0; + auto later_tokens = match_phrase.analyse_data_token("tags", ctx.get(), string_col.get(), 1, + nullptr, unused_offset); + ASSERT_EQ(later_tokens.size(), 1); + EXPECT_EQ(later_tokens[0].position, 2); + + ColumnUInt8::Container result(1, 0); + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", "hello world", 1, string_col.get(), + ctx.get(), &offsets, result) + .ok()); + EXPECT_EQ(result[0], 0); +} + +TEST(FunctionMatchTest, long_unrelated_phrase_miss) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + std::string unrelated; + std::string matching; + for (int i = 0; i < 1000; ++i) { + unrelated += "gamma "; + matching += "alpha "; + } + auto string_col = ColumnString::create(); + string_col->insert_data(unrelated.data(), unrelated.size()); + string_col->insert_data(matching.data(), matching.size()); + string_col->insert_data(unrelated.data(), unrelated.size()); + ColumnUInt8::Container result(3, 0); + FunctionMatchPhrase match_phrase; + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", matching, 3, string_col.get(), + ctx.ctx.get(), nullptr, result) + .ok()); + EXPECT_EQ(result, (ColumnUInt8::Container {0, 1, 0})); +} + +TEST(FunctionMatchTest, long_phrase_with_many_short_rows) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + std::string query; + for (int i = 0; i < 1000; ++i) { + query += "alpha "; + } + auto string_col = ColumnString::create(); + for (int i = 0; i < 2000; ++i) { + string_col->insert_data("alpha", 5); + } + ColumnUInt8::Container result(2000, 0); + FunctionMatchPhrase match_phrase; + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", query, 2000, string_col.get(), + ctx.ctx.get(), nullptr, result) + .ok()); + EXPECT_EQ(std::count(result.begin(), result.end(), 1), 0); +} + +TEST(FunctionMatchTest, long_phrase_with_unique_start_miss) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + std::string value = "beta "; + for (int i = 0; i < 1000; ++i) { + value += "alpha "; + } + auto string_col = ColumnString::create(); + string_col->insert_data(value.data(), value.size()); + ColumnUInt8::Container result(1, 0); + FunctionMatchPhrase match_phrase; + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", value + "gamma", 1, string_col.get(), + ctx.ctx.get(), nullptr, result) + .ok()); + EXPECT_EQ(result[0], 0); +} + +TEST(FunctionMatchTest, keyword_array_many_elements) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_NONE); + + auto string_col = ColumnString::create(); + for (int i = 0; i < 1000; ++i) { + string_col->insert_data("alpha", 5); + } + string_col->insert_data("omega", 5); + ColumnArray::Offsets64 offsets = {1000, 1001}; + ColumnUInt8::Container result(2, 0); + FunctionMatchAny match_any; + ASSERT_TRUE(match_any + .execute_match(context.get(), "tags", "omega", 2, string_col.get(), + ctx.ctx.get(), &offsets, result) + .ok()); + EXPECT_EQ(result, (ColumnUInt8::Container {0, 1})); +} + +TEST(FunctionMatchTest, long_repeated_phrase_miss) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + std::string value; + for (int i = 0; i < 10000; ++i) { + value += "alpha "; + } + std::string query; + for (int i = 0; i < 999; ++i) { + query += "alpha "; + } + query += "beta"; + + auto string_col = ColumnString::create(); + string_col->insert_data(value.data(), value.size()); + ColumnUInt8::Container result(1, 0); + FunctionMatchPhrase match_phrase; + ASSERT_TRUE(match_phrase + .execute_match(context.get(), "tags", query, 1, string_col.get(), + ctx.ctx.get(), nullptr, result) + .ok()); + EXPECT_EQ(result[0], 0); +} + +TEST(FunctionMatchTest, long_phrase_modes_cross_bitset_word) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto context = FunctionContext::create_context(&runtime_state, {}, {}); + auto ctx = create_inverted_index_ctx(InvertedIndexParserType::PARSER_ENGLISH); + + std::string value; + std::string phrase; + std::string edge = "pha "; + for (int i = 0; i < 64; ++i) { + value += "alpha "; + phrase += "alpha "; + if (i > 0) { + edge += "alpha "; + } + } + value += "beta"; + phrase += "beta"; + edge += "bet"; + + auto string_col = ColumnString::create(); + string_col->insert_data(value.data(), value.size()); + auto check_match = [&](const FunctionMatchBase& function, const std::string& query) { + SCOPED_TRACE(function.get_name()); + ColumnUInt8::Container result(1, 0); + ASSERT_TRUE(function.execute_match(context.get(), "tags", query, 1, string_col.get(), + ctx.ctx.get(), nullptr, result) + .ok()); + EXPECT_EQ(result[0], 1); + }; + check_match(FunctionMatchPhrase {}, phrase); + check_match(FunctionMatchPhrasePrefix {}, phrase.substr(0, phrase.size() - 1)); + check_match(FunctionMatchPhraseEdge {}, edge); +} + +TEST(FunctionMatchTest, null_array_elements_do_not_match_hidden_payload) { + TQueryOptions query_options; + query_options.__set_enable_match_without_inverted_index(true); + RuntimeState runtime_state(query_options, TQueryGlobals {}); + auto array_type = + std::make_shared(make_nullable(std::make_shared())); + auto string_type = std::make_shared(); + auto context = FunctionContext::create_context( + &runtime_state, std::make_shared(), {array_type, string_type}); + auto analyzer_ctx = std::make_shared(); + analyzer_ctx->parser_type = InvertedIndexParserType::PARSER_NONE; + context->set_function_state(FunctionContext::FRAGMENT_LOCAL, analyzer_ctx); + + auto check_match = [&](const FunctionMatchBase& function) { + auto values = ColumnString::create(); + for (const std::string value : {"alpha", "tail", "alpha", "alpha", "tail", "alpha"}) { + values->insert_data(value.data(), value.size()); + } + auto null_map = ColumnUInt8::create(); + const uint8_t nulls[] = {1, 0, 1, 0, 0, 1}; + for (uint8_t is_null : nulls) { + null_map->insert_value(is_null); + } + auto offsets = ColumnArray::ColumnOffsets::create(); + for (uint64_t offset : {2, 4, 6}) { + offsets->insert_value(offset); + } + auto query = ColumnString::create(); + for (size_t row = 0; row < 3; ++row) { + query->insert_data("alpha", 5); + } + + Block block; + block.insert( + {ColumnArray::create(ColumnNullable::create(std::move(values), std::move(null_map)), + std::move(offsets)), + array_type, "tags"}); + block.insert({std::move(query), string_type, "query"}); + block.insert({nullptr, std::make_shared(), "result"}); + ASSERT_TRUE(function.execute_impl(context.get(), block, {0, 1}, 2, 3).ok()); + + const auto& result = assert_cast(*block.get_by_position(2).column); + SCOPED_TRACE(function.get_name()); + EXPECT_EQ(result.get_element(0), 0); + EXPECT_EQ(result.get_element(1), 1); + EXPECT_EQ(result.get_element(2), 0); + }; + + check_match(FunctionMatchAny {}); + check_match(FunctionMatchAll {}); + check_match(FunctionMatchRegexp {}); + check_match(FunctionMatchPhrase {}); + check_match(FunctionMatchPhrasePrefix {}); + check_match(FunctionMatchPhraseEdge {}); +} + // Test Unicode and special character handling TEST(FunctionMatchTest, unicode_and_special_chars) { FunctionMatchAny match_any;