[fix](be) Score regexp and wildcard search terms with BM25 (#69052)

### What problem does this PR solve? Issue Number: None Related PR: #68758 Problem Summary: Regexp and wildcard search() clauses return constant score 1 even when scoring is requested. The old RegexpWeight has been replaced by ExpandWeight in master's unified MATCH/SEARCH engine. Pass the scoring flag through ExpandQuery. For regexp and wildcard expansions, create a BM25 scorer for each matching term using the bound IndexSource's statistics, including physical token counts from docs-only SNII sources and a unit-length fallback only when legacy token metadata is absent and sum contributions from terms matching the same document. Scored weights use scorers rather than the row-listing shortcut, which discards frequencies. Queries without scoring retain the batched bitmap path. This branch merges #68758 and moves the fix and its tests onto the new interfaces. Existing prefix/phrase-edge scoring and the pure-wildcard field:* existence rewrite are preserved. Tests with repeated and multiple matching terms reject the previous constant-score behavior. SNII source fixtures provide explicit scoring statistics and assert the expected BM25 score; pattern-only helpers remain unscored.
2026-11-04 07:31:17 +00:00 · 2026-10-18 20:32:20 +08:00
1 parent 677fc65e0f
commit 31085da6b3
@@ -38,9 +38,8 @@ public:
              _pattern(std::move(pattern)) {}
    ~ExpandQuery() override = default;

-    // Scores are constant, so scoring changes nothing.
-    WeightPtr weight(bool /*enable_scoring*/) override {
-        return std::make_shared<ExpandWeight>(_context, _field, _kind, _pattern);
+    WeightPtr weight(bool enable_scoring) override {
+        return std::make_shared<ExpandWeight>(_context, _field, _kind, _pattern, enable_scoring);
    }

 private:
@@ -17,6 +17,7 @@

 #include "storage/index/inverted/query/engine/expand_query/expand_weight.h"

+#include <cmath>
 #include <memory>
 #include <roaring/roaring.hh>
 #include <vector>
@@ -25,10 +26,14 @@
 #include "storage/index/inverted/query/engine/bit_set_query/bit_set_scorer.h"
 #include "storage/index/inverted/query/engine/const_score_query/const_score_scorer.h"
 #include "storage/index/inverted/query/engine/nullable_scorer.h"
+#include "storage/index/inverted/query/engine/score_combiner.h"
+#include "storage/index/inverted/query/engine/term_query/term_scorer.h"
+#include "storage/index/inverted/query/engine/union/buffered_union.h"
 #include "storage/index/inverted/query/exec/block_doc_set.h"
 #include "storage/index/inverted/query/exec/collect_postings.h"
 #include "storage/index/inverted/query/exec/roaring_docid_sink.h"
 #include "storage/index/inverted/query/exec/term_waves.h"
+#include "storage/index/inverted/query/scoring/bm25_similarity.h"
 #include "storage/index/inverted/query/spi/postings_cursor.h"

 namespace doris::segment_v2::inverted_index::query_v2 {
@@ -58,11 +63,15 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term
 }

 ExpandWeight::ExpandWeight(IndexQueryContextPtr context, std::wstring field,
-                           index_query::TermPatternKind kind, std::string pattern)
+                           index_query::TermPatternKind kind, std::string pattern,
+                           bool enable_scoring)
        : _context(std::move(context)),
          _field(std::move(field)),
          _kind(kind),
-          _pattern(std::move(pattern)) {}
+          _pattern(std::move(pattern)),
+          // Prefix and phrase-edge expansions retain their constant-score behavior.
+          _enable_scoring(enable_scoring && (kind == index_query::TermPatternKind::kRegexp ||
+                                             kind == index_query::TermPatternKind::kWildcard)) {}

 std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const QueryExecutionContext& context,
                                                      const std::string& binding_key,
@@ -85,12 +94,68 @@ std::shared_ptr<roaring::Roaring> ExpandWeight::_rows(const QueryExecutionContex

 ScorerPtr ExpandWeight::scorer(const QueryExecutionContext& context,
                               const std::string& binding_key) {
+    // The listing fast path discards term frequencies, so scored patterns need term scorers.
+    if (_enable_scoring) {
+        return make_nullable_scorer(_scored_terms(context, binding_key),
+                                    logical_field_or_fallback(context, binding_key, _field),
+                                    context.null_resolver);
+    }
    auto scorer = std::make_shared<ConstScoreScorer<BitSetScorerPtr>>(
            std::make_shared<BitSetScorer>(_rows(context, binding_key, nullptr)));
    return make_nullable_scorer(scorer, logical_field_or_fallback(context, binding_key, _field),
                                context.null_resolver);
 }

+ScorerPtr ExpandWeight::_scored_terms(const QueryExecutionContext& context,
+                                      const std::string& binding_key) {
+    index_query::TermPattern pattern;
+    THROW_IF_ERROR(index_query::TermPattern::create(_kind, _pattern, &pattern));
+    auto source = lookup_source(_field, context, binding_key);
+    // An unbound field has no postings and contributes no matches.
+    if (source == nullptr) {
+        return std::make_shared<EmptyScorer>();
+    }
+    std::vector<std::string> terms;
+    THROW_IF_ERROR(source->expand_terms(
+            pattern, index_query::expansion_limit(_kind, index_query::max_expansions(*_context)),
+            &terms));
+    // A valid pattern may match no dictionary terms.
+    if (terms.empty()) {
+        return std::make_shared<EmptyScorer>();
+    }
+    // Expansion terms are absent from collection statistics; use the bound source's statistics.
+    index_query::IndexScoringStats stats;
+    THROW_IF_ERROR(source->scoring_stats(&stats));
+    DORIS_CHECK_GT(stats.doc_count, 0);
+    // Docs-only SNII still reports its physical token count. Only legacy sources without
+    // that metadata report zero, so use unit average length only for those sources.
+    const uint64_t token_count = stats.token_count != 0 ? stats.token_count : stats.doc_count;
+    const float avgdl = static_cast<float>(token_count) / stats.doc_count;
+    DORIS_CHECK_GT(avgdl, 0.0F);
+    std::vector<ScorerPtr> scorers;
+    scorers.reserve(terms.size());
+    auto logical_field = logical_field_or_fallback(context, binding_key, _field);
+    for (const auto& term : terms) {
+        uint64_t doc_freq = 0;
+        THROW_IF_ERROR(source->doc_freq(term, &doc_freq));
+        // The immutable source just enumerated this term and must have its postings.
+        DCHECK_GT(doc_freq, 0);
+        const auto idf = static_cast<float>(std::log1p(
+                (static_cast<double>(stats.doc_count) - doc_freq + 0.5) / (doc_freq + 0.5)));
+        auto similarity = std::make_shared<BM25Similarity>(idf, avgdl);
+        auto postings = open_postings(*source, term, /*positions=*/false, /*enable_scoring=*/true,
+                                      similarity);
+        DCHECK(postings != nullptr);
+        scorers.emplace_back(std::make_shared<TermScorer>(std::move(postings),
+                                                          std::move(similarity), logical_field));
+    }
+    // One expanded term needs no union; multiple terms add their per-document BM25 scores.
+    if (scorers.size() == 1) {
+        return std::move(scorers.front());
+    }
+    return make_buffered_union(scorers, std::make_shared<SumCombiner>());
+}
+
 index_query::TruthSet ExpandWeight::listed_rows(const QueryExecutionContext& context,
                                                const std::string& binding_key,
                                                const roaring::Roaring* candidates) {
@@ -29,23 +29,25 @@ Status collect_expanded_rows(index_query::IndexSource& source, index_query::Term
                             int32_t max_expansions, const roaring::Roaring* candidates,
                             roaring::Roaring* rows);

-// Every document that holds a term the pattern expands to, with a constant score.
+// Every document that holds an expanded term; scored regexp and wildcard clauses sum BM25.
 class ExpandWeight : public Weight {
 public:
    ExpandWeight(IndexQueryContextPtr context, std::wstring field,
-                 index_query::TermPatternKind kind, std::string pattern);
+                 index_query::TermPatternKind kind, std::string pattern,
+                 bool enable_scoring = false);
    ~ExpandWeight() override = default;

    ScorerPtr scorer(const QueryExecutionContext& context, const std::string& binding_key) override;

    bool lists_rows(const QueryExecutionContext&, const std::string&) const override {
-        return true;
+        return !_enable_scoring;
    }
    index_query::TruthSet listed_rows(const QueryExecutionContext& context,
                                      const std::string& binding_key,
                                      const roaring::Roaring* candidates) override;

 private:
+    ScorerPtr _scored_terms(const QueryExecutionContext& context, const std::string& binding_key);
    std::shared_ptr<roaring::Roaring> _rows(const QueryExecutionContext& context,
                                            const std::string& binding_key,
                                            const roaring::Roaring* candidates);
@@ -53,6 +55,7 @@ private:
    std::wstring _field;
    index_query::TermPatternKind _kind;
    std::string _pattern;
+    bool _enable_scoring;
 };

 } // namespace doris::segment_v2::inverted_index::query_v2
@@ -21,6 +21,7 @@
 #include <gtest/gtest.h>

 #include <chrono>
+#include <cmath>
 #include <initializer_list>
 #include <map>
 #include <memory>
@@ -1854,6 +1855,8 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
            std::make_shared<RecordingNativeInvertedIndexReader>(&decoy_meta, decoy_file_reader);
    auto selected_reader = std::make_shared<RecordingNativeInvertedIndexReader>(
            &selected_meta, selected_file_reader);
+    selected_reader->fake_source->set_doc_count(4);
+    selected_reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 3});
    selected_reader->fake_source->add("alpha", {0, 2});
    selected_reader->fake_source->add("gamma", {1});
    decoy_reader->fake_source->add("alpha", {1});
@@ -1902,7 +1905,8 @@ TEST_F(FunctionSearchTest, TestBuildLeafQueryExecutesSelectedSniiWildcardReader)
                                 binding_key);
    ASSERT_NE(nullptr, scorer);
    EXPECT_EQ(0U, scorer->doc());
-    EXPECT_FLOAT_EQ(1.0F, scorer->score());
+    // The source has three tokens over four rows, even without stored norms.
+    EXPECT_NEAR(0.88F * std::log(2.0F), scorer->score(), 1e-6F);
    expect_bitmap_eq(collect_docs(scorer), {0, 2});
    // The pattern was lowercased by the selected analyzer and enumerated on the selected
    // index only; a leading wildcard has no prefix to enumerate from.
@@ -2502,7 +2506,8 @@ static void search_snii_pattern(const std::map<std::string, std::string>& proper
    ASSERT_TRUE(status.ok()) << status.to_string();
    EXPECT_EQ(0, reader->query_calls) << value;
    // The scorer enumerates the stem on the source; the reader itself is never asked.
-    auto weight = query->weight(true);
+    // This helper checks normalization and routing, without requesting scoring statistics.
+    auto weight = query->weight(false);
    ASSERT_NE(nullptr, weight);
    auto scorer = weight->scorer(build_variant_search_query_execution_context(4, resolver, nullptr),
                                 binding_key);
@@ -2733,13 +2738,12 @@ TEST_F(FunctionSearchTest, TestSniiScoredLeafRanksByTheEngineBm25) {
    }
 }

-// A leaf for which the reader publishes no per-document score must keep the constant that the
-// CLucene path also gives its own constant-score leaves, so that routing scored clauses through a
-// new scorer does not quietly re-rank the unscored ones. The fake reader is what withholds the
-// scores here, so this pins the "nothing published -> BitSetQuery -> 1.0" behaviour; it does not
-// pin which query types the production is_need_similarity_score gate rejects.
-TEST_F(FunctionSearchTest, TestSniiNativeLeafWithoutPublishedScoresKeepsConstantScore) {
+// Scored wildcard leaves read frequencies and statistics through the unified source,
+// even when the old reader query API publishes no per-document scores.
+TEST_F(FunctionSearchTest, TestSniiNativeWildcardUsesSourceScoringStatistics) {
    SniiScoringFixture fixture(43, 4);
+    fixture.reader->fake_source->set_doc_count(4);
+    fixture.reader->fake_source->set_scoring_stats({.doc_count = 4, .token_count = 2});
    fixture.reader->fake_source->add("alpha", {0, 2});

    inverted_index::query_v2::QueryPtr query;
@@ -2756,7 +2760,8 @@ TEST_F(FunctionSearchTest, TestSniiNativeLeafWithoutPublishedScoresKeepsConstant
    auto scorer = weight->scorer(exec_ctx, binding_key);
    ASSERT_NE(nullptr, scorer);
    EXPECT_EQ(0U, scorer->doc());
-    EXPECT_FLOAT_EQ(1.0F, scorer->score());
+    // avgdl = 2/4 and implicit length = 1: BM25 is idf * 2.2 / 3.1.
+    EXPECT_NEAR((2.2F / 3.1F) * std::log(2.0F), scorer->score(), 1e-6F);
    expect_bitmap_eq(collect_docs(scorer), {0, 2});
 }

@@ -18,6 +18,7 @@
 #include <CLucene.h>
 #include <gtest/gtest.h>

+#include <cmath>
 #include <memory>
 #include <roaring/roaring.hh>
 #include <string>
@@ -32,6 +33,8 @@
 #include "storage/index/inverted/inverted_index_reader.h"
 #include "storage/index/inverted/query/engine/expand_query/expand_query.h"
 #include "storage/index/inverted/query/logical/node.h"
+#include "storage/index/inverted/query/scoring/bm25_similarity.h"
+#include "storage/index/inverted/query/scoring/collection_similarity.h"
 #include "storage/index/inverted/query/spi/fake_index_source.h"
 #include "storage/olap_common.h"

@@ -492,6 +495,8 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
        auto source = std::make_shared<index_query::testing::FakeIndexSource>();
        source->batches = batches;
        source->set_doc_count(10);
+        // Scored regexp and wildcard leaves need statistics from their bound source.
+        source->set_scoring_stats({.doc_count = 10, .token_count = 10});
        source->add("apple", {0, 4});
        source->add("application", {1});
        source->add("apply", {2, 4});
@@ -509,12 +514,30 @@ TEST(ExpansionLeafTest, KeepsPatternLimitsAndLeavesRootCandidatesToTheCaller) {
                expected.add(2);
            }
            for (const bool scoring : {false, true}) {
+                context->collection_similarity =
+                        scoring ? std::make_shared<CollectionSimilarity>() : nullptr;
                auto result = std::make_shared<roaring::Roaring>();
                result->add(9);
                const auto status = run_leaf(context, L"content", leaf, &candidates, scoring,
                                             source, source->doc_count(), result);
                ASSERT_TRUE(status.ok()) << status;
                EXPECT_EQ(*result, expected) << pattern << " scoring=" << scoring;
+                if (scoring) {
+                    const auto& scores = context->collection_similarity->_bm25_scores;
+                    EXPECT_EQ(scores.size(), expected.cardinality() - 1);
+                    const bool bm25 = kind == logical::ExpandKind::kWildcard ||
+                                      kind == logical::ExpandKind::kRegexp;
+                    for (const auto& [doc, score] : scores) {
+                        // Row 9 predates this leaf and must not acquire a score from it.
+                        EXPECT_NE(doc, 9U);
+                        EXPECT_TRUE(expected.contains(doc));
+                        const float df = doc == 1 ? 1.0F : 2.0F;
+                        const float idf = std::log1p((10.0F - df + 0.5F) / (df + 0.5F));
+                        const float expected_score =
+                                bm25 ? BM25Similarity(idf, 1.0F).score(1.0F, 1) : 1.0F;
+                        EXPECT_FLOAT_EQ(score, expected_score) << pattern << " doc=" << doc;
+                    }
+                }
            }
        }
    }
@@ -50,13 +50,15 @@ public:
        st = io::global_local_filesystem()->create_directory(kTestDir);
        ASSERT_TRUE(st.ok()) << st;
        std::string field_name = "content";
-        create_test_index(
-                field_name, kTestDir,
-                {"apple123",   "apple456",     "banana789",     "test123abc", "pattern456",
-                 "regex999",   "match123",     "search456",     "prefix123",  "suffix789",
-                 "apple_test", "banana_data",  "test_pattern",  "data_regex", "apple_banana",
-                 "test_match", "pattern_test", "prefix_suffix", "abc123xyz",  "def456ghi"},
-                "standard");
+        create_test_index(field_name, kTestDir,
+                          {"apple123",     "apple456",     "banana789",
+                           "test123abc",   "pattern456",   "regex999",
+                           "match123",     "search456",    "prefix123",
+                           "suffix789",    "apple_test",   "banana_data",
+                           "test_pattern", "data_regex",   "apple_banana",
+                           "test_match",   "pattern_test", "prefix_suffix",
+                           "abc123xyz",    "def456ghi",    "apple789 apple789 apple456"},
+                          "standard");
    }

    void TearDown() override {
@@ -180,6 +182,36 @@ TEST_F(RegexpQueryV2Test, test_regexp_query_with_scoring) {
    ASSERT_NE(regexp_weight, nullptr);
 }

+TEST_F(RegexpQueryV2Test, test_regexp_query_scores_matching_terms) {
+    auto context = std::make_shared<IndexQueryContext>();
+    context->collection_statistics = std::make_shared<CollectionStatistics>();
+    context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+    auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+    auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
+    ASSERT_NE(reader_holder, nullptr);
+
+    std::wstring field = StringHelper::to_wstring("content");
+    auto query = std::make_shared<query_v2::ExpandQuery>(
+            context, field, index_query::TermPatternKind::kRegexp, anchored("apple.*"));
+    auto weight = query->weight(true);
+
+    query_v2::QueryExecutionContext exec_ctx;
+    exec_ctx.segment_num_rows = reader_holder->maxDoc();
+    exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)};
+    exec_ctx.field_sources.emplace(field, exec_ctx.sources.front());
+
+    auto scorer = weight->scorer(exec_ctx);
+    ASSERT_EQ(scorer->seek(0), 0);
+    float single_term_score = scorer->score();
+    EXPECT_NE(single_term_score, 1.0F);
+    uint32_t last_doc = reader_holder->maxDoc() - 1;
+    ASSERT_EQ(scorer->seek(last_doc), last_doc);
+    EXPECT_GT(scorer->score(), single_term_score);
+
+    _CLDECDELETE(dir);
+}
+
 // Test regexp query execution
 TEST_F(RegexpQueryV2Test, test_regexp_query_execution) {
    auto context = std::make_shared<IndexQueryContext>();
@@ -628,7 +660,7 @@ TEST_F(RegexpQueryV2Test, test_whole_term_match_any_term) {
        doc = scorer->advance();
    }

-    EXPECT_EQ(result.cardinality(), 20);
+    EXPECT_EQ(result.cardinality(), 21);

    _CLDECDELETE(dir);
 }
@@ -59,14 +59,20 @@ public:

 private:
    void create_test_index(const std::string& field_name, const std::string& dir) {
-        std::vector<std::string> test_data = {
-                "apple",     "application", "apply",       "apricot", "banana",     "band",
-                "bandana",   "can",         "candy",       "cat",     "catalog",    "dog",
-                "document",  "door",        "hello",       "help",    "helpful",    "test",
-                "testing",   "testcase",    "wildcard",    "wild",    "wilderness", "card",
-                "cardboard", "abc",         "abcd",        "abcde",   "prefix123",  "123suffix",
-                "pre123",    "123suf",      "both123both", "star",    "start",      "started",
-                "starter",   "question",    "quest",       "query",   "queries"};
+        std::vector<std::string> test_data = {"apple",     "application", "apply",
+                                              "apricot",   "banana",      "band",
+                                              "bandana",   "can",         "candy",
+                                              "cat",       "catalog",     "dog",
+                                              "document",  "door",        "hello",
+                                              "help",      "helpful",     "test",
+                                              "testing",   "testcase",    "wildcard",
+                                              "wild",      "wilderness",  "card",
+                                              "cardboard", "abc",         "abcd",
+                                              "abcde",     "prefix123",   "123suffix",
+                                              "pre123",    "123suf",      "both123both",
+                                              "star",      "start",       "started",
+                                              "starter",   "question",    "quest",
+                                              "query",     "queries",     "test test testing"};

        CustomAnalyzerConfig::Builder builder;
        builder.with_tokenizer_config("standard", {});
@@ -159,6 +165,36 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_with_scoring) {
    ASSERT_NE(wildcard_weight, nullptr);
 }

+TEST_F(WildcardQueryV2Test, test_wildcard_query_scores_matching_terms) {
+    auto context = std::make_shared<IndexQueryContext>();
+    context->collection_statistics = std::make_shared<CollectionStatistics>();
+    context->collection_similarity = std::make_shared<CollectionSimilarity>();
+
+    auto* dir = FSDirectory::getDirectory(kTestDir.c_str());
+    auto reader_holder = make_shared_reader(lucene::index::IndexReader::open(dir, true));
+    ASSERT_NE(reader_holder, nullptr);
+
+    std::wstring field = StringHelper::to_wstring("field");
+    auto query = std::make_shared<query_v2::ExpandQuery>(
+            context, field, index_query::TermPatternKind::kWildcard, "test*");
+    auto weight = query->weight(true);
+
+    query_v2::QueryExecutionContext exec_ctx;
+    exec_ctx.segment_num_rows = reader_holder->maxDoc();
+    exec_ctx.sources = {clucene_index_source(reader_holder, field, nullptr)};
+    exec_ctx.field_sources.emplace(field, exec_ctx.sources.front());
+
+    auto scorer = weight->scorer(exec_ctx);
+    ASSERT_EQ(scorer->seek(17), 17);
+    float single_term_score = scorer->score();
+    EXPECT_NE(single_term_score, 1.0F);
+    uint32_t last_doc = reader_holder->maxDoc() - 1;
+    ASSERT_EQ(scorer->seek(last_doc), last_doc);
+    EXPECT_GT(scorer->score(), single_term_score);
+
+    _CLDECDELETE(dir);
+}
+
 // Test wildcard query with asterisk prefix pattern
 TEST_F(WildcardQueryV2Test, test_wildcard_query_prefix_pattern) {
    auto context = std::make_shared<IndexQueryContext>();
@@ -540,4 +576,4 @@ TEST_F(WildcardQueryV2Test, test_wildcard_query_move_semantics) {
    ASSERT_NE(weight2, nullptr);
 }

-} // namespace doris::segment_v2
+} // namespace doris::segment_v2
@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <map>
 #include <memory>
+#include <optional>
 #include <set>
 #include <span>
 #include <string>
@@ -214,6 +215,17 @@ public:
    void remove(uint32_t doc) { _deleted.insert(doc); }
    void set_doc_count(uint32_t doc_count) { _doc_count = doc_count; }

+    void set_scoring_stats(IndexScoringStats stats) { _scoring_stats = stats; }
+
+    Status scoring_stats(IndexScoringStats* out) const override {
+        // Most fixtures do not score expansions and deliberately provide no source statistics.
+        if (!_scoring_stats.has_value()) {
+            return IndexSource::scoring_stats(out);
+        }
+        *out = *_scoring_stats;
+        return Status::OK();
+    }
+
    uint32_t doc_count() const override { return _doc_count; }
    bool batches_reads() const override { return batches; }

@@ -331,6 +343,7 @@ private:
    std::map<std::string, std::vector<Posting>> _terms;
    std::set<uint32_t> _deleted;
    uint32_t _doc_count = 0;
+    std::optional<IndexScoringStats> _scoring_stats;
 };

 } // namespace doris::index_query::testing