From 7aa1e9b7558611eba4d7b2f7c8a7a62805fffbb5 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Wed, 28 Oct 2026 17:13:47 +0800 Subject: [PATCH] [fix](search) Support keyword terms and quoted parentheses in SEARCH DSL (#69460) --- .../doris/nereids/search/SearchLexer.g4 | 16 ++- .../doris/nereids/search/SearchParser.g4 | 10 +- .../functions/scalar/SearchDslParser.java | 32 +++-- .../functions/scalar/SearchDslParserTest.java | 94 +++++++++++--- ...st_search_dsl_keyword_quoted_syntax.groovy | 116 ++++++++++++++++++ .../suites/search/test_search_escape.groovy | 41 +++---- 6 files changed, 253 insertions(+), 56 deletions(-) create mode 100644 regression-test/suites/search/test_search_dsl_keyword_quoted_syntax.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchLexer.g4 index 3c601ae7810..3bad12cec87 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchLexer.g4 @@ -42,9 +42,12 @@ fragment QUOTED_CHAR // ============== Default lexer rules ============== -AND : 'AND' ; -OR : 'OR' ; -NOT : 'NOT' | '!' ; +// Boolean operators are case-insensitive, matching the query string syntax used by +// Lucene/Elasticsearch: `a:x or b:y` is the same as `a:x OR b:y`. The parser also accepts +// these tokens in value position, so `field:and` keeps searching for the term "and". +AND : [Aa][Nn][Dd] ; +OR : [Oo][Rr] ; +NOT : [Nn][Oo][Tt] | '!' ; LPAREN : '(' ; RPAREN : ')' ; @@ -94,6 +97,13 @@ LIST_WS : [ \t\r\n\u3000]+ -> skip ; mode STRING_MODE; +// A quoted section is tokenized as one unit so that parentheses inside the quotes do not +// terminate the clause early, e.g. ANY("All Quiet on the Western Front (2022 film)"). +// It is a separate rule (instead of an alternative inside STRING_CONTENT) because a negated +// set inside a closure lets the lexer run past the closing parenthesis when another ANY/ALL +// appears later in the same DSL. +STRING_QUOTED : '"' (ESCAPED_CHAR | ~["\\])* '"' ; + STRING_CONTENT : (ESCAPED_CHAR | NON_ESCAPED)+ ; STRING_RPAREN : ')' -> popMode ; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchParser.g4 index ee5e2bf644f..5df6bbcd567 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/search/SearchParser.g4 @@ -38,7 +38,11 @@ nestedQuery : NESTED_LPAREN NESTED_PATH NESTED_COMMA clause RPAREN ; fieldGroupQuery : fieldPath COLON LPAREN clause RPAREN ; // Support for variant subcolumn paths (e.g., field.subcolumn, field.sub1.sub2) -fieldQuery : fieldPath COLON searchValue ; +fieldQuery : fieldPath COLON fieldValue ; +// Boolean keywords are also valid terms in value position, e.g. field:and. Keeping this +// separate from searchValue leaves the bare query rule free of operator tokens, so lowercase +// `or` between two clauses is still parsed as the OR operator. +fieldValue : searchValue | AND | OR | NOT ; // Bare query without field prefix - uses default_field bareQuery : searchValue ; @@ -65,5 +69,5 @@ rangeValue rangeEndpoint : RANGE_NUMBER | RANGE_STAR ; listValue : IN_LPAREN LIST_TERM* LIST_RPAREN ; -anyAllValue : (ANY_LPAREN | ALL_LPAREN) STRING_CONTENT? STRING_RPAREN ; -exactValue : EXACT_LPAREN STRING_CONTENT? STRING_RPAREN ; +anyAllValue : (ANY_LPAREN | ALL_LPAREN) (STRING_CONTENT | STRING_QUOTED)* STRING_RPAREN ; +exactValue : EXACT_LPAREN (STRING_CONTENT | STRING_QUOTED)* STRING_RPAREN ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java index d8ebc7f1fb5..6af680b880d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParser.java @@ -930,10 +930,16 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) currentFieldName = fieldPath; try { - if (ctx.searchValue() == null) { + if (ctx.fieldValue() == null) { throw new RuntimeException("Invalid field query: missing search value"); } - QsNode result = visit(ctx.searchValue()); + QsNode result; + if (ctx.fieldValue().searchValue() != null) { + result = visit(ctx.fieldValue().searchValue()); + } else { + // AND/OR/NOT in value position are ordinary terms: `field:and`. + result = createTermNode(fieldPath, ctx.fieldValue().getText()); + } if (result == null) { throw new RuntimeException("Invalid search value"); } @@ -1084,7 +1090,9 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) private QsNode createAnyAllNode(String fieldName, String anyAllText) { // Extract content between parentheses String innerContent = extractParenthesesContent(anyAllText); - String sanitizedContent = stripOuterQuotes(innerContent); + // Strip DSL-level quotes first, then resolve escapes such as \" or \), so that a + // search for a value containing quote characters stays expressible. + String sanitizedContent = unescapeTermValue(stripOuterQuotes(innerContent)); if (anyAllText.toUpperCase().startsWith("ANY(") || anyAllText.toLowerCase().startsWith("any(")) { return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent); @@ -1101,9 +1109,10 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) } private QsNode createExactNode(String fieldName, String exactText) { - // Extract content between parentheses + // Extract content between parentheses and resolve escapes (for example ABC\) is the + // value "ABC)"), otherwise the backslash is looked up as part of the indexed term. String innerContent = extractParenthesesContent(exactText); - return new QsNode(QsClauseType.EXACT, fieldName, innerContent); + return new QsNode(QsClauseType.EXACT, fieldName, unescapeTermValue(innerContent)); } private String extractParenthesesContent(String text) { @@ -2537,7 +2546,14 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) currentFieldName = fieldPath; try { - QsNode result = visit(ctx.searchValue()); + QsNode result; + if (ctx.fieldValue().searchValue() != null) { + result = visit(ctx.fieldValue().searchValue()); + } else { + // AND/OR/NOT in value position are ordinary terms: `field:and`. + result = new QsNode(QsClauseType.TERM, fieldPath, + unescapeTermValue(ctx.fieldValue().getText())); + } // Mark as explicit field - user wrote "field:term" syntax result.setExplicitField(true); return result; @@ -2646,7 +2662,7 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) if (ctx.anyAllValue() != null) { String text = ctx.anyAllValue().getText(); String innerContent = extractParenthesesContent(text); - String sanitizedContent = stripOuterQuotes(innerContent); + String sanitizedContent = unescapeTermValue(stripOuterQuotes(innerContent)); if (text.toUpperCase().startsWith("ANY(")) { return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent); } @@ -2654,7 +2670,7 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting) } if (ctx.exactValue() != null) { String innerContent = extractParenthesesContent(ctx.exactValue().getText()); - return new QsNode(QsClauseType.EXACT, fieldName, innerContent); + return new QsNode(QsClauseType.EXACT, fieldName, unescapeTermValue(innerContent)); } return new QsNode(QsClauseType.TERM, fieldName, unescapeTermValue(ctx.getText())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java index cb8b74aef58..2a4f3f5ac2b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SearchDslParserTest.java @@ -157,6 +157,68 @@ public class SearchDslParserTest { Assertions.assertEquals("Mandy Patinkin", plan.getRoot().getValue()); } + @Test + public void testBooleanKeywordAsFieldValue() { + // DORIS-23194: AND/OR/NOT are case-insensitive keywords, but in value position they are + // ordinary terms and must not be rejected by the parser. + String[][] cases = { + {"ch:AND", "AND"}, {"ch:and", "and"}, {"ch:And", "And"}, + {"ch:OR", "OR"}, {"ch:or", "or"}, + {"ch:NOT", "NOT"}, {"ch:not", "not"}}; + for (String[] testCase : cases) { + QsPlan plan = SearchDslParser.parseDsl(testCase[0]); + Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType(), testCase[0]); + Assertions.assertEquals("ch", plan.getRoot().getField(), testCase[0]); + Assertions.assertEquals(testCase[1], plan.getRoot().getValue(), testCase[0]); + } + } + + @Test + public void testLowercaseBooleanOperators() { + // DORIS-23236: lowercase boolean operators must behave like their uppercase form. + QsPlan orPlan = SearchDslParser.parseDsl("ch:foo or ch:bar", "{\"mode\":\"standard\"}"); + Assertions.assertEquals(QsClauseType.OR, orPlan.getRoot().getType()); + Assertions.assertEquals(2, orPlan.getRoot().getChildren().size()); + + QsPlan andPlan = SearchDslParser.parseDsl("ch:foo and ch:bar", "{\"mode\":\"standard\"}"); + Assertions.assertEquals(QsClauseType.AND, andPlan.getRoot().getType()); + Assertions.assertEquals(2, andPlan.getRoot().getChildren().size()); + + QsPlan notPlan = SearchDslParser.parseDsl("ch:foo and not ch:bar", "{\"mode\":\"standard\"}"); + Assertions.assertEquals(QsClauseType.AND, notPlan.getRoot().getType()); + Assertions.assertEquals(QsClauseType.NOT, notPlan.getRoot().getChildren().get(1).getType()); + } + + @Test + public void testQuotedParenthesesInAnyAll() { + // DORIS-23236: parentheses inside a quoted value must not terminate ANY/ALL early. + QsPlan anyPlan = SearchDslParser.parseDsl( + "ch:ANY(\"All Quiet on the Western Front (2022 film)\")"); + Assertions.assertEquals(QsClauseType.ANY, anyPlan.getRoot().getType()); + Assertions.assertEquals("All Quiet on the Western Front (2022 film)", + anyPlan.getRoot().getValue()); + + QsPlan bothPlan = SearchDslParser.parseDsl( + "ch:ANY(\"All Quiet on the Western Front (2022 film)\") " + + "or ch:ALL(\"F. Murray Abraham\")", "{\"mode\":\"standard\"}"); + Assertions.assertEquals(QsClauseType.OR, bothPlan.getRoot().getType()); + Assertions.assertEquals(2, bothPlan.getRoot().getChildren().size()); + Assertions.assertEquals(QsClauseType.ANY, bothPlan.getRoot().getChildren().get(0).getType()); + Assertions.assertEquals(QsClauseType.ALL, bothPlan.getRoot().getChildren().get(1).getType()); + } + + @Test + public void testEscapedCharactersInExactAndAny() { + // DORIS-23239: escapes are resolved before the term is looked up. + QsPlan exactPlan = SearchDslParser.parseDsl("ch:exact(ABC\\))"); + Assertions.assertEquals(QsClauseType.EXACT, exactPlan.getRoot().getType()); + Assertions.assertEquals("ABC)", exactPlan.getRoot().getValue()); + + QsPlan anyPlan = SearchDslParser.parseDsl("ch:any(\\\"ABC\\\")"); + Assertions.assertEquals(QsClauseType.ANY, anyPlan.getRoot().getType()); + Assertions.assertEquals("\"ABC\"", anyPlan.getRoot().getValue()); + } + @Test public void testAndQuery() { String dsl = "title:hello AND content:world"; @@ -1181,13 +1243,12 @@ public class SearchDslParserTest { @Test public void testLowercaseAndOperator() { - // Lowercase 'and' is NOT an operator in ANTLR grammar (case-sensitive). - // With bareQuery rule, it's parsed as a bare term without field. - // Without default_field, bare term throws exception. + // Boolean operators are case-insensitive, matching Lucene/ES query string syntax. String dsl = "field:a and field:b"; - Assertions.assertThrows(RuntimeException.class, () -> { - SearchDslParser.parseDsl(dsl); - }); + QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}"); + + Assertions.assertEquals(QsClauseType.AND, plan.getRoot().getType()); + Assertions.assertEquals(2, plan.getRoot().getChildren().size()); } @Test @@ -1203,13 +1264,12 @@ public class SearchDslParserTest { @Test public void testLowercaseOrOperator() { - // Lowercase 'or' is NOT an operator in ANTLR grammar (case-sensitive). - // With bareQuery rule, it's parsed as a bare term without field. - // Without default_field, bare term throws exception. + // Boolean operators are case-insensitive, matching Lucene/ES query string syntax. String dsl = "field:a or field:b"; - Assertions.assertThrows(RuntimeException.class, () -> { - SearchDslParser.parseDsl(dsl); - }); + QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}"); + + Assertions.assertEquals(QsClauseType.OR, plan.getRoot().getType()); + Assertions.assertEquals(2, plan.getRoot().getChildren().size()); } @Test @@ -1224,13 +1284,11 @@ public class SearchDslParserTest { @Test public void testLowercaseNotOperator() { - // Lowercase 'not' is NOT an operator in ANTLR grammar (case-sensitive). - // With bareQuery rule, it's parsed as a bare term without field. - // Without default_field, bare term throws exception. + // Boolean operators are case-insensitive, matching Lucene/ES query string syntax. String dsl = "not field:spam"; - Assertions.assertThrows(RuntimeException.class, () -> { - SearchDslParser.parseDsl(dsl); - }); + QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}"); + + Assertions.assertEquals(QsClauseType.NOT, plan.getRoot().getType()); } @Test diff --git a/regression-test/suites/search/test_search_dsl_keyword_quoted_syntax.groovy b/regression-test/suites/search/test_search_dsl_keyword_quoted_syntax.groovy new file mode 100644 index 00000000000..ca37b636d29 --- /dev/null +++ b/regression-test/suites/search/test_search_dsl_keyword_quoted_syntax.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_search_dsl_keyword_quoted_syntax", "p0") { + // SEARCH DSL parses boolean keywords case-insensitively. In value position they stay + // ordinary terms, so `field:and` keeps searching for the term "and" (DORIS-23194). + def keywordTable = "test_search_dsl_keyword_values" + sql "DROP TABLE IF EXISTS ${keywordTable}" + sql """ + CREATE TABLE ${keywordTable} ( + id INT, + ch VARCHAR(64), + INDEX idx_ch (ch) USING INVERTED PROPERTIES("lower_case" = "true") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """INSERT INTO ${keywordTable} VALUES + (1, 'and'), (2, 'or'), (3, 'not'), (4, 'AND'), (5, 'hello')""" + sql "sync" + + // Every casing parses and resolves to the same term as the MATCH_ANY control; the keyword + // index compares terms case-sensitively, so the expected ids differ per casing. + for (String keyword : ["AND", "and", "And", "OR", "or", "NOT", "not"]) { + def searchRows = sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:${keyword}') ORDER BY id") + .collect { it[0] as int } + def matchRows = sql("SELECT id FROM ${keywordTable} WHERE ch MATCH_ANY '${keyword}' ORDER BY id") + .collect { it[0] as int } + assertEquals(matchRows, searchRows, "SEARCH('ch:${keyword}') must match MATCH_ANY") + } + assertEquals([4], sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:AND') ORDER BY id") + .collect { it[0] as int }) + assertEquals([1], sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:and') ORDER BY id") + .collect { it[0] as int }) + // Lowercase OR still combines two clauses instead of becoming a term. + assertEquals([1, 2], sql(""" + SELECT id FROM ${keywordTable} WHERE SEARCH('ch:and or ch:or') ORDER BY id""") + .collect { it[0] as int }) + + // Lowercase operators combine clauses instead of being parsed as a bare term + // (DORIS-23236). Parentheses inside a quoted value must not terminate ANY/ALL early. + def phraseTable = "test_search_dsl_quoted_paren" + sql "DROP TABLE IF EXISTS ${phraseTable}" + sql """ + CREATE TABLE ${phraseTable} ( + id INT, + ch VARCHAR(1024), + INDEX idx_ch (ch) USING INVERTED PROPERTIES( + "lower_case" = "true", "parser" = "unicode", "support_phrase" = "true") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """INSERT INTO ${phraseTable} VALUES + (1, 'All Quiet on the Western Front (2022 film)'), + (2, 'F. Murray Abraham'), + (3, 'unrelated text')""" + sql "sync" + + assertEquals([1], sql(""" + SELECT id FROM ${phraseTable} + WHERE SEARCH('ch:ANY("All Quiet on the Western Front (2022 film)")') + ORDER BY id""").collect { it[0] as int }) + assertEquals([2], sql(""" + SELECT id FROM ${phraseTable} + WHERE SEARCH('ch:ALL("F. Murray Abraham")') + ORDER BY id""").collect { it[0] as int }) + assertEquals([1, 2], sql(""" + SELECT id FROM ${phraseTable} + WHERE SEARCH('ch:ANY("All Quiet on the Western Front (2022 film)") or ch:ALL("F. Murray Abraham")') + ORDER BY id""").collect { it[0] as int }) + + // Escapes are resolved before the term is looked up, so quoted values and closing + // parentheses remain searchable (DORIS-23239). + def specialTable = "test_search_dsl_special_chars" + sql "DROP TABLE IF EXISTS ${specialTable}" + sql """ + CREATE TABLE ${specialTable} ( + id INT, + ch VARCHAR(1024), + INDEX idx_ch (ch) USING INVERTED PROPERTIES("lower_case" = "false") + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + sql """INSERT INTO ${specialTable} VALUES (1, '\"ABC\"'), (2, 'ABC)'), (3, 'ABC')""" + sql "sync" + + // The keyword index stores the value including its quote characters, so the term has to + // keep the escaped quotes: SEARCH('ch:exact(\"ABC\")'). + assertEquals([1], sql("SELECT id FROM ${specialTable} WHERE ch MATCH_ANY '\"ABC\"'") + .collect { it[0] as int }) + assertEquals([1], sql(""" + SELECT id FROM ${specialTable} WHERE SEARCH('ch:exact(\\"ABC\\")')""") + .collect { it[0] as int }) + assertEquals([2], sql(""" + SELECT id FROM ${specialTable} WHERE SEARCH('ch:exact(ABC\\\\))')""") + .collect { it[0] as int }) +} diff --git a/regression-test/suites/search/test_search_escape.groovy b/regression-test/suites/search/test_search_escape.groovy index 0a72a81bc2b..e4d5fea789d 100644 --- a/regression-test/suites/search/test_search_escape.groovy +++ b/regression-test/suites/search/test_search_escape.groovy @@ -146,31 +146,24 @@ suite("test_search_escape", "p0") { ORDER BY id """ - // ============ Test 9: Lowercase 'and' should cause parse error ============ - // Per requirement: Only uppercase AND/OR/NOT are operators - // Lowercase 'and' is treated as a bare term (no field), causing error - test { - sql """ - SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, content - FROM ${tableName} - WHERE search('content:first and content:fruit') - ORDER BY id - """ - exception "No field specified and no default_field configured" - } + // ============ Test 9: Lowercase 'and' is an operator ============ + // Boolean operators are case-insensitive, matching Lucene/ES query string syntax. + def lowercaseAndRows = sql """ + SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id + FROM ${tableName} + WHERE search('content:first and content:fruit') + ORDER BY id + """ + assertEquals([[7]], lowercaseAndRows) - // ============ Test 10: Lowercase 'or' should cause parse error ============ - // Per requirement: Only uppercase AND/OR/NOT are operators - // Lowercase 'or' is treated as a bare term (no field), causing error - test { - sql """ - SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, content - FROM ${tableName} - WHERE search('content:first or content:second') - ORDER BY id - """ - exception "No field specified and no default_field configured" - } + // ============ Test 10: Lowercase 'or' is an operator ============ + def lowercaseOrRows = sql """ + SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id + FROM ${tableName} + WHERE search('content:first or content:second') + ORDER BY id + """ + assertEquals([[1], [2], [7], [8]], lowercaseOrRows) // ============ Test 11: Exclamation NOT operator ============ qt_exclamation_not """