[fix](search) Support keyword terms and quoted parentheses in SEARCH DSL (#69460)

This commit is contained in:
lihangyu authored and Your Name committed 2026-10-28 09:30:31 +00:00
1 parent 9f63be85ce
commit 7aa1e9b755
6 files changed
+253 -56

No files matched your search

@@ -42,9 +42,12 @@ fragment QUOTED_CHAR
// ============== Default lexer rules ==============
AND : 'AND' ;
OR : 'OR' ;
NOT : 'NOT' | '!' ;
// Boolean operators are case-insensitive, matching the query string syntax used by
// Lucene/Elasticsearch: `a:x or b:y` is the same as `a:x OR b:y`. The parser also accepts
// these tokens in value position, so `field:and` keeps searching for the term "and".
AND : [Aa][Nn][Dd] ;
OR : [Oo][Rr] ;
NOT : [Nn][Oo][Tt] | '!' ;
LPAREN : '(' ;
RPAREN : ')' ;
@@ -94,6 +97,13 @@ LIST_WS : [ \t\r\n\u3000]+ -> skip ;
mode STRING_MODE;
// A quoted section is tokenized as one unit so that parentheses inside the quotes do not
// terminate the clause early, e.g. ANY("All Quiet on the Western Front (2022 film)").
// It is a separate rule (instead of an alternative inside STRING_CONTENT) because a negated
// set inside a closure lets the lexer run past the closing parenthesis when another ANY/ALL
// appears later in the same DSL.
STRING_QUOTED : '"' (ESCAPED_CHAR | ~["\\])* '"' ;
STRING_CONTENT : (ESCAPED_CHAR | NON_ESCAPED)+ ;
STRING_RPAREN : ')' -> popMode ;
@@ -38,7 +38,11 @@ nestedQuery : NESTED_LPAREN NESTED_PATH NESTED_COMMA clause RPAREN ;
fieldGroupQuery : fieldPath COLON LPAREN clause RPAREN ;
// Support for variant subcolumn paths (e.g., field.subcolumn, field.sub1.sub2)
fieldQuery : fieldPath COLON searchValue ;
fieldQuery : fieldPath COLON fieldValue ;
// Boolean keywords are also valid terms in value position, e.g. field:and. Keeping this
// separate from searchValue leaves the bare query rule free of operator tokens, so lowercase
// `or` between two clauses is still parsed as the OR operator.
fieldValue : searchValue | AND | OR | NOT ;
// Bare query without field prefix - uses default_field
bareQuery : searchValue ;
@@ -65,5 +69,5 @@ rangeValue
rangeEndpoint : RANGE_NUMBER | RANGE_STAR ;
listValue : IN_LPAREN LIST_TERM* LIST_RPAREN ;
anyAllValue : (ANY_LPAREN | ALL_LPAREN) STRING_CONTENT? STRING_RPAREN ;
exactValue : EXACT_LPAREN STRING_CONTENT? STRING_RPAREN ;
anyAllValue : (ANY_LPAREN | ALL_LPAREN) (STRING_CONTENT | STRING_QUOTED)* STRING_RPAREN ;
exactValue : EXACT_LPAREN (STRING_CONTENT | STRING_QUOTED)* STRING_RPAREN ;
@@ -930,10 +930,16 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
currentFieldName = fieldPath;
try {
if (ctx.searchValue() == null) {
if (ctx.fieldValue() == null) {
throw new RuntimeException("Invalid field query: missing search value");
}
QsNode result = visit(ctx.searchValue());
QsNode result;
if (ctx.fieldValue().searchValue() != null) {
result = visit(ctx.fieldValue().searchValue());
} else {
// AND/OR/NOT in value position are ordinary terms: `field:and`.
result = createTermNode(fieldPath, ctx.fieldValue().getText());
}
if (result == null) {
throw new RuntimeException("Invalid search value");
}
@@ -1084,7 +1090,9 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
private QsNode createAnyAllNode(String fieldName, String anyAllText) {
// Extract content between parentheses
String innerContent = extractParenthesesContent(anyAllText);
String sanitizedContent = stripOuterQuotes(innerContent);
// Strip DSL-level quotes first, then resolve escapes such as \" or \), so that a
// search for a value containing quote characters stays expressible.
String sanitizedContent = unescapeTermValue(stripOuterQuotes(innerContent));
if (anyAllText.toUpperCase().startsWith("ANY(") || anyAllText.toLowerCase().startsWith("any(")) {
return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent);
@@ -1101,9 +1109,10 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
}
private QsNode createExactNode(String fieldName, String exactText) {
// Extract content between parentheses
// Extract content between parentheses and resolve escapes (for example ABC\) is the
// value "ABC)"), otherwise the backslash is looked up as part of the indexed term.
String innerContent = extractParenthesesContent(exactText);
return new QsNode(QsClauseType.EXACT, fieldName, innerContent);
return new QsNode(QsClauseType.EXACT, fieldName, unescapeTermValue(innerContent));
}
private String extractParenthesesContent(String text) {
@@ -2537,7 +2546,14 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
currentFieldName = fieldPath;
try {
QsNode result = visit(ctx.searchValue());
QsNode result;
if (ctx.fieldValue().searchValue() != null) {
result = visit(ctx.fieldValue().searchValue());
} else {
// AND/OR/NOT in value position are ordinary terms: `field:and`.
result = new QsNode(QsClauseType.TERM, fieldPath,
unescapeTermValue(ctx.fieldValue().getText()));
}
// Mark as explicit field - user wrote "field:term" syntax
result.setExplicitField(true);
return result;
@@ -2646,7 +2662,7 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
if (ctx.anyAllValue() != null) {
String text = ctx.anyAllValue().getText();
String innerContent = extractParenthesesContent(text);
String sanitizedContent = stripOuterQuotes(innerContent);
String sanitizedContent = unescapeTermValue(stripOuterQuotes(innerContent));
if (text.toUpperCase().startsWith("ANY(")) {
return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent);
}
@@ -2654,7 +2670,7 @@ MATCH_ALL_DOCS, // Matches all documents (used for pure NOT query rewriting)
}
if (ctx.exactValue() != null) {
String innerContent = extractParenthesesContent(ctx.exactValue().getText());
return new QsNode(QsClauseType.EXACT, fieldName, innerContent);
return new QsNode(QsClauseType.EXACT, fieldName, unescapeTermValue(innerContent));
}
return new QsNode(QsClauseType.TERM, fieldName, unescapeTermValue(ctx.getText()));
@@ -157,6 +157,68 @@ public class SearchDslParserTest {
Assertions.assertEquals("Mandy Patinkin", plan.getRoot().getValue());
}
@Test
public void testBooleanKeywordAsFieldValue() {
// DORIS-23194: AND/OR/NOT are case-insensitive keywords, but in value position they are
// ordinary terms and must not be rejected by the parser.
String[][] cases = {
{"ch:AND", "AND"}, {"ch:and", "and"}, {"ch:And", "And"},
{"ch:OR", "OR"}, {"ch:or", "or"},
{"ch:NOT", "NOT"}, {"ch:not", "not"}};
for (String[] testCase : cases) {
QsPlan plan = SearchDslParser.parseDsl(testCase[0]);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType(), testCase[0]);
Assertions.assertEquals("ch", plan.getRoot().getField(), testCase[0]);
Assertions.assertEquals(testCase[1], plan.getRoot().getValue(), testCase[0]);
}
}
@Test
public void testLowercaseBooleanOperators() {
// DORIS-23236: lowercase boolean operators must behave like their uppercase form.
QsPlan orPlan = SearchDslParser.parseDsl("ch:foo or ch:bar", "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.OR, orPlan.getRoot().getType());
Assertions.assertEquals(2, orPlan.getRoot().getChildren().size());
QsPlan andPlan = SearchDslParser.parseDsl("ch:foo and ch:bar", "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.AND, andPlan.getRoot().getType());
Assertions.assertEquals(2, andPlan.getRoot().getChildren().size());
QsPlan notPlan = SearchDslParser.parseDsl("ch:foo and not ch:bar", "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.AND, notPlan.getRoot().getType());
Assertions.assertEquals(QsClauseType.NOT, notPlan.getRoot().getChildren().get(1).getType());
}
@Test
public void testQuotedParenthesesInAnyAll() {
// DORIS-23236: parentheses inside a quoted value must not terminate ANY/ALL early.
QsPlan anyPlan = SearchDslParser.parseDsl(
"ch:ANY(\"All Quiet on the Western Front (2022 film)\")");
Assertions.assertEquals(QsClauseType.ANY, anyPlan.getRoot().getType());
Assertions.assertEquals("All Quiet on the Western Front (2022 film)",
anyPlan.getRoot().getValue());
QsPlan bothPlan = SearchDslParser.parseDsl(
"ch:ANY(\"All Quiet on the Western Front (2022 film)\") "
+ "or ch:ALL(\"F. Murray Abraham\")", "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.OR, bothPlan.getRoot().getType());
Assertions.assertEquals(2, bothPlan.getRoot().getChildren().size());
Assertions.assertEquals(QsClauseType.ANY, bothPlan.getRoot().getChildren().get(0).getType());
Assertions.assertEquals(QsClauseType.ALL, bothPlan.getRoot().getChildren().get(1).getType());
}
@Test
public void testEscapedCharactersInExactAndAny() {
// DORIS-23239: escapes are resolved before the term is looked up.
QsPlan exactPlan = SearchDslParser.parseDsl("ch:exact(ABC\\))");
Assertions.assertEquals(QsClauseType.EXACT, exactPlan.getRoot().getType());
Assertions.assertEquals("ABC)", exactPlan.getRoot().getValue());
QsPlan anyPlan = SearchDslParser.parseDsl("ch:any(\\\"ABC\\\")");
Assertions.assertEquals(QsClauseType.ANY, anyPlan.getRoot().getType());
Assertions.assertEquals("\"ABC\"", anyPlan.getRoot().getValue());
}
@Test
public void testAndQuery() {
String dsl = "title:hello AND content:world";
@@ -1181,13 +1243,12 @@ public class SearchDslParserTest {
@Test
public void testLowercaseAndOperator() {
// Lowercase 'and' is NOT an operator in ANTLR grammar (case-sensitive).
// With bareQuery rule, it's parsed as a bare term without field.
// Without default_field, bare term throws exception.
// Boolean operators are case-insensitive, matching Lucene/ES query string syntax.
String dsl = "field:a and field:b";
Assertions.assertThrows(RuntimeException.class, () -> {
SearchDslParser.parseDsl(dsl);
});
QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.AND, plan.getRoot().getType());
Assertions.assertEquals(2, plan.getRoot().getChildren().size());
}
@Test
@@ -1203,13 +1264,12 @@ public class SearchDslParserTest {
@Test
public void testLowercaseOrOperator() {
// Lowercase 'or' is NOT an operator in ANTLR grammar (case-sensitive).
// With bareQuery rule, it's parsed as a bare term without field.
// Without default_field, bare term throws exception.
// Boolean operators are case-insensitive, matching Lucene/ES query string syntax.
String dsl = "field:a or field:b";
Assertions.assertThrows(RuntimeException.class, () -> {
SearchDslParser.parseDsl(dsl);
});
QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.OR, plan.getRoot().getType());
Assertions.assertEquals(2, plan.getRoot().getChildren().size());
}
@Test
@@ -1224,13 +1284,11 @@ public class SearchDslParserTest {
@Test
public void testLowercaseNotOperator() {
// Lowercase 'not' is NOT an operator in ANTLR grammar (case-sensitive).
// With bareQuery rule, it's parsed as a bare term without field.
// Without default_field, bare term throws exception.
// Boolean operators are case-insensitive, matching Lucene/ES query string syntax.
String dsl = "not field:spam";
Assertions.assertThrows(RuntimeException.class, () -> {
SearchDslParser.parseDsl(dsl);
});
QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"mode\":\"standard\"}");
Assertions.assertEquals(QsClauseType.NOT, plan.getRoot().getType());
}
@Test
@@ -0,0 +1,116 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_search_dsl_keyword_quoted_syntax", "p0") {
// SEARCH DSL parses boolean keywords case-insensitively. In value position they stay
// ordinary terms, so `field:and` keeps searching for the term "and" (DORIS-23194).
def keywordTable = "test_search_dsl_keyword_values"
sql "DROP TABLE IF EXISTS ${keywordTable}"
sql """
CREATE TABLE ${keywordTable} (
id INT,
ch VARCHAR(64),
INDEX idx_ch (ch) USING INVERTED PROPERTIES("lower_case" = "true")
) ENGINE=OLAP
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_allocation" = "tag.location.default: 1")
"""
sql """INSERT INTO ${keywordTable} VALUES
(1, 'and'), (2, 'or'), (3, 'not'), (4, 'AND'), (5, 'hello')"""
sql "sync"
// Every casing parses and resolves to the same term as the MATCH_ANY control; the keyword
// index compares terms case-sensitively, so the expected ids differ per casing.
for (String keyword : ["AND", "and", "And", "OR", "or", "NOT", "not"]) {
def searchRows = sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:${keyword}') ORDER BY id")
.collect { it[0] as int }
def matchRows = sql("SELECT id FROM ${keywordTable} WHERE ch MATCH_ANY '${keyword}' ORDER BY id")
.collect { it[0] as int }
assertEquals(matchRows, searchRows, "SEARCH('ch:${keyword}') must match MATCH_ANY")
}
assertEquals([4], sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:AND') ORDER BY id")
.collect { it[0] as int })
assertEquals([1], sql("SELECT id FROM ${keywordTable} WHERE SEARCH('ch:and') ORDER BY id")
.collect { it[0] as int })
// Lowercase OR still combines two clauses instead of becoming a term.
assertEquals([1, 2], sql("""
SELECT id FROM ${keywordTable} WHERE SEARCH('ch:and or ch:or') ORDER BY id""")
.collect { it[0] as int })
// Lowercase operators combine clauses instead of being parsed as a bare term
// (DORIS-23236). Parentheses inside a quoted value must not terminate ANY/ALL early.
def phraseTable = "test_search_dsl_quoted_paren"
sql "DROP TABLE IF EXISTS ${phraseTable}"
sql """
CREATE TABLE ${phraseTable} (
id INT,
ch VARCHAR(1024),
INDEX idx_ch (ch) USING INVERTED PROPERTIES(
"lower_case" = "true", "parser" = "unicode", "support_phrase" = "true")
) ENGINE=OLAP
DUPLICATE KEY(id)
DISTRIBUTED BY RANDOM BUCKETS AUTO
PROPERTIES ("replication_allocation" = "tag.location.default: 1")
"""
sql """INSERT INTO ${phraseTable} VALUES
(1, 'All Quiet on the Western Front (2022 film)'),
(2, 'F. Murray Abraham'),
(3, 'unrelated text')"""
sql "sync"
assertEquals([1], sql("""
SELECT id FROM ${phraseTable}
WHERE SEARCH('ch:ANY("All Quiet on the Western Front (2022 film)")')
ORDER BY id""").collect { it[0] as int })
assertEquals([2], sql("""
SELECT id FROM ${phraseTable}
WHERE SEARCH('ch:ALL("F. Murray Abraham")')
ORDER BY id""").collect { it[0] as int })
assertEquals([1, 2], sql("""
SELECT id FROM ${phraseTable}
WHERE SEARCH('ch:ANY("All Quiet on the Western Front (2022 film)") or ch:ALL("F. Murray Abraham")')
ORDER BY id""").collect { it[0] as int })
// Escapes are resolved before the term is looked up, so quoted values and closing
// parentheses remain searchable (DORIS-23239).
def specialTable = "test_search_dsl_special_chars"
sql "DROP TABLE IF EXISTS ${specialTable}"
sql """
CREATE TABLE ${specialTable} (
id INT,
ch VARCHAR(1024),
INDEX idx_ch (ch) USING INVERTED PROPERTIES("lower_case" = "false")
) ENGINE=OLAP
DUPLICATE KEY(id)
DISTRIBUTED BY RANDOM BUCKETS AUTO
PROPERTIES ("replication_allocation" = "tag.location.default: 1")
"""
sql """INSERT INTO ${specialTable} VALUES (1, '\"ABC\"'), (2, 'ABC)'), (3, 'ABC')"""
sql "sync"
// The keyword index stores the value including its quote characters, so the term has to
// keep the escaped quotes: SEARCH('ch:exact(\"ABC\")').
assertEquals([1], sql("SELECT id FROM ${specialTable} WHERE ch MATCH_ANY '\"ABC\"'")
.collect { it[0] as int })
assertEquals([1], sql("""
SELECT id FROM ${specialTable} WHERE SEARCH('ch:exact(\\"ABC\\")')""")
.collect { it[0] as int })
assertEquals([2], sql("""
SELECT id FROM ${specialTable} WHERE SEARCH('ch:exact(ABC\\\\))')""")
.collect { it[0] as int })
}
@@ -146,31 +146,24 @@ suite("test_search_escape", "p0") {
ORDER BY id
"""
// ============ Test 9: Lowercase 'and' should cause parse error ============
// Per requirement: Only uppercase AND/OR/NOT are operators
// Lowercase 'and' is treated as a bare term (no field), causing error
test {
sql """
SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, content
FROM ${tableName}
WHERE search('content:first and content:fruit')
ORDER BY id
"""
exception "No field specified and no default_field configured"
}
// ============ Test 9: Lowercase 'and' is an operator ============
// Boolean operators are case-insensitive, matching Lucene/ES query string syntax.
def lowercaseAndRows = sql """
SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id
FROM ${tableName}
WHERE search('content:first and content:fruit')
ORDER BY id
"""
assertEquals([[7]], lowercaseAndRows)
// ============ Test 10: Lowercase 'or' should cause parse error ============
// Per requirement: Only uppercase AND/OR/NOT are operators
// Lowercase 'or' is treated as a bare term (no field), causing error
test {
sql """
SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id, content
FROM ${tableName}
WHERE search('content:first or content:second')
ORDER BY id
"""
exception "No field specified and no default_field configured"
}
// ============ Test 10: Lowercase 'or' is an operator ============
def lowercaseOrRows = sql """
SELECT /*+SET_VAR(enable_segment_limit_pushdown=true) */ id
FROM ${tableName}
WHERE search('content:first or content:second')
ORDER BY id
"""
assertEquals([[1], [2], [7], [8]], lowercaseOrRows)
// ============ Test 11: Exclamation NOT operator ============
qt_exclamation_not """