[fix](search) Fix slash character in search query_string terms (#61599)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

The ANTLR lexer in the search() DSL parser excluded `/` from
`TERM_CHAR`, causing terms like `AC/DC` to be incorrectly tokenized. The
slash was silently skipped by ANTLR's default error recovery, splitting
`AC/DC` into two separate terms `AC` and `DC` instead of treating it as
a single term.

This caused inconsistent behavior compared to Elasticsearch's
query_string parsing, where `AC\/DC` (escaped slash) is handled as a
single analyzed term.

**Fix**: Add `/` to the `TERM_CHAR` fragment in `SearchLexer.g4`. This
allows `/` to appear within terms (e.g., `AC/DC` -> single term) while
regex patterns like `/[a-z]+/` still work correctly since `/` remains
excluded from `TERM_START_CHAR`.
This commit is contained in:
Jack authored and GitHub committed 2026-03-23 14:51:29 +08:00
1 parent c9ee1936ff
commit 574b84fa9b
4 files changed
+233

No files matched your search

@@ -32,6 +32,7 @@ fragment TERM_CHAR
: TERM_START_CHAR
| '-'
| '+'
| '/'
;
fragment QUOTED_CHAR
@@ -1094,6 +1094,81 @@ public class SearchDslParserTest {
Assertions.assertEquals("path\\to\\file", plan.getRoot().getValue());
}
@Test
public void testSlashInTerm() {
// DORIS-24624: slash within a term should be treated as a regular character
// e.g., AC/DC should parse as a single term, not trigger regex parsing
String dsl = "title:AC/DC";
QsPlan plan = SearchDslParser.parseDsl(dsl);
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType());
Assertions.assertEquals("title", plan.getRoot().getField());
Assertions.assertEquals("AC/DC", plan.getRoot().getValue());
}
@Test
public void testSlashInTermBareQuery() {
// DORIS-24624: slash within a bare term (using default_field)
String dsl = "AC/DC";
QsPlan plan = SearchDslParser.parseDsl(dsl, "{\"default_field\":\"title\",\"default_operator\":\"OR\"}");
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType());
Assertions.assertEquals("title", plan.getRoot().getField());
Assertions.assertEquals("AC/DC", plan.getRoot().getValue());
}
@Test
public void testSlashInTermLuceneMode() {
// DORIS-24624: slash within a bare term in Lucene mode
String dsl = "AC/DC";
QsPlan plan = SearchDslParser.parseDsl(dsl,
"{\"default_field\":\"title\",\"default_operator\":\"OR\",\"minimum_should_match\":0}");
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType());
Assertions.assertEquals("title", plan.getRoot().getField());
Assertions.assertEquals("AC/DC", plan.getRoot().getValue());
}
@Test
public void testEscapedSlashInTerm() {
// Escaped slash should also work and produce same result as unescaped
String dsl = "title:AC\\/DC";
QsPlan plan = SearchDslParser.parseDsl(dsl);
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType());
Assertions.assertEquals("title", plan.getRoot().getField());
// After unescape: AC\/DC -> AC/DC
Assertions.assertEquals("AC/DC", plan.getRoot().getValue());
}
@Test
public void testMultipleSlashesInTerm() {
// Multiple slashes within a term
String dsl = "path:foo/bar/baz";
QsPlan plan = SearchDslParser.parseDsl(dsl);
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.TERM, plan.getRoot().getType());
Assertions.assertEquals("path", plan.getRoot().getField());
Assertions.assertEquals("foo/bar/baz", plan.getRoot().getValue());
}
@Test
public void testSlashDoesNotBreakRegexp() {
// Regex pattern /pattern/ should still work correctly
String dsl = "title:/[a-z]+/";
QsPlan plan = SearchDslParser.parseDsl(dsl);
Assertions.assertNotNull(plan);
Assertions.assertEquals(QsClauseType.REGEXP, plan.getRoot().getType());
Assertions.assertEquals("title", plan.getRoot().getField());
Assertions.assertEquals("[a-z]+", plan.getRoot().getValue());
}
@Test
public void testUppercaseAndOperator() {
// Test: uppercase AND should be treated as operator
@@ -0,0 +1,32 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !slash_in_term --
1 AC/DC is a rock band
2 AC power supply
3 DC comics
-- !escaped_slash_in_term --
1 AC/DC is a rock band
2 AC power supply
3 DC comics
-- !slash_bare_lucene --
1 AC/DC is a rock band
2 AC power supply
3 DC comics
-- !escaped_slash_bare_lucene --
1 AC/DC is a rock band
2 AC power supply
3 DC comics
-- !multi_slash --
4 path/to/file
-- !regex_still_works --
1 AC/DC is a rock band
-- !slash_standard_mode --
1 AC/DC is a rock band
2 AC power supply
3 DC comics
@@ -0,0 +1,125 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
/**
* DORIS-24624: Tests for slash (/) character handling in search() function.
*
* The slash character is used as a regex delimiter in Lucene query_string syntax
* (e.g., /pattern/). However, when it appears in the middle of a term (e.g., AC/DC),
* it should be treated as a regular character, not as a regex delimiter.
*
* This test verifies that:
* 1. Slash within a term (AC/DC) is parsed correctly as a single term
* 2. Escaped slash (AC\/DC) produces the same result
* 3. Regex patterns (/pattern/) still work correctly
* 4. Both standard and lucene modes handle slashes consistently
*/
suite("test_search_slash_in_term", "p0") {
def tableName = "search_slash_in_term_test"
sql """ set enable_common_expr_pushdown = true """
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
CREATE TABLE ${tableName} (
id INT,
title VARCHAR(200),
content VARCHAR(500),
INDEX idx_title(title) USING INVERTED PROPERTIES("parser" = "standard"),
INDEX idx_content(content) USING INVERTED PROPERTIES("parser" = "standard")
) ENGINE=OLAP
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_allocation" = "tag.location.default: 1")
"""
sql """INSERT INTO ${tableName} VALUES
(1, 'AC/DC is a rock band', 'rock music'),
(2, 'AC power supply', 'electrical engineering'),
(3, 'DC comics', 'entertainment'),
(4, 'path/to/file', 'file system'),
(5, 'a/b/c/d', 'multi slash path'),
(6, 'hello world', 'greeting'),
(7, 'acdc together', 'no slash')
"""
// Wait for index building
Thread.sleep(3000)
// ============ Test 1: Slash in term with field prefix ============
// title:AC/DC should parse as single term, standard analyzer tokenizes to "ac" and "dc"
// With default OR operator, matches rows containing "ac" or "dc" in title
order_qt_slash_in_term """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('title:AC/DC')
ORDER BY id
"""
// ============ Test 2: Escaped slash should produce same result ============
// title:AC\/DC should produce the same result as title:AC/DC
// Groovy: \\\\/ -> SQL: \\/ -> DSL: \/ -> unescaped: /
order_qt_escaped_slash_in_term """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('title:AC\\\\/DC')
ORDER BY id
"""
// ============ Test 3: Slash in term with default_field (lucene mode) ============
// Bare AC/DC with default_field should work
order_qt_slash_bare_lucene """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('AC/DC', '{"default_field":"title","default_operator":"OR","minimum_should_match":0}')
ORDER BY id
"""
// ============ Test 4: Escaped slash with default_field should match ============
order_qt_escaped_slash_bare_lucene """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('AC\\\\/DC', '{"default_field":"title","default_operator":"OR","minimum_should_match":0}')
ORDER BY id
"""
// ============ Test 5: Multiple slashes in term ============
order_qt_multi_slash """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('title:path/to/file')
ORDER BY id
"""
// ============ Test 6: Regex pattern still works ============
// /[a-z]+/ should be parsed as regex, not as term with slashes
order_qt_regex_still_works """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('title:/rock/')
ORDER BY id
"""
// ============ Test 7: Slash in term with standard mode ============
order_qt_slash_standard_mode """
SELECT /*+SET_VAR(enable_common_expr_pushdown=true) */ id, title
FROM ${tableName}
WHERE search('AC/DC', '{"default_field":"title","mode":"standard"}')
ORDER BY id
"""
}