nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 07:31:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

[fix](fe) Preserve non-UTF-8 prepared statement parameters (#69805)

Browse Source 
### What problem does this PR solve?

Related PR: #69314

JDBC `setBytes()` and Connector/ODBC binary bindings can send
`MYSQL_TYPE_VAR_STRING` / `MYSQL_TYPE_STRING` rather than BLOB. FE
decoded those bytes with Java's replacement UTF-8 decoder, silently
changing arbitrary binary data. For example, `0080FF4100C328` became
`00EFBFBDEFBFBD4100EFBFBD28` in `SELECT HEX(?)`. The BLOB handling added
by #69314 did not cover these wire types.

Decode string parameters strictly and retain non-UTF-8 payloads as
binary literals. Bind those payloads through a BE-side cast to the
original string category, preserving string function and comparison
typing without replacing bytes. Valid UTF-8 retains the existing text
literal types, and BLOB types retain their existing binary behavior. Use
value-based binary literal child hashes so equivalent casts remain equal
after rebinding and forwarding.

### Release note

Preserve non-UTF-8 bytes in MySQL server-prepared string parameters,
including JDBC `setBytes()` and Connector/ODBC binary bindings.

### Check List (For Author)

- Test:
- [x] Regression test: generated and replayed
`prepared_stmt_p0/binary_parameters`; replayed
`prepared_stmt_p0/send_long_data`. Covers all byte values, malformed
UTF-8, parameter reuse, stored HEX/MD5, string expression predicates,
NULL/empty values and ordinary Unicode text.
- [x] Unit Test: 17 tests passed across `MysqlParameterLiteralTest` and
`MysqlConnectProcessorLongDataTest`, including string wire aliases,
cached types, uploads and forwarding.
- [x] Manual test: Connector/J 8.2.0 server-prepared `setBytes`,
`setBinaryStream` and `setBlob` preserve `0080FF4100C328`.
- [x] Manual test: Linux Connector/ODBC 5.3.14 Unicode, 17 cases passed,
including 1-MiB binary/Unicode inputs. An additional 17-case
proxy-rewritten VARCHAR wire-type probe also passed.
    - [x] FE build and Checkstyle passed.
- Behavior changed:
- [x] Yes. Non-UTF-8 string-coded parameters preserve the input bytes
instead of substituting U+FFFD. Valid UTF-8 text behavior is unchanged.
- Does this need documentation?
    - [x] No.

Runtime regression checks used the newly built master FE with an
existing 4.1.4 BE and condition cache disabled. The predicate regression
uses `CONCAT(value, '') = ?`; direct constant predicate pushdown on a
current master BE was not validated. No BE, storage-format or
protocol-field changes are included.
This commit is contained in:
Calvin Kirs authored and GitHub committed 2026-11-03 17:04:13 +08:00
1 parent a35707da74
commit 1aa505cb0b
9 files changed
+402 -23
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
+13 -18
View File
@@ -58,7 +58,6 @@ import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.Optional;
@@ -600,10 +599,8 @@ public abstract class Literal extends Expression implements LeafExpression {
break;
case MYSQL_TYPE_STRING:
case MYSQL_TYPE_VARSTRING:
literal = handleStringLiteral(data);
break;
case MYSQL_TYPE_VARCHAR:
literal = handleVarcharLiteral(data);
literal = handleStringLiteral(mysqlType, data);
break;
case MYSQL_TYPE_TINY_BLOB:
case MYSQL_TYPE_MEDIUM_BLOB:
@@ -714,24 +711,22 @@ public abstract class Literal extends Expression implements LeafExpression {
}
}
private static Literal handleStringLiteral(ByteBuffer data) {
private static Literal handleStringLiteral(MysqlColType mysqlType, ByteBuffer data) {
int strLen = getParmLen(data);
strLen = Math.min(strLen, data.remaining());
byte[] bytes = new byte[strLen];
data.get(bytes);
// ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in
// different environment
return new StringLiteral(new String(bytes, StandardCharsets.UTF_8));
}
private static Literal handleVarcharLiteral(ByteBuffer data) {
int strLen = getParmLen(data);
strLen = Math.min(strLen, data.remaining());
byte[] bytes = new byte[strLen];
data.get(bytes);
// ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in
// different environment
return new VarcharLiteral(new String(bytes, StandardCharsets.UTF_8));
// Connector/J setBytes and Connector/ODBC binary bindings also use the string wire types.
// Preserve bytes that cannot be represented as UTF-8 instead of replacing them with U+FFFD.
// Valid UTF-8 retains the existing text type because the wire type cannot distinguish it
// from a text binding.
VarBinaryLiteral binary = new VarBinaryLiteral(bytes);
Optional<String> text = binary.toUtf8String();
if (!text.isPresent()) {
return binary;
}
return mysqlType == MysqlColType.MYSQL_TYPE_VARCHAR
? new VarcharLiteral(text.get()) : new StringLiteral(text.get());
}
/**convertToTypedLiteral*/
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java
+5
View File
@@ -175,4 +175,9 @@ public class VarBinaryLiteral extends Literal implements ComparableLiteral {
// equals compares byte contents independently of the declared length, never array identity.
return Arrays.hashCode(byteValues);
}
@Override
public int fastChildrenHashCode() {
return computeHashCode();
}
}
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
+9 -2
View File
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.PlaceholderId;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
@@ -148,8 +149,12 @@ public class ExecuteCommand extends Command {
LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(logicalPlan, statementContext);
executor.setStatementContext(statementContext);
executor.setParsedStmt(planAdapter);
// These fast paths serialize literals without expression evaluation. Non-UTF-8 string
// parameters contain a binary-to-string Cast that must be evaluated by the normal plan.
boolean hasLiteralBindings = statementContext.getIdToPlaceholderRealExpr().values().stream()
.allMatch(Literal.class::isInstance);
boolean hasShortCircuitContext = preparedStmtCtx.shortCircuitQueryContext.isPresent();
boolean shortCircuitContextReusable = hasShortCircuitContext
boolean shortCircuitContextReusable = hasLiteralBindings && hasShortCircuitContext
&& preparedStmtCtx.shortCircuitQueryContext.get().isReusable(ctx);
// Reuse the cached short-circuit plan only when table metadata is unchanged and the statement
// has no nondeterministic functions. Otherwise fall back to the normal execution path below.
@@ -165,7 +170,7 @@ public class ExecuteCommand extends Command {
PointQueryExecutor.directExecuteShortCircuitQuery(executor, preparedStmtCtx, statementContext);
return;
}
if (ctx.getSessionVariable().enableGroupCommitFullPrepare) {
if (hasLiteralBindings && ctx.getSessionVariable().enableGroupCommitFullPrepare) {
if (preparedStmtCtx.groupCommitPlanner.isPresent()) {
OlapGroupCommitInsertExecutor.fastAnalyzeGroupCommit(ctx, prepareCommand);
} else {
@@ -180,6 +185,8 @@ public class ExecuteCommand extends Command {
if (statementContext.isShortCircuitQuery() && hasShortCircuitContext && !shortCircuitContextReusable) {
statementContext = refreshPreparedPlan(preparedStmtCtx, executor, prepareCommand, statementContext);
} else {
// A failed execution can leave the flag set without publishing a cached context.
statementContext.setShortCircuitQuery(false);
statementContext.setShortCircuitQueryContext(null);
}
// Drop the previously cached short-circuit context: either it was reusable and returned
fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
+22 -1
View File
@@ -42,11 +42,16 @@ import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.mysql.protocol.MysqlProtocolAdapter;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral;
import org.apache.doris.nereids.trees.plans.PlaceholderId;
import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand;
import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.thrift.TUniqueId;
import com.google.common.base.Preconditions;
@@ -214,7 +219,7 @@ public class MysqlConnectProcessor extends ConnectProcessor {
MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType();
boolean isUnsigned = prepareCommand.getPlaceholders().get(i).isUnsigned();
byte[] longData = prepCtx.getLongData(i);
Literal l;
Expression l;
if (longData != null) {
switch (type) {
case MYSQL_TYPE_STRING:
@@ -240,6 +245,22 @@ public class MysqlConnectProcessor extends ConnectProcessor {
packetBuf.position() - start);
}
}
// A string wire type still has string semantics, including in predicates. BE can
// cast its original bytes to a string without the lossy Java UTF-8 decoding.
if (l instanceof VarBinaryLiteral binary) {
switch (type) {
case MYSQL_TYPE_STRING:
case MYSQL_TYPE_VARSTRING:
l = new Cast(binary, StringType.INSTANCE);
break;
case MYSQL_TYPE_VARCHAR:
l = new Cast(binary,
VarcharType.createVarcharType(((byte[]) binary.getValue()).length));
break;
default:
break;
}
}
statementContext.getIdToPlaceholderRealExpr().put(exprId, l);
}
if (forwarded != null && !ctx.isProxy()) {
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/MysqlParameterLiteralTest.java
+88
View File
@@ -0,0 +1,88 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.literal;
import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.mysql.MysqlSerializer;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
public class MysqlParameterLiteralTest {
private static final MysqlColType[] STRING_TYPES = {MysqlColType.MYSQL_TYPE_STRING,
MysqlColType.MYSQL_TYPE_VARSTRING, MysqlColType.MYSQL_TYPE_VARCHAR};
@Test
public void testStringWireTypesPreserveInvalidUtf8() {
byte[] allBytes = new byte[256];
for (int i = 0; i < allBytes.length; i++) {
allBytes[i] = (byte) i;
}
byte[][] values = {allBytes, {0, (byte) 0x80, (byte) 0xff, 65, 0, (byte) 0xc3, 40},
{(byte) 0xc3}, {(byte) 0xc0, (byte) 0x80}, {(byte) 0xed, (byte) 0xa0, (byte) 0x80},
{(byte) 0xf4, (byte) 0x90, (byte) 0x80, (byte) 0x80}};
for (MysqlColType type : STRING_TYPES) {
for (byte[] bytes : values) {
Literal literal = bind(type, bytes);
Assertions.assertEquals(new VarBinaryLiteral(bytes), literal);
Assertions.assertArrayEquals(bytes,
((org.apache.doris.analysis.VarBinaryLiteral) literal.toLegacyLiteral()).getValue());
}
}
}
@Test
public void testValidUtf8RetainsTextTypesAndBytes() {
// A real U+FFFD is valid text; it must not be mistaken for a decoding replacement.
for (String text : new String[] {"", "ASCII", "中文", "emoji: 🙂", "a\0b", "�"}) {
byte[] bytes = text.getBytes(StandardCharsets.UTF_8);
for (MysqlColType type : STRING_TYPES) {
Literal literal = bind(type, bytes);
Assertions.assertEquals(type == MysqlColType.MYSQL_TYPE_VARCHAR
? new VarcharLiteral(text) : new StringLiteral(text), literal);
Assertions.assertEquals(type == MysqlColType.MYSQL_TYPE_VARCHAR
? VarcharLiteral.class : StringLiteral.class, literal.getClass());
Assertions.assertArrayEquals(bytes, ((String) literal.getValue()).getBytes(StandardCharsets.UTF_8));
}
}
}
@Test
public void testBlobRemainsBinaryForValidUtf8() {
for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_TINY_BLOB,
MysqlColType.MYSQL_TYPE_MEDIUM_BLOB, MysqlColType.MYSQL_TYPE_LONG_BLOB,
MysqlColType.MYSQL_TYPE_BLOB}) {
Assertions.assertEquals(new VarBinaryLiteral("中文".getBytes(StandardCharsets.UTF_8)),
bind(type, "中文".getBytes(StandardCharsets.UTF_8)));
}
}
private Literal bind(MysqlColType type, byte[] bytes) {
MysqlSerializer serializer = MysqlSerializer.newInstance();
serializer.writeLenEncodedBytes(bytes);
serializer.writeInt1(42); // The next parameter must not be consumed by UTF-8 validation.
ByteBuffer packet = serializer.toByteBuffer();
Literal literal = Literal.getLiteralByMysqlType(type, false, packet);
Assertions.assertEquals(1, packet.remaining());
Assertions.assertEquals(42, packet.get());
return literal;
}
}
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java
+134
View File
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.TableScanParams;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.MysqlColType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
@@ -31,14 +32,24 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.datasource.mvcc.MvccTable;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral;
import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.planner.GroupCommitPlanner;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.PointQueryExecutor;
import org.apache.doris.qe.PreparedStatementContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.ShortCircuitQueryContext;
@@ -46,8 +57,10 @@ import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.thrift.TQueryOptions;
import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import java.util.ArrayList;
@@ -59,6 +72,127 @@ import java.util.concurrent.atomic.AtomicInteger;
public class ExecuteCommandTest {
@AfterEach
public void tearDown() {
ConnectContext.remove();
}
@Test
public void testExpressionBindingRefreshesCachedPointQuery() throws Exception {
ConnectContext context = new ConnectContext();
PreparedStatementContext prepared = prepareParameterStatement(context, "select * from tbl where k = ?");
StatementContext initialContext = prepared.getStatementContext();
initialContext.setShortCircuitQuery(true);
ShortCircuitQueryContext cached = Mockito.mock(ShortCircuitQueryContext.class);
Mockito.when(cached.isReusable(context)).thenReturn(true);
prepared.shortCircuitQueryContext = Optional.of(cached);
StmtExecutor executor = parameterExecutor(context);
try (MockedStatic<PointQueryExecutor> pointQueries = Mockito.mockStatic(PointQueryExecutor.class)) {
bindAndExecute(context, prepared, executor, new StringLiteral("ASCII"));
pointQueries.verify(() -> PointQueryExecutor.directExecuteShortCircuitQuery(
executor, prepared, prepared.getStatementContext()));
Expression binary = new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE);
bindAndExecute(context, prepared, executor, binary);
StatementContext refreshed = prepared.getStatementContext();
Assertions.assertFalse(refreshed.isShortCircuitQuery());
Assertions.assertNull(refreshed.getShortCircuitQueryContext());
Assertions.assertTrue(prepared.shortCircuitQueryContext.isEmpty());
Assertions.assertNotSame(initialContext, refreshed);
Assertions.assertEquals(binary, refreshed.getIdToPlaceholderRealExpr().get(
prepared.command.getPlaceholders().get(0).getPlaceholderId()));
Assertions.assertEquals(MysqlColType.MYSQL_TYPE_VARSTRING,
prepared.command.getPlaceholders().get(0).getMysqlColType());
Mockito.verify(executor).execute();
pointQueries.verifyNoMoreInteractions();
bindAndExecute(context, prepared, executor, new StringLiteral("ASCII again"));
Mockito.verify(executor, Mockito.times(2)).execute();
}
}
@Test
public void testExpressionBindingClearsUncachedPointQueryFlag() throws Exception {
ConnectContext context = new ConnectContext();
PreparedStatementContext prepared = prepareParameterStatement(context, "select * from tbl where k = ?");
// A point-query execution can fail before publishing its cached context.
prepared.getStatementContext().setShortCircuitQuery(true);
StmtExecutor executor = parameterExecutor(context);
bindAndExecute(context, prepared, executor,
new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE));
Assertions.assertFalse(prepared.getStatementContext().isShortCircuitQuery());
Assertions.assertNull(prepared.getStatementContext().getShortCircuitQueryContext());
Mockito.verify(executor).execute();
}
@Test
public void testExpressionBindingSkipsPreparedGroupCommit() throws Exception {
ConnectContext context = new ConnectContext();
context.getSessionVariable().groupCommit = "sync_mode";
Assertions.assertTrue(context.getSessionVariable().enableGroupCommitFullPrepare);
PreparedStatementContext prepared = prepareParameterStatement(context, "insert into test.tbl values (?)");
// A previous literal execution has already populated the full-prepare cache.
GroupCommitPlanner cached = Mockito.mock(GroupCommitPlanner.class);
prepared.groupCommitPlanner = Optional.of(cached);
StmtExecutor executor = parameterExecutor(context);
try (MockedStatic<GroupCommitPlanner> groupCommit = Mockito.mockStatic(GroupCommitPlanner.class)) {
bindAndExecute(context, prepared, executor, new StringLiteral("ASCII"));
groupCommit.verify(() -> GroupCommitPlanner.executeGroupCommitInsert(
context, prepared, prepared.getStatementContext()));
groupCommit.clearInvocations();
Expression binary = new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE);
bindAndExecute(context, prepared, executor, binary);
Mockito.verify(executor).execute();
Assertions.assertEquals(binary, prepared.getStatementContext().getIdToPlaceholderRealExpr().get(
prepared.command.getPlaceholders().get(0).getPlaceholderId()));
groupCommit.verifyNoInteractions();
bindAndExecute(context, prepared, executor, new NullLiteral());
groupCommit.verify(() -> GroupCommitPlanner.executeGroupCommitInsert(
context, prepared, prepared.getStatementContext()));
Assertions.assertSame(cached, prepared.groupCommitPlanner.orElseThrow());
}
}
private PreparedStatementContext prepareParameterStatement(ConnectContext context, String sql) throws Exception {
context.setThreadLocalInfo();
LogicalPlanAdapter parsed = (LogicalPlanAdapter) new NereidsParser().parseSQL(sql).get(0);
StatementContext statementContext = parsed.getStatementContext();
statementContext.setConnectContext(context);
List<Placeholder> placeholders = statementContext.getPlaceholders().stream()
.map(placeholder -> placeholder.withNewMysqlColType(MysqlColType.MYSQL_TYPE_VARSTRING.getCode()))
.toList();
statementContext.setPlaceholders(placeholders);
PrepareCommand command = new PrepareCommand("stmt", parsed.getLogicalPlan(), placeholders,
new OriginStatement(sql, 0));
PreparedStatementContext prepared = new PreparedStatementContext(command, context, statementContext, "stmt");
context.setStatementContext(statementContext);
context.addPreparedStatementContext("stmt", prepared);
return prepared;
}
private StmtExecutor parameterExecutor(ConnectContext context) {
StmtExecutor executor = Mockito.mock(StmtExecutor.class);
Mockito.when(executor.getContext()).thenReturn(context);
Mockito.doAnswer(invocation -> {
context.setStatementContext(invocation.getArgument(0));
return null;
}).when(executor).setStatementContext(Mockito.any(StatementContext.class));
return executor;
}
private void bindAndExecute(ConnectContext context, PreparedStatementContext prepared,
StmtExecutor executor, Expression value) throws Exception {
context.setGroupCommit(false);
prepared.getStatementContext().getIdToPlaceholderRealExpr().put(
prepared.command.getPlaceholders().get(0).getPlaceholderId(), value);
new ExecuteCommand("stmt", prepared.command, prepared.getStatementContext()).run(context, executor);
}
@Test
public void testResolvedScanOptionsAreResetForEveryExecute() throws Exception {
String sql = "select * from p@options('scan.mode'='latest')";
fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java
+33 -2
View File
@@ -24,6 +24,8 @@ import org.apache.doris.mysql.protocol.MysqlProtocolAdapter;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -34,6 +36,8 @@ import org.apache.doris.nereids.trees.plans.PlaceholderId;
import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.thrift.TUniqueId;
import org.junit.jupiter.api.Assertions;
@@ -254,6 +258,33 @@ public class MysqlConnectProcessorLongDataTest {
}
}
@Test
public void testStringWireTypesPreserveBinaryUploadsAndForwarding() throws Exception {
for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_STRING,
MysqlColType.MYSQL_TYPE_VARSTRING, MysqlColType.MYSQL_TYPE_VARCHAR}) {
Harness h = new Harness(type, MysqlColType.MYSQL_TYPE_LONG);
byte[] bytes = new byte[] {0, (byte) 0x80, (byte) 0xff, 65, (byte) 0xc3, 40};
h.upload(7, 0, new byte[] {0, (byte) 0x80});
h.upload(7, 0, new byte[] {(byte) 0xff, 65, (byte) 0xc3, 40});
h.execute(new byte[] {0, 0, 42, 0, 0, 0});
Expression expected = new Cast(new VarBinaryLiteral(bytes), type == MysqlColType.MYSQL_TYPE_VARCHAR
? VarcharType.createVarcharType(bytes.length) : StringType.INSTANCE);
Assertions.assertEquals(expected, h.bound(0));
Assertions.assertEquals(new IntegerLiteral(42), h.bound(1));
assertForwardedParameters(h);
MysqlSerializer inline = MysqlSerializer.newInstance();
inline.writeInt1(0);
inline.writeInt1(0); // Reuse the parameter types cached by the upload execution.
inline.writeLenEncodedBytes(bytes);
inline.writeInt4(43);
h.execute(inline.toArray());
Assertions.assertEquals(expected, h.bound(0));
Assertions.assertEquals(new IntegerLiteral(43), h.bound(1));
assertForwardedParameters(h);
}
}
@Test
public void testTruncatedBlobFailsBinding() {
Assertions.assertThrows(IllegalArgumentException.class,
@@ -360,8 +391,8 @@ public class MysqlConnectProcessorLongDataTest {
}
}
Literal bound(int parameter) {
return (Literal) prepared.getStatementContext().getIdToPlaceholderRealExpr()
Expression bound(int parameter) {
return prepared.getStatementContext().getIdToPlaceholderRealExpr()
.get(prepared.command.getPlaceholders().get(parameter).getPlaceholderId());
}
}
regression-test/data/prepared_stmt_p0/binary_parameters.out
+32
View File
@@ -0,0 +1,32 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !binary_parameter --
0080FF4100C328 abb21d6f67b520e215563c6fda2bd1c0
-- !binary_parameter --
000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F808182838485868788898A8B8C8D8E8F909192939495969798999A9B9C9D9E9FA0A1A2A3A4A5A6A7A8A9AAABACADAEAFB0B1B2B3B4B5B6B7B8B9BABBBCBDBEBFC0C1C2C3C4C5C6C7C8C9CACBCCCDCECFD0D1D2D3D4D5D6D7D8D9DADBDCDDDEDFE0E1E2E3E4E5E6E7E8E9EAEBECEDEEEFF0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF e2c865db4162bed963bfaa9ef6ac18f0
-- !binary_parameter --
E4B8ADE69687F09F9982 138a2c2516fcf4f9555f4bc54c3a0ae9
-- !binary_parameter --
d41d8cd98f00b204e9800998ecf8427e
-- !binary_parameter --
\N \N
-- !binary_parameter --
C3 d78276f56f8ec8d4f8cca375e4534366
-- !binary_parameter --
4100FF 93d15884da3d7122ebd5942e630719ab
-- !stored_bytes --
1 0080FF4100C328 abb21d6f67b520e215563c6fda2bd1c0
2 E4B8ADE69687F09F9982 138a2c2516fcf4f9555f4bc54c3a0ae9
-- !binary_predicate --
1
-- !text_parameters --
中文🙂 text! 3
regression-test/suites/prepared_stmt_p0/binary_parameters.groovy
+66
View File
@@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
import java.nio.charset.StandardCharsets
suite("binary_parameters") {
String url = getServerPrepareJdbcUrl(context.config.jdbcUrl, "regression_test_prepared_stmt_p0") +
"&emulateUnsupportedPstmts=false&socketTimeout=10000"
connect(context.config.jdbcUser, context.config.jdbcPassword, url) {
def connection = context.getConnection()
// setBytes uses MYSQL_TYPE_VAR_STRING, while setBlob/setBinaryStream use MYSQL_TYPE_BLOB.
connection.prepareStatement("SELECT HEX(?), MD5(?)").withCloseable { prepared ->
assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, prepared.class)
byte[] allBytes = new byte[256]
for (int i = 0; i < allBytes.length; i++) {
allBytes[i] = (byte) i
}
[new byte[] {0, -128, -1, 65, 0, -61, 40}, allBytes,
"中文🙂".getBytes(StandardCharsets.UTF_8), new byte[0], null,
new byte[] {-61}, new byte[] {65, 0, -1}].each { bytes ->
prepared.setBytes(1, bytes)
prepared.setBytes(2, bytes)
qe_binary_parameter prepared
}
}
sql "DROP TABLE IF EXISTS prepared_binary_parameters"
sql """CREATE TABLE prepared_binary_parameters (id INT, value STRING)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ('replication_num' = '1')"""
connection.prepareStatement("INSERT INTO prepared_binary_parameters VALUES (?, ?)").withCloseable { prepared ->
prepared.setInt(1, 1)
prepared.setBytes(2, new byte[] {0, -128, -1, 65, 0, -61, 40})
prepared.executeUpdate()
prepared.setInt(1, 2)
prepared.setString(2, "中文🙂")
prepared.executeUpdate()
}
order_qt_stored_bytes "SELECT id, HEX(value), MD5(value) FROM prepared_binary_parameters"
connection.prepareStatement("SELECT id FROM prepared_binary_parameters WHERE CONCAT(value, '') = ?")
.withCloseable { prepared ->
prepared.setBytes(1, new byte[] {0, -128, -1, 65, 0, -61, 40})
qe_binary_predicate prepared
}
connection.prepareStatement("SELECT ?, CONCAT(?, '!'), LENGTH(?)").withCloseable { prepared ->
prepared.setString(1, "中文🙂")
prepared.setString(2, "text")
prepared.setString(3, "a\u0000b")
qe_text_parameters prepared
}
}
}