diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 1a33e0d52f3..73326bed8d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -58,7 +58,6 @@ import java.math.BigInteger; import java.math.MathContext; import java.math.RoundingMode; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.Objects; import java.util.Optional; @@ -600,10 +599,8 @@ public abstract class Literal extends Expression implements LeafExpression { break; case MYSQL_TYPE_STRING: case MYSQL_TYPE_VARSTRING: - literal = handleStringLiteral(data); - break; case MYSQL_TYPE_VARCHAR: - literal = handleVarcharLiteral(data); + literal = handleStringLiteral(mysqlType, data); break; case MYSQL_TYPE_TINY_BLOB: case MYSQL_TYPE_MEDIUM_BLOB: @@ -714,24 +711,22 @@ public abstract class Literal extends Expression implements LeafExpression { } } - private static Literal handleStringLiteral(ByteBuffer data) { + private static Literal handleStringLiteral(MysqlColType mysqlType, ByteBuffer data) { int strLen = getParmLen(data); strLen = Math.min(strLen, data.remaining()); byte[] bytes = new byte[strLen]; data.get(bytes); - // ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in - // different environment - return new StringLiteral(new String(bytes, StandardCharsets.UTF_8)); - } - - private static Literal handleVarcharLiteral(ByteBuffer data) { - int strLen = getParmLen(data); - strLen = Math.min(strLen, data.remaining()); - byte[] bytes = new byte[strLen]; - data.get(bytes); - // ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in - // different environment - return new VarcharLiteral(new String(bytes, StandardCharsets.UTF_8)); + // Connector/J setBytes and Connector/ODBC binary bindings also use the string wire types. + // Preserve bytes that cannot be represented as UTF-8 instead of replacing them with U+FFFD. + // Valid UTF-8 retains the existing text type because the wire type cannot distinguish it + // from a text binding. + VarBinaryLiteral binary = new VarBinaryLiteral(bytes); + Optional text = binary.toUtf8String(); + if (!text.isPresent()) { + return binary; + } + return mysqlType == MysqlColType.MYSQL_TYPE_VARCHAR + ? new VarcharLiteral(text.get()) : new StringLiteral(text.get()); } /**convertToTypedLiteral*/ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java index 7acfbad76da..7065d9b9980 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java @@ -175,4 +175,9 @@ public class VarBinaryLiteral extends Literal implements ComparableLiteral { // equals compares byte contents independently of the declared length, never array identity. return Arrays.hashCode(byteValues); } + + @Override + public int fastChildrenHashCode() { + return computeHashCode(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index f7f129e23cf..3bcf0817423 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; @@ -148,8 +149,12 @@ public class ExecuteCommand extends Command { LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(logicalPlan, statementContext); executor.setStatementContext(statementContext); executor.setParsedStmt(planAdapter); + // These fast paths serialize literals without expression evaluation. Non-UTF-8 string + // parameters contain a binary-to-string Cast that must be evaluated by the normal plan. + boolean hasLiteralBindings = statementContext.getIdToPlaceholderRealExpr().values().stream() + .allMatch(Literal.class::isInstance); boolean hasShortCircuitContext = preparedStmtCtx.shortCircuitQueryContext.isPresent(); - boolean shortCircuitContextReusable = hasShortCircuitContext + boolean shortCircuitContextReusable = hasLiteralBindings && hasShortCircuitContext && preparedStmtCtx.shortCircuitQueryContext.get().isReusable(ctx); // Reuse the cached short-circuit plan only when table metadata is unchanged and the statement // has no nondeterministic functions. Otherwise fall back to the normal execution path below. @@ -165,7 +170,7 @@ public class ExecuteCommand extends Command { PointQueryExecutor.directExecuteShortCircuitQuery(executor, preparedStmtCtx, statementContext); return; } - if (ctx.getSessionVariable().enableGroupCommitFullPrepare) { + if (hasLiteralBindings && ctx.getSessionVariable().enableGroupCommitFullPrepare) { if (preparedStmtCtx.groupCommitPlanner.isPresent()) { OlapGroupCommitInsertExecutor.fastAnalyzeGroupCommit(ctx, prepareCommand); } else { @@ -180,6 +185,8 @@ public class ExecuteCommand extends Command { if (statementContext.isShortCircuitQuery() && hasShortCircuitContext && !shortCircuitContextReusable) { statementContext = refreshPreparedPlan(preparedStmtCtx, executor, prepareCommand, statementContext); } else { + // A failed execution can leave the flag set without publishing a cached context. + statementContext.setShortCircuitQuery(false); statementContext.setShortCircuitQueryContext(null); } // Drop the previously cached short-circuit context: either it was reusable and returned diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index c5a0931df1e..5e3babf50d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -42,11 +42,16 @@ import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.protocol.MysqlProtocolAdapter; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Preconditions; @@ -214,7 +219,7 @@ public class MysqlConnectProcessor extends ConnectProcessor { MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType(); boolean isUnsigned = prepareCommand.getPlaceholders().get(i).isUnsigned(); byte[] longData = prepCtx.getLongData(i); - Literal l; + Expression l; if (longData != null) { switch (type) { case MYSQL_TYPE_STRING: @@ -240,6 +245,22 @@ public class MysqlConnectProcessor extends ConnectProcessor { packetBuf.position() - start); } } + // A string wire type still has string semantics, including in predicates. BE can + // cast its original bytes to a string without the lossy Java UTF-8 decoding. + if (l instanceof VarBinaryLiteral binary) { + switch (type) { + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_VARSTRING: + l = new Cast(binary, StringType.INSTANCE); + break; + case MYSQL_TYPE_VARCHAR: + l = new Cast(binary, + VarcharType.createVarcharType(((byte[]) binary.getValue()).length)); + break; + default: + break; + } + } statementContext.getIdToPlaceholderRealExpr().put(exprId, l); } if (forwarded != null && !ctx.isProxy()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/MysqlParameterLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/MysqlParameterLiteralTest.java new file mode 100644 index 00000000000..cbf15f4f9f6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/MysqlParameterLiteralTest.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.mysql.MysqlSerializer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +public class MysqlParameterLiteralTest { + private static final MysqlColType[] STRING_TYPES = {MysqlColType.MYSQL_TYPE_STRING, + MysqlColType.MYSQL_TYPE_VARSTRING, MysqlColType.MYSQL_TYPE_VARCHAR}; + + @Test + public void testStringWireTypesPreserveInvalidUtf8() { + byte[] allBytes = new byte[256]; + for (int i = 0; i < allBytes.length; i++) { + allBytes[i] = (byte) i; + } + byte[][] values = {allBytes, {0, (byte) 0x80, (byte) 0xff, 65, 0, (byte) 0xc3, 40}, + {(byte) 0xc3}, {(byte) 0xc0, (byte) 0x80}, {(byte) 0xed, (byte) 0xa0, (byte) 0x80}, + {(byte) 0xf4, (byte) 0x90, (byte) 0x80, (byte) 0x80}}; + for (MysqlColType type : STRING_TYPES) { + for (byte[] bytes : values) { + Literal literal = bind(type, bytes); + Assertions.assertEquals(new VarBinaryLiteral(bytes), literal); + Assertions.assertArrayEquals(bytes, + ((org.apache.doris.analysis.VarBinaryLiteral) literal.toLegacyLiteral()).getValue()); + } + } + } + + @Test + public void testValidUtf8RetainsTextTypesAndBytes() { + // A real U+FFFD is valid text; it must not be mistaken for a decoding replacement. + for (String text : new String[] {"", "ASCII", "δΈ­ζ–‡", "emoji: πŸ™‚", "a\0b", "οΏ½"}) { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + for (MysqlColType type : STRING_TYPES) { + Literal literal = bind(type, bytes); + Assertions.assertEquals(type == MysqlColType.MYSQL_TYPE_VARCHAR + ? new VarcharLiteral(text) : new StringLiteral(text), literal); + Assertions.assertEquals(type == MysqlColType.MYSQL_TYPE_VARCHAR + ? VarcharLiteral.class : StringLiteral.class, literal.getClass()); + Assertions.assertArrayEquals(bytes, ((String) literal.getValue()).getBytes(StandardCharsets.UTF_8)); + } + } + } + + @Test + public void testBlobRemainsBinaryForValidUtf8() { + for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_TINY_BLOB, + MysqlColType.MYSQL_TYPE_MEDIUM_BLOB, MysqlColType.MYSQL_TYPE_LONG_BLOB, + MysqlColType.MYSQL_TYPE_BLOB}) { + Assertions.assertEquals(new VarBinaryLiteral("δΈ­ζ–‡".getBytes(StandardCharsets.UTF_8)), + bind(type, "δΈ­ζ–‡".getBytes(StandardCharsets.UTF_8))); + } + } + + private Literal bind(MysqlColType type, byte[] bytes) { + MysqlSerializer serializer = MysqlSerializer.newInstance(); + serializer.writeLenEncodedBytes(bytes); + serializer.writeInt1(42); // The next parameter must not be consumed by UTF-8 validation. + ByteBuffer packet = serializer.toByteBuffer(); + Literal literal = Literal.getLiteralByMysqlType(type, false, packet); + Assertions.assertEquals(1, packet.remaining()); + Assertions.assertEquals(42, packet.get()); + return literal; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java index 7905b31b5ef..419120179e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.Queriable; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; @@ -31,14 +32,24 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral; import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.planner.GroupCommitPlanner; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.Planner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.PointQueryExecutor; import org.apache.doris.qe.PreparedStatementContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.ShortCircuitQueryContext; @@ -46,8 +57,10 @@ import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TQueryOptions; import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.util.ArrayList; @@ -59,6 +72,127 @@ import java.util.concurrent.atomic.AtomicInteger; public class ExecuteCommandTest { + @AfterEach + public void tearDown() { + ConnectContext.remove(); + } + + @Test + public void testExpressionBindingRefreshesCachedPointQuery() throws Exception { + ConnectContext context = new ConnectContext(); + PreparedStatementContext prepared = prepareParameterStatement(context, "select * from tbl where k = ?"); + StatementContext initialContext = prepared.getStatementContext(); + initialContext.setShortCircuitQuery(true); + ShortCircuitQueryContext cached = Mockito.mock(ShortCircuitQueryContext.class); + Mockito.when(cached.isReusable(context)).thenReturn(true); + prepared.shortCircuitQueryContext = Optional.of(cached); + StmtExecutor executor = parameterExecutor(context); + + try (MockedStatic pointQueries = Mockito.mockStatic(PointQueryExecutor.class)) { + bindAndExecute(context, prepared, executor, new StringLiteral("ASCII")); + pointQueries.verify(() -> PointQueryExecutor.directExecuteShortCircuitQuery( + executor, prepared, prepared.getStatementContext())); + + Expression binary = new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE); + bindAndExecute(context, prepared, executor, binary); + StatementContext refreshed = prepared.getStatementContext(); + Assertions.assertFalse(refreshed.isShortCircuitQuery()); + Assertions.assertNull(refreshed.getShortCircuitQueryContext()); + Assertions.assertTrue(prepared.shortCircuitQueryContext.isEmpty()); + Assertions.assertNotSame(initialContext, refreshed); + Assertions.assertEquals(binary, refreshed.getIdToPlaceholderRealExpr().get( + prepared.command.getPlaceholders().get(0).getPlaceholderId())); + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_VARSTRING, + prepared.command.getPlaceholders().get(0).getMysqlColType()); + Mockito.verify(executor).execute(); + pointQueries.verifyNoMoreInteractions(); + + bindAndExecute(context, prepared, executor, new StringLiteral("ASCII again")); + Mockito.verify(executor, Mockito.times(2)).execute(); + } + } + + @Test + public void testExpressionBindingClearsUncachedPointQueryFlag() throws Exception { + ConnectContext context = new ConnectContext(); + PreparedStatementContext prepared = prepareParameterStatement(context, "select * from tbl where k = ?"); + // A point-query execution can fail before publishing its cached context. + prepared.getStatementContext().setShortCircuitQuery(true); + StmtExecutor executor = parameterExecutor(context); + + bindAndExecute(context, prepared, executor, + new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE)); + + Assertions.assertFalse(prepared.getStatementContext().isShortCircuitQuery()); + Assertions.assertNull(prepared.getStatementContext().getShortCircuitQueryContext()); + Mockito.verify(executor).execute(); + } + + @Test + public void testExpressionBindingSkipsPreparedGroupCommit() throws Exception { + ConnectContext context = new ConnectContext(); + context.getSessionVariable().groupCommit = "sync_mode"; + Assertions.assertTrue(context.getSessionVariable().enableGroupCommitFullPrepare); + PreparedStatementContext prepared = prepareParameterStatement(context, "insert into test.tbl values (?)"); + // A previous literal execution has already populated the full-prepare cache. + GroupCommitPlanner cached = Mockito.mock(GroupCommitPlanner.class); + prepared.groupCommitPlanner = Optional.of(cached); + StmtExecutor executor = parameterExecutor(context); + try (MockedStatic groupCommit = Mockito.mockStatic(GroupCommitPlanner.class)) { + bindAndExecute(context, prepared, executor, new StringLiteral("ASCII")); + groupCommit.verify(() -> GroupCommitPlanner.executeGroupCommitInsert( + context, prepared, prepared.getStatementContext())); + groupCommit.clearInvocations(); + + Expression binary = new Cast(new VarBinaryLiteral(new byte[] {(byte) 0xff}), StringType.INSTANCE); + bindAndExecute(context, prepared, executor, binary); + Mockito.verify(executor).execute(); + Assertions.assertEquals(binary, prepared.getStatementContext().getIdToPlaceholderRealExpr().get( + prepared.command.getPlaceholders().get(0).getPlaceholderId())); + groupCommit.verifyNoInteractions(); + + bindAndExecute(context, prepared, executor, new NullLiteral()); + groupCommit.verify(() -> GroupCommitPlanner.executeGroupCommitInsert( + context, prepared, prepared.getStatementContext())); + Assertions.assertSame(cached, prepared.groupCommitPlanner.orElseThrow()); + } + } + + private PreparedStatementContext prepareParameterStatement(ConnectContext context, String sql) throws Exception { + context.setThreadLocalInfo(); + LogicalPlanAdapter parsed = (LogicalPlanAdapter) new NereidsParser().parseSQL(sql).get(0); + StatementContext statementContext = parsed.getStatementContext(); + statementContext.setConnectContext(context); + List placeholders = statementContext.getPlaceholders().stream() + .map(placeholder -> placeholder.withNewMysqlColType(MysqlColType.MYSQL_TYPE_VARSTRING.getCode())) + .toList(); + statementContext.setPlaceholders(placeholders); + PrepareCommand command = new PrepareCommand("stmt", parsed.getLogicalPlan(), placeholders, + new OriginStatement(sql, 0)); + PreparedStatementContext prepared = new PreparedStatementContext(command, context, statementContext, "stmt"); + context.setStatementContext(statementContext); + context.addPreparedStatementContext("stmt", prepared); + return prepared; + } + + private StmtExecutor parameterExecutor(ConnectContext context) { + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Mockito.when(executor.getContext()).thenReturn(context); + Mockito.doAnswer(invocation -> { + context.setStatementContext(invocation.getArgument(0)); + return null; + }).when(executor).setStatementContext(Mockito.any(StatementContext.class)); + return executor; + } + + private void bindAndExecute(ConnectContext context, PreparedStatementContext prepared, + StmtExecutor executor, Expression value) throws Exception { + context.setGroupCommit(false); + prepared.getStatementContext().getIdToPlaceholderRealExpr().put( + prepared.command.getPlaceholders().get(0).getPlaceholderId(), value); + new ExecuteCommand("stmt", prepared.command, prepared.getStatementContext()).run(context, executor); + } + @Test public void testResolvedScanOptionsAreResetForEveryExecute() throws Exception { String sql = "select * from p@options('scan.mode'='latest')"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java index 9c214a76d78..e5cfb9f9795 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java @@ -24,6 +24,8 @@ import org.apache.doris.mysql.protocol.MysqlProtocolAdapter; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -34,6 +36,8 @@ import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.thrift.TUniqueId; import org.junit.jupiter.api.Assertions; @@ -254,6 +258,33 @@ public class MysqlConnectProcessorLongDataTest { } } + @Test + public void testStringWireTypesPreserveBinaryUploadsAndForwarding() throws Exception { + for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_STRING, + MysqlColType.MYSQL_TYPE_VARSTRING, MysqlColType.MYSQL_TYPE_VARCHAR}) { + Harness h = new Harness(type, MysqlColType.MYSQL_TYPE_LONG); + byte[] bytes = new byte[] {0, (byte) 0x80, (byte) 0xff, 65, (byte) 0xc3, 40}; + h.upload(7, 0, new byte[] {0, (byte) 0x80}); + h.upload(7, 0, new byte[] {(byte) 0xff, 65, (byte) 0xc3, 40}); + h.execute(new byte[] {0, 0, 42, 0, 0, 0}); + Expression expected = new Cast(new VarBinaryLiteral(bytes), type == MysqlColType.MYSQL_TYPE_VARCHAR + ? VarcharType.createVarcharType(bytes.length) : StringType.INSTANCE); + Assertions.assertEquals(expected, h.bound(0)); + Assertions.assertEquals(new IntegerLiteral(42), h.bound(1)); + assertForwardedParameters(h); + + MysqlSerializer inline = MysqlSerializer.newInstance(); + inline.writeInt1(0); + inline.writeInt1(0); // Reuse the parameter types cached by the upload execution. + inline.writeLenEncodedBytes(bytes); + inline.writeInt4(43); + h.execute(inline.toArray()); + Assertions.assertEquals(expected, h.bound(0)); + Assertions.assertEquals(new IntegerLiteral(43), h.bound(1)); + assertForwardedParameters(h); + } + } + @Test public void testTruncatedBlobFailsBinding() { Assertions.assertThrows(IllegalArgumentException.class, @@ -360,8 +391,8 @@ public class MysqlConnectProcessorLongDataTest { } } - Literal bound(int parameter) { - return (Literal) prepared.getStatementContext().getIdToPlaceholderRealExpr() + Expression bound(int parameter) { + return prepared.getStatementContext().getIdToPlaceholderRealExpr() .get(prepared.command.getPlaceholders().get(parameter).getPlaceholderId()); } } diff --git a/regression-test/data/prepared_stmt_p0/binary_parameters.out b/regression-test/data/prepared_stmt_p0/binary_parameters.out new file mode 100644 index 00000000000..deae9d43cab --- /dev/null +++ b/regression-test/data/prepared_stmt_p0/binary_parameters.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !binary_parameter -- +0080FF4100C328 abb21d6f67b520e215563c6fda2bd1c0 + +-- !binary_parameter -- +000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F808182838485868788898A8B8C8D8E8F909192939495969798999A9B9C9D9E9FA0A1A2A3A4A5A6A7A8A9AAABACADAEAFB0B1B2B3B4B5B6B7B8B9BABBBCBDBEBFC0C1C2C3C4C5C6C7C8C9CACBCCCDCECFD0D1D2D3D4D5D6D7D8D9DADBDCDDDEDFE0E1E2E3E4E5E6E7E8E9EAEBECEDEEEFF0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF e2c865db4162bed963bfaa9ef6ac18f0 + +-- !binary_parameter -- +E4B8ADE69687F09F9982 138a2c2516fcf4f9555f4bc54c3a0ae9 + +-- !binary_parameter -- + d41d8cd98f00b204e9800998ecf8427e + +-- !binary_parameter -- +\N \N + +-- !binary_parameter -- +C3 d78276f56f8ec8d4f8cca375e4534366 + +-- !binary_parameter -- +4100FF 93d15884da3d7122ebd5942e630719ab + +-- !stored_bytes -- +1 0080FF4100C328 abb21d6f67b520e215563c6fda2bd1c0 +2 E4B8ADE69687F09F9982 138a2c2516fcf4f9555f4bc54c3a0ae9 + +-- !binary_predicate -- +1 + +-- !text_parameters -- +δΈ­ζ–‡πŸ™‚ text! 3 + diff --git a/regression-test/suites/prepared_stmt_p0/binary_parameters.groovy b/regression-test/suites/prepared_stmt_p0/binary_parameters.groovy new file mode 100644 index 00000000000..b60b903680c --- /dev/null +++ b/regression-test/suites/prepared_stmt_p0/binary_parameters.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.nio.charset.StandardCharsets + +suite("binary_parameters") { + String url = getServerPrepareJdbcUrl(context.config.jdbcUrl, "regression_test_prepared_stmt_p0") + + "&emulateUnsupportedPstmts=false&socketTimeout=10000" + connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + def connection = context.getConnection() + // setBytes uses MYSQL_TYPE_VAR_STRING, while setBlob/setBinaryStream use MYSQL_TYPE_BLOB. + connection.prepareStatement("SELECT HEX(?), MD5(?)").withCloseable { prepared -> + assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, prepared.class) + byte[] allBytes = new byte[256] + for (int i = 0; i < allBytes.length; i++) { + allBytes[i] = (byte) i + } + [new byte[] {0, -128, -1, 65, 0, -61, 40}, allBytes, + "δΈ­ζ–‡πŸ™‚".getBytes(StandardCharsets.UTF_8), new byte[0], null, + new byte[] {-61}, new byte[] {65, 0, -1}].each { bytes -> + prepared.setBytes(1, bytes) + prepared.setBytes(2, bytes) + qe_binary_parameter prepared + } + } + + sql "DROP TABLE IF EXISTS prepared_binary_parameters" + sql """CREATE TABLE prepared_binary_parameters (id INT, value STRING) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ('replication_num' = '1')""" + connection.prepareStatement("INSERT INTO prepared_binary_parameters VALUES (?, ?)").withCloseable { prepared -> + prepared.setInt(1, 1) + prepared.setBytes(2, new byte[] {0, -128, -1, 65, 0, -61, 40}) + prepared.executeUpdate() + prepared.setInt(1, 2) + prepared.setString(2, "δΈ­ζ–‡πŸ™‚") + prepared.executeUpdate() + } + order_qt_stored_bytes "SELECT id, HEX(value), MD5(value) FROM prepared_binary_parameters" + connection.prepareStatement("SELECT id FROM prepared_binary_parameters WHERE CONCAT(value, '') = ?") + .withCloseable { prepared -> + prepared.setBytes(1, new byte[] {0, -128, -1, 65, 0, -61, 40}) + qe_binary_predicate prepared + } + connection.prepareStatement("SELECT ?, CONCAT(?, '!'), LENGTH(?)").withCloseable { prepared -> + prepared.setString(1, "δΈ­ζ–‡πŸ™‚") + prepared.setString(2, "text") + prepared.setString(3, "a\u0000b") + qe_text_parameters prepared + } + } +}