diff --git a/be/src/core/data_type_serde/complex_type_deserialize_util.h b/be/src/core/data_type_serde/complex_type_deserialize_util.h index b1320b4b073..7ac43568fab 100644 --- a/be/src/core/data_type_serde/complex_type_deserialize_util.h +++ b/be/src/core/data_type_serde/complex_type_deserialize_util.h @@ -78,33 +78,14 @@ struct ComplexTypeDeserializeUtil { return elements; } - static bool is_null_string(const StringRef& str) { - if (str.size == 4) { - // null - return str.data[0] == 'n' && str.data[1] == 'u' && str.data[2] == 'l' && - str.data[3] == 'l'; - } - return false; - } - template static Status process_column(const DataTypeSerDeSPtr& serde, IColumn& column, StringRef& str, const DataTypeSerDe::FormatOptions& options) { DCHECK(column.is_nullable()) << "Column must be nullable but got " << column.get_name(); - if (is_null_string(str)) { - auto& column_nullable = assert_cast(column); - column_nullable.insert_default(); - return Status::OK(); - } - auto str_without_quote = str.trim_quote(); - if constexpr (is_strict_mode) { - return serde->from_string_strict_mode(str_without_quote, column, options); - } else { - auto st = serde->from_string(str_without_quote, column, options); - DCHECK(st.ok()) << "no strict mode, so should not return error"; - return st; - } + auto status = serde->from_string_as_nested(str, column, options, is_strict_mode); + DCHECK(is_strict_mode || status.ok()) << "no strict mode, so should not return error"; + return status; } }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp index f2eeaa6c219..bf9252342e9 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp @@ -298,6 +298,20 @@ Status DataTypeJsonbSerDe::from_string(StringRef& str, IColumn& column, return deserialize_one_cell_from_json(column, slice, options); } +Status DataTypeJsonbSerDe::from_string_as_nested(StringRef& str, IColumn& column, + const FormatOptions& options, + bool is_strict_mode) const { + // Quotes belong to the JSON value: stripping them turns "123" into a number + // and "null" into JSON null, losing the original type. + auto token = str; + // SQL apostrophes are an optional outer wrapper, e.g. '{"a":1}' or '"123"'. + if (token.size >= 2 && token.front() == '\'' && token.back() == '\'') { + token = token.substring(1, token.size - 2); + } + return is_strict_mode ? from_string_strict_mode(token, column, options) + : from_string(token, column, options); +} + void DataTypeJsonbSerDe::write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars, int64_t row_num) const { @@ -341,14 +355,10 @@ void DataTypeJsonbSerDe::to_string(const IColumn& column, size_t row_num, Buffer const auto& col = assert_cast(column); const auto& data_ref = col.get_data_at(row_num); if (data_ref.size > 0) { - if (_nesting_level > 1) { - bw.write('"'); - } + // JSON text already quotes strings and escapes their contents. Wrapping a nested + // object again would produce invalid text such as ["{"a":1}"]. std::string str = JsonbToJson::jsonb_to_json_string(data_ref.data, data_ref.size); bw.write(str.c_str(), str.size()); - if (_nesting_level > 1) { - bw.write('"'); - } } else { bw.write("NULL", 4); } diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.h b/be/src/core/data_type_serde/data_type_jsonb_serde.h index b97de7fac77..3a1e5b6df77 100644 --- a/be/src/core/data_type_serde/data_type_jsonb_serde.h +++ b/be/src/core/data_type_serde/data_type_jsonb_serde.h @@ -37,6 +37,12 @@ public: std::string get_name() const override { return "JSONB"; } + // JSON null is a value, so a nested SQL NULL needs a distinct spelling. + StringRef get_nested_null_format() const override { return {"NULL", 4}; } + + Status from_string_as_nested(StringRef& str, IColumn& column, const FormatOptions& options, + bool is_strict_mode) const override; + Status from_string(StringRef& str, IColumn& column, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index e354d423f0b..fef85bf9187 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -580,8 +580,10 @@ void DataTypeNullableSerDe::to_string(const IColumn& column, size_t row_num, Buf const FormatOptions& options) const { const auto& col_null = assert_cast(column); if (col_null.is_null_at(row_num)) { + // A child may reserve lowercase null for a value; use its paired SQL-null marker. if (_nesting_level > 1) { - bw.write("null", 4); + const auto null_format = nested_serde->get_nested_null_format(); + bw.write(null_format.data, null_format.size); } else { bw.write("NULL", 4); } @@ -594,6 +596,28 @@ std::string DataTypeNullableSerDe::to_olap_string(const Field& field) const { return nested_serde->to_olap_string(field); } +Status DataTypeNullableSerDe::from_string_as_nested(StringRef& str, IColumn& column, + const FormatOptions& options, + bool is_strict_mode) const { + auto& null_column = assert_cast(column); + // Match the unquoted marker before parsing the value: for JSON, null and + // "null" are non-SQL-null values, whereas NULL is the SQL-null marker. + if (str == nested_serde->get_nested_null_format()) { + null_column.insert_default(); + return Status::OK(); + } + auto status = nested_serde->from_string_as_nested(str, null_column.get_nested_column(), options, + is_strict_mode); + // Malformed children become SQL NULL only in non-strict mode, just as in from_string. + if (!status.ok() && !is_strict_mode) { + null_column.insert_default(); + return Status::OK(); + } + RETURN_IF_ERROR(status); + null_column.get_null_map_data().push_back(0); + return Status::OK(); +} + // In non-strict mode, from string will handle errors by inserting a null value. Status DataTypeNullableSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h b/be/src/core/data_type_serde/data_type_nullable_serde.h index f5ec2d1973f..e43da468806 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.h +++ b/be/src/core/data_type_serde/data_type_nullable_serde.h @@ -36,6 +36,9 @@ public: std::string get_name() const override { return "Nullable(" + nested_serde->get_name() + ")"; } + Status from_string_as_nested(StringRef& str, IColumn& column, const FormatOptions& options, + bool is_strict_mode) const override; + Status from_string(StringRef& str, IColumn& column, const FormatOptions& options) const override; Status from_zonemap_string(const std::string& str, Field& field) const override; diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index 760d7109a08..ab95bd7f0be 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -40,6 +40,14 @@ namespace doris { DataTypeSerDe::~DataTypeSerDe() = default; +Status DataTypeSerDe::from_string_as_nested(StringRef& str, IColumn& column, + const FormatOptions& options, + bool is_strict_mode) const { + auto unquoted = str.trim_quote(); + return is_strict_mode ? from_string_strict_mode(unquoted, column, options) + : from_string(unquoted, column, options); +} + bool decoded_column_view_can_null_on_conversion_failure(const DecodedColumnView& view) { return !view.enable_strict_mode && view.conversion_failure_null_map != nullptr; } diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index 2a57adaf5d3..7618322300e 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -143,6 +143,7 @@ class DataTypeSerDe { public: // return type name , such as "BOOL", "BIGINT", "ARRAY" virtual std::string get_name() const = 0; + // Text serialization/deserialization of data types depend on some settings witch we define // in formatOptions. struct FormatOptions { @@ -339,6 +340,14 @@ public: return from_string(str, column, options); } + // Parse a complete child token from ARRAY/MAP/STRUCT text. Scalar parsers normally + // receive unquoted text; types whose syntax owns the quotes override this entry point. + virtual Status from_string_as_nested(StringRef& str, IColumn& column, + const FormatOptions& options, bool is_strict_mode) const; + + // Nullable uses the same marker when reading and writing a child SQL NULL. + virtual StringRef get_nested_null_format() const { return {"null", 4}; } + // Override functions with _batch suffix only when performance is critical // Only used for basic data types, such as Ip, Date, Number, etc. virtual Status from_string_batch(const ColumnString& str, ColumnNullable& column, diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp index d5779725272..5f65573b389 100644 --- a/be/src/format/jni/jni_data_bridge.cpp +++ b/be/src/format/jni/jni_data_bridge.cpp @@ -39,11 +39,36 @@ #include "core/data_type/primitive_type.h" #include "core/types.h" #include "core/value/decimalv2_value.h" +#include "core/value/jsonb_value.h" #include "util/string_util.h" #include "util/url_coding.h" namespace doris { +namespace { + +Status fill_jsonb_column(JniDataBridge::TableMetaAddress& address, MutableColumnPtr& column, + const bool* null_map, size_t num_rows) { + const auto* offsets = static_cast(address.next_meta_as_ptr()); + const auto* chars = static_cast(address.next_meta_as_ptr()); + JsonBinaryValue value; + size_t start = 0; + for (size_t i = 0; i < num_rows; ++i) { + // JNI carries JSON as UTF-8 text, not JSONB bytes. SQL NULL has no text to parse, + // whereas the non-null text "null" must become a JSON null value. + if (null_map[i]) { + column->insert_default(); + } else { + RETURN_IF_ERROR(value.from_json_string(chars + start, offsets[i] - start)); + column->insert_data(value.value(), value.size()); + } + start = offsets[i]; + } + return Status::OK(); +} + +} // namespace + #define FOR_FIXED_LENGTH_TYPES(M) \ M(PrimitiveType::TYPE_TINYINT, ColumnInt8, Int8) \ M(PrimitiveType::TYPE_BOOLEAN, ColumnUInt8, UInt8) \ @@ -139,6 +164,10 @@ Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_co case PrimitiveType::TYPE_VARCHAR: status = _fill_string_column(address, data_column, num_rows); break; + case PrimitiveType::TYPE_JSONB: + status = fill_jsonb_column(address, data_column, static_cast(null_map_ptr), + num_rows); + break; case PrimitiveType::TYPE_ARRAY: status = _fill_array_column(address, data_column, data_type, num_rows); break; diff --git a/be/test/core/data_type_serde/data_type_jsonb_serde_test.cpp b/be/test/core/data_type_serde/data_type_jsonb_serde_test.cpp index 176369da58f..2bacf10d233 100644 --- a/be/test/core/data_type_serde/data_type_jsonb_serde_test.cpp +++ b/be/test/core/data_type_serde/data_type_jsonb_serde_test.cpp @@ -37,9 +37,14 @@ #include "core/data_type/common_data_type_serder_test.h" #include "core/data_type/common_data_type_test.h" #include "core/data_type/data_type.h" +#include "core/data_type/data_type_array.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_jsonb.h" +#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/string_buffer.hpp" #include "core/types.h" #include "storage/olap_common.h" #include "testutil/test_util.h" @@ -321,4 +326,160 @@ TEST_F(DataTypeJsonbSerDeTest, FixedSizeBinaryReadColumnFromArrowWithNonZeroStar } } +TEST(DataTypeJsonbNestedSerDeTest, JsonValuesKeepTheirOwnQuotesAtEveryNestingLevel) { + const std::array values {R"({"a":1})", R"("x\"y")", "null", + "[]", "123.125", "true"}; + DataTypeSerDe::FormatOptions options; + for (int level : {1, 2, 3}) { + DataTypeJsonbSerDe serde(level); + for (const auto& text : values) { + auto column = ColumnString::create(); + Slice slice(text.data(), text.size()); + const auto status = serde.deserialize_one_cell_from_json(*column, slice, options); + ASSERT_TRUE(status.ok()) << status; + auto output = ColumnString::create(); + BufferWritable writer(*output); + serde.to_string(*column, 0, writer, options); + writer.commit(); + EXPECT_EQ(output->get_data_at(0).to_string(), text) << "nesting level " << level; + } + } +} + +TEST(DataTypeJsonbNestedSerDeTest, TypedComplexRoundTripPreservesJsonAndSqlNull) { + auto json = make_nullable(std::make_shared()); + auto string = make_nullable(std::make_shared()); + const std::array types { + std::make_shared(json), std::make_shared(string, json), + std::make_shared(DataTypes {json}, Strings {"payload"})}; + const std::array values {R"({"a":[1,"two"]})", + R"("text")", + R"("123")", + R"("null")", + R"("true")", + R"("x\"y\\z")", + "null", + "NULL", + "[]", + "123.125", + "true"}; + DataTypeSerDe::FormatOptions options; + for (size_t i = 0; i < types.size(); ++i) { + const auto serde = types[i]->get_serde(); + for (const auto& value : values) { + const auto text = i == 0 ? "[" + value + "]" + : i == 1 ? "{\"key\":" + value + "}" + : "{\"payload\":" + value + "}"; + for (bool strict : {false, true}) { + auto column = types[i]->create_column(); + StringRef input(text); + const auto status = strict ? serde->from_string_strict_mode(input, *column, options) + : serde->from_string(input, *column, options); + ASSERT_TRUE(status.ok()) << status << ": " << text << ", strict=" << strict; + auto output = ColumnString::create(); + BufferWritable writer(*output); + serde->to_string(*column, 0, writer, options); + writer.commit(); + EXPECT_EQ(output->get_data_at(0).to_string(), text) + << types[i]->get_name() << ", strict=" << strict; + } + } + } +} + +TEST(DataTypeJsonbNestedSerDeTest, MultipleNestingLevelsPreserveChildNullMarkers) { + auto json = make_nullable(std::make_shared()); + auto string = make_nullable(std::make_shared()); + auto array = make_nullable(std::make_shared(json)); + const std::array types { + std::make_shared(array), std::make_shared(string, array), + std::make_shared(DataTypes {array, string}, + Strings {"payload", "label"})}; + const std::string payload = + R"(["123", "null", "true", null, NULL, {"a":[1,"two"]}, [], 123.125, true, "x\"y\\z"])"; + const std::array texts { + "[" + payload + ", null, []]", + "{\"key\":" + payload + ", \"missing\":null, \"empty\":[]}", + "{\"payload\":" + payload + ", \"label\":\"null\"}"}; + DataTypeSerDe::FormatOptions options; + for (size_t i = 0; i < types.size(); ++i) { + const auto serde = types[i]->get_serde(); + for (bool strict : {false, true}) { + auto column = types[i]->create_column(); + StringRef input(texts[i]); + const auto status = strict ? serde->from_string_strict_mode(input, *column, options) + : serde->from_string(input, *column, options); + ASSERT_TRUE(status.ok()) << status << ", strict=" << strict; + auto output = ColumnString::create(); + BufferWritable writer(*output); + serde->to_string(*column, 0, writer, options); + writer.commit(); + EXPECT_EQ(output->get_data_at(0).to_string(), texts[i]) + << types[i]->get_name() << ", strict=" << strict; + } + } +} + +TEST(DataTypeJsonbNestedSerDeTest, InvalidChildRespectsStrictMode) { + auto json = make_nullable(std::make_shared()); + auto string = make_nullable(std::make_shared()); + const std::array types { + std::make_shared(json), std::make_shared(string, json), + std::make_shared(DataTypes {json}, Strings {"payload"})}; + const std::array inputs {"[invalid]", "{\"key\":invalid}", + "{\"payload\":invalid}"}; + const std::array outputs {"[NULL]", "{\"key\":NULL}", "{\"payload\":NULL}"}; + DataTypeSerDe::FormatOptions options; + for (size_t i = 0; i < types.size(); ++i) { + const auto serde = types[i]->get_serde(); + for (bool strict : {false, true}) { + auto column = types[i]->create_column(); + StringRef input(inputs[i]); + const auto status = strict ? serde->from_string_strict_mode(input, *column, options) + : serde->from_string(input, *column, options); + ASSERT_EQ(status.ok(), !strict) << status << ": " << types[i]->get_name(); + if (!strict) { + auto output = ColumnString::create(); + BufferWritable writer(*output); + serde->to_string(*column, 0, writer, options); + writer.commit(); + EXPECT_EQ(output->get_data_at(0).to_string(), outputs[i]) << types[i]->get_name(); + } + } + } +} + +TEST(DataTypeJsonbNestedSerDeTest, SingleQuotedChildrenPreserveJsonSyntax) { + auto json = make_nullable(std::make_shared()); + auto string = make_nullable(std::make_shared()); + const std::array types { + std::make_shared(json), std::make_shared(string, json), + std::make_shared(DataTypes {json}, Strings {"payload"})}; + const std::array values { + R"({"a":[1,"two"]})", R"("123")", R"("null")", R"("true")", + R"("x\"y\\z")", "null", "123.125", "true"}; + DataTypeSerDe::FormatOptions options; + for (size_t i = 0; i < types.size(); ++i) { + const auto serde = types[i]->get_serde(); + const std::string prefix = i == 0 ? "[" : i == 1 ? "{\"key\":" : "{\"payload\":"; + const std::string suffix = i == 0 ? "]" : "}"; + for (const auto& value : values) { + const auto text = prefix + "'" + value + "'" + suffix; + for (bool strict : {false, true}) { + auto column = types[i]->create_column(); + StringRef input(text); + const auto status = strict ? serde->from_string_strict_mode(input, *column, options) + : serde->from_string(input, *column, options); + ASSERT_TRUE(status.ok()) << status << ": " << text << ", strict=" << strict; + auto output = ColumnString::create(); + BufferWritable writer(*output); + serde->to_string(*column, 0, writer, options); + writer.commit(); + EXPECT_EQ(output->get_data_at(0).to_string(), prefix + value + suffix) + << types[i]->get_name() << ", strict=" << strict; + } + } + } +} + } // namespace doris diff --git a/be/test/format/jni/jni_data_bridge_test.cpp b/be/test/format/jni/jni_data_bridge_test.cpp index d6418c0b97e..d7112a5f3c9 100644 --- a/be/test/format/jni/jni_data_bridge_test.cpp +++ b/be/test/format/jni/jni_data_bridge_test.cpp @@ -23,9 +23,12 @@ #include #include "core/column/column_nullable.h" +#include "core/column/column_string.h" #include "core/column/variant_v2/column_variant_v2.h" +#include "core/data_type/data_type_jsonb.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_variant_v2.h" +#include "util/jsonb_utils.h" namespace doris { namespace { @@ -85,5 +88,58 @@ TEST(JniDataBridgeTest, FillNullableVariantV2FromEncodedPaimonLayout) { EXPECT_FALSE(variant_column.get_value_ref(2).get_bool()); } +TEST(JniDataBridgeTest, FillJsonPreservesSqlNullAndJsonNullAcrossBatches) { + std::array null_map {false, true, false, false, false}; + std::array offsets {7, 7, 11, 13, 19}; + std::string text = R"({"a":1}null[]"text")"; + std::array meta {address_of(null_map.data()), address_of(offsets.data()), + address_of(text.data())}; + DataTypePtr type = make_nullable(std::make_shared()); + ColumnPtr column = type->create_column(); + for (int batch = 0; batch < 2; ++batch) { + JniDataBridge::TableMetaAddress address(address_of(meta.data())); + const auto status = JniDataBridge::fill_column(address, column, type, null_map.size()); + ASSERT_TRUE(status.ok()) << status; + } + const auto& nullable = assert_cast(*column); + EXPECT_EQ(nullable.get_null_map_data(), NullMap({0, 1, 0, 0, 0, 0, 1, 0, 0, 0})); + const auto& values = assert_cast(nullable.get_nested_column()); + const std::array expected {R"({"a":1})", "", "null", "[]", R"("text")"}; + for (size_t i = 0; i < values.size(); ++i) { + auto value = values.get_data_at(i); + if (i % 5 == 1) { + EXPECT_EQ(value.size, 0); + } else { + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(value.data, value.size), expected[i % 5]); + } + } +} + +TEST(JniDataBridgeTest, FillJsonRejectsMalformedText) { + std::array null_map {false}; + std::array offsets {1}; + std::string text = "{"; + std::array meta {address_of(null_map.data()), address_of(offsets.data()), + address_of(text.data())}; + JniDataBridge::TableMetaAddress address(address_of(meta.data())); + DataTypePtr type = std::make_shared(); + ColumnPtr column = type->create_column(); + const auto status = JniDataBridge::fill_column(address, column, type, 1); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("parse"), std::string::npos) << status; +} + +TEST(JniDataBridgeTest, FillEmptyJsonBatchConsumesItsMetadata) { + bool null_map = false; + std::array meta {address_of(&null_map), 0, 0, 42}; + JniDataBridge::TableMetaAddress address(address_of(meta.data())); + DataTypePtr type = std::make_shared(); + ColumnPtr column = type->create_column(); + const auto status = JniDataBridge::fill_column(address, column, type, 0); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(column->size(), 0); + EXPECT_EQ(address.next_meta_as_long(), 42); +} + } // namespace } // namespace doris diff --git a/build.sh b/build.sh index 6a5044fdf62..5615b1423b5 100755 --- a/build.sh +++ b/build.sh @@ -1189,6 +1189,24 @@ function build_fe_modules() { return 1 } +function stamp_fe_build_version() { + local version_java="${DORIS_HOME}/fe/fe-common/target/generated-sources/build/org/apache/doris/common/Version.java" + local version_classes="${DORIS_HOME}/fe/fe-common/target/build-version" + local common_jar + + # A Maven cache hit can restore fe-common with another commit's Version.class. + # Stamp only the deployed jar so volatile build metadata does not invalidate the FE cache. + # Cached outputs can restore Version.java too, so regenerate after restoration. + rm -f "${version_java}" + bash "${DORIS_HOME}/gensrc/script/gen_build_version.sh" + mkdir -p "${version_classes}" + # env.sh also accepts Java 17 from PATH with JAVA_HOME unset; use the same selection here. + "${JAVA_HOME:+${JAVA_HOME}/bin/}javac" --release 17 -encoding UTF-8 -d "${version_classes}" "${version_java}" + for common_jar in "$1"/fe-common-*.jar; do + "${JAVA_HOME:+${JAVA_HOME}/bin/}jar" uf "${common_jar}" -C "${version_classes}" org/apache/doris/common/Version.class + done +} + # FE UI must be built before building FE if [[ "${BUILD_FE}" -eq 1 ]]; then if [[ "${BUILD_UI}" -eq 1 ]]; then @@ -1203,6 +1221,11 @@ if [[ "${FE_MODULES}" != '' ]]; then if [[ "${CLEAN}" -eq 1 ]]; then clean_fe fi + if [[ "${BUILD_FE}" -eq 1 ]]; then + # Consumers must compile against nonconstant fields before their bytecode is cached. + rm -f "${DORIS_HOME}/fe/fe-common/target/generated-sources/build/org/apache/doris/common/Version.java" + bash "${DORIS_HOME}/gensrc/script/gen_build_version.sh" + fi build_fe_modules cd "${DORIS_HOME}" fi @@ -1256,6 +1279,8 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then shopt -u nullglob done + stamp_fe_build_version "${DORIS_OUTPUT}/fe/lib" + #cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/ # Third-party filesystem jars (JuiceFS, JindoFS) are packaged by post-build.sh diff --git a/docker/thirdparties/docker-compose/mongodb/.gitignore b/docker/thirdparties/docker-compose/mongodb/.gitignore new file mode 100644 index 00000000000..2f489569f8e --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/.gitignore @@ -0,0 +1,2 @@ +/mongodb.yaml +/mongodb.env diff --git a/docker/thirdparties/docker-compose/mongodb/README.md b/docker/thirdparties/docker-compose/mongodb/README.md new file mode 100644 index 00000000000..11609a834ec --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/README.md @@ -0,0 +1,55 @@ +# MongoDB regression fixture + +This optional standalone MongoDB 8.0.20 service supplies deterministic BSON data for the embedded Trino 435 MongoDB connector. It does not require a Trino server. Start it with: + +```bash +CONTAINER_UID=doris-mongodb-test- bash docker/thirdparties/run-thirdparties-docker.sh -c mongodb +``` + +Set `MONGODB_PORT` to override the published port. Restarting this stack recreates the synthetic database `doris_mongodb_test`; do not point it at a production database. Both MongoDB data directories use disposable tmpfs mounts, so recreating or stopping the stack leaves no anonymous data volumes. The service authenticates with the fixture account `mongodb_test` / `mongodb_test` in `admin`. Readiness requires the final network-facing MongoDB process and the completed fixtures. + +Install the complete MongoDB plugin ZIP (`io.trino:trino-mongodb:435:zip`, published on Maven Central), or its directory from the Trino 435 server distribution in the operator-controlled Trino plugin directory on every FE and BE. Use the same connector version on both sides; copying only `trino-mongodb-435.jar` omits its runtime dependencies and service-provider metadata. Configure `trino_connector_plugin_dir` before starting Doris, and restart the processes after changing plugins. These suites use the preinstalled plugin and do not download or replace the other Trino fixtures. + +Add these values to `regression-test/conf/regression-conf-custom.groovy`, using the reachable Docker host and published port: + +```groovy +enableMongoDBTest = true +externalEnvIp = "127.0.0.1" +mongodb_port = 27017 +mongodb_user = "mongodb_test" +mongodb_password = "mongodb_test" +``` + +Run the suites with the normal runner: + +```bash +bash run-regression-test.sh --run -d external_table_p0/trino_connector/mongodb +``` + +| Suite | Coverage | +|---|---| +| `test_trino_mongodb_metadata` | Discovery, explicit and inferred schemas, hidden `_id`, custom schema collections, refresh, missing tables and unsupported types | +| `test_trino_mongodb_types` | ObjectId and ordinary Binary, numerical/date/string values, unicode, null/missing fields, nullable nested rows, arrays, mixed numeric types and empty collections | +| `test_trino_mongodb_queries` | Scalar filters, null logic, residual predicates with Limit, projections, sort/offset, aggregates, joins, subqueries, CTEs, unions, windows, array expansion, views, joins with internal tables and hexadecimal materialization | +| `test_trino_mongodb_batches` | 2,081 documents across Mongo cursor and Trino page boundaries, exact byte conversion, aggregates and repeated early termination | +| `test_trino_mongodb_json` | Native JSON values and functions, BSON documents, JSON null versus SQL NULL, nested JSON in arrays/maps/rows, multi-page reads, views and internal-table materialization | + +ObjectId maps to `VARBINARY(12)`, retaining the original bytes. Use `from_binary(_id)` for a deterministic hexadecimal string in filters, joins and grouping. Native scalar VARBINARY comparisons, grouping and join keys retain their existing Doris restrictions and are covered by negative tests. These suites do not claim arbitrary BSON-to-VARIANT support, CDC, writes, transactions, replica-set failover or parallel range splitting. + +Most collections have explicit `_schema` records; `inferred_records` intentionally exercises automatic initialization. The connector may write to `_schema` even for read queries. The invalid schema fixture intentionally declares an unsupported interval column to verify error reporting. Metadata refreshes must account for both Doris and connector caches; refresh tests here re-read an unchanged schema and do not claim immediate invalidation of the connector cache after external changes. + +## JSON columns + +Fields explicitly declared as `json` in the connector's `_schema` collection map to native Doris `JSON`. This also applies to JSON fields inside Trino `array`, `map` and `row` types. JNI transfers the connector's UTF-8 JSON text and parses it into Doris JSONB; SQL NULL and the JSON value `null` remain distinct. Scalar JSON supports JSON functions and internal-table materialization. Nested JSON can be queried through the catalog; internal-table creation retains the engine's existing complex-type restrictions, including rejecting `ARRAY`. + +Casting a complex value containing JSON to `STRING` preserves each JSON value's own quotes. JSON children use lowercase `null` for JSON null and uppercase `NULL` for SQL NULL, so a cast back to the original array, map or struct type preserves the distinction. This typed SQL text can contain `NULL` and is therefore not always a standalone JSON document. + +For example, the fixture declares `json_records.payload` as `json`. Its BSON document is queryable with: + +```sql +SELECT seq, json_extract_string(payload, '$.city') +FROM mongodb_catalog.doris_mongodb_test.json_records +WHERE json_extract_int(payload, '$.n') = 7; +``` + +Automatic schema inference still maps BSON documents to Trino `row` / Doris `STRUCT`. Use an explicit `json` declaration when a field should be read as a whole JSON value. BSON documents use the connector's Extended JSON representation, so embedded ObjectIds appear as `{"$oid":"..."}` objects. For BSON string values declared as `json`, their contents must be valid JSON text; malformed text fails the query. The JSON suite covers both real BSON documents and valid JSON text, plus a malformed-text negative case. diff --git a/docker/thirdparties/docker-compose/mongodb/init/01-fixtures.js b/docker/thirdparties/docker-compose/mongodb/init/01-fixtures.js new file mode 100644 index 00000000000..80607235c12 --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/init/01-fixtures.js @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const fixture = db.getSiblingDB("doris_mongodb_test"); +fixture.dropDatabase(); +const oid = n => ObjectId(n.toString(16).padStart(24, "0")); +const special = ObjectId("00ff807f0100020304050607"); +function schema(table, fields, collection = "_schema") { + fixture.getCollection(collection).insertOne({table, fields: [ + {name: "_id", type: "ObjectId", hidden: true}, + ...fields.map(([name, type]) => ({name, type, hidden: false})) + ]}); +} + +fixture.records.insertMany([ + {_id: oid(1), seq: NumberLong("1"), name: "alpha", grp: "east", active: true, score: NumberInt(10), + amount: NumberDecimal("123.45"), ratio: 1.25, created: ISODate("2024-01-01T00:00:00.001Z"), + owner: oid(101), blob: BinData(0, "AP+A"), tags: ["red", "blue"], numbers: [NumberLong("1"), NumberLong("2")], + profile: {city: "A", rank: NumberLong("1"), manager: special}, refs: [oid(101), special], optional_note: "first"}, + {_id: oid(2), seq: NumberLong("2"), name: "beta", grp: "west", active: false, score: NumberLong("20"), + amount: NumberDecimal("-2.50"), ratio: -0.5, created: ISODate("2024-02-01T12:34:56.789Z"), + owner: oid(102), blob: BinData(0, ""), tags: [], numbers: [], + profile: {city: "B", rank: NumberLong("2"), manager: oid(102)}, refs: [], optional_note: null}, + {_id: oid(3), seq: NumberLong("3"), name: "gamma", grp: "east", active: true, score: null, + amount: NumberDecimal("0.00"), ratio: 0.0, created: ISODate("1970-01-01T00:00:00.000Z"), + owner: null, blob: null, tags: ["red", null], numbers: [NumberLong("-1"), null], + profile: {city: "C", rank: null, manager: null}, refs: [null, oid(101)]}, + {_id: oid(4), seq: NumberLong("4"), name: "delta", grp: "west", active: false, + amount: NumberDecimal("99999.99"), ratio: 2.5, created: ISODate("2038-01-19T03:14:07.000Z"), + tags: null, numbers: null, profile: null, refs: null}, + {_id: oid(5), seq: NumberLong("5"), name: "epsilon", grp: "east", active: true, score: NumberLong("-5"), + amount: NumberDecimal("5.05"), ratio: 3.5, created: ISODate("2024-01-03T00:00:00.000Z"), + owner: oid(101), blob: BinData(0, "YWJj"), tags: ["green"], numbers: [NumberLong("9223372036854775807")], + profile: {city: "A", rank: NumberLong("5"), manager: oid(101)}, refs: [special]}, + {_id: ObjectId("ffffffffffffffffffffffff"), seq: NumberLong("6"), name: "中文-é", grp: "north", active: true, + score: NumberLong("0"), amount: NumberDecimal("1.00"), ratio: 4.5, created: null, + owner: special, blob: BinData(0, "/wA="), tags: ["unicode", "中文"], numbers: [NumberLong("0")], + profile: {city: "D", rank: NumberLong("6"), manager: special}, refs: [special, special]} +]); +schema("records", [["seq", "bigint"], ["name", "varchar"], ["grp", "varchar"], ["active", "boolean"], + ["score", "bigint"], ["amount", "decimal(18,2)"], ["ratio", "double"], ["created", "timestamp(3)"], + ["owner", "ObjectId"], ["blob", "varbinary"], ["tags", "array(varchar)"], ["numbers", "array(bigint)"], + ["profile", 'row("city" varchar, "rank" bigint, "manager" ObjectId)'], + ["refs", "array(ObjectId)"], ["optional_note", "varchar"]]); +schema("records", [["seq", "bigint"], ["name", "varchar"]], "_schema_alternate"); + +fixture.owners.insertMany([ + {_id: oid(101), label: "team-a", weight: NumberLong("10")}, + {_id: oid(102), label: "team-b", weight: NumberLong("20")}, + {_id: special, label: "team-c", weight: NumberLong("30")} +]); +schema("owners", [["label", "varchar"], ["weight", "bigint"]]); +fixture.createCollection("empty_records"); +schema("empty_records", [["seq", "bigint"], ["owner", "ObjectId"]]); +fixture.inferred_records.insertMany([{_id: oid(201), seq: NumberLong("1"), label: "auto"}, + {_id: oid(202), seq: NumberLong("2"), label: "inferred"}]); +fixture.mixed_values.insertMany([{_id: oid(301), seq: NumberLong("1"), value: NumberLong("7")}, + {_id: oid(302), seq: NumberLong("2"), value: "wrong-type"}, + {_id: oid(303), seq: NumberLong("3"), value: null}, {_id: oid(304), seq: NumberLong("4")}]); +schema("mixed_values", [["seq", "bigint"], ["value", "bigint"]]); +const bulk = []; +for (let i = 0; i < 2081; i++) { + bulk.push({_id: oid(1000 + i), seq: NumberLong(String(i)), bucket: NumberLong(String(i % 7)), + owner: i % 7 === 0 ? null : special, payload: "row-" + i, + json_payload: i % 7 === 0 ? null : {seq: i, text: "row-" + i, nested: {active: i % 2 === 0}}}); +} +fixture.large_records.insertMany(bulk); +schema("large_records", [["seq", "bigint"], ["bucket", "bigint"], ["owner", "ObjectId"], + ["payload", "varchar"], ["json_payload", "json"]]); +fixture.invalid_schema.insertOne({_id: oid(401), payload: "unsupported"}); +schema("invalid_schema", [["payload", "interval day to second"]]); + +fixture.json_records.insertMany([ + {_id: oid(601), seq: NumberLong("1"), + payload: {city: "北京", n: NumberInt(7), active: true, items: [1, "two", null], + nested: {key: "value"}, empty_object: {}, empty_array: [], owner: oid(101)}, + json_array: [{x: 1}, '"text"', "null", null, "[1,2]"], + json_row: {payload: {x: 9}}, json_map: {a: {x: 3}, b: "null", c: null}}, + {_id: oid(602), seq: NumberLong("2"), payload: '[1,"two",null,{"ok":true}]', + json_array: [], json_row: null, json_map: {}}, + {_id: oid(603), seq: NumberLong("3"), payload: JSON.stringify('中文-é"\\\n\u0000'), + json_array: null, json_row: {payload: null}, json_map: null}, + {_id: oid(604), seq: NumberLong("4"), payload: "true"}, + {_id: oid(605), seq: NumberLong("5"), payload: "-9223372036854775808", + json_array: ['"123"', '"null"', '"true"', JSON.stringify('x"y\\z\n中文'), "123", "true"], + json_row: {payload: '"123"'}, + json_map: {number_string: '"123"', null_string: '"null"', bool_string: '"true"', + escaped: JSON.stringify('x"y\\z\n中文')}}, + {_id: oid(606), seq: NumberLong("6"), payload: "123.125"}, + {_id: oid(607), seq: NumberLong("7"), payload: "null"}, + {_id: oid(608), seq: NumberLong("8"), payload: null}, + {_id: oid(609), seq: NumberLong("9")}, + {_id: oid(610), seq: NumberLong("10"), payload: {}}, + {_id: oid(611), seq: NumberLong("11"), payload: "[]"}, + {_id: oid(612), seq: NumberLong("12"), payload: [{x: 1}, {x: 2}]} +]); +schema("json_records", [["seq", "bigint"], ["payload", "json"], ["json_array", "array(json)"], + ["json_row", 'row("payload" json)'], ["json_map", "map(varchar,json)"]]); +fixture.createCollection("empty_json_records"); +schema("empty_json_records", [["seq", "bigint"], ["payload", "json"], ["json_array", "array(json)"]]); +fixture.invalid_json_records.insertOne({_id: oid(620), seq: NumberLong("1"), payload: "{"}); +schema("invalid_json_records", [["seq", "bigint"], ["payload", "json"]]); + +fixture.readonly_inferred_records.insertOne({_id: oid(501), seq: NumberLong("1")}); +db.getSiblingDB("admin").createUser({user: "mongodb_readonly", pwd: "mongodb_readonly", + roles: [{role: "read", db: "doris_mongodb_test"}]}); + +fixture.numeric_limits.insertMany([ + {_id: ObjectId("000000000000000000000000"), seq: NumberLong("1"), i32: NumberInt(2147483647), + i64: NumberLong("9223372036854775807"), amount: NumberDecimal("999999999999999999999999.1234567890"), + ratio: 1.0e100, text: "", created: ISODate("1969-12-31T23:59:59.999Z")}, + {_id: ObjectId("ffffffffffffffffffffffff"), seq: NumberLong("2"), i32: NumberInt(-2147483648), + i64: NumberLong("-9223372036854775808"), amount: NumberDecimal("-999999999999999999999999.1234567890"), + ratio: -1.0e100, text: "a\\b\"c", created: ISODate("1900-01-01T00:00:00.000Z")} +]); +schema("numeric_limits", [["seq", "bigint"], ["i32", "bigint"], ["i64", "bigint"], + ["amount", "decimal(38,10)"], ["ratio", "double"], ["text", "varchar"], ["created", "timestamp(3)"]]); diff --git a/docker/thirdparties/docker-compose/mongodb/mongodb.env.tpl b/docker/thirdparties/docker-compose/mongodb/mongodb.env.tpl new file mode 100644 index 00000000000..8c9c238b1fd --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/mongodb.env.tpl @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MONGODB_PORT=${MONGODB_PORT} diff --git a/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl b/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl new file mode 100644 index 00000000000..07f5145cd6f --- /dev/null +++ b/docker/thirdparties/docker-compose/mongodb/mongodb.yaml.tpl @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: doris--mongodb + +services: + mongodb: + image: mongo:8.0.20 + container_name: doris--mongodb + ports: + - "${MONGODB_PORT:-27017}:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: mongodb_test + MONGO_INITDB_ROOT_PASSWORD: mongodb_test + volumes: + - ./init:/docker-entrypoint-initdb.d:ro + # Override both image-declared volumes so recreating this disposable fixture leaves no data volumes. + tmpfs: + - /data/db:size=1g + - /data/configdb:size=64m + healthcheck: + test: + - CMD + - mongosh + - --quiet + # The init server binds only loopback; service DNS reaches only the final network-facing server. + - --host + - mongodb + - --username + - mongodb_test + - --password + - mongodb_test + - --authenticationDatabase + - admin + - --eval + - >- + const fixture = db.getSiblingDB('doris_mongodb_test'); + quit(db.adminCommand('ping').ok && fixture.large_records.countDocuments() === 2081 + && fixture.getCollection('_schema').countDocuments({table: 'numeric_limits'}) === 1 ? 0 : 1); + interval: 5s + timeout: 5s + retries: 30 + start_period: 10s diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 1c5ec1d5f83..cae2e383e95 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -76,12 +76,12 @@ Usage: $0 --hive-modules comma separated hive modules to refresh All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,kerberos,ranger,polaris,minio,fluss + mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,iceberg-rest,hudi,kafka,mariadb,db2,oceanbase,kerberos,ranger,polaris,minio,fluss,mongodb " exit 1 } DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,hudi,db2,oceanbase,kerberos,minio" -ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,ranger,polaris,fluss" +ALL_COMPONENTS="${DEFAULT_COMPONENTS},kafka,ranger,polaris,fluss,mongodb" COMPONENTS=$2 HELP=0 STOP=0 @@ -258,6 +258,7 @@ RUN_MINIO=0 RUN_RANGER=0 RUN_POLARIS=0 RUN_FLUSS=0 +RUN_MONGODB=0 RESERVED_PORTS="65535" @@ -279,6 +280,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RESERVED_PORTS="${RESERVED_PORTS},50070,50075" # namenode and datanode ports elif [[ "${element}"x == "hive3"x ]]; then RUN_HIVE3=1 + elif [[ "${element}"x == "mongodb"x ]]; then + RUN_MONGODB=1 elif [[ "${element}"x == "kafka"x ]]; then RUN_KAFKA=1 elif [[ "${element}"x == "iceberg"x ]]; then @@ -968,6 +971,18 @@ start_mysql() { "${ROOT}/docker-compose/mysql/data" } +start_mongodb() { + # sudo may clear the caller's environment; pass the selected port through the rendered env file. + export MONGODB_PORT="${MONGODB_PORT:-27017}" + envsubst '${MONGODB_PORT}' <"${ROOT}/docker-compose/mongodb/mongodb.env.tpl" \ + >"${ROOT}/docker-compose/mongodb/mongodb.env" + start_rendered_compose_stack "mongodb" \ + "${ROOT}/docker-compose/mongodb/mongodb.yaml.tpl" \ + "${ROOT}/docker-compose/mongodb/mongodb.yaml" \ + "${ROOT}/docker-compose/mongodb/mongodb.env" \ + -d --wait -- +} + start_pg() { # pg 14 start_rendered_compose_stack "pg" \ @@ -1891,6 +1906,10 @@ if [[ "${RUN_MYSQL}" -eq 1 ]]; then launch_component "mysql" "${LOG_ROOT}/start_mysql.log" start_mysql fi +if [[ "${RUN_MONGODB}" -eq 1 ]]; then + launch_component "mongodb" "${LOG_ROOT}/start_mongodb.log" start_mongodb +fi + if [[ "${RUN_PG}" -eq 1 ]]; then launch_component "pg" "${LOG_ROOT}/start_pg.log" start_pg fi diff --git a/fe/be-java-extensions/trino-connector-scanner/pom.xml b/fe/be-java-extensions/trino-connector-scanner/pom.xml index cc89bc0e91c..e64e532620d 100644 --- a/fe/be-java-extensions/trino-connector-scanner/pom.xml +++ b/fe/be-java-extensions/trino-connector-scanner/pom.xml @@ -287,6 +287,18 @@ under the License. log4j-1.2-api test + + io.trino + trino-mongodb + ${trino.version} + test + + + * + * + + + diff --git a/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/MongoObjectIdColumnValueTest.java b/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/MongoObjectIdColumnValueTest.java new file mode 100644 index 00000000000..2a12efe3f14 --- /dev/null +++ b/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/MongoObjectIdColumnValueTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.trinoconnector; + +import org.apache.doris.jni.spi.vec.ColumnType; +import org.apache.doris.jni.spi.vec.ColumnValue; + +import io.airlift.slice.Slices; +import io.trino.plugin.mongodb.ObjectIdType; +import io.trino.spi.block.ArrayBlockBuilder; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.DictionaryBlock; +import io.trino.spi.block.RowBlockBuilder; +import io.trino.spi.block.RunLengthEncodedBlock; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MongoObjectIdColumnValueTest { + private static final byte[] FIRST = {0, -1, -128, 127, 1, 0, 2, 3, 4, 5, 6, 7}; + private static final byte[] SECOND = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + + private static TrinoConnectorColumnValue value(Block block, Type type, String dorisType, int position) { + TrinoConnectorColumnValue value = new TrinoConnectorColumnValue(); + value.setBlock(block); + value.setTrinoType(type); + value.setColumnType(ColumnType.parseType("value", dorisType)); + value.setPosition(position); + return value; + } + + @Test + public void preservesEveryByteAndNullAcrossEncodedBlocks() { + BlockBuilder builder = ObjectIdType.OBJECT_ID.createBlockBuilder(null, 3); + ObjectIdType.OBJECT_ID.writeSlice(builder, Slices.wrappedBuffer(FIRST)); + builder.appendNull(); + ObjectIdType.OBJECT_ID.writeSlice(builder, Slices.wrappedBuffer(SECOND)); + Block block = builder.build(); + Assertions.assertArrayEquals(FIRST, value(block, ObjectIdType.OBJECT_ID, "varbinary(12)", 0).getBytes()); + Assertions.assertTrue(value(block, ObjectIdType.OBJECT_ID, "varbinary(12)", 1).isNull()); + Assertions.assertArrayEquals(SECOND, value(block, ObjectIdType.OBJECT_ID, "varbinary(12)", 2).getBytes()); + Block dictionary = DictionaryBlock.create(4, block, new int[] {2, 1, 0, 2}); + Assertions.assertArrayEquals(SECOND, + value(dictionary, ObjectIdType.OBJECT_ID, "varbinary(12)", 0).getBytes()); + Assertions.assertTrue(value(dictionary, ObjectIdType.OBJECT_ID, "varbinary(12)", 1).isNull()); + Assertions.assertArrayEquals(FIRST, + value(dictionary, ObjectIdType.OBJECT_ID, "varbinary(12)", 2).getBytes()); + Block repeated = RunLengthEncodedBlock.create(block.getSingleValueBlock(0), 3); + Assertions.assertArrayEquals(FIRST, + value(repeated, ObjectIdType.OBJECT_ID, "varbinary(12)", 2).getBytes()); + } + + @Test + public void preservesObjectIdsInsideNullableArrays() { + ArrayType type = new ArrayType(ObjectIdType.OBJECT_ID); + ArrayBlockBuilder builder = (ArrayBlockBuilder) type.createBlockBuilder(null, 3); + builder.buildEntry(elements -> { + ObjectIdType.OBJECT_ID.writeSlice(elements, Slices.wrappedBuffer(FIRST)); + elements.appendNull(); + ObjectIdType.OBJECT_ID.writeSlice(elements, Slices.wrappedBuffer(SECOND)); + }); + builder.appendNull(); + builder.buildEntry(elements -> ObjectIdType.OBJECT_ID.writeSlice(elements, Slices.wrappedBuffer(SECOND))); + Block block = builder.build(); + List elements = new ArrayList<>(); + value(block, type, "array", 0).unpackArray(elements); + Assertions.assertArrayEquals(FIRST, elements.get(0).getBytes()); + Assertions.assertTrue(elements.get(1).isNull()); + Assertions.assertArrayEquals(SECOND, elements.get(2).getBytes()); + elements.clear(); + value(block, type, "array", 2).unpackArray(elements); + Assertions.assertArrayEquals(SECOND, elements.get(0).getBytes()); + } + + @Test + public void preservesObjectIdInRowAfterNullRow() { + RowType type = RowType.rowType(RowType.field("owner", ObjectIdType.OBJECT_ID)); + RowBlockBuilder builder = (RowBlockBuilder) type.createBlockBuilder(null, 3); + builder.buildEntry(fields -> ObjectIdType.OBJECT_ID.writeSlice(fields.get(0), Slices.wrappedBuffer(FIRST))); + builder.appendNull(); + builder.buildEntry(fields -> ObjectIdType.OBJECT_ID.writeSlice(fields.get(0), Slices.wrappedBuffer(SECOND))); + Block block = builder.build(); + List fields = new ArrayList<>(); + value(block, type, "struct", 2).unpackStruct(Collections.singletonList(0), fields); + Assertions.assertArrayEquals(SECOND, fields.get(0).getBytes()); + } +} diff --git a/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/TrinoJsonColumnValueTest.java b/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/TrinoJsonColumnValueTest.java new file mode 100644 index 00000000000..362f1a6d0e8 --- /dev/null +++ b/fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/TrinoJsonColumnValueTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.trinoconnector; + +import org.apache.doris.jni.spi.vec.ColumnType; +import org.apache.doris.jni.spi.vec.ColumnValue; + +import io.airlift.slice.Slices; +import io.trino.spi.block.ArrayBlockBuilder; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.DictionaryBlock; +import io.trino.spi.block.RowBlockBuilder; +import io.trino.spi.block.RunLengthEncodedBlock; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import io.trino.type.JsonType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class TrinoJsonColumnValueTest { + private static final String OBJECT = "{\"text\":\"中文-é\\n\\u0000\",\"items\":[1,true,null]}"; + + private static TrinoConnectorColumnValue value(Block block, Type type, String carrier, int position) { + TrinoConnectorColumnValue value = new TrinoConnectorColumnValue(); + value.setBlock(block); + value.setTrinoType(type); + value.setColumnType(ColumnType.parseType("value", carrier)); + value.setPosition(position); + return value; + } + + @Test + public void preservesJsonUtf8AndDistinguishesSqlNullAcrossEncodedBlocks() { + BlockBuilder builder = JsonType.JSON.createBlockBuilder(null, 3); + JsonType.JSON.writeSlice(builder, Slices.utf8Slice(OBJECT)); + builder.appendNull(); + JsonType.JSON.writeSlice(builder, Slices.utf8Slice("null")); + Block block = builder.build(); + Assertions.assertEquals(OBJECT, value(block, JsonType.JSON, "string", 0).getString()); + Assertions.assertArrayEquals(OBJECT.getBytes(StandardCharsets.UTF_8), + value(block, JsonType.JSON, "string", 0).getStringAsBytes()); + Assertions.assertTrue(value(block, JsonType.JSON, "string", 1).isNull()); + Assertions.assertFalse(value(block, JsonType.JSON, "string", 2).isNull()); + Assertions.assertEquals("null", value(block, JsonType.JSON, "string", 2).getString()); + Block dictionary = DictionaryBlock.create(4, block, new int[] {2, 1, 0, 2}); + Assertions.assertEquals("null", value(dictionary, JsonType.JSON, "string", 0).getString()); + Assertions.assertTrue(value(dictionary, JsonType.JSON, "string", 1).isNull()); + Assertions.assertEquals(OBJECT, value(dictionary, JsonType.JSON, "string", 2).getString()); + Block repeated = RunLengthEncodedBlock.create(block.getSingleValueBlock(0), 3); + Assertions.assertEquals(OBJECT, value(repeated, JsonType.JSON, "string", 2).getString()); + } + + @Test + public void preservesJsonInsideNullableArray() { + ArrayType type = new ArrayType(JsonType.JSON); + ArrayBlockBuilder builder = (ArrayBlockBuilder) type.createBlockBuilder(null, 3); + builder.buildEntry(elements -> { + JsonType.JSON.writeSlice(elements, Slices.utf8Slice(OBJECT)); + elements.appendNull(); + JsonType.JSON.writeSlice(elements, Slices.utf8Slice("null")); + }); + builder.appendNull(); + builder.buildEntry(elements -> JsonType.JSON.writeSlice(elements, Slices.utf8Slice("[]"))); + Block block = builder.build(); + List elements = new ArrayList<>(); + value(block, type, "array", 0).unpackArray(elements); + Assertions.assertEquals(OBJECT, elements.get(0).getString()); + Assertions.assertTrue(elements.get(1).isNull()); + Assertions.assertFalse(elements.get(2).isNull()); + Assertions.assertEquals("null", elements.get(2).getString()); + elements.clear(); + value(block, type, "array", 2).unpackArray(elements); + Assertions.assertEquals("[]", elements.get(0).getString()); + } + + @Test + public void preservesJsonInRowAfterSqlNullRow() { + RowType type = RowType.rowType(RowType.field("payload", JsonType.JSON)); + RowBlockBuilder builder = (RowBlockBuilder) type.createBlockBuilder(null, 3); + builder.buildEntry(fields -> JsonType.JSON.writeSlice(fields.get(0), Slices.utf8Slice(OBJECT))); + builder.appendNull(); + builder.buildEntry(fields -> JsonType.JSON.writeSlice(fields.get(0), Slices.utf8Slice("null"))); + List fields = new ArrayList<>(); + value(builder.build(), type, "struct", 2).unpackStruct(Collections.singletonList(0), fields); + Assertions.assertFalse(fields.get(0).isNull()); + Assertions.assertEquals("null", fields.get(0).getString()); + } +} diff --git a/fe/fe-connector/fe-connector-trino/pom.xml b/fe/fe-connector/fe-connector-trino/pom.xml index 89c19b1a465..c1bacc3ab1d 100644 --- a/fe/fe-connector/fe-connector-trino/pom.xml +++ b/fe/fe-connector/fe-connector-trino/pom.xml @@ -99,6 +99,19 @@ under the License. log4j-api + + io.trino + trino-mongodb + ${trino.version} + test + + + * + * + + + + org.junit.jupiter junit-jupiter diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java index 04eb6798728..99398193ee0 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorDorisMetadata.java @@ -204,7 +204,10 @@ public class TrinoConnectorDorisMetadata implements ConnectorMetadata { for (ColumnHandle columnHandle : columnHandles.values()) { ColumnMetadata colMeta = metadata.getColumnMetadata( connSession, trinoHandle.getTrinoTableHandle(), columnHandle); - if (colMeta.isHidden()) { + // MongoDB hides its primary identifier by default; Doris has no hidden-column lookup path. + boolean mongoIdentifier = "_id".equals(colMeta.getName()) + && TrinoTypeMapping.isObjectIdType(colMeta.getType()); + if (colMeta.isHidden() && !mongoIdentifier) { continue; } ConnectorType connType = TrinoTypeMapping.toConnectorType(colMeta.getType()); diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoPluginManager.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoPluginManager.java index 7ebd4cdc270..d14f48f856d 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoPluginManager.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoPluginManager.java @@ -121,6 +121,8 @@ public class TrinoPluginManager implements PluginInstaller { } private void installPluginInternal(Plugin plugin) { + // Register logical types before a connector resolves its schema through the shared TypeManager. + plugin.getTypes().forEach(typeRegistry::addType); for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) { LOG.info("Registering connector {}", connectorFactory.getName()); ConnectorFactory existing = connectorFactories.putIfAbsent( diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoTypeMapping.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoTypeMapping.java index 322fcf5f0a5..3196c24afe9 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoTypeMapping.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoTypeMapping.java @@ -36,6 +36,7 @@ import io.trino.spi.type.TinyintType; import io.trino.spi.type.Type; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; +import io.trino.type.JsonType; import java.util.ArrayList; import java.util.List; @@ -74,11 +75,17 @@ public final class TrinoTypeMapping { return new ConnectorType("CHAR"); } else if (type instanceof VarcharType) { return new ConnectorType("STRING"); + } else if (type instanceof JsonType) { + // Keep JSON logical so JNI parses its UTF-8 carrier into Doris JSONB, including nested values. + return new ConnectorType("JSONB"); } else if (type instanceof io.trino.spi.type.UuidType) { // Preserve the logical type instead of exposing its physical 16-byte storage. return ConnectorType.of("UUID"); } else if (type instanceof VarbinaryType) { return new ConnectorType("VARBINARY"); + } else if (isObjectIdType(type)) { + // ObjectId is twelve raw bytes, including zero and non-UTF-8 bytes. + return new ConnectorType("VARBINARY", 12, -1); } else if (type instanceof DecimalType) { DecimalType decimal = (DecimalType) type; return new ConnectorType("DECIMALV3", decimal.getPrecision(), decimal.getScale()); @@ -119,4 +126,9 @@ public final class TrinoTypeMapping { throw new IllegalArgumentException("Cannot transform unknown Trino type: " + type); } } + + static boolean isObjectIdType(Type type) { + // The optional MongoDB plugin lives in its own classloader, so do not link its implementation class. + return "ObjectId".equalsIgnoreCase(type.getTypeSignature().getBase()); + } } diff --git a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/MongoObjectIdTest.java b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/MongoObjectIdTest.java new file mode 100644 index 00000000000..654d7a987d0 --- /dev/null +++ b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/MongoObjectIdTest.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.trino; + +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; +import org.apache.doris.connector.spi.pushdown.ConnectorComparison; +import org.apache.doris.connector.spi.pushdown.ConnectorIsNull; +import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; +import org.apache.doris.trinoconnector.TrinoConnectorPluginManager; + +import io.trino.FeaturesConfig; +import io.trino.metadata.HandleResolver; +import io.trino.metadata.TypeRegistry; +import io.trino.plugin.mongodb.MongoPlugin; +import io.trino.plugin.mongodb.ObjectIdType; +import io.trino.spi.connector.CatalogHandle; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.transaction.IsolationLevel; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.TypeOperators; +import io.trino.spi.type.VarcharType; +import io.trino.testing.TestingSession; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class MongoObjectIdTest { + @Test + public void registersPluginDefinedTypesBeforeReadingSchemas() { + TypeRegistry registry = new TypeRegistry(new TypeOperators(), new FeaturesConfig()); + TrinoConnectorPluginManager manager = new TrinoConnectorPluginManager( + (loader, factory) -> { }, registry, new HandleResolver()); + manager.installPlugin(new MongoPlugin()); + + Assertions.assertSame(ObjectIdType.OBJECT_ID, + Assertions.assertDoesNotThrow(() -> registry.fromSqlType("ObjectId"))); + Assertions.assertEquals(new ArrayType(ObjectIdType.OBJECT_ID), registry.fromSqlType("array(ObjectId)")); + } + + @Test + public void registersFrontendPluginDefinedTypes() { + TypeRegistry registry = new TypeRegistry(new TypeOperators(), new FeaturesConfig()); + TrinoPluginManager manager = new TrinoPluginManager( + (loader, factory) -> { }, registry, new HandleResolver()); + manager.installPlugin(new MongoPlugin()); + + Assertions.assertSame(ObjectIdType.OBJECT_ID, + Assertions.assertDoesNotThrow(() -> registry.fromSqlType("ObjectId"))); + } + + @Test + public void preservesObjectIdAsTwelveBytes() { + ConnectorType type = Assertions.assertDoesNotThrow( + () -> TrinoTypeMapping.toConnectorType(ObjectIdType.OBJECT_ID)); + Assertions.assertEquals("VARBINARY", type.getTypeName()); + Assertions.assertEquals(12, type.getPrecision()); + } + + @Test + public void preservesObjectIdsInsideArraysAndRows() { + ConnectorType type = Assertions.assertDoesNotThrow(() -> TrinoTypeMapping.toConnectorType(RowType.rowType( + RowType.field("owner", ObjectIdType.OBJECT_ID), + RowType.field("members", new ArrayType(ObjectIdType.OBJECT_ID))))); + Assertions.assertEquals("VARBINARY", type.getChildren().get(0).getTypeName()); + Assertions.assertEquals(12, type.getChildren().get(0).getPrecision()); + Assertions.assertEquals("VARBINARY", type.getChildren().get(1).getChildren().get(0).getTypeName()); + Assertions.assertEquals(12, type.getChildren().get(1).getChildren().get(0).getPrecision()); + } + + @Test + public void exposesOnlyTheMongoIdentifierAmongHiddenColumns() { + Map fields = new LinkedHashMap<>(); + fields.put(new ColumnHandle() { }, ColumnMetadata.builder().setName("_id") + .setType(ObjectIdType.OBJECT_ID).setHidden(true).build()); + fields.put(new ColumnHandle() { }, ColumnMetadata.builder().setName("hidden_ref") + .setType(ObjectIdType.OBJECT_ID).setHidden(true).build()); + fields.put(new ColumnHandle() { }, ColumnMetadata.builder().setName("$path") + .setType(VarcharType.VARCHAR).setHidden(true).build()); + fields.put(new ColumnHandle() { }, new ColumnMetadata("name", VarcharType.VARCHAR)); + Map handles = new LinkedHashMap<>(); + fields.forEach((handle, metadata) -> handles.put(metadata.getName(), handle)); + ConnectorMetadata source = new ConnectorMetadata() { + @Override + public ColumnMetadata getColumnMetadata(ConnectorSession session, + ConnectorTableHandle table, ColumnHandle column) { + return fields.get(column); + } + }; + Connector connector = new Connector() { + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolation, + boolean readOnly, boolean autoCommit) { + return new ConnectorTransactionHandle() { }; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) { + return source; + } + + @Override + public void commit(ConnectorTransactionHandle transaction) { + } + }; + TrinoConnectorDorisMetadata metadata = new TrinoConnectorDorisMetadata(connector, + TestingSession.testSessionBuilder().build(), + CatalogHandle.createRootCatalogHandle("mongodb", new CatalogHandle.CatalogVersion("test"))); + ConnectorTableSchema schema = metadata.getTableSchema(null, + new TrinoTableHandle("fixture", "records", null, handles, null)); + List names = schema.getColumns().stream().map(column -> column.getName()).collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList("_id", "name"), names); + } + + @Test + public void nullPredicateRetainsTheSourceObjectIdType() { + ColumnHandle handle = new ColumnHandle() { }; + TrinoPredicateConverter converter = new TrinoPredicateConverter(Collections.singletonMap("owner", handle), + Collections.singletonMap("owner", new ColumnMetadata("owner", ObjectIdType.OBJECT_ID))); + ConnectorColumnRef column = new ConnectorColumnRef("owner", ConnectorType.of("VARBINARY")); + Assertions.assertEquals(TupleDomain.withColumnDomains(Collections.singletonMap(handle, + Domain.onlyNull(ObjectIdType.OBJECT_ID))), converter.convert(new ConnectorIsNull(column, false))); + } + + @Test + public void textBoundsAreNotPushedAsBsonObjectIds() { + ColumnHandle handle = new ColumnHandle() { }; + TrinoPredicateConverter converter = new TrinoPredicateConverter(Collections.singletonMap("owner", handle), + Collections.singletonMap("owner", new ColumnMetadata("owner", ObjectIdType.OBJECT_ID))); + ConnectorColumnRef column = new ConnectorColumnRef("owner", ConnectorType.of("VARBINARY")); + Assertions.assertTrue(converter.convert(new ConnectorComparison(ConnectorComparison.Operator.EQ, + column, ConnectorLiteral.ofString("000000000000000000000001"))).isAll()); + } + +} diff --git a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoTypeMappingTest.java b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoTypeMappingTest.java index 8acdbaf9f01..f8d7c27811f 100644 --- a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoTypeMappingTest.java +++ b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoTypeMappingTest.java @@ -37,6 +37,7 @@ import io.trino.spi.type.TypeOperators; import io.trino.spi.type.UuidType; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; +import io.trino.type.JsonType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -168,4 +169,22 @@ public class TrinoTypeMappingTest { // A native UUID must not lose its logical type at the connector boundary. Assertions.assertEquals("UUID", name(UuidType.UUID)); } + + @Test + public void testJsonMapsToNativeJson() { + Assertions.assertEquals("JSONB", Assertions.assertDoesNotThrow(() -> name(JsonType.JSON))); + } + + @Test + public void testNestedJsonRetainsItsLogicalType() { + ConnectorType array = Assertions.assertDoesNotThrow( + () -> TrinoTypeMapping.toConnectorType(new ArrayType(JsonType.JSON))); + Assertions.assertEquals("JSONB", array.getChildren().get(0).getTypeName()); + ConnectorType map = TrinoTypeMapping.toConnectorType( + new MapType(VarcharType.VARCHAR, JsonType.JSON, new TypeOperators())); + Assertions.assertEquals("JSONB", map.getChildren().get(1).getTypeName()); + ConnectorType row = TrinoTypeMapping.toConnectorType(RowType.rowType(RowType.field("payload", JsonType.JSON))); + Assertions.assertEquals(Collections.singletonList("payload"), row.getFieldNames()); + Assertions.assertEquals("JSONB", row.getChildren().get(0).getTypeName()); + } } diff --git a/fe/fe-trino-connector-common/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginManager.java b/fe/fe-trino-connector-common/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginManager.java index 86ec5687471..c380867773d 100644 --- a/fe/fe-trino-connector-common/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginManager.java +++ b/fe/fe-trino-connector-common/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginManager.java @@ -119,6 +119,8 @@ public class TrinoConnectorPluginManager implements PluginInstaller { } private void installPluginInternal(Plugin plugin) { + // Connector schemas and serialized handles can refer to plugin-defined types such as ObjectId. + plugin.getTypes().forEach(typeRegistry::addType); for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) { LOG.info("Registering connector {}", connectorFactory.getName()); ConnectorFactory existingConnectorFactory = connectorFactories.putIfAbsent( diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index f5be09bc78e..ef393483117 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -146,14 +146,13 @@ package org.apache.doris.common; public class Version { - // Use String.valueOf() to prevent javac compile-time constant inlining. - // Without this, changing Version.class alone (e.g. via fast-compile-fe.sh) - // has no effect because the old values are baked into every referencing class. + // Runtime initializers keep build metadata out of cached consumers' constant pools. + // Refreshing Version.class during packaging must update both strings and version numbers. public static final String DORIS_BUILD_VERSION_PREFIX = String.valueOf("${build_version_prefix}"); - public static final int DORIS_BUILD_VERSION_MAJOR = ${build_version_major}; - public static final int DORIS_BUILD_VERSION_MINOR = ${build_version_minor}; - public static final int DORIS_BUILD_VERSION_PATCH = ${build_version_patch}; - public static final int DORIS_BUILD_VERSION_HOTFIX = ${build_version_hotfix}; + public static final int DORIS_BUILD_VERSION_MAJOR = Integer.parseInt("${build_version_major}"); + public static final int DORIS_BUILD_VERSION_MINOR = Integer.parseInt("${build_version_minor}"); + public static final int DORIS_BUILD_VERSION_PATCH = Integer.parseInt("${build_version_patch}"); + public static final int DORIS_BUILD_VERSION_HOTFIX = Integer.parseInt("${build_version_hotfix}"); public static final String DORIS_BUILD_VERSION_RC_VERSION = String.valueOf("${build_version_rc_version}"); public static final String DORIS_FEATURE_LIST = String.valueOf("${doris_feature_list}"); diff --git a/gensrc/script/test_gen_build_version.sh b/gensrc/script/test_gen_build_version.sh new file mode 100755 index 00000000000..2549b8c0c65 --- /dev/null +++ b/gensrc/script/test_gen_build_version.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +test_root="$(mktemp -d)" +trap 'rm -rf "${test_root}"' EXIT +export DORIS_HOME="${test_root}" +export JAVA_HOME="${JAVA_HOME:?JAVA_HOME must point to a JDK}" + +# A cached consumer must observe refreshed metadata without being recompiled. +DORIS_BUILD_VERSION_MAJOR=1 DORIS_BUILD_VERSION_MINOR=2 \ +DORIS_BUILD_VERSION_PATCH=3 DORIS_BUILD_VERSION_HOTFIX=4 \ + bash "${script_dir}/gen_build_version.sh" +version_java="${test_root}/fe/fe-common/target/generated-sources/build/org/apache/doris/common/Version.java" +cat >"${test_root}/VersionReader.java" <<'JAVA' +import org.apache.doris.common.Version; +public class VersionReader { + public static void main(String[] args) { + System.out.println(Version.DORIS_BUILD_VERSION_MAJOR + "." + Version.DORIS_BUILD_VERSION_MINOR + + "." + Version.DORIS_BUILD_VERSION_PATCH + "." + Version.DORIS_BUILD_VERSION_HOTFIX); + } +} +JAVA +"${JAVA_HOME}/bin/javac" --release 17 -d "${test_root}/classes" \ + "${version_java}" "${test_root}/VersionReader.java" +rm -f "${version_java}" +DORIS_BUILD_VERSION_MAJOR=5 DORIS_BUILD_VERSION_MINOR=6 \ +DORIS_BUILD_VERSION_PATCH=7 DORIS_BUILD_VERSION_HOTFIX=8 \ + bash "${script_dir}/gen_build_version.sh" +"${JAVA_HOME}/bin/javac" --release 17 -d "${test_root}/classes" "${version_java}" +actual="$("${JAVA_HOME}/bin/java" -cp "${test_root}/classes" VersionReader)" +if [[ "${actual}" != "5.6.7.8" ]]; then + echo "Cached version consumer retained ${actual}; expected refreshed metadata 5.6.7.8" >&2 + exit 1 +fi +echo "Cached version consumer observes refreshed numeric metadata" + +# Exercise the packaging function with a JDK selected only through PATH. +eval "$(sed -n '/^function stamp_fe_build_version() {$/,/^}$/p' "${script_dir}/../../build.sh")" +mkdir -p "${test_root}/gensrc/script" "${test_root}/lib" "${test_root}/jdk-bin" +cp "${script_dir}/gen_build_version.sh" "${test_root}/gensrc/script/" +"${JAVA_HOME}/bin/jar" cf "${test_root}/lib/fe-common-test.jar" -C "${test_root}/classes" org/apache/doris/common/Version.class +export VERSION_TEST_JDK_BIN="${JAVA_HOME}/bin" +export VERSION_TEST_TOOL_LOG="${test_root}/jdk-tools.log" +for tool in java javac jar; do + cat >"${test_root}/jdk-bin/${tool}" <<'TOOL' +#!/usr/bin/env bash +printf '%s\n' "${0##*/}" >>"${VERSION_TEST_TOOL_LOG}" +exec "${VERSION_TEST_JDK_BIN}/${0##*/}" "$@" +TOOL + chmod +x "${test_root}/jdk-bin/${tool}" +done +unset JAVA_HOME JDK_17 +export PATH="${test_root}/jdk-bin:${PATH}" +( + # Match build.sh, which permits unset optional environment variables. + set +u + DORIS_BUILD_VERSION_MAJOR=9 DORIS_BUILD_VERSION_MINOR=10 \ + DORIS_BUILD_VERSION_PATCH=11 DORIS_BUILD_VERSION_HOTFIX=12 \ + stamp_fe_build_version "${test_root}/lib" +) +for tool in javac jar; do + if ! grep -qx "${tool}" "${VERSION_TEST_TOOL_LOG}"; then + echo "FE version stamping bypassed the PATH-selected ${tool}" >&2 + exit 1 + fi +done +actual="$(java -cp "${test_root}/lib/fe-common-test.jar:${test_root}/classes" VersionReader)" +if [[ "${actual}" != "9.10.11.12" ]]; then + echo "Stamped FE jar retained ${actual}; expected refreshed metadata 9.10.11.12" >&2 + exit 1 +fi +echo "FE version stamping uses the PATH-selected JDK and refreshes the deployed jar" diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index cb0a3a94d6f..48cce7da714 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -282,6 +282,12 @@ trino_jdbc_port = 8080 // trino-connector catalog test config enableTrinoConnectorTest = false +// Optional MongoDB fixture: run-thirdparties-docker.sh -c mongodb. +enableMongoDBTest = false +mongodb_port = 27017 +mongodb_user = "mongodb_test" +mongodb_password = "mongodb_test" + enableKerberosTest=false kerberosHmsPort=9883 kerberosHdfsPort=8820 diff --git a/regression-test/data/doc/sql-manual/sql-function/test_array_function.out b/regression-test/data/doc/sql-manual/sql-function/test_array_function.out index 3d380e1474b..36dd3ce532e 100644 --- a/regression-test/data/doc/sql-manual/sql-function/test_array_function.out +++ b/regression-test/data/doc/sql-manual/sql-function/test_array_function.out @@ -178,7 +178,7 @@ false [{"key1":1}] -- !sql -- -["{"a":1}"] +[{"a":1}] -- !sql -- [{"b":"c"}] @@ -262,7 +262,7 @@ false [{"key1":1}, {"key1":1}, {"key1":1}] -- !sql -- -["{"a":1}", "{"a":1}", "{"a":1}"] +[{"a":1}, {"a":1}, {"a":1}] -- !sql -- [{"b":"c"}, {"b":"c"}, {"b":"c"}] @@ -271,7 +271,7 @@ false [{"key1":1}] -- !sql -- -["{"a":1}"] +[{"a":1}] -- !sql -- [{"b":"c"}] diff --git a/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.out b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.out new file mode 100644 index 00000000000..fd6992d0561 --- /dev/null +++ b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.out @@ -0,0 +1,61 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !multi_page_count -- +2081 1783 0 2080 2164240 + +-- !multi_page_groups -- +0 298 0 2079 0 +1 298 1 2080 298 +2 297 2 2074 297 +3 297 3 2075 297 +4 297 4 2076 297 +5 297 5 2077 297 +6 297 6 2078 297 + +-- !batch_boundaries -- +0 0000000000000000000003E8 \N row-0 +1023 0000000000000000000007E7 00FF807F0100020304050607 row-1023 +1024 0000000000000000000007E8 00FF807F0100020304050607 row-1024 +1025 0000000000000000000007E9 00FF807F0100020304050607 row-1025 +126 000000000000000000000466 \N row-126 +127 000000000000000000000467 00FF807F0100020304050607 row-127 +128 000000000000000000000468 00FF807F0100020304050607 row-128 +2047 000000000000000000000BE7 00FF807F0100020304050607 row-2047 +2048 000000000000000000000BE8 00FF807F0100020304050607 row-2048 +2080 000000000000000000000C08 00FF807F0100020304050607 row-2080 + +-- !multi_page_bytes -- +1783 + +-- !multi_page_null -- +298 + +-- !multi_page_offset -- +1022 0000000000000000000007E6 +1023 0000000000000000000007E7 +1024 0000000000000000000007E8 +1025 0000000000000000000007E9 +1026 0000000000000000000007EA + +-- !multi_page_aggregate -- +24972 2081 + +-- !multi_page_empty -- + +-- !repeated_limit_0 -- +0 0000000000000000000003E8 +1 0000000000000000000003E9 +2 0000000000000000000003EA + +-- !repeated_limit_1 -- +0 0000000000000000000003E8 +1 0000000000000000000003E9 +2 0000000000000000000003EA + +-- !repeated_limit_2 -- +0 0000000000000000000003E8 +1 0000000000000000000003E9 +2 0000000000000000000003EA + +-- !count_after_limits -- +2081 + diff --git a/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.out b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.out new file mode 100644 index 00000000000..2539b3db2f9 --- /dev/null +++ b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.out @@ -0,0 +1,262 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !schema -- +_id varbinary(12) Yes true \N +json_array array Yes true \N +json_map map Yes true \N +json_row struct Yes true \N +payload json Yes true \N +seq bigint Yes true \N + +-- !all_columns -- +0x000000000000000000000259 1 {"active":true,"city":"北京","empty_array":[],"empty_object":{},"items":[1,"two",null],"n":7,"nested":{"key":"value"},"owner":{"$oid":"000000000000000000000065"}} [{"x":1}, "text", null, NULL, [1,2]] {"payload":{"x":9}} {"a":{"x":3}, "b":null, "c":NULL} +0x00000000000000000000025A 2 [1,"two",null,{"ok":true}] [] \N {} +0x00000000000000000000025B 3 "中文-é\\"\\\\\\n\\u0000" \N {"payload":NULL} \N +0x00000000000000000000025C 4 true \N \N \N +0x00000000000000000000025D 5 -9223372036854775808 ["123", "null", "true", "x\\"y\\\\z\\n中文", 123, true] {"payload":"123"} {"number_string":"123", "null_string":"null", "bool_string":"true", "escaped":"x\\"y\\\\z\\n中文"} +0x00000000000000000000025E 6 123.125 \N \N \N +0x00000000000000000000025F 7 null \N \N \N +0x000000000000000000000260 8 \N \N \N \N +0x000000000000000000000261 9 \N \N \N \N +0x000000000000000000000262 10 {} \N \N \N +0x000000000000000000000263 11 [] \N \N \N +0x000000000000000000000264 12 [{"x":1},{"x":2}] \N \N \N + +-- !shapes -- +1 object false +10 object false +11 array false +12 array false +2 array false +3 string false +4 bool false +5 bigint false +6 double false +7 null false +8 \N true +9 \N true + +-- !object -- +北京 7 true value 000000000000000000000065 + +-- !array -- +2 4 two true + +-- !unicode -- +3 E4B8ADE696872DC3A9225C0A00 + +-- !scalars -- +4 true bool +5 -9223372036854775808 bigint +6 123.125 double +7 \N null + +-- !sql_null -- +8 +9 + +-- !json_null -- +7 + +-- !not_null -- +1 +10 +11 +12 +2 +3 +4 +5 +6 +7 + +-- !empty_values -- +10 0 +11 0 +12 2 + +-- !filter -- +1 + +-- !residual_limit -- +2 +11 + +-- !nested_array -- +1 [{"x":1}, "text", null, NULL, [1,2]] object 1 null true +2 [] \N \N \N true +3 \N \N \N \N true +4 \N \N \N \N true + +-- !nested_row -- +1 {"payload":{"x":9}} 9 false false +2 \N \N true true +3 {"payload":NULL} \N false true +4 \N \N true true + +-- !nested_map -- +1 {"a":{"x":3}, "b":null, "c":NULL} 3 null true +2 {} \N \N true +3 \N \N \N true +4 \N \N \N true + +-- !empty_nested_projection -- +2 [] {} + +-- !null_nested_projection -- +3 \N {"payload":NULL} \N +4 \N \N \N + +-- !serialization_round_trip -- +5 string 123 123 true + +-- !typed_array_round_trip_false -- +1 [{"x":1}, "text", null, NULL, [1,2]] string null false true +2 [] \N \N true true +3 \N \N \N true true +4 \N \N \N true true +5 ["123", "null", "true", "x\\"y\\\\z\\n中文", 123, true] string string false false + +-- !typed_map_round_trip_false -- +1 {"a":{"x":3}, "b":null, "c":NULL} null false true \N \N \N \N +2 {} \N true true \N \N \N \N +3 \N \N true true \N \N \N \N +4 \N \N true true \N \N \N \N +5 {"number_string":"123", "null_string":"null", "bool_string":"true", "escaped":"x\\"y\\\\z\\n中文"} \N true true string string string 7822795C7A0AE4B8ADE69687 + +-- !typed_struct_round_trip_false -- +1 {"payload":{"x":9}} object false false +2 \N \N true true +3 {"payload":NULL} \N false true +4 \N \N true true +5 {"payload":"123"} string false false + +-- !typed_array_round_trip_true -- +1 [{"x":1}, "text", null, NULL, [1,2]] string null false true +2 [] \N \N true true +3 \N \N \N true true +4 \N \N \N true true +5 ["123", "null", "true", "x\\"y\\\\z\\n中文", 123, true] string string false false + +-- !typed_map_round_trip_true -- +1 {"a":{"x":3}, "b":null, "c":NULL} null false true \N \N \N \N +2 {} \N true true \N \N \N \N +3 \N \N true true \N \N \N \N +4 \N \N true true \N \N \N \N +5 {"number_string":"123", "null_string":"null", "bool_string":"true", "escaped":"x\\"y\\\\z\\n中文"} \N true true string string string 7822795C7A0AE4B8ADE69687 + +-- !typed_struct_round_trip_true -- +1 {"payload":{"x":9}} object false false +2 \N \N true true +3 {"payload":NULL} \N false true +4 \N \N true true +5 {"payload":"123"} string false false + +-- !explode -- +1 \N \N +1 array [1,2] +1 null \N +1 object {"x":1} +1 string text +5 bool true +5 int 123 +5 string 123 +5 string null +5 string true +5 string x"y\\z\n中文 + +-- !aggregate -- +\N 2 +array 3 +bigint 1 +bool 1 +double 1 +null 1 +object 2 +string 1 + +-- !join -- +1 1 + +-- !empty_schema -- +_id varbinary(12) Yes true \N +json_array array Yes true \N +payload json Yes true \N +seq bigint Yes true \N + +-- !empty_projection -- + +-- !empty_aggregate -- +0 0 + +-- !multi_page -- +2081 1783 1854469 + +-- !batch_boundaries -- +0 \N true +1023 {"nested":{"active":false},"seq":1023,"text":"row-1023"} false +1024 {"nested":{"active":true},"seq":1024,"text":"row-1024"} false +1025 {"nested":{"active":false},"seq":1025,"text":"row-1025"} false +126 \N true +127 {"nested":{"active":false},"seq":127,"text":"row-127"} false +128 {"nested":{"active":true},"seq":128,"text":"row-128"} false +2047 {"nested":{"active":false},"seq":2047,"text":"row-2047"} false +2048 {"nested":{"active":true},"seq":2048,"text":"row-2048"} false +2080 {"nested":{"active":true},"seq":2080,"text":"row-2080"} false + +-- !multi_page_offset -- +1022 \N +1023 row-1023 +1024 row-1024 +1025 row-1025 +1026 row-1026 + +-- !view_schema -- +payload json Yes false \N +seq bigint Yes false \N + +-- !view_data -- +1 {"active":true,"city":"北京","empty_array":[],"empty_object":{},"items":[1,"two",null],"n":7,"nested":{"key":"value"},"owner":{"$oid":"000000000000000000000065"}} +10 {} +11 [] +12 [{"x":1},{"x":2}] +2 [1,"two",null,{"ok":true}] +3 "中文-é\\"\\\\\\n\\u0000" +4 true +5 -9223372036854775808 +6 123.125 +7 null +8 \N +9 \N + +-- !materialized_schema -- +payload json Yes false \N NONE +seq bigint Yes true \N + +-- !materialized_data -- +1 {"active":true,"city":"北京","empty_array":[],"empty_object":{},"items":[1,"two",null],"n":7,"nested":{"key":"value"},"owner":{"$oid":"000000000000000000000065"}} +10 {} +11 [] +12 [{"x":1},{"x":2}] +2 [1,"two",null,{"ok":true}] +3 "中文-é\\"\\\\\\n\\u0000" +4 true +5 -9223372036854775808 +6 123.125 +7 null +8 \N +9 \N + +-- !materialized_functions -- +1 object 北京 +10 object \N +11 array \N +12 array \N +2 array \N +3 string \N +4 bool \N +5 bigint \N +6 double \N +7 null \N +8 \N \N +9 \N \N + diff --git a/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.out b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.out new file mode 100644 index 00000000000..85b19dd82f8 --- /dev/null +++ b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.out @@ -0,0 +1,93 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !databases -- +doris_mongodb_test +information_schema +mysql + +-- !tables -- +_schema_alternate +empty_json_records +empty_records +inferred_records +invalid_json_records +invalid_schema +json_records +large_records +mixed_values +numeric_limits +owners +readonly_inferred_records +records + +-- !inferred_schema -- +_id varbinary(12) Yes true \N +label text Yes true \N +seq bigint Yes true \N + +-- !inferred_data -- +0000000000000000000000C9 1 auto +0000000000000000000000CA 2 inferred + +-- !explicit_schema -- +_id varbinary(12) Yes true \N +label text Yes true \N +weight bigint Yes true \N + +-- !explicit_data -- +000000000000000000000065 team-a 10 +000000000000000000000066 team-b 20 +00FF807F0100020304050607 team-c 30 + +-- !refresh_before -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !refresh_table -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !refresh_database -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !refresh_catalog -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !alternate_schema -- +_id varbinary(12) Yes true \N +name text Yes true \N +seq bigint Yes true \N + +-- !alternate_data -- +000000000000000000000001 1 alpha +000000000000000000000002 2 beta +000000000000000000000003 3 gamma +000000000000000000000004 4 delta +000000000000000000000005 5 epsilon +FFFFFFFFFFFFFFFFFFFFFFFF 6 中文-é + +-- !readonly_explicit_schema -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + diff --git a/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.out b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.out new file mode 100644 index 00000000000..cad17e82392 --- /dev/null +++ b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.out @@ -0,0 +1,277 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !count -- +6 4 4 3 + +-- !numeric_filter -- +2 +3 +4 +5 + +-- !boolean_filter -- +1 +3 +5 +6 + +-- !string_filter -- +1 alpha +2 beta + +-- !in_filter -- +1 +3 +6 + +-- !not_in_filter -- +2 +4 +5 + +-- !between_filter -- +1 10 +5 -5 +6 0 + +-- !or_filter -- +1 +4 +6 + +-- !compound_filter -- +1 +4 + +-- !null_logic -- +3 +4 +5 +6 + +-- !null_safe_equal -- +3 +4 + +-- !like_filter -- +1 +2 +4 + +-- !residual_with_limit -- +1 +3 + +-- !cast_objectid_filter -- +1 000000000000000000000001 + +-- !cast_objectid_range -- +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !cast_objectid_in -- +1 +5 +6 + +-- !case_expression -- +1 000000000000000000000065 +2 000000000000000000000066 +3 missing +4 missing +5 000000000000000000000065 +6 00FF807F0100020304050607 + +-- !coalesce -- +1 10 first +2 20 none +3 -99 none +4 -99 none +5 -5 none +6 0 none + +-- !arithmetic_projection -- +1 11 61.725000 +2 22 -1.250000 +3 \N 0.000000 +4 \N 49999.995000 +5 0 2.525000 +6 6 0.500000 + +-- !distinct -- +east +north +west + +-- !group_by -- +east 3 2 5 1 5 128.50 +north 1 1 0 6 6 1.00 +west 2 1 20 2 4 99997.49 + +-- !having -- +east 3 +west 2 + +-- !objectid_group -- +\N 2 +000000000000000000000065 2 +000000000000000000000066 1 +00FF807F0100020304050607 1 + +-- !topn -- +2 20 +1 10 +6 0 + +-- !offset -- +3 +4 + +-- !limit_zero -- + +-- !no_matches -- + +-- !join_numeric -- +1 1 +2 2 +3 3 + +-- !join_objectid -- +1 team-a +2 team-b +5 team-a +6 team-c + +-- !left_join_objectid -- +1 team-a +2 team-b +3 \N +4 \N +5 team-a +6 team-c + +-- !full_join -- +\N team-a +\N team-b +\N team-c +1 \N +2 \N +3 \N +4 \N +5 \N +6 \N + +-- !semi_join -- +1 +2 +5 +6 + +-- !anti_join -- +3 +4 + +-- !cross_join -- +1 team-a +1 team-b +1 team-c + +-- !exists -- +1 +2 +5 +6 + +-- !not_exists -- +3 +4 + +-- !in_subquery -- +1 +2 +5 + +-- !scalar_subquery -- +4 +5 +6 + +-- !cte -- +east 3 +west 2 + +-- !union_all -- +1 000000000000000000000001 +2 000000000000000000000002 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !union_distinct -- +east +north +west + +-- !window -- +1 east 1 9 +2 west 1 6 +3 east 2 9 +4 west 2 6 +5 east 3 9 +6 north 1 6 + +-- !array_unnest -- +1 blue +1 red +3 \N +3 red +5 green +6 unicode +6 中文 + +-- !objectid_array_unnest -- +1 000000000000000000000065 +1 00FF807F0100020304050607 +3 \N +3 000000000000000000000065 +5 00FF807F0100020304050607 +6 00FF807F0100020304050607 +6 00FF807F0100020304050607 + +-- !null_filtered_limit -- +3 + +-- !view_schema -- +_id varbinary(12) Yes false \N +owner varbinary(12) Yes false \N +seq bigint Yes false \N + +-- !view_data -- +1 000000000000000000000001 000000000000000000000065 +2 000000000000000000000002 000000000000000000000066 +3 000000000000000000000003 \N +4 000000000000000000000004 \N +5 000000000000000000000005 000000000000000000000065 +6 FFFFFFFFFFFFFFFFFFFFFFFF 00FF807F0100020304050607 + +-- !materialized_data -- +1 000000000000000000000001 000000000000000000000065 east +2 000000000000000000000002 000000000000000000000066 west +3 000000000000000000000003 \N east +4 000000000000000000000004 \N west +5 000000000000000000000005 000000000000000000000065 east +6 FFFFFFFFFFFFFFFFFFFFFFFF 00FF807F0100020304050607 north + +-- !internal_join -- +1 000000000000000000000001 +2 000000000000000000000002 +3 000000000000000000000003 +4 000000000000000000000004 +5 000000000000000000000005 +6 FFFFFFFFFFFFFFFFFFFFFFFF + +-- !internal_subquery -- +1 +2 +3 + diff --git a/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.out b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.out new file mode 100644 index 00000000000..9dc1dfbf3a2 --- /dev/null +++ b/regression-test/data/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.out @@ -0,0 +1,156 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !schema -- +_id varbinary(12) Yes true \N +active boolean Yes true \N +amount decimal(18,2) Yes true \N +blob varbinary(2147483647) Yes true \N +created datetime(3) Yes true \N +grp text Yes true \N +name text Yes true \N +numbers array Yes true \N +optional_note text Yes true \N +owner varbinary(12) Yes true \N +profile struct Yes true \N +ratio double Yes true \N +refs array Yes true \N +score bigint Yes true \N +seq bigint Yes true \N +tags array Yes true \N + +-- !all_columns -- +0x000000000000000000000001 1 alpha east true 10 123.45 1.25 2024-01-01T00:00:00.001 0x000000000000000000000065 0x00FF80 ["red", "blue"] [1, 2] {"city":"A", "rank":1, "manager":0x00FF807F0100020304050607} [0x000000000000000000000065, 0x00FF807F0100020304050607] first +0x000000000000000000000002 2 beta west false 20 -2.50 -0.5 2024-02-01T12:34:56.789 0x000000000000000000000066 0x [] [] {"city":"B", "rank":2, "manager":0x000000000000000000000066} [] \N +0x000000000000000000000003 3 gamma east true \N 0.00 0 1970-01-01T00:00 \N \N ["red", null] [-1, null] {"city":"C", "rank":null, "manager":null} [null, 0x000000000000000000000065] \N +0x000000000000000000000004 4 delta west false \N 99999.99 2.5 2038-01-19T03:14:07 \N \N \N \N \N \N \N +0x000000000000000000000005 5 epsilon east true -5 5.05 3.5 2024-01-03T00:00 0x000000000000000000000065 0x616263 ["green"] [9223372036854775807] {"city":"A", "rank":5, "manager":0x000000000000000000000065} [0x00FF807F0100020304050607] \N +0xFFFFFFFFFFFFFFFFFFFFFFFF 6 中文-é north true 0 1.00 4.5 \N 0x00FF807F0100020304050607 0xFF00 ["unicode", "中文"] [0] {"city":"D", "rank":6, "manager":0x00FF807F0100020304050607} [0x00FF807F0100020304050607, 0x00FF807F0100020304050607] \N + +-- !objectids -- +1 000000000000000000000001 000000000000000000000065 12 +2 000000000000000000000002 000000000000000000000066 12 +3 000000000000000000000003 \N 12 +4 000000000000000000000004 \N 12 +5 000000000000000000000005 000000000000000000000065 12 +6 FFFFFFFFFFFFFFFFFFFFFFFF 00FF807F0100020304050607 12 + +-- !ordinary_binary -- +1 00FF80 3 +2 0 +3 \N \N +4 \N \N +5 616263 3 +6 FF00 2 + +-- !primitive_values -- +1 alpha east true 10 123.45 1.25 2024-01-01T00:00:00.001 first +2 beta west false 20 -2.50 -0.5 2024-02-01T12:34:56.789 \N +3 gamma east true \N 0.00 0 1970-01-01T00:00 \N +4 delta west false \N 99999.99 2.5 2038-01-19T03:14:07 \N +5 epsilon east true -5 5.05 3.5 2024-01-03T00:00 \N +6 中文-é north true 0 1.00 4.5 \N \N + +-- !null_missing -- +1 false false false +2 false false true +3 true true true +4 true true true +5 false false true +6 false false true + +-- !objectid_null -- +3 \N +4 \N + +-- !objectid_not_null -- +1 000000000000000000000065 +2 000000000000000000000066 +5 000000000000000000000065 +6 00FF807F0100020304050607 + +-- !arrays -- +1 ["red", "blue"] [1, 2] [0x000000000000000000000065, 0x00FF807F0100020304050607] +2 [] [] [] +3 ["red", null] [-1, null] [null, 0x000000000000000000000065] +4 \N \N \N +5 ["green"] [9223372036854775807] [0x00FF807F0100020304050607] +6 ["unicode", "中文"] [0] [0x00FF807F0100020304050607, 0x00FF807F0100020304050607] + +-- !objectid_array_elements -- +1 000000000000000000000065 00FF807F0100020304050607 +2 \N \N +3 \N 000000000000000000000065 +4 \N \N +5 00FF807F0100020304050607 \N +6 00FF807F0100020304050607 00FF807F0100020304050607 + +-- !array_operations -- +1 2 2 true +2 0 0 false +3 2 2 true +4 \N \N \N +5 1 1 false +6 2 2 false + +-- !nested_row -- +1 A 1 00FF807F0100020304050607 +2 B 2 000000000000000000000066 +3 C \N \N +4 \N \N \N +5 A 5 000000000000000000000065 +6 D 6 00FF807F0100020304050607 + +-- !nested_projection_filter -- +1 A +5 A + +-- !nullable_row -- +3 {"city":"C", "rank":null, "manager":null} +4 \N +5 {"city":"A", "rank":5, "manager":0x000000000000000000000065} +6 {"city":"D", "rank":6, "manager":0x00FF807F0100020304050607} + +-- !nested_objectid_null -- +3 +4 + +-- !unicode -- +6 中文-é ["unicode", "中文"] + +-- !decimal_arithmetic -- +1 123.45 246.90 1.25 +2 -2.50 -5.00 -0.5 +3 0.00 0.00 0 +4 99999.99 199999.98 2.5 +5 5.05 10.10 3.5 +6 1.00 2.00 4.5 + +-- !dates -- +1 2024-01-01T00:00:00.001 2024 1 +2 2024-02-01T12:34:56.789 2024 2 +4 2038-01-19T03:14:07 2038 1 +5 2024-01-03T00:00 2024 1 + +-- !mixed_numeric_types -- +1 7 false +2 \N true +3 \N true +4 \N true + +-- !empty_schema -- +_id varbinary(12) Yes true \N +owner varbinary(12) Yes true \N +seq bigint Yes true \N + +-- !empty_projection -- + +-- !empty_aggregate -- +0 0 \N \N + +-- !numeric_limits -- +1 000000000000000000000000 2147483647 9223372036854775807 999999999999999999999999.1234567890 1e+100 1969-12-31T23:59:59.999 +2 FFFFFFFFFFFFFFFFFFFFFFFF -2147483648 -9223372036854775808 -999999999999999999999999.1234567890 -1e+100 a\\b"c 1900-01-01T00:00 + +-- !numeric_limit_predicates -- +1 +2 + diff --git a/regression-test/data/external_table_p2/es/test_es_query.out b/regression-test/data/external_table_p2/es/test_es_query.out index b80b954e165..f1ba3d9d316 100644 --- a/regression-test/data/external_table_p2/es/test_es_query.out +++ b/regression-test/data/external_table_p2/es/test_es_query.out @@ -115,15 +115,15 @@ text_ignore_above_10 2022-08-08T20:10:10 -- !sql_5_27 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_5_28 -- value1 value2 @@ -269,15 +269,15 @@ text_ignore_above_10 2022-08-08T20:10:10 -- !sql_6_27 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_6_28 -- value1 value2 @@ -465,15 +465,15 @@ text_ignore_above_10 1660191010000 -- !sql_7_34 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_7_35 -- string1 text#1 @@ -669,15 +669,15 @@ text_ignore_above_10 1660191010000 -- !sql_8_32 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_8_33 -- string1 text#1 @@ -825,15 +825,15 @@ text_ignore_above_10 2022-08-08T20:10:10 -- !sql_5_27 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_5_28 -- value1 value2 @@ -979,15 +979,15 @@ text_ignore_above_10 2022-08-08T20:10:10 -- !sql_6_27 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_6_28 -- value1 value2 @@ -1175,15 +1175,15 @@ text_ignore_above_10 1660191010000 -- !sql_7_34 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_7_35 -- string1 text#1 @@ -1379,15 +1379,15 @@ text_ignore_above_10 1660191010000 -- !sql_8_32 -- -Andy [100] ["soccer"] ["{"club":"Manchester","position":"forward","salary":50000}"] -Betty [90] ["pingpong ball"] ["{"club":"Beijing","position":"attacker","salary":30000}"] -Cindy [89] ["武术"] ["{"club":"Shaolin","position":"fighter","salary":25000}"] -David [77] ["volleyball"] ["{"club":"Tokyo","position":"spiker","salary":40000}"] -Emily [56, 78, 99] ["baseball", "golf", "hockey"] ["{"club":"Yankees","position":"pitcher","salary":80000}", "{"club":"PGA","position":"driver","salary":120000}", "{"club":"Rangers","position":"center","salary":90000}"] -Frank [45, 67, 88] ["rugby", "cricket", "boxing"] ["{"club":"Leicester","position":"scrum-half","salary":60000}", "{"club":"Mumbai","position":"batsman","salary":70000}", "{"club":"Vegas","position":"heavyweight","salary":150000}"] -Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] ["{"club":"China","position":"attacker","salary":35000}", "{"club":"Denmark","position":"singles","salary":45000}", "{"club":"Jamaica","position":"sprinter","salary":85000}"] -Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] ["{"club":"Korea","position":"recurve","salary":20000}", "{"club":"France","position":"epee","salary":30000}", "{"club":"Bulgaria","position":"heavyweight","salary":55000}"] -Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] ["{"club":"Japan","position":"lightweight","salary":25000}", "{"club":"Okinawa","position":"kata","salary":22000}", "{"club":"Korea","position":"middleweight","salary":28000}"] +Andy [100] ["soccer"] [{"club":"Manchester","position":"forward","salary":50000}] +Betty [90] ["pingpong ball"] [{"club":"Beijing","position":"attacker","salary":30000}] +Cindy [89] ["武术"] [{"club":"Shaolin","position":"fighter","salary":25000}] +David [77] ["volleyball"] [{"club":"Tokyo","position":"spiker","salary":40000}] +Emily [56, 78, 99] ["baseball", "golf", "hockey"] [{"club":"Yankees","position":"pitcher","salary":80000}, {"club":"PGA","position":"driver","salary":120000}, {"club":"Rangers","position":"center","salary":90000}] +Frank [45, 67, 88] ["rugby", "cricket", "boxing"] [{"club":"Leicester","position":"scrum-half","salary":60000}, {"club":"Mumbai","position":"batsman","salary":70000}, {"club":"Vegas","position":"heavyweight","salary":150000}] +Grace [34, 56, 78] ["table tennis", "badminton", "athletics"] [{"club":"China","position":"attacker","salary":35000}, {"club":"Denmark","position":"singles","salary":45000}, {"club":"Jamaica","position":"sprinter","salary":85000}] +Henry [23, 45, 67] ["archery", "fencing", "weightlifting"] [{"club":"Korea","position":"recurve","salary":20000}, {"club":"France","position":"epee","salary":30000}, {"club":"Bulgaria","position":"heavyweight","salary":55000}] +Ivy [12, 34, 56] ["judo", "karate", "taekwondo"] [{"club":"Japan","position":"lightweight","salary":25000}, {"club":"Okinawa","position":"kata","salary":22000}, {"club":"Korea","position":"middleweight","salary":28000}] -- !sql_8_33 -- string1 text#1 diff --git a/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.groovy b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.groovy new file mode 100644 index 00000000000..d1e35ed4109 --- /dev/null +++ b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_batches.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_trino_mongodb_batches", "p0,external") { + String enabled = context.config.otherConfigs.get("enableMongoDBTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + String encodedUser = java.net.URLEncoder.encode(user, "UTF-8").replace("+", "%20") + String encodedPassword = java.net.URLEncoder.encode(password, "UTF-8").replace("+", "%20") + sql "DROP CATALOG IF EXISTS mongodb_batches_test" + sql """CREATE CATALOG mongodb_batches_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.cursor-batch-size" = "127" + )""" + sql "SWITCH mongodb_batches_test" + sql "USE doris_mongodb_test" + + order_qt_multi_page_count """SELECT count(*), count(owner), min(seq), max(seq), sum(seq) FROM large_records""" + + order_qt_multi_page_groups """SELECT bucket, count(*), min(seq), max(seq), count(owner) FROM large_records GROUP BY bucket""" + + order_qt_batch_boundaries """SELECT seq, from_binary(_id), from_binary(owner), payload FROM large_records WHERE seq IN (0,126,127,128,1023,1024,1025,2047,2048,2080)""" + + order_qt_multi_page_bytes """SELECT count(*) FROM large_records WHERE owner IS NOT NULL AND from_binary(owner) = '00FF807F0100020304050607'""" + + order_qt_multi_page_null """SELECT count(*) FROM large_records WHERE owner IS NULL""" + + qt_multi_page_offset """SELECT seq, from_binary(_id) FROM large_records ORDER BY seq LIMIT 5 OFFSET 1022""" + + order_qt_multi_page_aggregate """SELECT sum(length(cast(_id AS STRING))), count(DISTINCT payload) FROM large_records""" + + order_qt_multi_page_empty """SELECT seq FROM large_records WHERE seq >= 2081""" + + // Repeated early termination must not poison the next cursor or scan. + for (int i = 0; i < 3; i++) { + "qt_repeated_limit_${i}" "SELECT seq, from_binary(_id) FROM large_records ORDER BY seq LIMIT 3" + } + order_qt_count_after_limits "SELECT count(*) FROM large_records" +} diff --git a/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.groovy b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.groovy new file mode 100644 index 00000000000..d8131f5a742 --- /dev/null +++ b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_json.groovy @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_trino_mongodb_json", "p0,external") { + String enabled = context.config.otherConfigs.get("enableMongoDBTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + String encodedUser = java.net.URLEncoder.encode(user, "UTF-8").replace("+", "%20") + String encodedPassword = java.net.URLEncoder.encode(password, "UTF-8").replace("+", "%20") + sql "DROP CATALOG IF EXISTS mongodb_json_test" + sql """CREATE CATALOG mongodb_json_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.cursor-batch-size" = "127" + )""" + sql "SWITCH mongodb_json_test" + sql "USE doris_mongodb_test" + + order_qt_schema "DESC json_records" + qt_all_columns "SELECT * FROM json_records ORDER BY seq" + order_qt_shapes '''SELECT seq, json_type(payload, '$'), payload IS NULL FROM json_records''' + order_qt_object '''SELECT json_extract_string(payload, '$.city'), json_extract_int(payload, '$.n'), + json_extract_bool(payload, '$.active'), json_extract_string(payload, '$.nested.key'), + json_extract_string(payload, '$.owner."$oid"') FROM json_records WHERE seq = 1''' + order_qt_array '''SELECT seq, json_length(payload), json_extract_string(payload, '$[1]'), + json_extract_bool(payload, '$[3].ok') FROM json_records WHERE seq = 2''' + order_qt_unicode '''SELECT seq, hex(json_extract_string(payload, '$')) + FROM json_records WHERE seq = 3''' + order_qt_scalars '''SELECT seq, cast(payload AS STRING), json_type(payload, '$') + FROM json_records WHERE seq BETWEEN 4 AND 7''' + order_qt_sql_null "SELECT seq FROM json_records WHERE payload IS NULL" + order_qt_json_null '''SELECT seq FROM json_records + WHERE json_type(payload, '$') = 'null' + ''' + order_qt_not_null "SELECT seq FROM json_records WHERE payload IS NOT NULL" + order_qt_empty_values "SELECT seq, json_length(payload) FROM json_records WHERE seq IN (10, 11, 12)" + order_qt_filter '''SELECT seq FROM json_records WHERE json_extract_int(payload, '$.n') = 7''' + qt_residual_limit '''SELECT seq FROM json_records WHERE json_type(payload, '$') = 'array' + ORDER BY seq LIMIT 2''' + order_qt_nested_array '''SELECT seq, json_array, json_type(json_array[1], '$'), + json_extract_int(json_array[1], '$.x'), json_type(json_array[3], '$'), json_array[4] IS NULL + FROM json_records WHERE seq <= 4''' + order_qt_nested_row '''SELECT seq, json_row, json_extract_int(json_row.payload, '$.x'), + json_row IS NULL, json_row.payload IS NULL FROM json_records WHERE seq <= 4''' + order_qt_nested_map '''SELECT seq, json_map, json_extract_int(json_map['a'], '$.x'), + json_type(json_map['b'], '$'), json_map['c'] IS NULL FROM json_records WHERE seq <= 4''' + order_qt_empty_nested_projection "SELECT seq, json_array, json_map FROM json_records WHERE seq = 2" + order_qt_null_nested_projection '''SELECT seq, json_array, json_row, json_map + FROM json_records WHERE seq IN (3, 4)''' + order_qt_serialization_round_trip '''SELECT seq, + json_type(json_parse(cast(json_array AS STRING)), '$[0]'), + json_extract_string(json_parse(cast(json_array AS STRING)), '$[0]'), + json_extract_string(json_parse(cast(json_row AS STRING)), '$.payload'), + json_extract_string(json_parse(cast(json_map AS STRING)), '$.bool_string') + FROM json_records WHERE seq = 5''' + // Typed SQL text uses NULL for SQL nulls; JSON null retains its lowercase spelling. + for (String strict : ["false", "true"]) { + quickTest("typed_array_round_trip_${strict}", """SELECT /*+ SET_VAR(enable_strict_cast=${strict}) */ + seq, a, json_type(a[2], '\$'), json_type(a[3], '\$'), a[3] IS NULL, a[4] IS NULL + FROM (SELECT seq, cast(cast(json_array AS STRING) AS ARRAY) a + FROM json_records WHERE seq <= 5) t ORDER BY seq""") + quickTest("typed_map_round_trip_${strict}", """SELECT /*+ SET_VAR(enable_strict_cast=${strict}) */ + seq, m, json_type(m['b'], '\$'), m['b'] IS NULL, m['c'] IS NULL, + json_type(m['number_string'], '\$'), json_type(m['null_string'], '\$'), + json_type(m['bool_string'], '\$'), hex(json_extract_string(m['escaped'], '\$')) + FROM (SELECT seq, cast(cast(json_map AS STRING) AS MAP) m + FROM json_records WHERE seq <= 5) t ORDER BY seq""") + quickTest("typed_struct_round_trip_${strict}", """SELECT /*+ SET_VAR(enable_strict_cast=${strict}) */ + seq, r, json_type(r.payload, '\$'), r IS NULL, r.payload IS NULL + FROM (SELECT seq, cast(cast(json_row AS STRING) AS STRUCT) r + FROM json_records WHERE seq <= 5) t ORDER BY seq""") + } + order_qt_explode '''SELECT seq, json_type(item, '$'), cast(item AS STRING) + FROM json_records LATERAL VIEW explode(json_array) t AS item''' + order_qt_aggregate '''SELECT json_type(payload, '$'), count(*) FROM json_records GROUP BY json_type(payload, '$')''' + order_qt_join '''SELECT a.seq, b.seq FROM json_records a JOIN json_records b + ON json_extract_int(a.payload, '$.n') = json_extract_int(b.payload, '$.n')''' + order_qt_empty_schema "DESC empty_json_records" + order_qt_empty_projection "SELECT payload, json_array FROM empty_json_records" + order_qt_empty_aggregate "SELECT count(*), count(payload) FROM empty_json_records" + order_qt_multi_page '''SELECT count(*), count(json_payload), + sum(json_extract_bigint(json_payload, '$.seq')) FROM large_records''' + order_qt_batch_boundaries '''SELECT seq, json_payload, json_payload IS NULL FROM large_records + WHERE seq IN (0,126,127,128,1023,1024,1025,2047,2048,2080)''' + qt_multi_page_offset '''SELECT seq, json_extract_string(json_payload, '$.text') + FROM large_records ORDER BY seq LIMIT 5 OFFSET 1022''' + + sql "CREATE DATABASE IF NOT EXISTS internal.mongodb_json_regression" + sql "DROP VIEW IF EXISTS internal.mongodb_json_regression.json_view" + sql '''CREATE VIEW internal.mongodb_json_regression.json_view AS + SELECT seq, payload FROM mongodb_json_test.doris_mongodb_test.json_records''' + order_qt_view_schema "DESC internal.mongodb_json_regression.json_view" + order_qt_view_data "SELECT * FROM internal.mongodb_json_regression.json_view" + sql "DROP TABLE IF EXISTS internal.mongodb_json_regression.json_materialized" + sql '''CREATE TABLE internal.mongodb_json_regression.json_materialized + DISTRIBUTED BY HASH(seq) BUCKETS 1 PROPERTIES ("replication_num" = "1") AS + SELECT seq, payload FROM json_records''' + order_qt_materialized_schema "DESC internal.mongodb_json_regression.json_materialized" + order_qt_materialized_data "SELECT * FROM internal.mongodb_json_regression.json_materialized" + order_qt_materialized_functions '''SELECT seq, json_type(payload, '$'), + json_extract_string(payload, '$.city') + FROM internal.mongodb_json_regression.json_materialized''' + sql "DROP TABLE IF EXISTS internal.mongodb_json_regression.nested_json_materialized" + test { + sql '''CREATE TABLE internal.mongodb_json_regression.nested_json_materialized + DISTRIBUTED BY HASH(seq) BUCKETS 1 PROPERTIES ("replication_num" = "1") AS + SELECT seq, json_array FROM json_records''' + exception "ARRAY unsupported sub-type: json" + } + test { + sql "SELECT payload FROM invalid_json_records" + exception "JSON" + } +} diff --git a/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.groovy b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.groovy new file mode 100644 index 00000000000..2d49bac017a --- /dev/null +++ b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_metadata.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_trino_mongodb_metadata", "p0,external") { + String enabled = context.config.otherConfigs.get("enableMongoDBTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + String encodedUser = java.net.URLEncoder.encode(user, "UTF-8").replace("+", "%20") + String encodedPassword = java.net.URLEncoder.encode(password, "UTF-8").replace("+", "%20") + sql "DROP CATALOG IF EXISTS mongodb_metadata_test" + sql """CREATE CATALOG mongodb_metadata_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.cursor-batch-size" = "127" + )""" + sql "SWITCH mongodb_metadata_test" + sql "USE doris_mongodb_test" + + order_qt_databases """SHOW DATABASES""" + + order_qt_tables """SHOW TABLES""" + + order_qt_inferred_schema """DESC inferred_records""" + + order_qt_inferred_data """SELECT from_binary(_id), seq, label FROM inferred_records""" + + order_qt_explicit_schema """DESC owners""" + + order_qt_explicit_data """SELECT from_binary(_id), label, weight FROM owners""" + + order_qt_refresh_before """SELECT seq, from_binary(_id) FROM records""" + + sql "REFRESH TABLE records" + order_qt_refresh_table "SELECT seq, from_binary(_id) FROM records" + sql "REFRESH DATABASE mongodb_metadata_test.doris_mongodb_test" + order_qt_refresh_database "SELECT seq, from_binary(_id) FROM records" + sql "REFRESH CATALOG mongodb_metadata_test" + order_qt_refresh_catalog "SELECT seq, from_binary(_id) FROM records" + test { + sql "SELECT payload FROM invalid_schema" + exception "Cannot transform unknown Trino type" + } + test { + sql "SELECT * FROM missing_collection" + exception "does not exist" + } + // A separate metadata collection changes the declared SQL schema without changing BSON data. + sql "DROP CATALOG IF EXISTS mongodb_alternate_schema_test" + sql """CREATE CATALOG mongodb_alternate_schema_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.schema-collection" = "_schema_alternate" + )""" + order_qt_alternate_schema "DESC mongodb_alternate_schema_test.doris_mongodb_test.records" + order_qt_alternate_data """SELECT from_binary(_id), seq, name + FROM mongodb_alternate_schema_test.doris_mongodb_test.records""" + + sql "DROP CATALOG IF EXISTS mongodb_readonly_test" + sql """CREATE CATALOG mongodb_readonly_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://mongodb_readonly:mongodb_readonly@${host}:${port}/?authSource=admin" + )""" + order_qt_readonly_explicit_schema """SELECT seq, from_binary(_id) + FROM mongodb_readonly_test.doris_mongodb_test.records""" + test { + sql "SELECT * FROM mongodb_readonly_test.doris_mongodb_test.readonly_inferred_records" + exception "not authorized" + } + sql "DROP CATALOG IF EXISTS mongodb_bad_auth_test" + sql """CREATE CATALOG mongodb_bad_auth_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://mongodb_test:incorrect@${host}:${port}/?authSource=admin&serverSelectionTimeoutMS=2000" + )""" + test { + sql "SHOW TABLES FROM mongodb_bad_auth_test.doris_mongodb_test" + exception "authenticating MongoCredential" + } + sql "DROP CATALOG mongodb_bad_auth_test" +} diff --git a/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.groovy b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.groovy new file mode 100644 index 00000000000..cc2d0def822 --- /dev/null +++ b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_queries.groovy @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_trino_mongodb_queries", "p0,external") { + String enabled = context.config.otherConfigs.get("enableMongoDBTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + String encodedUser = java.net.URLEncoder.encode(user, "UTF-8").replace("+", "%20") + String encodedPassword = java.net.URLEncoder.encode(password, "UTF-8").replace("+", "%20") + sql "DROP CATALOG IF EXISTS mongodb_queries_test" + sql """CREATE CATALOG mongodb_queries_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.cursor-batch-size" = "127" + )""" + sql "SWITCH mongodb_queries_test" + sql "USE doris_mongodb_test" + + order_qt_count """SELECT count(*), count(owner), count(score), count(DISTINCT grp) FROM records""" + + order_qt_numeric_filter """SELECT seq FROM records WHERE seq >= 2 AND seq < 6""" + + order_qt_boolean_filter """SELECT seq FROM records WHERE active""" + + order_qt_string_filter """SELECT seq, name FROM records WHERE name IN ('alpha', 'beta', 'missing')""" + + order_qt_in_filter """SELECT seq FROM records WHERE seq IN (1, 3, 6)""" + + order_qt_not_in_filter """SELECT seq FROM records WHERE seq NOT IN (1, 3, 6)""" + + order_qt_between_filter """SELECT seq, score FROM records WHERE score BETWEEN -5 AND 10""" + + order_qt_or_filter """SELECT seq FROM records WHERE seq = 1 OR seq = 4 OR grp = 'north'""" + + order_qt_compound_filter """SELECT seq FROM records WHERE (grp = 'east' AND score > 0) OR (NOT active AND seq > 2)""" + + order_qt_null_logic """SELECT seq FROM records WHERE NOT (score > 0) OR score IS NULL""" + + order_qt_null_safe_equal """SELECT seq FROM records WHERE score <=> NULL""" + + order_qt_like_filter """SELECT seq FROM records WHERE name LIKE '%a%' AND name NOT LIKE 'g%'""" + + qt_residual_with_limit """SELECT seq FROM records WHERE length(name) > 4 ORDER BY seq LIMIT 2""" + + order_qt_cast_objectid_filter """SELECT seq, from_binary(_id) FROM records WHERE from_binary(_id) = '000000000000000000000001'""" + + order_qt_cast_objectid_range """SELECT seq, from_binary(_id) FROM records WHERE from_binary(_id) >= '000000000000000000000003'""" + + order_qt_cast_objectid_in """SELECT seq FROM records WHERE from_binary(owner) IN ('000000000000000000000065', '00FF807F0100020304050607')""" + + order_qt_case_expression """SELECT seq, CASE WHEN owner IS NULL THEN 'missing' ELSE from_binary(owner) END FROM records""" + + order_qt_coalesce """SELECT seq, coalesce(score, -99), coalesce(optional_note, 'none') FROM records""" + + order_qt_arithmetic_projection """SELECT seq, score + seq, amount / 2 FROM records""" + + order_qt_distinct """SELECT DISTINCT grp FROM records""" + + order_qt_group_by """SELECT grp, count(*), count(score), sum(score), min(seq), max(seq), sum(amount) FROM records GROUP BY grp""" + + order_qt_having """SELECT grp, count(*) FROM records GROUP BY grp HAVING count(*) > 1""" + + order_qt_objectid_group """SELECT from_binary(owner), count(*) FROM records GROUP BY from_binary(owner)""" + + qt_topn """SELECT seq, score FROM records ORDER BY score DESC, seq LIMIT 3""" + + qt_offset """SELECT seq FROM records ORDER BY seq LIMIT 2 OFFSET 2""" + + order_qt_limit_zero """SELECT _id, owner FROM records LIMIT 0""" + + order_qt_no_matches """SELECT _id, owner FROM records WHERE seq < 0""" + + order_qt_join_numeric """SELECT a.seq, b.seq FROM records a JOIN records b ON a.seq = b.seq WHERE a.seq <= 3""" + + order_qt_join_objectid """SELECT r.seq, o.label FROM records r JOIN owners o ON from_binary(r.owner) = from_binary(o._id)""" + + order_qt_left_join_objectid """SELECT r.seq, o.label FROM records r LEFT JOIN owners o ON from_binary(r.owner) = from_binary(o._id)""" + + order_qt_full_join """SELECT r.seq, o.label FROM records r FULL OUTER JOIN owners o ON r.seq = o.weight""" + + order_qt_semi_join """SELECT r.seq FROM records r LEFT SEMI JOIN owners o ON from_binary(r.owner) = from_binary(o._id)""" + + order_qt_anti_join """SELECT r.seq FROM records r LEFT ANTI JOIN owners o ON from_binary(r.owner) = from_binary(o._id)""" + + order_qt_cross_join """SELECT r.seq, o.label FROM records r CROSS JOIN owners o WHERE r.seq = 1""" + + order_qt_exists """SELECT r.seq FROM records r WHERE EXISTS (SELECT 1 FROM owners o WHERE from_binary(o._id) = from_binary(r.owner))""" + + order_qt_not_exists """SELECT r.seq FROM records r WHERE NOT EXISTS (SELECT 1 FROM owners o WHERE from_binary(o._id) = from_binary(r.owner))""" + + order_qt_in_subquery """SELECT seq FROM records WHERE from_binary(owner) IN (SELECT from_binary(_id) FROM owners WHERE weight <= 20)""" + + order_qt_scalar_subquery """SELECT seq FROM records WHERE seq > (SELECT avg(seq) FROM records)""" + + order_qt_cte """WITH t AS (SELECT grp, count(*) AS n FROM records GROUP BY grp) SELECT grp, n FROM t WHERE n > 1""" + + order_qt_union_all """SELECT seq, from_binary(_id) FROM records WHERE seq <= 2 UNION ALL SELECT seq, from_binary(_id) FROM records WHERE seq >= 5""" + + order_qt_union_distinct """SELECT grp FROM records WHERE seq < 4 UNION SELECT grp FROM records WHERE seq > 2""" + + order_qt_window """SELECT seq, grp, row_number() OVER (PARTITION BY grp ORDER BY seq), sum(seq) OVER (PARTITION BY grp) FROM records""" + + order_qt_array_unnest """SELECT seq, tag FROM records LATERAL VIEW explode(tags) t AS tag""" + + order_qt_objectid_array_unnest """SELECT seq, from_binary(ref) FROM records LATERAL VIEW explode(refs) t AS ref""" + + qt_null_filtered_limit """SELECT seq FROM records WHERE owner IS NULL ORDER BY seq LIMIT 1""" + + sql "CREATE DATABASE IF NOT EXISTS internal.mongodb_objectid_regression" + sql "DROP VIEW IF EXISTS internal.mongodb_objectid_regression.objectid_view" + sql """CREATE VIEW internal.mongodb_objectid_regression.objectid_view AS + SELECT _id, seq, owner FROM mongodb_queries_test.doris_mongodb_test.records""" + order_qt_view_schema "DESC internal.mongodb_objectid_regression.objectid_view" + order_qt_view_data """SELECT seq, from_binary(_id), from_binary(owner) + FROM internal.mongodb_objectid_regression.objectid_view""" + sql "DROP TABLE IF EXISTS internal.mongodb_objectid_regression.materialized_records" + sql """CREATE TABLE internal.mongodb_objectid_regression.materialized_records + DISTRIBUTED BY HASH(seq) BUCKETS 1 PROPERTIES ("replication_num" = "1") AS + SELECT seq, from_binary(_id) AS id_hex, from_binary(owner) AS owner_hex, grp + FROM records""" + order_qt_materialized_data "SELECT * FROM internal.mongodb_objectid_regression.materialized_records" + order_qt_internal_join """SELECT r.seq, t.id_hex FROM records r + JOIN internal.mongodb_objectid_regression.materialized_records t ON from_binary(r._id) = t.id_hex""" + order_qt_internal_subquery """SELECT seq FROM records WHERE from_binary(_id) IN + (SELECT id_hex FROM internal.mongodb_objectid_regression.materialized_records WHERE seq <= 3)""" + sql "DROP TABLE IF EXISTS internal.mongodb_objectid_regression.raw_objectid_ctas" + test { + sql """CREATE TABLE internal.mongodb_objectid_regression.raw_objectid_ctas + DISTRIBUTED BY HASH(seq) BUCKETS 1 PROPERTIES ("replication_num" = "1") AS + SELECT seq, _id FROM records""" + exception "varbinary" + } + + // Native binary operators retain their existing analysis restrictions. + test { + sql "SELECT seq FROM records WHERE _id = X'000000000000000000000001'" + exception "varbinary" + } + test { + sql "SELECT owner, count(*) FROM records GROUP BY owner" + exception "don't support filter, group by or order by" + } + test { + sql "SELECT r.seq FROM records r JOIN owners o ON r.owner = o._id" + exception "varbinary" + } +} diff --git a/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.groovy b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.groovy new file mode 100644 index 00000000000..577a8c91e74 --- /dev/null +++ b/regression-test/suites/external_table_p0/trino_connector/mongodb/test_trino_mongodb_types.groovy @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_trino_mongodb_types", "p0,external") { + String enabled = context.config.otherConfigs.get("enableMongoDBTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + String host = context.config.otherConfigs.get("externalEnvIp") + String port = context.config.otherConfigs.get("mongodb_port") + String user = context.config.otherConfigs.get("mongodb_user") + String password = context.config.otherConfigs.get("mongodb_password") + String encodedUser = java.net.URLEncoder.encode(user, "UTF-8").replace("+", "%20") + String encodedPassword = java.net.URLEncoder.encode(password, "UTF-8").replace("+", "%20") + sql "DROP CATALOG IF EXISTS mongodb_types_test" + sql """CREATE CATALOG mongodb_types_test PROPERTIES ( + "type" = "trino-connector", + "trino.connector.name" = "mongodb", + "trino.mongodb.connection-url" = + "mongodb://${encodedUser}:${encodedPassword}@${host}:${port}/?authSource=admin", + "trino.mongodb.cursor-batch-size" = "127" + )""" + sql "SWITCH mongodb_types_test" + sql "USE doris_mongodb_test" + + order_qt_schema """DESC records""" + + qt_all_columns """SELECT * FROM records ORDER BY seq""" + + order_qt_objectids """SELECT seq, from_binary(_id), from_binary(owner), length(cast(_id AS STRING)) FROM records""" + + order_qt_ordinary_binary """SELECT seq, from_binary(blob), length(cast(blob AS STRING)) FROM records""" + + order_qt_primitive_values """SELECT seq, name, grp, active, score, amount, ratio, created, optional_note FROM records""" + + order_qt_null_missing """SELECT seq, owner IS NULL, score IS NULL, optional_note IS NULL FROM records""" + + order_qt_objectid_null """SELECT seq, from_binary(owner) FROM records WHERE owner IS NULL""" + + order_qt_objectid_not_null """SELECT seq, from_binary(owner) FROM records WHERE owner IS NOT NULL""" + + order_qt_arrays """SELECT seq, tags, numbers, refs FROM records""" + + order_qt_objectid_array_elements """SELECT seq, from_binary(refs[1]), from_binary(refs[2]) FROM records""" + + order_qt_array_operations """SELECT seq, array_size(refs), array_size(tags), array_contains(tags, 'red') FROM records""" + + order_qt_nested_row """SELECT seq, profile.city, profile.rank, from_binary(profile.manager) FROM records""" + + order_qt_nested_projection_filter """SELECT seq, profile.city FROM records WHERE profile.city = 'A'""" + + order_qt_nullable_row """SELECT seq, profile FROM records WHERE seq >= 3""" + + order_qt_nested_objectid_null """SELECT seq FROM records WHERE profile.manager IS NULL""" + + order_qt_unicode """SELECT seq, name, tags FROM records WHERE name LIKE '%é' OR array_contains(tags, '中文')""" + + order_qt_decimal_arithmetic """SELECT seq, amount, amount * 2, round(ratio, 2) FROM records""" + + order_qt_dates """SELECT seq, created, year(created), month(created) FROM records WHERE created >= '2024-01-01'""" + + order_qt_mixed_numeric_types """SELECT seq, value, value IS NULL FROM mixed_values""" + + order_qt_empty_schema """DESC empty_records""" + + order_qt_empty_projection """SELECT _id, seq, owner FROM empty_records""" + + order_qt_empty_aggregate """SELECT count(*), count(owner), min(seq), max(seq) FROM empty_records""" + + order_qt_numeric_limits """SELECT seq, from_binary(_id), i32, i64, amount, ratio, text, created + FROM numeric_limits""" + order_qt_numeric_limit_predicates """SELECT seq FROM numeric_limits + WHERE i64 = 9223372036854775807 OR created < '1969-12-31'""" +} diff --git a/regression-test/suites/external_table_p0/tvf/test_http_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_http_tvf.groovy index 485d267e48a..2dd4fc4070e 100644 --- a/regression-test/suites/external_table_p0/tvf/test_http_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_http_tvf.groovy @@ -239,14 +239,14 @@ suite("test_http_tvf", "p0") { ); """ - // The Spark INT96 fixture stores timestamps normalized to UTC; the golden uses Asia/Shanghai. - // Specify the INT96 timezone explicitly instead of relying on the reader's default semantics. + // The golden retains the fixture's stored INT96 wall-clock values. Use UTC to avoid + // shifting timestamp map keys by eight hours when the session uses Asia/Shanghai. qt_sql08 """ select * from http( "uri" = "${httpUrl("external_table_p0/tvf/t.parquet")}", "format" = "parquet", - "hive.parquet.time-zone" = "Asia/Shanghai" + "hive.parquet.time-zone" = "UTC" ) order by id limit 10; """ @@ -255,7 +255,7 @@ suite("test_http_tvf", "p0") { http( "uri" = "${httpUrl("external_table_p0/tvf/t.parquet")}", "format" = "parquet", - "hive.parquet.time-zone" = "Asia/Shanghai" + "hive.parquet.time-zone" = "UTC" ) order by id limit 10; """ @@ -264,7 +264,7 @@ suite("test_http_tvf", "p0") { http( "uri" = "${httpUrl("external_table_p0/tvf/t.parquet")}", "format" = "parquet", - "hive.parquet.time-zone" = "Asia/Shanghai" + "hive.parquet.time-zone" = "UTC" ); """ diff --git a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy index 6f10c2ca96b..d643cbd964e 100644 --- a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy +++ b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy @@ -38,6 +38,10 @@ suite("bucket_shuffle_set_operation") { // backend count / parallelism of the environment running this suite sql "set bucket_shuffle_downgrade_ratio=0" + // Spill rewrites INTERSECT/EXCEPT into joins. Keep the physical set operations + // whose bucket distribution is checked by this suite, including in spill-enabled CI. + sql "set disable_nereids_rules='REWRITE_INTERSECT_TO_JOIN_WHEN_SPILL,REWRITE_EXCEPT_TO_JOIN_WHEN_SPILL'" + def checkShapeAndResult = { String tag, String sqlStr -> quickTest(tag + "_shape", "explain shape plan " + sqlStr) quickTest(tag + "_result", sqlStr, true)