nprasad2077/doris
1
0 Fork 0
mirror of https://github.com/apache/doris.git synced 2026-11-04 17:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
699680e6f8d0decd07f1c49cc0fe3aeb34775d87
doris/be/test/exprs/function/function_hash_test.cpp
T

425 lines
16 KiB
C++
Raw Blame History

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "exprs/function/function_hash.h"
#include <stdint.h>
#include <string>
#include <vector>
#include "common/status.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/types.h"
#include "core/value/large_int_value.h"
#include "exprs/function/function_test_util.h"
#include "gtest/gtest_pred_impl.h"
#include "testutil/any_type.h"
#include "util/hash/murmur_hash3.h"
namespace doris {
using namespace ut_type;
namespace {
__int128_t pack_murmur_hash3_128_for_test(uint64_t h1, uint64_t h2) {
static_assert(sizeof(__int128_t) == sizeof(uint64_t) * 2);
const auto value =
(static_cast<unsigned __int128>(h2) << 64) | static_cast<unsigned __int128>(h1);
return static_cast<__int128_t>(value);
}
__int128_t murmur_hash3_128_for_test(const std::vector<std::string>& values) {
uint64_t h1 = 0;
uint64_t h2 = 0;
for (const std::string& value : values) {
murmur_hash3_x64_process(value.data(), value.size(), h1, h2);
}
return pack_murmur_hash3_128_for_test(h1, h2);
}
std::string murmur_hash3_u128_for_test(const std::vector<std::string>& values) {
return LargeIntValue::to_string(static_cast<__uint128_t>(murmur_hash3_128_for_test(values)));
}
} // namespace
TEST(HashFunctionTest, murmur_hash_3_test) {
std::string func_name = "murmur_hash3_32";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{Null()}, Null()}, {{std::string("hello")}, (int32_t)1321743225}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world")}, (int32_t)984713481},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
(int32_t)-666935433},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
}
TEST(HashFunctionTest, murmur_hash_3_64_test) {
std::string func_name = "murmur_hash3_64";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{Null()}, Null()},
{{std::string("hello")}, (int64_t)-3215607508166160593}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
{{std::string("hello"), std::string("world")}, (int64_t)3583109472027628045},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
(int64_t)1887828212617890932},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
}
TEST(HashFunctionTest, murmur_hash_3_64_v2_test) {
std::string func_name = "murmur_hash3_64_v2";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("1000209601_1756808272")}, (int64_t)4038800892574899471},
{{std::string("hello world")}, (int64_t)5998619086395760910},
{{std::string("apache doris")}, (int64_t)3669213779466221743}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
}
TEST(HashFunctionTest, murmur_hash_3_128_test) {
std::string func_name = "murmur_hash3_128";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
{{Null()}, Null()},
{{std::string("hello world")},
pack_murmur_hash3_128_for_test(5998619086395760910ULL, 12364428806279881649ULL)}};
static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world")},
murmur_hash3_128_for_test({"hello", "world"})},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt128, true>(func_name, input_types, data_set));
check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
murmur_hash3_128_for_test({"hello", "world", "!"})},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
check_function_all_arg_comb<DataTypeInt128, true>(func_name, input_types, data_set);
};
}
TEST(HashFunctionTest, murmur_hash_3_128_empty_arguments_test) {
auto check_empty_arguments = [](const std::string& func_name, const DataTypePtr& return_type) {
Block block;
FunctionBasePtr func = SimpleFunctionFactory::instance().get_function(
func_name, block.get_columns_with_type_and_name(), return_type);
ASSERT_TRUE(func != nullptr);
FunctionUtils fn_utils(return_type, {}, false);
auto* fn_ctx = fn_utils.get_fn_ctx();
ASSERT_TRUE(func->open(fn_ctx, FunctionContext::FRAGMENT_LOCAL).ok());
ASSERT_TRUE(func->open(fn_ctx, FunctionContext::THREAD_LOCAL).ok());
block.insert({nullptr, return_type, "result"});
const auto st = func->execute(fn_ctx, block, {}, 0, 1);
EXPECT_FALSE(st.ok());
EXPECT_NE(st.to_string().find("requires at least one argument"), std::string::npos);
static_cast<void>(func->close(fn_ctx, FunctionContext::THREAD_LOCAL));
static_cast<void>(func->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
};
check_empty_arguments("murmur_hash3_128", std::make_shared<DataTypeInt128>());
check_empty_arguments("murmur_hash3_u128", std::make_shared<DataTypeString>());
}
TEST(HashFunctionTest, murmur_hash_3_u128_test) {
std::string func_name = "murmur_hash3_u128";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
{{Null()}, Null()},
{{std::string("")}, std::string("0")},
{{std::string("hello world")}, murmur_hash3_u128_for_test({"hello world"})}};
static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world")},
murmur_hash3_u128_for_test({"hello", "world"})},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
murmur_hash3_u128_for_test({"hello", "world", "!"})},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
check_function_all_arg_comb<DataTypeString, true>(func_name, input_types, data_set);
};
}
TEST(HashFunctionTest, murmur_hash_3_u128_unsigned_range_test) {
std::string func_name = "murmur_hash3_u128";
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
{{std::string("hello world")}, std::string("228083453807047072434243676435732455694")}};
static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
}
TEST(HashFunctionTest, murmur_hash_get_name_test) {
EXPECT_STREQ(murmur_hash3_get_name_type_int_for_test(), "murmur_hash3_32");
EXPECT_STREQ(murmur_hash3_get_name_type_bigint_for_test(), "murmur_hash3_64");
EXPECT_STREQ(murmur_hash3_get_name_type_bigint_v2_for_test(), "murmur_hash3_64_v2");
}
TEST(HashFunctionTest, xxhash_32_test) {
std::string func_name = "xxhash_32";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{Null()}, Null()}, {{std::string("hello")}, (int32_t)-83855367}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world")}, (int32_t)-920844969},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
(int32_t)352087701},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {{{Null()}, Null()}, {{VARBINARY("hello")}, (int32_t)-83855367}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world")}, (int32_t)-920844969},
{{VARBINARY("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY,
PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {
{{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")}, (int32_t)352087701},
{{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt32, true>(func_name, input_types, data_set));
};
}
TEST(HashFunctionTest, xxhash_64_test) {
std::string func_name = "xxhash_64";
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{Null()}, Null()},
{{std::string("hello")}, (int64_t)-7685981735718036227}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {
{{std::string("hello"), std::string("world")}, (int64_t)7001965798170371843},
{{std::string("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR, PrimitiveType::TYPE_VARCHAR,
PrimitiveType::TYPE_VARCHAR};
DataSet data_set = {{{std::string("hello"), std::string("world"), std::string("!")},
(int64_t)6796829678999971400},
{{std::string("hello"), std::string("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {{{Null()}, Null()},
{{VARBINARY("hello")}, (int64_t)-7685981735718036227}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {
{{VARBINARY("hello"), VARBINARY("world")}, (int64_t)7001965798170371843},
{{VARBINARY("hello"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
{
InputTypeSet input_types = {PrimitiveType::TYPE_VARBINARY, PrimitiveType::TYPE_VARBINARY,
PrimitiveType::TYPE_VARBINARY};
DataSet data_set = {{{VARBINARY("hello"), VARBINARY("world"), VARBINARY("!")},
(int64_t)6796829678999971400},
{{VARBINARY("hello"), VARBINARY("world"), Null()}, Null()}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
};
}
TEST(HashFunctionTest, murmur_hash3_helper_functions_test) {
{
std::string input = "hello world";
uint64_t h1 = 0;
uint64_t h2 = 0;
murmur_hash3_x64_process(input.data(), input.size(), h1, h2);
EXPECT_EQ(h1, 5998619086395760910ULL);
EXPECT_EQ(h2, 12364428806279881649ULL);
}
{
std::string input = "hello world";
uint64_t out[2] = {0, 0};
murmur_hash3_x64_128(input.data(), input.size(), 0, out);
EXPECT_TRUE(out[0] == 5998619086395760910ULL && out[1] == 12364428806279881649ULL);
}
{
std::string input = "hello world";
uint64_t out = 0;
murmur_hash3_x64_64_shared(input.data(), input.size(), 0, &out);
EXPECT_EQ(out, 5998619086395760910ULL);
}
{
std::string input = "hello";
uint64_t out = 0;
murmur_hash3_x64_64(input.data(), input.size(), 0, &out);
EXPECT_EQ(out, static_cast<uint64_t>(-3215607508166160593LL));
}
{
std::string input = "";
uint64_t h1 = 0, h2 = 0;
murmur_hash3_x64_process(input.data(), input.size(), h1, h2);
EXPECT_EQ(h1, 0ULL);
EXPECT_EQ(h2, 0ULL);
}
}
} // namespace doris
Reference in new issue View Git Blame Copy Permalink