mirror of
https://github.com/apache/doris.git
synced 2026-11-04 07:31:17 +00:00
Fix incompatibility with arm architecture in util and gutil (#2650)
1. upgrade gutil code from imapla to new verison， include `cpuinfo`, `spinlock` and `linux_syscall_support ` 2. impliments arm version utf8 check code 3. remove incompatible code from stopwatch
This commit is contained in:
yangzhg authored and 
ZHAO Chun committed
1 parent
c6badaec91
commit
de4d1778c6
15 files changed
+1738
-2021
No files matched your search
+23
@@ -599,3 +599,26 @@ be/src/util/minizip/* : Condition of use and distribution are the same than zli
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
be/src/util/utf8_check.cpp: MIT license
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 Yibo Cai
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+142
-155
@@ -4,10 +4,14 @@
|
||||
|
||||
#include "gutil/cpu.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <utility>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "gutil/integral_types.h"
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <streambuf>
|
||||
#include <string>
|
||||
|
||||
#if defined(__x86_64__)
|
||||
#if defined(_MSC_VER)
|
||||
@@ -17,7 +21,39 @@
|
||||
#endif
|
||||
|
||||
namespace base {
|
||||
|
||||
#if defined(ARCH_CPU_X86_FAMILY)
|
||||
namespace internal {
|
||||
std::tuple<int, int, int, int> ComputeX86FamilyAndModel(
|
||||
const std::string& vendor,
|
||||
int signature) {
|
||||
int family = (signature >> 8) & 0xf;
|
||||
int model = (signature >> 4) & 0xf;
|
||||
int ext_family = 0;
|
||||
int ext_model = 0;
|
||||
// The "Intel 64 and IA-32 Architectures Developer's Manual: Vol. 2A"
|
||||
// specifies the Extended Model is defined only when the Base Family is
|
||||
// 06h or 0Fh.
|
||||
// The "AMD CPUID Specification" specifies that the Extended Model is
|
||||
// defined only when Base Family is 0Fh.
|
||||
// Both manuals define the display model as
|
||||
// {ExtendedModel[3:0],BaseModel[3:0]} in that case.
|
||||
if (family == 0xf || (family == 0x6 && vendor == "GenuineIntel")) {
|
||||
ext_model = (signature >> 16) & 0xf;
|
||||
model += ext_model << 4;
|
||||
}
|
||||
// Both the "Intel 64 and IA-32 Architectures Developer's Manual: Vol. 2A"
|
||||
// and the "AMD CPUID Specification" specify that the Extended Family is
|
||||
// defined only when the Base Family is 0Fh.
|
||||
// Both manuals define the display family as {0000b,BaseFamily[3:0]} +
|
||||
// ExtendedFamily[7:0] in that case.
|
||||
if (family == 0xf) {
|
||||
ext_family = (signature >> 20) & 0xff;
|
||||
family += ext_family;
|
||||
}
|
||||
return {family, model, ext_family, ext_model};
|
||||
}
|
||||
} // namespace internal
|
||||
#endif // defined(ARCH_CPU_X86_FAMILY)
|
||||
CPU::CPU()
|
||||
: signature_(0),
|
||||
type_(0),
|
||||
@@ -33,61 +69,53 @@ CPU::CPU()
|
||||
has_ssse3_(false),
|
||||
has_sse41_(false),
|
||||
has_sse42_(false),
|
||||
has_popcnt_(false),
|
||||
has_avx_(false),
|
||||
has_avx2_(false),
|
||||
has_aesni_(false),
|
||||
has_non_stop_time_stamp_counter_(false),
|
||||
has_broken_neon_(false),
|
||||
is_running_in_vm_(false),
|
||||
cpu_vendor_("unknown") {
|
||||
Initialize();
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
#if defined(__x86_64__)
|
||||
#ifndef _MSC_VER
|
||||
|
||||
#if defined(ARCH_CPU_X86_FAMILY)
|
||||
#if !defined(COMPILER_MSVC)
|
||||
#if defined(__pic__) && defined(__i386__)
|
||||
|
||||
void __cpuid(int cpu_info[4], int info_type) {
|
||||
__asm__ volatile (
|
||||
"mov %%ebx, %%edi\n"
|
||||
"cpuid\n"
|
||||
"xchg %%edi, %%ebx\n"
|
||||
: "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
|
||||
: "a"(info_type)
|
||||
);
|
||||
__asm__ volatile(
|
||||
"mov %%ebx, %%edi\n"
|
||||
"cpuid\n"
|
||||
"xchg %%edi, %%ebx\n"
|
||||
: "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]),
|
||||
"=d"(cpu_info[3])
|
||||
: "a"(info_type), "c"(0));
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void __cpuid(int cpu_info[4], int info_type) {
|
||||
__asm__ volatile (
|
||||
"cpuid\n"
|
||||
: "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
|
||||
: "a"(info_type), "c"(0)
|
||||
);
|
||||
__asm__ volatile("cpuid\n"
|
||||
: "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]),
|
||||
"=d"(cpu_info[3])
|
||||
: "a"(info_type), "c"(0));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// _xgetbv returns the value of an Intel Extended Control Register (XCR).
|
||||
#endif // !defined(COMPILER_MSVC)
|
||||
// xgetbv returns the value of an Intel Extended Control Register (XCR).
|
||||
// Currently only XCR0 is defined by Intel so |xcr| should always be zero.
|
||||
uint64 _xgetbv(uint32 xcr) {
|
||||
uint32 eax, edx;
|
||||
|
||||
uint64_t xgetbv(uint32_t xcr) {
|
||||
#if defined(COMPILER_MSVC)
|
||||
return _xgetbv(xcr);
|
||||
#else
|
||||
uint32_t eax, edx;
|
||||
__asm__ volatile (
|
||||
"xgetbv" : "=a"(eax), "=d"(edx) : "c"(xcr));
|
||||
return (static_cast<uint64>(edx) << 32) | eax;
|
||||
return (static_cast<uint64_t>(edx) << 32) | eax;
|
||||
#endif // defined(COMPILER_MSVC)
|
||||
}
|
||||
|
||||
#endif // !_MSC_VER
|
||||
#endif // __x86_64__
|
||||
|
||||
#endif // ARCH_CPU_X86_FAMILY
|
||||
#if defined(ARCH_CPU_ARM_FAMILY) && (defined(OS_ANDROID) || defined(OS_LINUX))
|
||||
class LazyCpuInfoValue {
|
||||
public:
|
||||
LazyCpuInfoValue() : has_broken_neon_(false) {
|
||||
std::string* CpuInfoBrand() {
|
||||
static std::string* brand = []() {
|
||||
// This function finds the value from /proc/cpuinfo under the key "model
|
||||
// name" or "Processor". "model name" is used in Linux 3.8 and later (3.7
|
||||
// and later for arm64) and is shown once per CPU. "Processor" is used in
|
||||
@@ -95,112 +123,48 @@ class LazyCpuInfoValue {
|
||||
// regardless of the number CPUs.
|
||||
const char kModelNamePrefix[] = "model name\t: ";
|
||||
const char kProcessorPrefix[] = "Processor\t: ";
|
||||
|
||||
// This function also calculates whether we believe that this CPU has a
|
||||
// broken NEON unit based on these fields from cpuinfo:
|
||||
unsigned implementer = 0, architecture = 0, variant = 0, part = 0,
|
||||
revision = 0;
|
||||
const struct {
|
||||
const char key[17];
|
||||
unsigned int* result;
|
||||
} kUnsignedValues[] = {
|
||||
{"CPU implementer", &implementer},
|
||||
{"CPU architecture", &architecture},
|
||||
{"CPU variant", &variant},
|
||||
{"CPU part", &part},
|
||||
{"CPU revision", &revision},
|
||||
};
|
||||
|
||||
std::ifstream info("/proc/cpuinfo");
|
||||
std::string contents;
|
||||
ReadFileToString(FilePath("/proc/cpuinfo"), &contents);
|
||||
DCHECK(!contents.empty());
|
||||
if (contents.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
contents.assign(std::istreambuf_iterator<char>(info), std::istreambuf_iterator<char>());
|
||||
std::istringstream iss(contents);
|
||||
std::string line;
|
||||
while (std::getline(iss, line)) {
|
||||
if (brand_.empty() &&
|
||||
(line.compare(0, strlen(kModelNamePrefix), kModelNamePrefix) == 0 ||
|
||||
line.compare(0, strlen(kProcessorPrefix), kProcessorPrefix) == 0)) {
|
||||
brand_.assign(line.substr(strlen(kModelNamePrefix)));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < arraysize(kUnsignedValues); i++) {
|
||||
const char *key = kUnsignedValues[i].key;
|
||||
const size_t len = strlen(key);
|
||||
|
||||
if (line.compare(0, len, key) == 0 &&
|
||||
line.size() >= len + 1 &&
|
||||
(line[len] == '\t' || line[len] == ' ' || line[len] == ':')) {
|
||||
size_t colon_pos = line.find(':', len);
|
||||
if (colon_pos == std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const StringPiece line_sp(line);
|
||||
StringPiece value_sp = line_sp.substr(colon_pos + 1);
|
||||
while (!value_sp.empty() &&
|
||||
(value_sp[0] == ' ' || value_sp[0] == '\t')) {
|
||||
value_sp = value_sp.substr(1);
|
||||
}
|
||||
|
||||
// The string may have leading "0x" or not, so we use strtoul to
|
||||
// handle that.
|
||||
char* endptr;
|
||||
std::string value(value_sp.as_string());
|
||||
unsigned long int result = strtoul(value.c_str(), &endptr, 0);
|
||||
if (*endptr == 0 && result <= UINT_MAX) {
|
||||
*kUnsignedValues[i].result = result;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (line.compare(0, strlen(kModelNamePrefix), kModelNamePrefix) == 0)
|
||||
return new std::string(line.substr(strlen(kModelNamePrefix)));
|
||||
if (line.compare(0, strlen(kProcessorPrefix), kProcessorPrefix) == 0)
|
||||
return new std::string(line.substr(strlen(kProcessorPrefix)));
|
||||
}
|
||||
|
||||
has_broken_neon_ =
|
||||
implementer == 0x51 &&
|
||||
architecture == 7 &&
|
||||
variant == 1 &&
|
||||
part == 0x4d &&
|
||||
revision == 0;
|
||||
}
|
||||
|
||||
const std::string& brand() const { return brand_; }
|
||||
bool has_broken_neon() const { return has_broken_neon_; }
|
||||
|
||||
private:
|
||||
std::string brand_;
|
||||
bool has_broken_neon_;
|
||||
DISALLOW_COPY_AND_ASSIGN(LazyCpuInfoValue);
|
||||
};
|
||||
|
||||
base::LazyInstance<LazyCpuInfoValue>::Leaky g_lazy_cpuinfo =
|
||||
LAZY_INSTANCE_INITIALIZER;
|
||||
|
||||
return new std::string();
|
||||
}();
|
||||
return brand;
|
||||
}
|
||||
#endif // defined(ARCH_CPU_ARM_FAMILY) && (defined(OS_ANDROID) ||
|
||||
// defined(OS_LINUX))
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
} // namespace
|
||||
void CPU::Initialize() {
|
||||
#if defined(__x86_64__)
|
||||
#if defined(ARCH_CPU_X86_FAMILY)
|
||||
int cpu_info[4] = {-1};
|
||||
char cpu_string[48];
|
||||
|
||||
// This array is used to temporarily hold the vendor name and then the brand
|
||||
// name. Thus it has to be big enough for both use cases. There are
|
||||
// static_asserts below for each of the use cases to make sure this array is
|
||||
// big enough.
|
||||
char cpu_string[sizeof(cpu_info) * 3 + 1];
|
||||
// __cpuid with an InfoType argument of 0 returns the number of
|
||||
// valid Ids in CPUInfo[0] and the CPU identification string in
|
||||
// the other three array elements. The CPU identification string is
|
||||
// not in linear order. The code below arranges the information
|
||||
// in a human readable form. The human readable order is CPUInfo[1] |
|
||||
// CPUInfo[3] | CPUInfo[2]. CPUInfo[2] and CPUInfo[3] are swapped
|
||||
// before using memcpy to copy these three array elements to cpu_string.
|
||||
// before using memcpy() to copy these three array elements to |cpu_string|.
|
||||
__cpuid(cpu_info, 0);
|
||||
int num_ids = cpu_info[0];
|
||||
std::swap(cpu_info[2], cpu_info[3]);
|
||||
memcpy(cpu_string, &cpu_info[1], 3 * sizeof(cpu_info[1]));
|
||||
cpu_vendor_.assign(cpu_string, 3 * sizeof(cpu_info[1]));
|
||||
|
||||
static constexpr size_t kVendorNameSize = 3 * sizeof(cpu_info[1]);
|
||||
static_assert(kVendorNameSize < sizeof(cpu_string) / sizeof(cpu_string[0]),
|
||||
"cpu_string too small");
|
||||
memcpy(cpu_string, &cpu_info[1], kVendorNameSize);
|
||||
cpu_string[kVendorNameSize] = '\0';
|
||||
cpu_vendor_ = cpu_string;
|
||||
// Interpret CPU feature information.
|
||||
if (num_ids > 0) {
|
||||
int cpu_info7[4] = {0};
|
||||
@@ -210,11 +174,9 @@ void CPU::Initialize() {
|
||||
}
|
||||
signature_ = cpu_info[0];
|
||||
stepping_ = cpu_info[0] & 0xf;
|
||||
model_ = ((cpu_info[0] >> 4) & 0xf) + ((cpu_info[0] >> 12) & 0xf0);
|
||||
family_ = (cpu_info[0] >> 8) & 0xf;
|
||||
type_ = (cpu_info[0] >> 12) & 0x3;
|
||||
ext_model_ = (cpu_info[0] >> 16) & 0xf;
|
||||
ext_family_ = (cpu_info[0] >> 20) & 0xff;
|
||||
std::tie(family_, model_, ext_family_, ext_model_) =
|
||||
internal::ComputeX86FamilyAndModel(cpu_vendor_, signature_);
|
||||
has_mmx_ = (cpu_info[3] & 0x00800000) != 0;
|
||||
has_sse_ = (cpu_info[3] & 0x02000000) != 0;
|
||||
has_sse2_ = (cpu_info[3] & 0x04000000) != 0;
|
||||
@@ -222,6 +184,13 @@ void CPU::Initialize() {
|
||||
has_ssse3_ = (cpu_info[2] & 0x00000200) != 0;
|
||||
has_sse41_ = (cpu_info[2] & 0x00080000) != 0;
|
||||
has_sse42_ = (cpu_info[2] & 0x00100000) != 0;
|
||||
has_popcnt_ = (cpu_info[2] & 0x00800000) != 0;
|
||||
// "Hypervisor Present Bit: Bit 31 of ECX of CPUID leaf 0x1."
|
||||
// See https://lwn.net/Articles/301888/
|
||||
// This is checking for any hypervisor. Hypervisors may choose not to
|
||||
// announce themselves. Hypervisors trap CPUID and sometimes return
|
||||
// different results to underlying hardware.
|
||||
is_running_in_vm_ = (cpu_info[2] & 0x80000000) != 0;
|
||||
// AVX instructions will generate an illegal instruction exception unless
|
||||
// a) they are supported by the CPU,
|
||||
// b) XSAVE is supported by the CPU and
|
||||
@@ -236,41 +205,60 @@ void CPU::Initialize() {
|
||||
(cpu_info[2] & 0x10000000) != 0 &&
|
||||
(cpu_info[2] & 0x04000000) != 0 /* XSAVE */ &&
|
||||
(cpu_info[2] & 0x08000000) != 0 /* OSXSAVE */ &&
|
||||
(_xgetbv(0) & 6) == 6 /* XSAVE enabled by kernel */;
|
||||
(xgetbv(0) & 6) == 6 /* XSAVE enabled by kernel */;
|
||||
has_aesni_ = (cpu_info[2] & 0x02000000) != 0;
|
||||
has_avx2_ = has_avx_ && (cpu_info7[1] & 0x00000020) != 0;
|
||||
}
|
||||
|
||||
// Get the brand string of the cpu.
|
||||
__cpuid(cpu_info, 0x80000000);
|
||||
const int parameter_end = 0x80000004;
|
||||
int max_parameter = cpu_info[0];
|
||||
|
||||
if (cpu_info[0] >= parameter_end) {
|
||||
char* cpu_string_ptr = cpu_string;
|
||||
|
||||
for (int parameter = 0x80000002; parameter <= parameter_end &&
|
||||
cpu_string_ptr < &cpu_string[sizeof(cpu_string)]; parameter++) {
|
||||
const int max_parameter = cpu_info[0];
|
||||
static constexpr int kParameterStart = 0x80000002;
|
||||
static constexpr int kParameterEnd = 0x80000004;
|
||||
static constexpr int kParameterSize = kParameterEnd - kParameterStart + 1;
|
||||
static_assert(kParameterSize * sizeof(cpu_info) + 1 == sizeof(cpu_string) / sizeof(cpu_string[0]),
|
||||
"cpu_string has wrong size");
|
||||
if (max_parameter >= kParameterEnd) {
|
||||
size_t i = 0;
|
||||
for (int parameter = kParameterStart; parameter <= kParameterEnd;
|
||||
++parameter) {
|
||||
__cpuid(cpu_info, parameter);
|
||||
memcpy(cpu_string_ptr, cpu_info, sizeof(cpu_info));
|
||||
cpu_string_ptr += sizeof(cpu_info);
|
||||
memcpy(&cpu_string[i], cpu_info, sizeof(cpu_info));
|
||||
i += sizeof(cpu_info);
|
||||
}
|
||||
cpu_brand_.assign(cpu_string, cpu_string_ptr - cpu_string);
|
||||
cpu_string[i] = '\0';
|
||||
cpu_brand_ = cpu_string;
|
||||
}
|
||||
|
||||
const int parameter_containing_non_stop_time_stamp_counter = 0x80000007;
|
||||
if (max_parameter >= parameter_containing_non_stop_time_stamp_counter) {
|
||||
__cpuid(cpu_info, parameter_containing_non_stop_time_stamp_counter);
|
||||
static constexpr int kParameterContainingNonStopTimeStampCounter = 0x80000007;
|
||||
if (max_parameter >= kParameterContainingNonStopTimeStampCounter) {
|
||||
__cpuid(cpu_info, kParameterContainingNonStopTimeStampCounter);
|
||||
has_non_stop_time_stamp_counter_ = (cpu_info[3] & (1 << 8)) != 0;
|
||||
}
|
||||
#elif defined(ARCH_CPU_ARM_FAMILY) && (defined(OS_ANDROID) || defined(OS_LINUX))
|
||||
cpu_brand_.assign(g_lazy_cpuinfo.Get().brand());
|
||||
has_broken_neon_ = g_lazy_cpuinfo.Get().has_broken_neon();
|
||||
#else
|
||||
#error unknown architecture
|
||||
if (!has_non_stop_time_stamp_counter_ && is_running_in_vm_) {
|
||||
int cpu_info_hv[4] = {};
|
||||
__cpuid(cpu_info_hv, 0x40000000);
|
||||
if (cpu_info_hv[1] == 0x7263694D && // Micr
|
||||
cpu_info_hv[2] == 0x666F736F && // osof
|
||||
cpu_info_hv[3] == 0x76482074) { // t Hv
|
||||
// If CPUID says we have a variant TSC and a hypervisor has identified
|
||||
// itself and the hypervisor says it is Microsoft Hyper-V, then treat
|
||||
// TSC as invariant.
|
||||
//
|
||||
// Microsoft Hyper-V hypervisor reports variant TSC as there are some
|
||||
// scenarios (eg. VM live migration) where the TSC is variant, but for
|
||||
// our purposes we can treat it as invariant.
|
||||
has_non_stop_time_stamp_counter_ = true;
|
||||
}
|
||||
}
|
||||
#elif defined(ARCH_CPU_ARM_FAMILY)
|
||||
#if (defined(OS_ANDROID) || defined(OS_LINUX))
|
||||
cpu_brand_ = *CpuInfoBrand();
|
||||
#elif defined(OS_WIN)
|
||||
// Windows makes high-resolution thread timing information available in
|
||||
// user-space.
|
||||
has_non_stop_time_stamp_counter_ = true;
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
CPU::IntelMicroArchitecture CPU::GetIntelMicroArchitecture() const {
|
||||
if (has_avx2()) return AVX2;
|
||||
if (has_avx()) return AVX;
|
||||
@@ -282,5 +270,4 @@ CPU::IntelMicroArchitecture CPU::GetIntelMicroArchitecture() const {
|
||||
if (has_sse()) return SSE;
|
||||
return PENTIUM;
|
||||
}
|
||||
|
||||
} // namespace base
|
||||
+47
-17
@@ -1,20 +1,57 @@
|
||||
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef BASE_CPU_H_
|
||||
#define BASE_CPU_H_
|
||||
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#define OS_MACOSX 1
|
||||
#elif defined(__ANDROID__)
|
||||
#define OS_ANDROID 1
|
||||
#elif defined(__linux__)
|
||||
#define OS_LINUX 1
|
||||
#elif defined(_WIN32)
|
||||
#define OS_WIN 1
|
||||
#endif
|
||||
|
||||
#if defined(_M_X64) || defined(__x86_64__)
|
||||
#define ARCH_CPU_X86_FAMILY 1
|
||||
#define ARCH_CPU_X86_64 1
|
||||
#define ARCH_CPU_64_BITS 1
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define ARCH_CPU_X86_FAMILY 1
|
||||
#define ARCH_CPU_X86 1
|
||||
#define ARCH_CPU_32_BITS 1
|
||||
#elif defined(__ARMEL__)
|
||||
#define ARCH_CPU_ARM_FAMILY 1
|
||||
#define ARCH_CPU_ARMEL 1
|
||||
#define ARCH_CPU_32_BITS 1
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define ARCH_CPU_ARM_FAMILY 1
|
||||
#define ARCH_CPU_ARM64 1
|
||||
#define ARCH_CPU_64_BITS 1
|
||||
#endif
|
||||
|
||||
#if defined(OS_MACOSX) || defined(OS_LINUX) || defined(OS_ANDROID)
|
||||
#define OS_POSIX 1
|
||||
#endif
|
||||
|
||||
namespace base {
|
||||
|
||||
#if defined(ARCH_CPU_X86_FAMILY)
|
||||
namespace internal {
|
||||
// Compute the CPU family and model based on the vendor and CPUID signature.
|
||||
// Returns in order: family, model, extended family, extended model.
|
||||
std::tuple<int, int, int, int> ComputeX86FamilyAndModel(
|
||||
const std::string& vendor,
|
||||
int signature);
|
||||
} // namespace internal
|
||||
#endif // defined(ARCH_CPU_X86_FAMILY)
|
||||
// Query information about the processor.
|
||||
class CPU {
|
||||
class CPU final {
|
||||
public:
|
||||
// Constructor
|
||||
CPU();
|
||||
|
||||
enum IntelMicroArchitecture {
|
||||
PENTIUM,
|
||||
SSE,
|
||||
@@ -27,7 +64,6 @@ class CPU {
|
||||
AVX2,
|
||||
MAX_INTEL_MICRO_ARCHITECTURE
|
||||
};
|
||||
|
||||
// Accessors for CPU information.
|
||||
const std::string& vendor_name() const { return cpu_vendor_; }
|
||||
int signature() const { return signature_; }
|
||||
@@ -44,24 +80,19 @@ class CPU {
|
||||
bool has_ssse3() const { return has_ssse3_; }
|
||||
bool has_sse41() const { return has_sse41_; }
|
||||
bool has_sse42() const { return has_sse42_; }
|
||||
bool has_popcnt() const { return has_popcnt_; }
|
||||
bool has_avx() const { return has_avx_; }
|
||||
bool has_avx2() const { return has_avx2_; }
|
||||
bool has_aesni() const { return has_aesni_; }
|
||||
bool has_non_stop_time_stamp_counter() const {
|
||||
return has_non_stop_time_stamp_counter_;
|
||||
}
|
||||
// has_broken_neon is only valid on ARM chips. If true, it indicates that we
|
||||
// believe that the NEON unit on the current CPU is flawed and cannot execute
|
||||
// some code. See https://code.google.com/p/chromium/issues/detail?id=341598
|
||||
bool has_broken_neon() const { return has_broken_neon_; }
|
||||
|
||||
bool is_running_in_vm() const { return is_running_in_vm_; }
|
||||
IntelMicroArchitecture GetIntelMicroArchitecture() const;
|
||||
const std::string& cpu_brand() const { return cpu_brand_; }
|
||||
|
||||
private:
|
||||
// Query the processor for CPUID information.
|
||||
void Initialize();
|
||||
|
||||
int signature_; // raw form of type, family, model, and stepping
|
||||
int type_; // process type
|
||||
int family_; // family of the processor
|
||||
@@ -76,15 +107,14 @@ class CPU {
|
||||
bool has_ssse3_;
|
||||
bool has_sse41_;
|
||||
bool has_sse42_;
|
||||
bool has_popcnt_;
|
||||
bool has_avx_;
|
||||
bool has_avx2_;
|
||||
bool has_aesni_;
|
||||
bool has_non_stop_time_stamp_counter_;
|
||||
bool has_broken_neon_;
|
||||
bool is_running_in_vm_;
|
||||
std::string cpu_vendor_;
|
||||
std::string cpu_brand_;
|
||||
};
|
||||
|
||||
} // namespace base
|
||||
|
||||
#endif // BASE_CPU_H_
|
||||
+1023
-1790
File diff suppressed because it is too large.
Load diff
@@ -51,15 +51,10 @@ static struct InitModule {
|
||||
int x = 0;
|
||||
// futexes are ints, so we can use them only when
|
||||
// that's the same size as the lockword_ in SpinLock.
|
||||
#ifdef __arm__
|
||||
// ARM linux doesn't support sys_futex1(void*, int, int, struct timespec*);
|
||||
have_futex = 0;
|
||||
#else
|
||||
have_futex = (sizeof (Atomic32) == sizeof (int) &&
|
||||
sys_futex(&x, FUTEX_WAKE, 1, 0) >= 0);
|
||||
#endif
|
||||
sys_futex(&x, FUTEX_WAKE, 1, NULL, NULL, 0) >= 0);
|
||||
if (have_futex &&
|
||||
sys_futex(&x, FUTEX_WAKE | futex_private_flag, 1, 0) < 0) {
|
||||
sys_futex(&x, FUTEX_WAKE | futex_private_flag, 1, NULL, NULL, 0) < 0) {
|
||||
futex_private_flag = 0;
|
||||
}
|
||||
}
|
||||
@@ -85,7 +80,8 @@ void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) {
|
||||
tm.tv_nsec *= 16; // increase the delay; we expect explicit wakeups
|
||||
sys_futex(reinterpret_cast<int *>(const_cast<Atomic32 *>(w)),
|
||||
FUTEX_WAIT | futex_private_flag,
|
||||
value, reinterpret_cast<struct kernel_timespec *>(&tm));
|
||||
value, reinterpret_cast<struct kernel_timespec *>(&tm),
|
||||
NULL, 0);
|
||||
} else {
|
||||
nanosleep(&tm, NULL);
|
||||
}
|
||||
@@ -96,7 +92,8 @@ void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) {
|
||||
void SpinLockWake(volatile Atomic32 *w, bool all) {
|
||||
if (have_futex) {
|
||||
sys_futex(reinterpret_cast<int *>(const_cast<Atomic32 *>(w)),
|
||||
FUTEX_WAKE | futex_private_flag, all? INT_MAX : 1, 0);
|
||||
FUTEX_WAKE | futex_private_flag, all? INT_MAX : 1,
|
||||
NULL, NULL, 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ set(UTIL_FILES
|
||||
minizip/ioapi.c
|
||||
minizip/unzip.c
|
||||
zip_util.cpp
|
||||
utf8_check.cpp
|
||||
)
|
||||
|
||||
if (WITH_MYSQL)
|
||||
|
||||
@@ -17,11 +17,23 @@
|
||||
|
||||
#include "util/cpu_info.h"
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <sys/sysctl.h>
|
||||
#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
|
||||
/* GCC-compatible compiler, targeting x86/x86-64 */
|
||||
#include <x86intrin.h>
|
||||
#elif defined(__GNUC__) && defined(__ARM_NEON__)
|
||||
/* GCC-compatible compiler, targeting ARM with NEON */
|
||||
#include <arm_neon.h>
|
||||
#elif defined(__GNUC__) && defined(__IWMMXT__)
|
||||
/* GCC-compatible compiler, targeting ARM with WMMX */
|
||||
#include <mmintrin.h>
|
||||
#elif (defined(__GNUC__) || defined(__xlC__)) && (defined(__VEC__) || defined(__ALTIVEC__))
|
||||
/* XLC or GCC-compatible compiler, targeting PowerPC with VMX/VSX */
|
||||
#include <altivec.h>
|
||||
#elif defined(__GNUC__) && defined(__SPE__)
|
||||
/* GCC-compatible compiler, targeting PowerPC with SPE */
|
||||
#include <spe.h>
|
||||
#endif
|
||||
|
||||
#include <mmintrin.h>
|
||||
#include <sched.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -23,50 +23,6 @@
|
||||
|
||||
namespace doris {
|
||||
|
||||
// Utility class to measure time. This is measured using the cpu tick counter which
|
||||
// is very low overhead but can be inaccurate if the thread is switched away. This
|
||||
// is useful for measuring cpu time at the row batch level (too much overhead at the
|
||||
// row granularity).
|
||||
class StopWatch {
|
||||
public:
|
||||
StopWatch() {
|
||||
_total_time = 0;
|
||||
_running = false;
|
||||
}
|
||||
|
||||
void start() {
|
||||
if (!_running) {
|
||||
_start = rdtsc();
|
||||
_running = true;
|
||||
}
|
||||
}
|
||||
|
||||
void stop() {
|
||||
if (_running) {
|
||||
_total_time += rdtsc() - _start;
|
||||
_running = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Returns time in cpu ticks.
|
||||
uint64_t elapsed_time() const {
|
||||
return _running ? rdtsc() - _start : _total_time;
|
||||
}
|
||||
|
||||
static uint64_t rdtsc() {
|
||||
uint32_t lo, hi;
|
||||
__asm__ __volatile__(
|
||||
"xorl %%eax,%%eax \n cpuid"
|
||||
::: "%rax", "%rbx", "%rcx", "%rdx");
|
||||
__asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi));
|
||||
return (uint64_t)hi << 32 | lo;
|
||||
}
|
||||
|
||||
private:
|
||||
uint64_t _start, _total_time;
|
||||
bool _running;
|
||||
};
|
||||
|
||||
// Stop watch for reporting elapsed time in nanosec based on CLOCK_MONOTONIC.
|
||||
// It is as fast as Rdtsc.
|
||||
// It is also accurate because it not affected by cpu frequency changes and
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright (c) cyb70289(https://github.com/cyb70289). All rights reserved.
|
||||
// Use of this source code is governed by a MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
/*
|
||||
* These functions are used for validating utf8 string.
|
||||
* Details can be seen here: https://github.com/cyb70289/utf8/
|
||||
*/
|
||||
|
||||
#include "util/utf8_check.h"
|
||||
|
||||
#include <cstdlib>
|
||||
|
||||
#if defined(__i386) || defined(__x86_64__)
|
||||
#include "util/simdutf8check.h"
|
||||
#elif defined(__aarch64__)
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
/*
|
||||
* http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
|
||||
*
|
||||
* Table 3-7. Well-Formed UTF-8 Byte Sequences
|
||||
*
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0000..U+007F | 00..7F | | | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0080..U+07FF | C2..DF | 80..BF | | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
*/
|
||||
namespace doris {
|
||||
bool validate_utf8_naive(const char *data, size_t len) {
|
||||
while (len) {
|
||||
int bytes;
|
||||
const unsigned char byte1 = data[0];
|
||||
|
||||
/* 00..7F */
|
||||
if (byte1 <= 0x7F) {
|
||||
bytes = 1;
|
||||
/* C2..DF, 80..BF */
|
||||
} else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
|
||||
(signed char)data[1] <= (signed char)0xBF) {
|
||||
bytes = 2;
|
||||
} else if (len >= 3) {
|
||||
const unsigned char byte2 = data[1];
|
||||
|
||||
/* Is byte2, byte3 between 0x80 ~ 0xBF */
|
||||
const int byte2_ok = (signed char)byte2 <= (signed char)0xBF;
|
||||
const int byte3_ok = (signed char)data[2] <= (signed char)0xBF;
|
||||
|
||||
if (byte2_ok && byte3_ok &&
|
||||
/* E0, A0..BF, 80..BF */
|
||||
((byte1 == 0xE0 && byte2 >= 0xA0) ||
|
||||
/* E1..EC, 80..BF, 80..BF */
|
||||
(byte1 >= 0xE1 && byte1 <= 0xEC) ||
|
||||
/* ED, 80..9F, 80..BF */
|
||||
(byte1 == 0xED && byte2 <= 0x9F) ||
|
||||
/* EE..EF, 80..BF, 80..BF */
|
||||
(byte1 >= 0xEE && byte1 <= 0xEF))) {
|
||||
bytes = 3;
|
||||
} else if (len >= 4) {
|
||||
/* Is byte4 between 0x80 ~ 0xBF */
|
||||
const int byte4_ok = (signed char)data[3] <= (signed char)0xBF;
|
||||
|
||||
if (byte2_ok && byte3_ok && byte4_ok &&
|
||||
/* F0, 90..BF, 80..BF, 80..BF */
|
||||
((byte1 == 0xF0 && byte2 >= 0x90) ||
|
||||
/* F1..F3, 80..BF, 80..BF, 80..BF */
|
||||
(byte1 >= 0xF1 && byte1 <= 0xF3) ||
|
||||
/* F4, 80..8F, 80..BF, 80..BF */
|
||||
(byte1 == 0xF4 && byte2 <= 0x8F))) {
|
||||
bytes = 4;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
len -= bytes;
|
||||
data += bytes;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#if defined(__i386) || defined(__x86_64__)
|
||||
bool validate_utf8(const char *src, size_t len) {
|
||||
return validate_utf8_fast(src, len);
|
||||
}
|
||||
#elif defined(__aarch64__)
|
||||
/*
|
||||
* Map high nibble of "First Byte" to legal character length minus 1
|
||||
* 0x00 ~ 0xBF --> 0
|
||||
* 0xC0 ~ 0xDF --> 1
|
||||
* 0xE0 ~ 0xEF --> 2
|
||||
* 0xF0 ~ 0xFF --> 3
|
||||
*/
|
||||
const uint8_t _first_len_tbl[] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
|
||||
};
|
||||
|
||||
/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
|
||||
static const uint8_t _first_range_tbl[] = {
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
|
||||
};
|
||||
|
||||
/*
|
||||
* Range table, map range index to min and max values
|
||||
* Index 0 : 00 ~ 7F (First Byte, ascii)
|
||||
* Index 1,2,3: 80 ~ BF (Second, Third, Fourth Byte)
|
||||
* Index 4 : A0 ~ BF (Second Byte after E0)
|
||||
* Index 5 : 80 ~ 9F (Second Byte after ED)
|
||||
* Index 6 : 90 ~ BF (Second Byte after F0)
|
||||
* Index 7 : 80 ~ 8F (Second Byte after F4)
|
||||
* Index 8 : C2 ~ F4 (First Byte, non ascii)
|
||||
* Index 9~15 : illegal: u >= 255 && u <= 0
|
||||
*/
|
||||
static const uint8_t _range_min_tbl[] = {
|
||||
0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
|
||||
0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
};
|
||||
static const uint8_t _range_max_tbl[] = {
|
||||
0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
|
||||
0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
};
|
||||
|
||||
/*
|
||||
* This table is for fast handling four special First Bytes(E0,ED,F0,F4), after
|
||||
* which the Second Byte are not 80~BF. It contains "range index adjustment".
|
||||
* - The idea is to minus byte with E0, use the result(0~31) as the index to
|
||||
* lookup the "range index adjustment". Then add the adjustment to original
|
||||
* range index to get the correct range.
|
||||
* - Range index adjustment
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | First Byte | original range| range adjustment | adjusted range |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | E0 | 2 | 2 | 4 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | ED | 2 | 3 | 5 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | F0 | 3 | 3 | 6 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | F4 | 4 | 4 | 8 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* - Below is a uint8x16x2 table, data is interleaved in NEON register. So I'm
|
||||
* putting it vertically. 1st column is for E0~EF, 2nd column for F0~FF.
|
||||
*/
|
||||
static const uint8_t _range_adjust_tbl[] = {
|
||||
/* index -> 0~15 16~31 <- index */
|
||||
/* E0 -> */ 2, 3, /* <- F0 */
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 4, /* <- F4 */
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
/* ED -> */ 3, 0,
|
||||
0, 0,
|
||||
0, 0,
|
||||
};
|
||||
|
||||
/* 2x ~ 4x faster than naive method */
|
||||
/* Return true on success, false on error */
|
||||
bool utf8_range(const char *data, size_t len)
|
||||
{
|
||||
if (len >= 16) {
|
||||
uint8x16_t prev_input = vdupq_n_u8(0);
|
||||
uint8x16_t prev_first_len = vdupq_n_u8(0);
|
||||
|
||||
/* Cached tables */
|
||||
const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl);
|
||||
const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl);
|
||||
const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl);
|
||||
const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl);
|
||||
const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl);
|
||||
|
||||
/* Cached values */
|
||||
const uint8x16_t const_1 = vdupq_n_u8(1);
|
||||
const uint8x16_t const_2 = vdupq_n_u8(2);
|
||||
const uint8x16_t const_e0 = vdupq_n_u8(0xE0);
|
||||
|
||||
uint8x16_t error = vdupq_n_u8(0);
|
||||
|
||||
while (len >= 16) {
|
||||
const uint8x16_t input = vld1q_u8((const uint8_t*)data);
|
||||
|
||||
/* high_nibbles = input >> 4 */
|
||||
const uint8x16_t high_nibbles = vshrq_n_u8(input, 4);
|
||||
|
||||
/* first_len = legal character length minus 1 */
|
||||
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
|
||||
/* first_len = first_len_tbl[high_nibbles] */
|
||||
const uint8x16_t first_len =
|
||||
vqtbl1q_u8(first_len_tbl, high_nibbles);
|
||||
|
||||
/* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
|
||||
/* range = first_range_tbl[high_nibbles] */
|
||||
uint8x16_t range = vqtbl1q_u8(first_range_tbl, high_nibbles);
|
||||
|
||||
/* Second Byte: set range index to first_len */
|
||||
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
|
||||
/* range |= (first_len, prev_first_len) << 1 byte */
|
||||
range =
|
||||
vorrq_u8(range, vextq_u8(prev_first_len, first_len, 15));
|
||||
|
||||
/* Third Byte: set range index to saturate_sub(first_len, 1) */
|
||||
/* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
|
||||
uint8x16_t tmp1, tmp2;
|
||||
/* tmp1 = saturate_sub(first_len, 1) */
|
||||
tmp1 = vqsubq_u8(first_len, const_1);
|
||||
/* tmp2 = saturate_sub(prev_first_len, 1) */
|
||||
tmp2 = vqsubq_u8(prev_first_len, const_1);
|
||||
/* range |= (tmp1, tmp2) << 2 bytes */
|
||||
range = vorrq_u8(range, vextq_u8(tmp2, tmp1, 14));
|
||||
|
||||
/* Fourth Byte: set range index to saturate_sub(first_len, 2) */
|
||||
/* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
|
||||
/* tmp1 = saturate_sub(first_len, 2) */
|
||||
tmp1 = vqsubq_u8(first_len, const_2);
|
||||
/* tmp2 = saturate_sub(prev_first_len, 2) */
|
||||
tmp2 = vqsubq_u8(prev_first_len, const_2);
|
||||
/* range |= (tmp1, tmp2) << 3 bytes */
|
||||
range = vorrq_u8(range, vextq_u8(tmp2, tmp1, 13));
|
||||
|
||||
/*
|
||||
* Now we have below range indices caluclated
|
||||
* Correct cases:
|
||||
* - 8 for C0~FF
|
||||
* - 3 for 1st byte after F0~FF
|
||||
* - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
|
||||
* - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
|
||||
* 3rd byte after F0~FF
|
||||
* - 0 for others
|
||||
* Error cases:
|
||||
* 9,10,11 if non ascii First Byte overlaps
|
||||
* E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
|
||||
*/
|
||||
|
||||
/* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
|
||||
/* See _range_adjust_tbl[] definition for details */
|
||||
/* Overlaps lead to index 9~15, which are illegal in range table */
|
||||
uint8x16_t shift1 = vextq_u8(prev_input, input, 15);
|
||||
uint8x16_t pos = vsubq_u8(shift1, const_e0);
|
||||
range = vaddq_u8(range, vqtbl2q_u8(range_adjust_tbl, pos));
|
||||
|
||||
/* Load min and max values per calculated range index */
|
||||
uint8x16_t minv = vqtbl1q_u8(range_min_tbl, range);
|
||||
uint8x16_t maxv = vqtbl1q_u8(range_max_tbl, range);
|
||||
|
||||
/* Check value range */
|
||||
error = vorrq_u8(error, vcltq_u8(input, minv));
|
||||
error = vorrq_u8(error, vcgtq_u8(input, maxv));
|
||||
|
||||
prev_input = input;
|
||||
prev_first_len = first_len;
|
||||
|
||||
data += 16;
|
||||
len -= 16;
|
||||
}
|
||||
|
||||
/* Delay error check till loop ends */
|
||||
if (vmaxvq_u8(error))
|
||||
return false;
|
||||
|
||||
/* Find previous token (not 80~BF) */
|
||||
uint32_t token4;
|
||||
vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3);
|
||||
|
||||
const int8_t *token = (const int8_t *)&token4;
|
||||
int lookahead = 0;
|
||||
if (token[3] > (int8_t)0xBF)
|
||||
lookahead = 1;
|
||||
else if (token[2] > (int8_t)0xBF)
|
||||
lookahead = 2;
|
||||
else if (token[1] > (int8_t)0xBF)
|
||||
lookahead = 3;
|
||||
|
||||
data -= lookahead;
|
||||
len += lookahead;
|
||||
}
|
||||
|
||||
/* Check remaining bytes with naive method */
|
||||
return validate_utf8_naive(data, len);
|
||||
}
|
||||
|
||||
bool validate_utf8(const char *src, size_t len) {
|
||||
return utf8_range(src, len);
|
||||
}
|
||||
#else
|
||||
bool validate_utf8(const char *src, size_t len) {
|
||||
return validate_utf8_naive(src, len);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#ifndef DORIS_BE_SRC_UTIL_UTF8_CHECK_H
|
||||
#define DORIS_BE_SRC_UTIL_UTF8_CHECK_H
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace doris {
|
||||
// check utf8 code using simd instructions
|
||||
// Return true - success, false fail
|
||||
bool validate_utf8(const char *src, size_t len);
|
||||
// chech utf8 use naive c++
|
||||
bool validate_utf8_naive(const char *data, size_t len);
|
||||
} // namespce doris
|
||||
|
||||
#endif // DORIS_BE_SRC_UTIL_UTF8_CHECK_H
|
||||
@@ -136,8 +136,8 @@ TEST_F(GeoFunctionsTest, st_from_wkt) {
|
||||
GeoPoint point;
|
||||
auto res = point.decode_from(str2.ptr, str2.len);
|
||||
ASSERT_TRUE(res);
|
||||
ASSERT_EQ(10.1, point.x());
|
||||
ASSERT_EQ(20.2, point.y());
|
||||
ASSERT_DOUBLE_EQ(10.1, point.x());
|
||||
ASSERT_DOUBLE_EQ(20.2, point.y());
|
||||
GeoFunctions::st_from_wkt_close(ctx, FunctionContext::FRAGMENT_LOCAL);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,3 +53,4 @@ ADD_BE_TEST(frame_of_reference_coding_test)
|
||||
ADD_BE_TEST(bit_stream_utils_test)
|
||||
ADD_BE_TEST(radix_sort_test)
|
||||
ADD_BE_TEST(zip_util_test)
|
||||
ADD_BE_TEST(utf8_check_test)
|
||||
@@ -110,7 +110,7 @@ void test_float_value_is_nan(const std::string& s, StringParser::ParseResult exp
|
||||
EXPECT_EQ(exp_result, result);
|
||||
|
||||
if (exp_result == StringParser::PARSE_SUCCESS && result == exp_result) {
|
||||
EXPECT_TRUE(isnan(val));
|
||||
EXPECT_TRUE(std::isnan(val));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "util/utf8_check.h"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
namespace doris {
|
||||
|
||||
struct test {
|
||||
const char *data;
|
||||
int len;
|
||||
};
|
||||
|
||||
class Utf8CheckTest : public testing::Test {
|
||||
public:
|
||||
Utf8CheckTest() { }
|
||||
virtual ~Utf8CheckTest() { }
|
||||
private:
|
||||
/* positive tests */
|
||||
std::vector<test> pos = {
|
||||
{"", 0},
|
||||
{"\x00", 1},
|
||||
{"\x66", 1},
|
||||
{"\x7F", 1},
|
||||
{"\x00\x7F", 2},
|
||||
{"\x7F\x00", 2},
|
||||
{"\xC2\x80", 2},
|
||||
{"\xDF\xBF", 2},
|
||||
{"\xE0\xA0\x80", 3},
|
||||
{"\xE0\xA0\xBF", 3},
|
||||
{"\xED\x9F\x80", 3},
|
||||
{"\xEF\x80\xBF", 3},
|
||||
{"\xF0\x90\xBF\x80", 4},
|
||||
{"\xF2\x81\xBE\x99", 4},
|
||||
{"\xF4\x8F\x88\xAA", 4}
|
||||
};
|
||||
|
||||
/* negative tests */
|
||||
std::vector<test> neg = {
|
||||
{"\x80", 1},
|
||||
{"\xBF", 1},
|
||||
{"\xC0\x80", 2},
|
||||
{"\xC1\x00", 2},
|
||||
{"\xC2\x7F", 2},
|
||||
{"\xDF\xC0", 2},
|
||||
{"\xE0\x9F\x80", 3},
|
||||
{"\xE0\xC2\x80", 3},
|
||||
{"\xED\xA0\x80", 3},
|
||||
{"\xED\x7F\x80", 3},
|
||||
{"\xEF\x80\x00", 3},
|
||||
{"\xF0\x8F\x80\x80", 4},
|
||||
{"\xF0\xEE\x80\x80", 4},
|
||||
{"\xF2\x90\x91\x7F", 4},
|
||||
{"\xF4\x90\x88\xAA", 4},
|
||||
{"\xF4\x00\xBF\xBF", 4},
|
||||
{"\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2" \
|
||||
"\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
|
||||
32},
|
||||
{"\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00",
|
||||
16},
|
||||
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
||||
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80",
|
||||
32},
|
||||
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
||||
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1",
|
||||
32},
|
||||
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
||||
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80" \
|
||||
"\x80", 33},
|
||||
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
||||
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80" \
|
||||
"\xC2\x80", 34},
|
||||
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
||||
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0" \
|
||||
"\x80\x80\x80", 35}
|
||||
};
|
||||
};
|
||||
TEST_F(Utf8CheckTest, empty) {
|
||||
ASSERT_TRUE(validate_utf8(pos[0].data, pos[0].len));
|
||||
}
|
||||
|
||||
TEST_F(Utf8CheckTest, normal) {
|
||||
for (int i = 0; i < sizeof(pos)/sizeof(pos[0]); ++i) {
|
||||
ASSERT_TRUE(validate_utf8(pos[i].data, pos[i].len));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(Utf8CheckTest, abnormal) {
|
||||
for (int i = 0; i < sizeof(neg)/sizeof(neg[0]); ++i) {
|
||||
ASSERT_FALSE(validate_utf8(neg[i].data, neg[i].len));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(Utf8CheckTest, naive) {
|
||||
for (int i = 0; i < sizeof(pos)/sizeof(pos[0]); ++i) {
|
||||
ASSERT_TRUE(validate_utf8_naive(pos[i].data, pos[i].len));
|
||||
}
|
||||
for (int i = 0; i < sizeof(neg)/sizeof(neg[0]); ++i) {
|
||||
std::cout << validate_utf8_naive(neg[i].data, neg[i].len) << std::endl;
|
||||
ASSERT_FALSE(validate_utf8_naive(neg[i].data, neg[i].len));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -164,6 +164,7 @@ ${DORIS_TEST_BINARY_DIR}/util/counter_cond_variable_test
|
||||
${DORIS_TEST_BINARY_DIR}/util/bit_stream_utils_test
|
||||
${DORIS_TEST_BINARY_DIR}/util/frame_of_reference_coding_test
|
||||
${DORIS_TEST_BINARY_DIR}/util/zip_util_test
|
||||
${DORIS_TEST_BINARY_DIR}/util/utf8_check_test
|
||||
|
||||
# Running common Unittest
|
||||
${DORIS_TEST_BINARY_DIR}/common/resource_tls_test
|
||||
|
||||
Reference in new issue
Block a user