mirror of
https://github.com/apache/doris.git
synced 2026-11-04 16:03:23 +00:00
Refrain
07ba0d1e44
Problem Summary: S3 endpoint scheme handling was inconsistent across FE, BE, and Cloud, and the previous implementation distinguished Doris internal buckets from user-configured external buckets. That classification leaked into S3Conf construction and client-cache identity even though the desired transport rule is the same for every AWS/S3-compatible endpoint. This PR applies one rule across S3 client construction and Resource/Storage Vault persistence boundaries: - Keep the existing FE, BE, and Cloud s3_client_http_scheme compatibility setting. - Default every endpoint without an explicit scheme to HTTPS. - If s3_client_http_scheme is explicitly set to http, use HTTP for endpoints without a scheme. - Preserve endpoints that already contain an explicit http:// or https:// scheme. - Remove the internal/external bucket classification from BE and Cloud S3 configuration. - Apply the same behavior to FE native S3 clients, the S3 filesystem plugin, connectivity checks, stages, resources, and Iceberg S3 FileIO handoffs. - Normalize scheme-less S3 Resource and Storage Vault endpoint metadata before persistence using the same configured default. Azure is intentionally unchanged in this PR. Its existing provider-specific behavior already defaults a bare endpoint to HTTPS and preserves an explicit scheme; whether Azure should share the S3 compatibility setting is a separate policy decision. ### Release note S3 endpoints without an explicit scheme now use HTTPS by default across FE, BE, and Cloud. Set s3_client_http_scheme = http for compatibility with scheme-less HTTP endpoints. Explicit HTTP and HTTPS endpoint schemes are unchanged.
98 lines
3.5 KiB
C++
98 lines
3.5 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include "aws_common.h"
|
|
|
|
#include <aws/core/client/ClientConfiguration.h>
|
|
#include <glog/logging.h>
|
|
|
|
namespace doris {
|
|
|
|
CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB cred_provider_type) {
|
|
switch (cred_provider_type) {
|
|
case cloud::CredProviderTypePB::DEFAULT:
|
|
return CredProviderType::Default;
|
|
case cloud::CredProviderTypePB::SIMPLE:
|
|
return CredProviderType::Simple;
|
|
case cloud::CredProviderTypePB::INSTANCE_PROFILE:
|
|
return CredProviderType::InstanceProfile;
|
|
case cloud::CredProviderTypePB::ENV:
|
|
return CredProviderType::Env;
|
|
case cloud::CredProviderTypePB::SYSTEM_PROPERTIES:
|
|
return CredProviderType::SystemProperties;
|
|
case cloud::CredProviderTypePB::WEB_IDENTITY:
|
|
return CredProviderType::WebIdentity;
|
|
case cloud::CredProviderTypePB::CONTAINER:
|
|
return CredProviderType::Container;
|
|
case cloud::CredProviderTypePB::ANONYMOUS:
|
|
return CredProviderType::Anonymous;
|
|
default:
|
|
__builtin_unreachable();
|
|
LOG(WARNING) << "Invalid CredProviderTypePB value: " << cred_provider_type
|
|
<< ", use default instead.";
|
|
return CredProviderType::Default;
|
|
}
|
|
}
|
|
|
|
CredProviderType cred_provider_type_from_string(const std::string& type) {
|
|
if (type.empty() || type == "DEFAULT") {
|
|
return CredProviderType::Default;
|
|
}
|
|
if (type == "SIMPLE") {
|
|
return CredProviderType::Simple;
|
|
}
|
|
if (type == "INSTANCE_PROFILE") {
|
|
return CredProviderType::InstanceProfile;
|
|
}
|
|
if (type == "ENV") {
|
|
return CredProviderType::Env;
|
|
}
|
|
if (type == "SYSTEM_PROPERTIES") {
|
|
return CredProviderType::SystemProperties;
|
|
}
|
|
if (type == "WEB_IDENTITY") {
|
|
return CredProviderType::WebIdentity;
|
|
}
|
|
if (type == "CONTAINER") {
|
|
return CredProviderType::Container;
|
|
}
|
|
if (type == "ANONYMOUS") {
|
|
return CredProviderType::Anonymous;
|
|
}
|
|
LOG(WARNING) << "Unknown credentials provider type: " << type << ", use default instead.";
|
|
return CredProviderType::Default;
|
|
}
|
|
|
|
std::string get_valid_ca_cert_path(const std::vector<std::string>& ca_cert_file_paths) {
|
|
for (const auto& path : ca_cert_file_paths) {
|
|
if (std::filesystem::exists(path)) {
|
|
return path;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
void set_s3_client_default_http_scheme(Aws::Client::ClientConfiguration& client_config,
|
|
const std::string& scheme) {
|
|
if (client_config.endpointOverride.starts_with("http://") ||
|
|
client_config.endpointOverride.starts_with("https://")) {
|
|
return;
|
|
}
|
|
client_config.scheme = scheme == "http" ? Aws::Http::Scheme::HTTP : Aws::Http::Scheme::HTTPS;
|
|
}
|
|
} // namespace doris
|