mirror of
https://github.com/apache/doris.git
synced 2026-11-04 18:02:19 +00:00
Yixuan Wang
533f02b970
### What problem does this PR solve?
Support configuring the AWS S3 credentials provider type for cloud
storage vaults and S3 storage parameters. This allows IAM role based
access to use providers other than instance profile, such as
environment, system
properties, web identity, container, default chain, and anonymous
credentials. The credential provider type is propagated through FE
properties, thrift/proto definitions, meta service storage vault
handling, and cloud
recycler S3 accessor creation.
```
CREATE storage vault IF NOT EXISTS s3_vault_2
PROPERTIES (
"type"="S3",
"s3.endpoint"="xxx",
"s3.role_arn" = "xxx",
"s3.region" = "us-west-1",
"s3.root.path" = "xxx",
"s3.bucket" = "xxx",
"provider" = "S3",
"use_path_style" = "false",
"s3.credentials_provider_type" = "WEB_IDENTITY",
"s3_validity_check" = "false"
);
```
```
curl "127.0.0.1:5678/MetaService/http/get_obj_store_info
```
```
{
"id": "3",
"name": "s3_vault_2",
"obj_info": {
"ctime": "1777011933",
"mtime": "1777011933",
"id": "3",
"bucket": "xxx",
"prefix": "wyx_test",
"endpoint": "xxx",
"region": "us-west-1",
"provider": "S3",
"external_endpoint": "",
"sse_enabled": true,
"use_path_style": false,
"cred_provider_type": "WEB_IDENTITY",
"role_arn": "xxx",
"external_id": ""
}
}
```
88 lines
3.0 KiB
C++
88 lines
3.0 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include "aws_common.h"
|
|
|
|
#include <glog/logging.h>
|
|
|
|
namespace doris {
|
|
|
|
CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB cred_provider_type) {
|
|
switch (cred_provider_type) {
|
|
case cloud::CredProviderTypePB::DEFAULT:
|
|
return CredProviderType::Default;
|
|
case cloud::CredProviderTypePB::SIMPLE:
|
|
return CredProviderType::Simple;
|
|
case cloud::CredProviderTypePB::INSTANCE_PROFILE:
|
|
return CredProviderType::InstanceProfile;
|
|
case cloud::CredProviderTypePB::ENV:
|
|
return CredProviderType::Env;
|
|
case cloud::CredProviderTypePB::SYSTEM_PROPERTIES:
|
|
return CredProviderType::SystemProperties;
|
|
case cloud::CredProviderTypePB::WEB_IDENTITY:
|
|
return CredProviderType::WebIdentity;
|
|
case cloud::CredProviderTypePB::CONTAINER:
|
|
return CredProviderType::Container;
|
|
case cloud::CredProviderTypePB::ANONYMOUS:
|
|
return CredProviderType::Anonymous;
|
|
default:
|
|
__builtin_unreachable();
|
|
LOG(WARNING) << "Invalid CredProviderTypePB value: " << cred_provider_type
|
|
<< ", use default instead.";
|
|
return CredProviderType::Default;
|
|
}
|
|
}
|
|
|
|
CredProviderType cred_provider_type_from_string(const std::string& type) {
|
|
if (type.empty() || type == "DEFAULT") {
|
|
return CredProviderType::Default;
|
|
}
|
|
if (type == "SIMPLE") {
|
|
return CredProviderType::Simple;
|
|
}
|
|
if (type == "INSTANCE_PROFILE") {
|
|
return CredProviderType::InstanceProfile;
|
|
}
|
|
if (type == "ENV") {
|
|
return CredProviderType::Env;
|
|
}
|
|
if (type == "SYSTEM_PROPERTIES") {
|
|
return CredProviderType::SystemProperties;
|
|
}
|
|
if (type == "WEB_IDENTITY") {
|
|
return CredProviderType::WebIdentity;
|
|
}
|
|
if (type == "CONTAINER") {
|
|
return CredProviderType::Container;
|
|
}
|
|
if (type == "ANONYMOUS") {
|
|
return CredProviderType::Anonymous;
|
|
}
|
|
LOG(WARNING) << "Unknown credentials provider type: " << type << ", use default instead.";
|
|
return CredProviderType::Default;
|
|
}
|
|
|
|
std::string get_valid_ca_cert_path(const std::vector<std::string>& ca_cert_file_paths) {
|
|
for (const auto& path : ca_cert_file_paths) {
|
|
if (std::filesystem::exists(path)) {
|
|
return path;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
}
|