Skip to content

Expose create_s3_header and add new API for S3AuthParams::ReadFrom #173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NiclasHaderer wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Expose create_s3_header and add new API for S3AuthParams::ReadFrom #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5fa6bf9
Expose httpfs function
NiclasHaderer Dec 9, 2025
069bcb6
Renamed create_s3_header
NiclasHaderer Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/include/s3fs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ struct S3AuthParams {
string oauth2_bearer_token; // OAuth2 bearer token for GCS

static S3AuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);
static S3AuthParams ReadFrom(KeyValueSecretReader& secret_reader, const std::string& file_path);
};

struct AWSEnvironmentCredentialsProvider {
Expand Down Expand Up @@ -255,4 +256,8 @@ struct AWSListObjectV2 {
static vector<string> ParseCommonPrefix(string &aws_response);
static string ParseContinuationToken(string &aws_response);
};

HTTPHeaders CreateS3Header(string url, string query, string host, string service, string method,
const S3AuthParams &auth_params, string date_now = "", string datetime_now = "",
string payload_hash = "", string content_type = "");
} // namespace duckdb
31 changes: 18 additions & 13 deletions src/s3fs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@

namespace duckdb {

static HTTPHeaders create_s3_header(string url, string query, string host, string service, string method,
const S3AuthParams &auth_params, string date_now = "", string datetime_now = "",
string payload_hash = "", string content_type = "") {
HTTPHeaders CreateS3Header(string url, string query, string host, string service, string method,
const S3AuthParams &auth_params, string date_now, string datetime_now, string payload_hash,
string content_type) {

HTTPHeaders res;
res["Host"] = host;
Expand Down Expand Up @@ -184,16 +184,21 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
}

S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info) {
auto result = S3AuthParams();

// Without a FileOpener we can not access settings nor secrets: return empty auth params
if (!opener) {
return result;
return {};
}

const char *secret_types[] = {"s3", "r2", "gcs", "aws"};
KeyValueSecretReader secret_reader(*opener, info, secret_types, 3);

return ReadFrom(secret_reader, info.file_path);
}

S3AuthParams S3AuthParams::ReadFrom(KeyValueSecretReader &secret_reader, const std::string &file_path) {
auto result = S3AuthParams();

// These settings we just set or leave to their S3AuthParams default value
secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region);
secret_reader.TryGetSecretKeyOrSetting("key_id", "s3_access_key_id", result.access_key_id);
Expand All @@ -210,7 +215,7 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
auto endpoint_result = secret_reader.TryGetSecretKeyOrSetting("endpoint", "s3_endpoint", result.endpoint);
auto url_style_result = secret_reader.TryGetSecretKeyOrSetting("url_style", "s3_url_style", result.url_style);

if (StringUtil::StartsWith(info.file_path, "gcs://") || StringUtil::StartsWith(info.file_path, "gs://")) {
if (StringUtil::StartsWith(file_path, "gcs://") || StringUtil::StartsWith(file_path, "gs://")) {
// For GCS urls we force the endpoint and vhost path style, allowing only to be overridden by secrets
if (result.endpoint.empty() || endpoint_result.GetScope() != SettingScope::SECRET) {
result.endpoint = "storage.googleapis.com";
Expand Down Expand Up @@ -742,7 +747,7 @@ unique_ptr<HTTPResponse> S3FileSystem::PostRequest(FileHandle &handle, string ur
} else {
// Use existing S3 authentication
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
headers = CreateS3Header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
"", payload_hash, "application/octet-stream");
}

Expand All @@ -765,7 +770,7 @@ unique_ptr<HTTPResponse> S3FileSystem::PutRequest(FileHandle &handle, string url
} else {
// Use existing S3 authentication
auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
headers = CreateS3Header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
"", payload_hash, content_type);
}

Expand All @@ -785,7 +790,7 @@ unique_ptr<HTTPResponse> S3FileSystem::HeadRequest(FileHandle &handle, string s3
} else {
// Use existing S3 authentication
headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", "");
CreateS3Header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", "");
}

return HTTPFileSystem::HeadRequest(handle, http_url, headers);
Expand All @@ -804,7 +809,7 @@ unique_ptr<HTTPResponse> S3FileSystem::GetRequest(FileHandle &handle, string s3_
} else {
// Use existing S3 authentication
headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
CreateS3Header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
}

return HTTPFileSystem::GetRequest(handle, http_url, headers);
Expand All @@ -824,7 +829,7 @@ unique_ptr<HTTPResponse> S3FileSystem::GetRangeRequest(FileHandle &handle, strin
} else {
// Use existing S3 authentication
headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
CreateS3Header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
}

return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len);
Expand All @@ -843,7 +848,7 @@ unique_ptr<HTTPResponse> S3FileSystem::DeleteRequest(FileHandle &handle, string
} else {
// Use existing S3 authentication
headers =
create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", "");
CreateS3Header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", "");
}

return HTTPFileSystem::DeleteRequest(handle, http_url, headers);
Expand Down Expand Up @@ -1232,7 +1237,7 @@ string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthPar
string listobjectv2_url = req_path + "?" + req_params;

auto header_map =
create_s3_header(req_path, req_params, parsed_url.host, "s3", "GET", s3_auth_params, "", "", "", "");
CreateS3Header(req_path, req_params, parsed_url.host, "s3", "GET", s3_auth_params, "", "", "", "");

// Get requests use fresh connection
string full_host = parsed_url.http_proto + parsed_url.host;
Expand Down
Loading