From b893f89be5beb285659312b8ac25d696497b857b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 12 Nov 2025 11:21:27 +0100 Subject: [PATCH 01/13] Fixup order of operations, first set body, then send, THEN set body This has been discovered while moving to self-signing AWS requests to Iceberg catalogs --- src/httpfs_httplib_client.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/httpfs_httplib_client.cpp b/src/httpfs_httplib_client.cpp index 239a1124..84e7276f 100644 --- a/src/httpfs_httplib_client.cpp +++ b/src/httpfs_httplib_client.cpp @@ -102,8 +102,12 @@ class HTTPFSClient : public HTTPClient { info.buffer_out += string(data, data_length); return true; }; + // First assign body, this is the body that will be uploaded req.body.assign(const_char_ptr_cast(info.buffer_in), info.buffer_in_len); - return TransformResult(client->send(req)); + auto transformed_req = TransformResult(client->send(req)); + // Then, after actual re-quest, re-assign body to the response value of the POST request + transformed_req->body.assign(const_char_ptr_cast(info.buffer_in), info.buffer_in_len); + return std::move(transformed_req); } private: From 0b60ba349fed50c0b746534b8a0e5499064cc023 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sat, 15 Nov 2025 13:12:19 +0100 Subject: [PATCH 02/13] Check reads from cached buffers using the known length --- src/httpfs.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index a11af95b..270adc72 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -438,6 +438,9 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b if (!hfh.cached_file_handle->Initialized()) { throw InternalException("Cached file not initialized properly"); } + if (hfh.cached_file_handle->GetSize() < location + nr_bytes) { + throw InternalException("Cached file length can't satisfy the requested Read"); + } memcpy(buffer, hfh.cached_file_handle->GetData() + location, nr_bytes); DUCKDB_LOG_FILE_SYSTEM_READ(handle, nr_bytes, location); hfh.file_offset = location + nr_bytes; From 78eda7f12f549aab1b0682a97132bf754d95e40d Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 2 Dec 2025 15:09:15 +0100 Subject: [PATCH 03/13] fix http header merging --- src/create_secret_functions.cpp | 14 +++++++ src/httpfs.cpp | 25 ++++++++++++- src/httpfs_extension.cpp | 3 ++ src/httpfs_httplib_client.cpp | 3 -- src/s3fs.cpp | 3 +- test/sql/copy/s3/http_secret.test | 62 +++++++++++++++++++++++++++++-- 6 files changed, 100 insertions(+), 10 deletions(-) diff --git a/src/create_secret_functions.cpp b/src/create_secret_functions.cpp index b94f7f24..be306433 100644 --- a/src/create_secret_functions.cpp +++ b/src/create_secret_functions.cpp @@ -118,6 +118,14 @@ unique_ptr CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli secret->secret_map["bearer_token"] = named_param.second.ToString(); // Mark it as sensitive secret->redact_keys.insert("bearer_token"); + } else if (lower_name == "http_proxy") { + secret->secret_map["http_proxy"] = named_param.second; + } else if (lower_name == "http_proxy_password") { + secret->secret_map["http_proxy_password"] = named_param.second; + } else if (lower_name == "http_proxy_username") { + secret->secret_map["http_proxy_username"] = named_param.second; + } else if (lower_name == "extra_http_headers") { + secret->secret_map["extra_http_headers"] = named_param.second; } else { throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " + lower_name); @@ -200,6 +208,12 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function, // Whether a secret refresh attempt should be made when the secret appears to be incorrect function.named_parameters["refresh"] = LogicalType::VARCHAR; + // Params for HTTP configuration + function.named_parameters["http_proxy"] = LogicalType::VARCHAR; + function.named_parameters["http_proxy_password"] = LogicalType::VARCHAR; + function.named_parameters["http_proxy_username"] = LogicalType::VARCHAR; + function.named_parameters["extra_http_headers"] = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + // Refresh Modes // - auto // - disabled diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 270adc72..83c2864d 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -65,8 +65,18 @@ unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr } } + // Secret types to look for HTTP settings in + const char *secret_types[] = {"s3", "r2", "gcs", "aws", "http"}; + idx_t secret_type_count = 5; + + Value merge_http_secret_into_s3_request; + if (FileOpener::TryGetCurrentSetting(opener, "merge_http_secret_into_s3_request", merge_http_secret_into_s3_request) && !merge_http_secret_into_s3_request.IsNull() && !merge_http_secret_into_s3_request.GetValue()) { + // Drop the http secret from the lookup + secret_type_count = 4; + } + // HTTP Secret lookups - KeyValueSecretReader settings_reader(*opener, info, "http"); + KeyValueSecretReader settings_reader(*opener, info, secret_types, secret_type_count); string proxy_setting; if (settings_reader.TryGetSecretKey("http_proxy", proxy_setting) && !proxy_setting.empty()) { @@ -115,6 +125,13 @@ static void AddUserAgentIfAvailable(HTTPFSParams &http_params, HTTPHeaders &head } } +static void AddHandleHeaders(HTTPFileHandle &handle, HTTPHeaders &header_map) { + // Inject headers from the http param extra_headers into the request + for (auto &header : handle.http_params.extra_headers) { + header_map[header.first] = header.second; + } +} + unique_ptr HTTPFileSystem::PostRequest(FileHandle &handle, string url, HTTPHeaders header_map, string &buffer_out, char *buffer_in, idx_t buffer_in_len, string params) { @@ -122,6 +139,7 @@ unique_ptr HTTPFileSystem::PostRequest(FileHandle &handle, string auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); PostRequestInfo post_request(url, header_map, hfh.http_params, const_data_ptr_cast(buffer_in), buffer_in_len); auto result = http_util.Request(post_request); @@ -135,6 +153,7 @@ unique_ptr HTTPFileSystem::PutRequest(FileHandle &handle, string u auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); string content_type = "application/octet-stream"; PutRequestInfo put_request(url, header_map, hfh.http_params, (const_data_ptr_t)buffer_in, buffer_in_len, @@ -147,6 +166,7 @@ unique_ptr HTTPFileSystem::HeadRequest(FileHandle &handle, string auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); auto http_client = hfh.GetClient(); @@ -162,6 +182,7 @@ unique_ptr HTTPFileSystem::DeleteRequest(FileHandle &handle, strin auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); auto http_client = hfh.GetClient(); DeleteRequestInfo delete_request(url, header_map, hfh.http_params); @@ -187,6 +208,7 @@ unique_ptr HTTPFileSystem::GetRequest(FileHandle &handle, string u auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); D_ASSERT(hfh.cached_file_handle); @@ -238,6 +260,7 @@ unique_ptr HTTPFileSystem::GetRangeRequest(FileHandle &handle, str auto &http_util = hfh.http_params.http_util; AddUserAgentIfAvailable(hfh.http_params, header_map); + AddHandleHeaders(hfh, header_map); // send the Range header to read only subset of file string range_expr = "bytes=" + to_string(file_offset) + "-" + to_string(file_offset + buffer_out_len - 1); diff --git a/src/httpfs_extension.cpp b/src/httpfs_extension.cpp index 9070621f..5b249504 100644 --- a/src/httpfs_extension.cpp +++ b/src/httpfs_extension.cpp @@ -96,6 +96,9 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests", LogicalType::UBIGINT, Value::UBIGINT(0)); + config.AddExtensionOption("merge_http_secret_into_s3_request", "Merges http secret params into S3 requests", LogicalType::BOOLEAN, + Value(true)); + auto callback_httpfs_client_implementation = [](ClientContext &context, SetScope scope, Value ¶meter) { auto &config = DBConfig::GetConfig(context); string value = StringValue::Get(parameter); diff --git a/src/httpfs_httplib_client.cpp b/src/httpfs_httplib_client.cpp index ef6ce98f..8266a948 100644 --- a/src/httpfs_httplib_client.cpp +++ b/src/httpfs_httplib_client.cpp @@ -120,9 +120,6 @@ class HTTPFSClient : public HTTPClient { for (auto &entry : header_map) { headers.insert(entry); } - for (auto &entry : params.extra_headers) { - headers.insert(entry); - } return headers; } diff --git a/src/s3fs.cpp b/src/s3fs.cpp index b4d6c741..db4ff510 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -807,6 +807,7 @@ unique_ptr S3FileSystem::GetRequest(FileHandle &handle, string s3_ create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); } + return HTTPFileSystem::GetRequest(handle, http_url, headers); } @@ -916,8 +917,6 @@ void S3FileHandle::Initialize(optional_ptr opener) { HTTPFileHandle::Initialize(opener); } - auto &s3fs = file_system.Cast(); - if (flags.OpenForWriting()) { auto aws_minimum_part_size = 5242880; // 5 MiB https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html auto max_part_count = config_params.max_parts_per_file; diff --git a/test/sql/copy/s3/http_secret.test b/test/sql/copy/s3/http_secret.test index ea526c6b..9b627cf5 100644 --- a/test/sql/copy/s3/http_secret.test +++ b/test/sql/copy/s3/http_secret.test @@ -1,5 +1,5 @@ # name: test/sql/copy/s3/http_secret.test -# description: Test http secret +# description: Test http secret params in s3 secret usage # group: [s3] require parquet @@ -28,17 +28,71 @@ COPY (SELECT 'value-1' as value) TO 's3://test-bucket/http-secret-test/test.parq statement ok PRAGMA enable_verification -# Create some wonky headers +# Add http secret header statement ok CREATE SECRET http3 ( - TYPE HTTP, + TYPE HTTP, EXTRA_HTTP_HEADERS MAP{ - 'Authorization': 'Im very important', 'CustomHeader': 'fliepflap' } ); +statement ok +call enable_logging('HTTP'); + +query I +FROM 's3://test-bucket/http-secret-test/test.parquet' +---- +value-1 + +# Note that this header is now added to all requests +query I +SELECT distinct request.headers['CustomHeader'] FROM duckdb_logs_parsed('HTTP') +---- +fliepflap + +statement ok +CALL truncate_duckdb_logs() + +# Disabling this setting will stop this +statement ok +set merge_http_secret_into_s3_request=false + +query I +FROM 's3://test-bucket/http-secret-test/test.parquet' +---- +value-1 + +query I +SELECT distinct request.headers['CustomHeader'] FROM duckdb_logs_parsed('HTTP') +---- +NULL + +statement ok +CALL truncate_duckdb_logs() + +# Header field can be set directly in S3 secret though +statement ok +CREATE SECRET ( + TYPE S3, + PROVIDER config, + KEY_ID '${AWS_ACCESS_KEY_ID}', + SECRET '${AWS_SECRET_ACCESS_KEY}', + REGION '${AWS_DEFAULT_REGION}', + ENDPOINT '${DUCKDB_S3_ENDPOINT}', + USE_SSL '${DUCKDB_S3_USE_SSL}', + EXTRA_HTTP_HEADERS MAP{ + 'CustomHeader': 'fliepflap' + } +) + query I FROM 's3://test-bucket/http-secret-test/test.parquet' ---- value-1 + +# Now header is back in the request logs +query I +SELECT distinct request.headers['CustomHeader'] FROM duckdb_logs_parsed('HTTP') +---- +fliepflap \ No newline at end of file From 11de015615fc2ae9b4475fbb9adbe6b2289e56d4 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 2 Dec 2025 15:20:57 +0100 Subject: [PATCH 04/13] format --- src/httpfs.cpp | 7 ++++-- src/httpfs_curl_client.cpp | 2 +- src/httpfs_extension.cpp | 4 +-- src/httpfs_httplib_client.cpp | 2 +- src/s3fs.cpp | 1 - test/sql/crypto/test_openssl_crypto.test | 4 +-- test/sql/tmp.test | 32 ++++++++++++++++++++++++ 7 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 test/sql/tmp.test diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 83c2864d..1c649bcf 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -70,7 +70,9 @@ unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr idx_t secret_type_count = 5; Value merge_http_secret_into_s3_request; - if (FileOpener::TryGetCurrentSetting(opener, "merge_http_secret_into_s3_request", merge_http_secret_into_s3_request) && !merge_http_secret_into_s3_request.IsNull() && !merge_http_secret_into_s3_request.GetValue()) { + if (FileOpener::TryGetCurrentSetting(opener, "merge_http_secret_into_s3_request", + merge_http_secret_into_s3_request) && + !merge_http_secret_into_s3_request.IsNull() && !merge_http_secret_into_s3_request.GetValue()) { // Drop the http secret from the lookup secret_type_count = 4; } @@ -755,7 +757,8 @@ void HTTPFileHandle::LoadFileInfo() { return; } else { // HEAD request fail, use Range request for another try (read only one byte) - if (flags.OpenForReading() && res->status != HTTPStatusCode::NotFound_404 && res->status != HTTPStatusCode::MovedPermanently_301) { + if (flags.OpenForReading() && res->status != HTTPStatusCode::NotFound_404 && + res->status != HTTPStatusCode::MovedPermanently_301) { auto range_res = hfs.GetRangeRequest(*this, path, {}, 0, nullptr, 2); if (range_res->status != HTTPStatusCode::PartialContent_206 && range_res->status != HTTPStatusCode::Accepted_202 && range_res->status != HTTPStatusCode::OK_200) { diff --git a/src/httpfs_curl_client.cpp b/src/httpfs_curl_client.cpp index a7e4637a..d4e39bf5 100644 --- a/src/httpfs_curl_client.cpp +++ b/src/httpfs_curl_client.cpp @@ -119,7 +119,7 @@ class HTTPFSCurlClient : public HTTPClient { Initialize(http_params); } void Initialize(HTTPParams &http_p) override { - HTTPFSParams &http_params = (HTTPFSParams&)http_p; + HTTPFSParams &http_params = (HTTPFSParams &)http_p; auto bearer_token = ""; if (!http_params.bearer_token.empty()) { bearer_token = http_params.bearer_token.c_str(); diff --git a/src/httpfs_extension.cpp b/src/httpfs_extension.cpp index 5b249504..b61e5154 100644 --- a/src/httpfs_extension.cpp +++ b/src/httpfs_extension.cpp @@ -96,8 +96,8 @@ static void LoadInternal(ExtensionLoader &loader) { config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests", LogicalType::UBIGINT, Value::UBIGINT(0)); - config.AddExtensionOption("merge_http_secret_into_s3_request", "Merges http secret params into S3 requests", LogicalType::BOOLEAN, - Value(true)); + config.AddExtensionOption("merge_http_secret_into_s3_request", "Merges http secret params into S3 requests", + LogicalType::BOOLEAN, Value(true)); auto callback_httpfs_client_implementation = [](ClientContext &context, SetScope scope, Value ¶meter) { auto &config = DBConfig::GetConfig(context); diff --git a/src/httpfs_httplib_client.cpp b/src/httpfs_httplib_client.cpp index 8266a948..fa890b95 100644 --- a/src/httpfs_httplib_client.cpp +++ b/src/httpfs_httplib_client.cpp @@ -12,7 +12,7 @@ class HTTPFSClient : public HTTPClient { Initialize(http_params); } void Initialize(HTTPParams &http_p) override { - HTTPFSParams &http_params = (HTTPFSParams&)http_p; + HTTPFSParams &http_params = (HTTPFSParams &)http_p; client->set_follow_location(http_params.follow_location); client->set_keep_alive(http_params.keep_alive); if (!http_params.ca_cert_file.empty()) { diff --git a/src/s3fs.cpp b/src/s3fs.cpp index db4ff510..727bf3db 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -807,7 +807,6 @@ unique_ptr S3FileSystem::GetRequest(FileHandle &handle, string s3_ create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); } - return HTTPFileSystem::GetRequest(handle, http_url, headers); } diff --git a/test/sql/crypto/test_openssl_crypto.test b/test/sql/crypto/test_openssl_crypto.test index 3af3924c..95270518 100644 --- a/test/sql/crypto/test_openssl_crypto.test +++ b/test/sql/crypto/test_openssl_crypto.test @@ -1,6 +1,6 @@ -# name: test/sql/attach/attach_encryption_fallback_readonly.test +# name: test/sql/crypto/test_openssl_crypto.test # description: Test the openssl based crypto util -# group: [attach] +# group: [crypto] require httpfs diff --git a/test/sql/tmp.test b/test/sql/tmp.test new file mode 100644 index 00000000..bf09acb2 --- /dev/null +++ b/test/sql/tmp.test @@ -0,0 +1,32 @@ +# name: test/sql/tmp.test +# group: [sql] + +require parquet + +require httpfs + +require json + +mode output_result + +statement ok +CREATE SECRET http_auth ( + TYPE http, + EXTRA_HTTP_HEADERS MAP { + 'Authorization': 'Bearer ' + } +); + +statement ok +call enable_logging('HTTP'); + +# Should do 4 paginated requests +query I +select count(*) FROM read_json_auto('https://api.github.com/orgs/duckdblabs/projectsV2/43/items?per_page=5') limit 20; +---- +20 + +mode output_result + +statement ok +SELECT count(*) as count, request.type, request.url FROM duckdb_logs_parsed('HTTP') GROUP BY ALL ORDER BY request.type; \ No newline at end of file From f8543415b30700216cb5543a949a755d97aed8a9 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 2 Dec 2025 16:13:52 +0100 Subject: [PATCH 05/13] check prefix to ensure http is still properly handled --- src/httpfs.cpp | 41 +++++++++++++++++++++++++---------------- src/include/s3fs.hpp | 3 +++ src/s3fs.cpp | 15 +++++++++++---- test/sql/tmp.test | 32 -------------------------------- 4 files changed, 39 insertions(+), 52 deletions(-) delete mode 100644 test/sql/tmp.test diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 1c649bcf..dfe845f7 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -20,6 +20,8 @@ #include #include +#include "s3fs.hpp" + namespace duckdb { shared_ptr HTTPFSUtil::GetHTTPUtil(optional_ptr opener) { @@ -34,7 +36,7 @@ unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr auto result = make_uniq(*this); result->Initialize(opener); - // No point in continueing without an opener + // No point in continuing without an opener if (!opener) { return std::move(result); } @@ -65,35 +67,42 @@ unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr } } - // Secret types to look for HTTP settings in - const char *secret_types[] = {"s3", "r2", "gcs", "aws", "http"}; - idx_t secret_type_count = 5; + unique_ptr settings_reader; + string s3_prefix = S3FileSystem::TryGetPrefix(info->file_path); + if (info && !s3_prefix.empty()) { + // This is an S3-type url, we should + const char *s3_secret_types[] = {"s3", "r2", "gcs", "aws", "http"}; + + idx_t secret_type_count = 5; + Value merge_http_secret_into_s3_request; + FileOpener::TryGetCurrentSetting(opener, "merge_http_secret_into_s3_request", + merge_http_secret_into_s3_request); - Value merge_http_secret_into_s3_request; - if (FileOpener::TryGetCurrentSetting(opener, "merge_http_secret_into_s3_request", - merge_http_secret_into_s3_request) && - !merge_http_secret_into_s3_request.IsNull() && !merge_http_secret_into_s3_request.GetValue()) { - // Drop the http secret from the lookup - secret_type_count = 4; + if (!merge_http_secret_into_s3_request.IsNull() && !merge_http_secret_into_s3_request.GetValue()) { + // Drop the http secret from the lookup + secret_type_count = 4; + } + settings_reader = make_uniq(*opener, info, s3_secret_types, secret_type_count); + } else { + settings_reader = make_uniq(*opener, info, "http"); } // HTTP Secret lookups - KeyValueSecretReader settings_reader(*opener, info, secret_types, secret_type_count); string proxy_setting; - if (settings_reader.TryGetSecretKey("http_proxy", proxy_setting) && !proxy_setting.empty()) { + if (settings_reader->TryGetSecretKey("http_proxy", proxy_setting) && !proxy_setting.empty()) { idx_t port; string host; HTTPUtil::ParseHTTPProxyHost(proxy_setting, host, port); result->http_proxy = host; result->http_proxy_port = port; } - settings_reader.TryGetSecretKey("http_proxy_username", result->http_proxy_username); - settings_reader.TryGetSecretKey("http_proxy_password", result->http_proxy_password); - settings_reader.TryGetSecretKey("bearer_token", result->bearer_token); + settings_reader->TryGetSecretKey("http_proxy_username", result->http_proxy_username); + settings_reader->TryGetSecretKey("http_proxy_password", result->http_proxy_password); + settings_reader->TryGetSecretKey("bearer_token", result->bearer_token); Value extra_headers; - if (settings_reader.TryGetSecretKey("extra_http_headers", extra_headers)) { + if (settings_reader->TryGetSecretKey("extra_http_headers", extra_headers)) { auto children = MapValue::GetChildren(extra_headers); for (const auto &child : children) { auto kv = StructValue::GetChildren(child); diff --git a/src/include/s3fs.hpp b/src/include/s3fs.hpp index a7e933ea..153dfeb5 100644 --- a/src/include/s3fs.hpp +++ b/src/include/s3fs.hpp @@ -212,6 +212,8 @@ class S3FileSystem : public HTTPFileSystem { static string UrlEncode(const string &input, bool encode_slash = false); static string UrlDecode(string input); + static string TryGetPrefix(const string &url); + // Uploads the contents of write_buffer to S3. // Note: caller is responsible to not call this method twice on the same buffer static void UploadBuffer(S3FileHandle &file_handle, shared_ptr write_buffer); @@ -238,6 +240,7 @@ class S3FileSystem : public HTTPFileSystem { protected: static void NotifyUploadsInProgress(S3FileHandle &file_handle); + static string GetPrefix(const string &url); duckdb::unique_ptr CreateHandle(const OpenFileInfo &file, FileOpenFlags flags, optional_ptr opener) override; diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 727bf3db..362b2649 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -625,15 +625,22 @@ void S3FileSystem::ReadQueryParams(const string &url_query_param, S3AuthParams & } } -static string GetPrefix(string url) { +string S3FileSystem::TryGetPrefix(const string &url) { const string prefixes[] = {"s3://", "s3a://", "s3n://", "gcs://", "gs://", "r2://"}; for (auto &prefix : prefixes) { - if (StringUtil::StartsWith(url, prefix)) { + if (StringUtil::StartsWith(StringUtil::Lower(url), prefix)) { return prefix; } } - throw IOException("URL needs to start with s3://, gcs:// or r2://"); - return string(); + return {}; +} + +string S3FileSystem::GetPrefix(const string &url) { + auto prefix = TryGetPrefix(url); + if (prefix.empty()) { + throw IOException("URL needs to start with s3://, gcs:// or r2://"); + } + return prefix; } ParsedS3Url S3FileSystem::S3UrlParse(string url, S3AuthParams ¶ms) { diff --git a/test/sql/tmp.test b/test/sql/tmp.test deleted file mode 100644 index bf09acb2..00000000 --- a/test/sql/tmp.test +++ /dev/null @@ -1,32 +0,0 @@ -# name: test/sql/tmp.test -# group: [sql] - -require parquet - -require httpfs - -require json - -mode output_result - -statement ok -CREATE SECRET http_auth ( - TYPE http, - EXTRA_HTTP_HEADERS MAP { - 'Authorization': 'Bearer ' - } -); - -statement ok -call enable_logging('HTTP'); - -# Should do 4 paginated requests -query I -select count(*) FROM read_json_auto('https://api.github.com/orgs/duckdblabs/projectsV2/43/items?per_page=5') limit 20; ----- -20 - -mode output_result - -statement ok -SELECT count(*) as count, request.type, request.url FROM duckdb_logs_parsed('HTTP') GROUP BY ALL ORDER BY request.type; \ No newline at end of file From dc1d6c87546e9e1ae4d0be1937b42cf947d24bdb Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 3 Dec 2025 11:11:26 +0100 Subject: [PATCH 06/13] avoid sending the same header twice --- src/httpfs.cpp | 1 + src/httpfs_httplib_client.cpp | 7 +++++++ src/include/httpfs_client.hpp | 2 ++ 3 files changed, 10 insertions(+) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index dfe845f7..382ed5db 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -141,6 +141,7 @@ static void AddHandleHeaders(HTTPFileHandle &handle, HTTPHeaders &header_map) { for (auto &header : handle.http_params.extra_headers) { header_map[header.first] = header.second; } + handle.http_params.pre_merged_headers = true; } unique_ptr HTTPFileSystem::PostRequest(FileHandle &handle, string url, HTTPHeaders header_map, diff --git a/src/httpfs_httplib_client.cpp b/src/httpfs_httplib_client.cpp index fa890b95..3a94cb82 100644 --- a/src/httpfs_httplib_client.cpp +++ b/src/httpfs_httplib_client.cpp @@ -116,10 +116,17 @@ class HTTPFSClient : public HTTPClient { private: duckdb_httplib_openssl::Headers TransformHeaders(const HTTPHeaders &header_map, const HTTPParams ¶ms) { + auto &httpfs_params = params.Cast(); + duckdb_httplib_openssl::Headers headers; for (auto &entry : header_map) { headers.insert(entry); } + if (!httpfs_params.pre_merged_headers) { + for (auto &entry : params.extra_headers) { + headers.insert(entry); + } + } return headers; } diff --git a/src/include/httpfs_client.hpp b/src/include/httpfs_client.hpp index ab462cd7..4fa9b17a 100644 --- a/src/include/httpfs_client.hpp +++ b/src/include/httpfs_client.hpp @@ -27,6 +27,8 @@ struct HTTPFSParams : public HTTPParams { bool unsafe_disable_etag_checks {false}; shared_ptr state; string user_agent = {""}; + bool pre_merged_headers = false; + // Additional fields needs to be appended at the end and need to be propagated to duckdb-wasm // TODO: make this unnecessary }; From 4cbc2a152f8c8836f6fcfd4e3c847fb4866839e9 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 3 Dec 2025 14:39:06 +0100 Subject: [PATCH 07/13] fix potential nullptr deref --- src/httpfs.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 382ed5db..c9221690 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -68,8 +68,7 @@ unique_ptr HTTPFSUtil::InitializeParameters(optional_ptr } unique_ptr settings_reader; - string s3_prefix = S3FileSystem::TryGetPrefix(info->file_path); - if (info && !s3_prefix.empty()) { + if (info && !S3FileSystem::TryGetPrefix(info->file_path).empty()) { // This is an S3-type url, we should const char *s3_secret_types[] = {"s3", "r2", "gcs", "aws", "http"}; From b34797ceacfbaecaae6f0d469b2f02fa5c667b37 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 9 Dec 2025 16:10:47 +0100 Subject: [PATCH 08/13] logarithmically grow read buffer --- src/httpfs.cpp | 48 +++++++++++++++++++++++++++++++++++++++--- src/include/httpfs.hpp | 18 ++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index c9221690..6a428d5c 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -433,14 +433,47 @@ unique_ptr HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file return std::move(handle); } +void HTTPFileHandle::AddStatistics(idx_t read_offset, idx_t read_length, idx_t read_duration) { + range_request_statistics.push_back({read_offset, read_length, read_duration}); +} + +void HTTPFileHandle::AdaptReadBufferSize(idx_t next_read_offset) { + if (range_request_statistics.empty()) { + return; // No requests yet - nothing to do + } + + const auto &last_read = range_request_statistics.back(); + if (last_read.offset + last_read.length != next_read_offset) { + return; // Not reading sequentially + } + + if (read_buffer.GetSize() >= MAXIMUM_READ_BUFFER_LEN) { + return; // Already at maximum size + } + + // Grow the buffer + // TODO: can use statistics to estimate per-byte and round-trip cost using least squares, and do something smarter + read_buffer = read_buffer.GetAllocator()->Allocate(read_buffer.GetSize() * 2); +} + bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) { + auto &hfh = handle.Cast(); + + const auto timestamp_before = Timestamp::GetCurrentTimestamp(); auto res = GetRangeRequest(handle, url, header_map, file_offset, buffer_out, buffer_out_len); if (res) { // Request succeeded TODO: fix upstream that 206 is not considered success if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 || res->status == HTTPStatusCode::Accepted_202) { + + if (hfh.flags.RequireParallelAccess()) { + // Update range request statistics + const auto duration = NumericCast(Timestamp::GetCurrentTimestamp().value - timestamp_before.value); + hfh.AddStatistics(file_offset, buffer_out_len, duration); + } + return true; } @@ -531,7 +564,7 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b } if (to_read > 0 && hfh.buffer_available == 0) { - auto new_buffer_available = MinValue(hfh.READ_BUFFER_LEN, hfh.length - start_offset); + auto new_buffer_available = MinValue(hfh.read_buffer.GetSize(), hfh.length - start_offset); // Bypass buffer if we read more than buffer size if (to_read > new_buffer_available) { @@ -544,6 +577,8 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b start_offset += to_read; break; } else { + hfh.AdaptReadBufferSize(start_offset); + new_buffer_available = MinValue(hfh.read_buffer.GetSize(), hfh.length - start_offset); if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(), new_buffer_available)) { return false; @@ -812,6 +847,13 @@ void HTTPFileHandle::TryAddLogger(FileOpener &opener) { } } +static AllocatedData AllocateReadBuffer(optional_ptr opener) { + if (opener && opener->TryGetClientContext()) { + return BufferAllocator::Get(*opener->TryGetClientContext()).Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); + } + return Allocator::DefaultAllocator().Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); +} + void HTTPFileHandle::Initialize(optional_ptr opener) { auto &hfs = file_system.Cast(); http_params.state = HTTPState::TryGetState(opener); @@ -842,7 +884,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { etag = value.etag; if (flags.OpenForReading()) { - read_buffer = duckdb::unique_ptr(new data_t[READ_BUFFER_LEN]); + read_buffer = AllocateReadBuffer(opener); } return; } @@ -861,7 +903,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { } // Initialize the read buffer now that we know the file exists - read_buffer = duckdb::unique_ptr(new data_t[READ_BUFFER_LEN]); + read_buffer = AllocateReadBuffer(opener); } // If we're writing to a file, we might as well remove it from the cache diff --git a/src/include/httpfs.hpp b/src/include/httpfs.hpp index 804968a1..5564f073 100644 --- a/src/include/httpfs.hpp +++ b/src/include/httpfs.hpp @@ -85,8 +85,13 @@ class HTTPFileHandle : public FileHandle { std::mutex mu; // Read buffer - duckdb::unique_ptr read_buffer; - constexpr static idx_t READ_BUFFER_LEN = 1000000; + AllocatedData read_buffer; + constexpr static idx_t INITIAL_READ_BUFFER_LEN = 1048576; + constexpr static idx_t MAXIMUM_READ_BUFFER_LEN = 33554432; + + // Adaptively resizes read_buffer based on range_request_statistics + void AddStatistics(idx_t read_offset, idx_t read_length, idx_t read_duration); + void AdaptReadBufferSize(idx_t next_read_offset); void AddHeaders(HTTPHeaders &map); @@ -95,6 +100,15 @@ class HTTPFileHandle : public FileHandle { // Return the client for re-use void StoreClient(unique_ptr client); +private: + // Statistics that are used to adaptively grow the read_buffer + struct RangeRequestStatistics { + idx_t offset; + idx_t length; + idx_t duration; + }; + vector range_request_statistics; + public: void Close() override { } From 99b2575c8e2aecc11b50dfd07b2b52f18fc268d6 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 9 Dec 2025 16:10:47 +0100 Subject: [PATCH 09/13] logarithmically grow read buffer --- src/httpfs.cpp | 48 +++++++++++++++++++++++++++++++++++++++--- src/include/httpfs.hpp | 18 ++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index c9221690..6a428d5c 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -433,14 +433,47 @@ unique_ptr HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file return std::move(handle); } +void HTTPFileHandle::AddStatistics(idx_t read_offset, idx_t read_length, idx_t read_duration) { + range_request_statistics.push_back({read_offset, read_length, read_duration}); +} + +void HTTPFileHandle::AdaptReadBufferSize(idx_t next_read_offset) { + if (range_request_statistics.empty()) { + return; // No requests yet - nothing to do + } + + const auto &last_read = range_request_statistics.back(); + if (last_read.offset + last_read.length != next_read_offset) { + return; // Not reading sequentially + } + + if (read_buffer.GetSize() >= MAXIMUM_READ_BUFFER_LEN) { + return; // Already at maximum size + } + + // Grow the buffer + // TODO: can use statistics to estimate per-byte and round-trip cost using least squares, and do something smarter + read_buffer = read_buffer.GetAllocator()->Allocate(read_buffer.GetSize() * 2); +} + bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) { + auto &hfh = handle.Cast(); + + const auto timestamp_before = Timestamp::GetCurrentTimestamp(); auto res = GetRangeRequest(handle, url, header_map, file_offset, buffer_out, buffer_out_len); if (res) { // Request succeeded TODO: fix upstream that 206 is not considered success if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 || res->status == HTTPStatusCode::Accepted_202) { + + if (hfh.flags.RequireParallelAccess()) { + // Update range request statistics + const auto duration = NumericCast(Timestamp::GetCurrentTimestamp().value - timestamp_before.value); + hfh.AddStatistics(file_offset, buffer_out_len, duration); + } + return true; } @@ -531,7 +564,7 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b } if (to_read > 0 && hfh.buffer_available == 0) { - auto new_buffer_available = MinValue(hfh.READ_BUFFER_LEN, hfh.length - start_offset); + auto new_buffer_available = MinValue(hfh.read_buffer.GetSize(), hfh.length - start_offset); // Bypass buffer if we read more than buffer size if (to_read > new_buffer_available) { @@ -544,6 +577,8 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b start_offset += to_read; break; } else { + hfh.AdaptReadBufferSize(start_offset); + new_buffer_available = MinValue(hfh.read_buffer.GetSize(), hfh.length - start_offset); if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(), new_buffer_available)) { return false; @@ -812,6 +847,13 @@ void HTTPFileHandle::TryAddLogger(FileOpener &opener) { } } +static AllocatedData AllocateReadBuffer(optional_ptr opener) { + if (opener && opener->TryGetClientContext()) { + return BufferAllocator::Get(*opener->TryGetClientContext()).Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); + } + return Allocator::DefaultAllocator().Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); +} + void HTTPFileHandle::Initialize(optional_ptr opener) { auto &hfs = file_system.Cast(); http_params.state = HTTPState::TryGetState(opener); @@ -842,7 +884,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { etag = value.etag; if (flags.OpenForReading()) { - read_buffer = duckdb::unique_ptr(new data_t[READ_BUFFER_LEN]); + read_buffer = AllocateReadBuffer(opener); } return; } @@ -861,7 +903,7 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { } // Initialize the read buffer now that we know the file exists - read_buffer = duckdb::unique_ptr(new data_t[READ_BUFFER_LEN]); + read_buffer = AllocateReadBuffer(opener); } // If we're writing to a file, we might as well remove it from the cache diff --git a/src/include/httpfs.hpp b/src/include/httpfs.hpp index 804968a1..5564f073 100644 --- a/src/include/httpfs.hpp +++ b/src/include/httpfs.hpp @@ -85,8 +85,13 @@ class HTTPFileHandle : public FileHandle { std::mutex mu; // Read buffer - duckdb::unique_ptr read_buffer; - constexpr static idx_t READ_BUFFER_LEN = 1000000; + AllocatedData read_buffer; + constexpr static idx_t INITIAL_READ_BUFFER_LEN = 1048576; + constexpr static idx_t MAXIMUM_READ_BUFFER_LEN = 33554432; + + // Adaptively resizes read_buffer based on range_request_statistics + void AddStatistics(idx_t read_offset, idx_t read_length, idx_t read_duration); + void AdaptReadBufferSize(idx_t next_read_offset); void AddHeaders(HTTPHeaders &map); @@ -95,6 +100,15 @@ class HTTPFileHandle : public FileHandle { // Return the client for re-use void StoreClient(unique_ptr client); +private: + // Statistics that are used to adaptively grow the read_buffer + struct RangeRequestStatistics { + idx_t offset; + idx_t length; + idx_t duration; + }; + vector range_request_statistics; + public: void Close() override { } From 897f9c559aebd9b636642ee8e4c4f5d04ba9d146 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Tue, 9 Dec 2025 16:20:17 +0100 Subject: [PATCH 10/13] negate --- src/httpfs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 6a428d5c..0b5d159c 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -468,7 +468,7 @@ bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 || res->status == HTTPStatusCode::Accepted_202) { - if (hfh.flags.RequireParallelAccess()) { + if (!hfh.flags.RequireParallelAccess()) { // Update range request statistics const auto duration = NumericCast(Timestamp::GetCurrentTimestamp().value - timestamp_before.value); hfh.AddStatistics(file_offset, buffer_out_len, duration); From 6d9494b3c13949f26871924d7464dd1e7f3c728c Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Wed, 10 Dec 2025 09:11:02 +0100 Subject: [PATCH 11/13] avoid allocating the read buffer if we know it won't be used --- src/httpfs.cpp | 28 ++++++++++++++++------------ src/include/httpfs.hpp | 7 +++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/httpfs.cpp b/src/httpfs.cpp index 0b5d159c..1f4c1329 100644 --- a/src/httpfs.cpp +++ b/src/httpfs.cpp @@ -438,6 +438,7 @@ void HTTPFileHandle::AddStatistics(idx_t read_offset, idx_t read_length, idx_t r } void HTTPFileHandle::AdaptReadBufferSize(idx_t next_read_offset) { + D_ASSERT(!SkipBuffer()); if (range_request_statistics.empty()) { return; // No requests yet - nothing to do } @@ -470,7 +471,8 @@ bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders if (!hfh.flags.RequireParallelAccess()) { // Update range request statistics - const auto duration = NumericCast(Timestamp::GetCurrentTimestamp().value - timestamp_before.value); + const auto duration = + NumericCast(Timestamp::GetCurrentTimestamp().value - timestamp_before.value); hfh.AddStatistics(file_offset, buffer_out_len, duration); } @@ -518,8 +520,7 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b idx_t buffer_offset = 0; // Don't buffer when DirectIO is set or when we are doing parallel reads - bool skip_buffer = hfh.flags.DirectIO() || hfh.flags.RequireParallelAccess(); - if (skip_buffer && to_read > 0) { + if (hfh.SkipBuffer() && to_read > 0) { if (!TryRangeRequest(hfh, hfh.path, {}, location, (char *)buffer, to_read)) { return false; } @@ -847,11 +848,12 @@ void HTTPFileHandle::TryAddLogger(FileOpener &opener) { } } -static AllocatedData AllocateReadBuffer(optional_ptr opener) { - if (opener && opener->TryGetClientContext()) { - return BufferAllocator::Get(*opener->TryGetClientContext()).Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); - } - return Allocator::DefaultAllocator().Allocate(HTTPFileHandle::INITIAL_READ_BUFFER_LEN); +void HTTPFileHandle::AllocateReadBuffer(optional_ptr opener) { + D_ASSERT(!SkipBuffer()); + D_ASSERT(!read_buffer.IsSet()); + auto &allocator = opener && opener->TryGetClientContext() ? BufferAllocator::Get(*opener->TryGetClientContext()) + : Allocator::DefaultAllocator(); + read_buffer = allocator.Allocate(INITIAL_READ_BUFFER_LEN); } void HTTPFileHandle::Initialize(optional_ptr opener) { @@ -883,8 +885,8 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { length = value.length; etag = value.etag; - if (flags.OpenForReading()) { - read_buffer = AllocateReadBuffer(opener); + if (flags.OpenForReading() && !SkipBuffer()) { + AllocateReadBuffer(opener); } return; } @@ -902,8 +904,10 @@ void HTTPFileHandle::Initialize(optional_ptr opener) { current_cache->Insert(path, {length, last_modified, etag}); } - // Initialize the read buffer now that we know the file exists - read_buffer = AllocateReadBuffer(opener); + if (!SkipBuffer()) { + // Initialize the read buffer now that we know the file exists + AllocateReadBuffer(opener); + } } // If we're writing to a file, we might as well remove it from the cache diff --git a/src/include/httpfs.hpp b/src/include/httpfs.hpp index 5564f073..c6baf131 100644 --- a/src/include/httpfs.hpp +++ b/src/include/httpfs.hpp @@ -100,7 +100,14 @@ class HTTPFileHandle : public FileHandle { // Return the client for re-use void StoreClient(unique_ptr client); + // Whether to bypass the read buffer + bool SkipBuffer() const { + return flags.DirectIO() || flags.RequireParallelAccess(); + } + private: + void AllocateReadBuffer(optional_ptr opener); + // Statistics that are used to adaptively grow the read_buffer struct RangeRequestStatistics { idx_t offset; From 9c57a4a79f1033dbed0f39b4707da2d4f46ff10e Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Wed, 10 Dec 2025 12:50:17 +0100 Subject: [PATCH 12/13] add test --- .../json/table/internal_issue_6807.test_slow | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/sql/json/table/internal_issue_6807.test_slow diff --git a/test/sql/json/table/internal_issue_6807.test_slow b/test/sql/json/table/internal_issue_6807.test_slow new file mode 100644 index 00000000..9241c8e9 --- /dev/null +++ b/test/sql/json/table/internal_issue_6807.test_slow @@ -0,0 +1,18 @@ +# name: test/sql/json/table/internal_issue_6807.test +# description: Test logarithmic growth of read buffer for sequential reads +# group: [table] + +require json + +require httpfs + +statement ok +CALL enable_logging('HTTP'); + +statement ok +CREATE TABLE T AS FROM 'https://data.gharchive.org/2023-02-08-0.json.gz'; + +query I +SELECT count(*) FROM duckdb_logs_parsed('HTTP') WHERE request.type = 'GET' GROUP BY request.type; +---- +9 From b03f9280070160bf5fea22473b9408ef142d12c3 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 11 Dec 2025 13:48:44 +0100 Subject: [PATCH 13/13] Bump to v1.4.3 --- .github/workflows/MainDistributionPipeline.yml | 12 ++++++------ duckdb | 2 +- extension-ci-tools | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index ba2e534f..69f0a58c 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,21 +14,21 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.3 with: extension_name: httpfs - duckdb_version: v1.4.2 - ci_tools_version: main + duckdb_version: v1.4.3 + ci_tools_version: v1.4.3 duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main + uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.3 secrets: inherit with: extension_name: httpfs - duckdb_version: v1.4.2 - ci_tools_version: main + duckdb_version: v1.4.3 + ci_tools_version: v1.4.3 deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') }} deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }} diff --git a/duckdb b/duckdb index 68d7555f..d1dc88f9 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 68d7555f68bd25c1a251ccca2e6338949c33986a +Subproject commit d1dc88f950d456d72493df452dabdcd13aa413dd diff --git a/extension-ci-tools b/extension-ci-tools index ba18d4f1..5b96f639 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ba18d4f106a6cc1d5597f442bac06a1d7db098ef +Subproject commit 5b96f6390b7eb51ef395c81e26b3627c2049ec28