Skip to content

Add credentials refresh in more locations #165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
J-Meyers wants to merge 1 commit into
base: v1.4-andium
Choose a base branch
from
Open

Add credentials refresh in more locations #165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions src/include/s3fs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,15 @@ struct S3AuthParams {
bool requester_pays = false;
string oauth2_bearer_token; // OAuth2 bearer token for GCS

// Store FileOpener and path for credential refresh
optional_ptr<FileOpener> opener;
string path;

static S3AuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);

//! Try to refresh credentials if they've expired
//! Returns true if refresh succeeded and credentials were updated
bool TryRefreshCredentials();
};

struct AWSEnvironmentCredentialsProvider {
Expand Down Expand Up @@ -236,6 +244,9 @@ class S3FileSystem : public HTTPFileSystem {
static string GetGCSAuthError(S3AuthParams &s3_auth_params);
static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url);

//! Helper method to attempt secret refresh for a given path
static bool TryRefreshSecret(const string &path, optional_ptr<FileOpener> opener);

protected:
static void NotifyUploadsInProgress(S3FileHandle &file_handle);
duckdb::unique_ptr<HTTPFileHandle> CreateHandle(const OpenFileInfo &file, FileOpenFlags flags,
Expand All @@ -245,6 +256,24 @@ class S3FileSystem : public HTTPFileSystem {
string GetPayloadHash(char *buffer, idx_t buffer_len);

HTTPException GetHTTPError(FileHandle &, const HTTPResponse &response, const string &url) override;

private:
template <typename RequestFunc>
auto ExecuteWithRefresh(S3AuthParams &auth_params, RequestFunc request_func)
-> decltype(request_func(auth_params)) {
try {
return request_func(auth_params);
} catch (std::exception &ex) {
ErrorData error(ex);
// Only attempt refresh for HTTP or IO exceptions (same logic as S3FileHandle::Initialize)
if (error.Type() == ExceptionType::IO || error.Type() == ExceptionType::HTTP) {
if (auth_params.TryRefreshCredentials()) {
return request_func(auth_params); // Retry
}
}
throw;
}
}
};

// Helper class to do s3 ListObjectV2 api call https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
Expand Down
Loading
Loading