diff --git a/internal/api/handlers/episode_qa_tos.go b/internal/api/handlers/episode_qa_tos.go index 1e885e9..13832be 100644 --- a/internal/api/handlers/episode_qa_tos.go +++ b/internal/api/handlers/episode_qa_tos.go @@ -17,19 +17,22 @@ import ( "io" "net/http" "net/url" + "path" "sort" "strconv" "strings" "time" - "archebase.com/keystone-edge/internal/config" - "archebase.com/keystone-edge/internal/logger" - "archebase.com/keystone-edge/internal/volcengineauth" + vtos "github.com/volcengine/ve-tos-golang-sdk/v2/tos" "github.com/volcengine/volcengine-go-sdk/service/sts" "github.com/volcengine/volcengine-go-sdk/volcengine" "github.com/volcengine/volcengine-go-sdk/volcengine/request" "github.com/volcengine/volcengine-go-sdk/volcengine/session" "github.com/volcengine/volcengine-go-sdk/volcengine/volcengineerr" + + "archebase.com/keystone-edge/internal/config" + "archebase.com/keystone-edge/internal/logger" + "archebase.com/keystone-edge/internal/volcengineauth" ) const ( @@ -151,6 +154,70 @@ func (r *episodeQATOSReader) OpenObject(ctx context.Context, bucket, objectName return resp.Body, nil } +func (r *episodeQATOSReader) presignGetObject(ctx context.Context, bucket, objectName string, ttl time.Duration) (string, error) { + downloadExpiresAt := time.Now().Add(ttl) + credentials, err := r.credentials(ctx, bucket, objectName) + if err != nil { + return "", err + } + ttl = time.Until(downloadExpiresAt) + if !credentials.expiration.IsZero() { + if credentialsTTL := time.Until(credentials.expiration); credentialsTTL < ttl { + ttl = credentialsTTL + } + } + if ttl <= 0 { + return "", fmt.Errorf("TOS presigned URL TTL must be positive") + } + expiresSeconds := int64(ttl / time.Second) + if expiresSeconds < 1 { + return "", fmt.Errorf("TOS credentials expire too soon to presign download") + } + + sdkCredentials := vtos.NewStaticCredentials(credentials.accessKeyID, credentials.accessKeySecret) + sdkCredentials.WithSecurityToken(credentials.securityToken) + client, err := vtos.NewClientV2( + publicTOSEndpoint(r.endpoint, r.useSSL), + vtos.WithCredentials(sdkCredentials), + vtos.WithRegion(r.region), + ) + if err != nil { + return "", fmt.Errorf("create TOS presign client: %w", err) + } + + output, err := client.PreSignedURL(&vtos.PreSignedURLInput{ + HTTPMethod: http.MethodGet, + Bucket: bucket, + Key: objectName, + Expires: expiresSeconds, + Query: map[string]string{ + "response-content-disposition": "attachment; filename*=UTF-8''" + url.PathEscape(path.Base(objectName)), + }, + }) + if err != nil { + return "", fmt.Errorf("presign TOS object: %w", err) + } + return output.SignedUrl, nil +} + +func publicTOSEndpoint(endpoint string, useSSL bool) string { + endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") + const ( + privateSuffix = ".ivolces.com" + publicSuffix = ".volces.com" + ) + if strings.HasSuffix(strings.ToLower(endpoint), privateSuffix) { + endpoint = endpoint[:len(endpoint)-len(privateSuffix)] + publicSuffix + } + if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { + return endpoint + } + if useSSL { + return "https://" + endpoint + } + return "http://" + endpoint +} + func (r *episodeQATOSReader) GetObjectWithMetadata(ctx context.Context, bucket, objectName string, byteRange *httpRange) (episodeQATOSObject, error) { headers := http.Header{} if byteRange != nil { diff --git a/internal/api/handlers/storage.go b/internal/api/handlers/storage.go index 47cc183..8c939d8 100644 --- a/internal/api/handlers/storage.go +++ b/internal/api/handlers/storage.go @@ -64,30 +64,31 @@ func (h *StorageHandler) requireBearerToken(c *gin.Context) bool { // authorizeGetObject allows either a valid Bearer JWT (e.g. Range reads from the SPA worker) // or a short-lived dl_token query parameter (e.g. navigation without custom headers). -func (h *StorageHandler) authorizeGetObject(c *gin.Context, bucket, objectName string) (usedDownloadToken bool, ok bool) { +func (h *StorageHandler) authorizeGetObject(c *gin.Context, bucket, objectName string) (usedDownloadToken bool, downloadTTL time.Duration, ok bool) { if h.authCfg == nil { logger.Printf("[S3] auth config is nil; refusing request") c.JSON(http.StatusInternalServerError, gin.H{"error": "auth is not configured"}) - return false, false + return false, 0, false } dl := strings.TrimSpace(c.Query("dl_token")) if dl != "" { - if err := auth.ParseStorageDownloadToken(dl, h.authCfg, bucket, objectName); err != nil { + claims, err := auth.ParseStorageDownloadTokenClaims(dl, h.authCfg, bucket, objectName) + if err != nil { if err == auth.ErrExpiredToken { c.JSON(http.StatusUnauthorized, gin.H{"error": "download token has expired"}) - return false, false + return false, 0, false } c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid download token"}) - return false, false + return false, 0, false } - return true, true + return true, time.Until(claims.ExpiresAt.Time), true } if !h.requireBearerToken(c) { - return false, false + return false, 0, false } - return false, true + return false, 0, true } // RegisterRoutes registers storage-related routes on the given router group. @@ -237,12 +238,24 @@ func (h *StorageHandler) GetObject(c *gin.Context) { return } - usedDownloadToken, authed := h.authorizeGetObject(c, bucket, objectName) + usedDownloadToken, downloadTTL, authed := h.authorizeGetObject(c, bucket, objectName) if !authed { return } if h.usesTOSBucket(bucket) { + // Browser downloads arrive without a Range header and can go directly to TOS. + // Keep ranged reads on the same-origin proxy so MCAP preview does not require TOS CORS. + if usedDownloadToken && strings.TrimSpace(c.GetHeader("Range")) == "" { + directURL, err := h.tos.presignGetObject(c.Request.Context(), bucket, objectName, downloadTTL) + if err != nil { + logger.Printf("[STORAGE] TOS direct download presign failed: bucket=%s, object=%s, err=%v", bucket, objectName, err) + c.JSON(http.StatusBadGateway, gin.H{"error": "failed to presign object download"}) + return + } + c.Redirect(http.StatusTemporaryRedirect, directURL) + return + } h.getTOSObject(c, bucket, objectName, usedDownloadToken) return } diff --git a/internal/api/handlers/storage_test.go b/internal/api/handlers/storage_test.go index a2c26e1..2c33f88 100644 --- a/internal/api/handlers/storage_test.go +++ b/internal/api/handlers/storage_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "strings" "testing" "time" @@ -37,16 +38,18 @@ func (fakeEpisodeQASTSClient) AssumeRoleWithContext(volcengine.Context, *sts.Ass SetAccessKeyId("temp-ak"). SetSecretAccessKey("temp-sk"). SetSessionToken("temp-token"). - SetExpiredTime("2026-07-15T08:00:00Z"), + SetExpiredTime(time.Now().Add(time.Hour).UTC().Format(time.RFC3339)), ), nil } func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) { gin.SetMode(gin.TestMode) - handler := NewStorageHandler(nil, nil, &config.StorageConfig{ + authCfg := &config.AuthConfig{JWTSecret: "test-secret"} + handler := NewStorageHandler(nil, authCfg, &config.StorageConfig{ Type: "tos", Endpoint: "tos-cn-beijing.volces.com", + Bucket: "bucket-a", Region: "cn-beijing", AccessKey: "test-ak", SecretKey: "test-sk", @@ -72,13 +75,21 @@ func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) { return resp, nil })} - req := httptest.NewRequest(http.MethodGet, "/api/v1/storage/object", nil) + token, err := auth.SignStorageDownloadToken("bucket-a", "device-uploads/capture.mcap", time.Minute, authCfg) + if err != nil { + t.Fatalf("sign download token: %v", err) + } + req := httptest.NewRequest( + http.MethodGet, + "/api/v1/storage/object?bucket=bucket-a&object=device-uploads/capture.mcap&dl_token="+url.QueryEscape(token), + nil, + ) req.Header.Set("Range", "bytes=0-0") w := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(w) ctx.Request = req - handler.getTOSObject(ctx, "bucket-a", "device-uploads/capture.mcap", false) + handler.GetObject(ctx) if w.Code != http.StatusPartialContent { t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusPartialContent, w.Body.String()) @@ -94,13 +105,13 @@ func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) { } } -func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) { +func TestStorageHandlerRedirectsTOSDownloadToPresignedURL(t *testing.T) { gin.SetMode(gin.TestMode) authCfg := &config.AuthConfig{JWTSecret: "test-secret"} handler := NewStorageHandler(nil, authCfg, &config.StorageConfig{ Type: "tos", - Endpoint: "tos-cn-beijing.volces.com", + Endpoint: "tos-cn-beijing.ivolces.com", Bucket: "tos-bucket", Region: "cn-beijing", AccessKey: "test-ak", @@ -110,8 +121,8 @@ func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) { }) handler.tos.stsClient = fakeEpisodeQASTSClient{} handler.tos.client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if got := req.URL.Host; got != "tos-bucket.tos-cn-beijing.volces.com" { - t.Fatalf("host = %q, want tos-bucket.tos-cn-beijing.volces.com", got) + if got := req.URL.Host; got != "tos-bucket.tos-cn-beijing.ivolces.com" { + t.Fatalf("host = %q, want tos-bucket.tos-cn-beijing.ivolces.com", got) } return &http.Response{ StatusCode: http.StatusOK, @@ -136,11 +147,34 @@ func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) { handler.GetObject(ctx) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusOK, w.Body.String()) + if w.Code != http.StatusTemporaryRedirect { + t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusTemporaryRedirect, w.Body.String()) + } + location, err := url.Parse(w.Header().Get("Location")) + if err != nil { + t.Fatalf("parse redirect location: %v", err) + } + if got := location.Scheme; got != "https" { + t.Fatalf("redirect scheme = %q, want https", got) + } + if got := location.Host; got != "tos-bucket.tos-cn-beijing.volces.com" { + t.Fatalf("redirect host = %q, want public TOS host", got) + } + if got := location.Path; got != "/device-uploads/capture.mcap" { + t.Fatalf("redirect path = %q, want object path", got) + } + query := location.Query() + for _, key := range []string{"X-Tos-Algorithm", "X-Tos-Credential", "X-Tos-Date", "X-Tos-Expires", "X-Tos-Security-Token", "X-Tos-Signature"} { + if query.Get(key) == "" { + t.Fatalf("redirect query missing %s: %s", key, location.String()) + } + } + expires, err := strconv.Atoi(query.Get("X-Tos-Expires")) + if err != nil || expires < 1 || expires > 60 { + t.Fatalf("X-Tos-Expires = %q, want 1..60 seconds", query.Get("X-Tos-Expires")) } - if got := w.Body.String(); got != "mcap" { - t.Fatalf("body = %q, want mcap", got) + if got := query.Get("response-content-disposition"); !strings.HasPrefix(got, "attachment;") { + t.Fatalf("response-content-disposition = %q, want attachment", got) } } diff --git a/internal/auth/storage_download_token.go b/internal/auth/storage_download_token.go index 8523259..1dde8b7 100644 --- a/internal/auth/storage_download_token.go +++ b/internal/auth/storage_download_token.go @@ -53,14 +53,14 @@ func SignStorageDownloadToken(bucket, object string, ttl time.Duration, cfg *con return tok.SignedString([]byte(cfg.JWTSecret)) } -// ParseStorageDownloadToken validates token signature, expiry, kind, and bucket/object binding. -func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) error { +// ParseStorageDownloadTokenClaims validates a token and returns its bucket/object-bound claims. +func ParseStorageDownloadTokenClaims(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) (*StorageDownloadClaims, error) { if cfg == nil || strings.TrimSpace(cfg.JWTSecret) == "" { - return ErrInvalidToken + return nil, ErrInvalidToken } tokenString = strings.TrimSpace(tokenString) if tokenString == "" { - return ErrInvalidToken + return nil, ErrInvalidToken } var claims StorageDownloadClaims @@ -72,18 +72,24 @@ func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantB }) if err != nil { if errors.Is(err, jwt.ErrTokenExpired) { - return ErrExpiredToken + return nil, ErrExpiredToken } - return ErrInvalidToken + return nil, ErrInvalidToken } if !token.Valid { - return ErrInvalidToken + return nil, ErrInvalidToken } - if claims.Kind != StorageDownloadTokenKind { - return ErrInvalidToken + if claims.Kind != StorageDownloadTokenKind || claims.ExpiresAt == nil { + return nil, ErrInvalidToken } if claims.Bucket != wantBucket || claims.Object != wantObject { - return ErrInvalidToken + return nil, ErrInvalidToken } - return nil + return &claims, nil +} + +// ParseStorageDownloadToken validates token signature, expiry, kind, and bucket/object binding. +func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) error { + _, err := ParseStorageDownloadTokenClaims(tokenString, cfg, wantBucket, wantObject) + return err }