perf(storage): redirect TOS downloads to signed URLs

internal/api/handlers/episode_qa_tos.go

@@ -17,19 +17,22 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	"archebase.com/keystone-edge/internal/config"
-	"archebase.com/keystone-edge/internal/logger"
-	"archebase.com/keystone-edge/internal/volcengineauth"
+	vtos "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
 	"github.com/volcengine/volcengine-go-sdk/service/sts"
 	"github.com/volcengine/volcengine-go-sdk/volcengine"
 	"github.com/volcengine/volcengine-go-sdk/volcengine/request"
 	"github.com/volcengine/volcengine-go-sdk/volcengine/session"
 	"github.com/volcengine/volcengine-go-sdk/volcengine/volcengineerr"
+
+	"archebase.com/keystone-edge/internal/config"
+	"archebase.com/keystone-edge/internal/logger"
+	"archebase.com/keystone-edge/internal/volcengineauth"
 )
 
 const (
@@ -151,6 +154,70 @@ func (r *episodeQATOSReader) OpenObject(ctx context.Context, bucket, objectName
 	return resp.Body, nil
 }
 
+func (r *episodeQATOSReader) presignGetObject(ctx context.Context, bucket, objectName string, ttl time.Duration) (string, error) {
+	downloadExpiresAt := time.Now().Add(ttl)
+	credentials, err := r.credentials(ctx, bucket, objectName)
+	if err != nil {
+		return "", err
+	}
+	ttl = time.Until(downloadExpiresAt)
+	if !credentials.expiration.IsZero() {
+		if credentialsTTL := time.Until(credentials.expiration); credentialsTTL < ttl {
+			ttl = credentialsTTL
+		}
+	}
+	if ttl <= 0 {
+		return "", fmt.Errorf("TOS presigned URL TTL must be positive")
+	}
+	expiresSeconds := int64(ttl / time.Second)
+	if expiresSeconds < 1 {
+		return "", fmt.Errorf("TOS credentials expire too soon to presign download")
+	}
+
+	sdkCredentials := vtos.NewStaticCredentials(credentials.accessKeyID, credentials.accessKeySecret)
+	sdkCredentials.WithSecurityToken(credentials.securityToken)
+	client, err := vtos.NewClientV2(
+		publicTOSEndpoint(r.endpoint, r.useSSL),
+		vtos.WithCredentials(sdkCredentials),
+		vtos.WithRegion(r.region),
+	)
+	if err != nil {
+		return "", fmt.Errorf("create TOS presign client: %w", err)
+	}
+
+	output, err := client.PreSignedURL(&vtos.PreSignedURLInput{
+		HTTPMethod: http.MethodGet,
+		Bucket:     bucket,
+		Key:        objectName,
+		Expires:    expiresSeconds,
+		Query: map[string]string{
+			"response-content-disposition": "attachment; filename*=UTF-8''" + url.PathEscape(path.Base(objectName)),
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("presign TOS object: %w", err)
+	}
+	return output.SignedUrl, nil
+}
+
+func publicTOSEndpoint(endpoint string, useSSL bool) string {
+	endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
+	const (
+		privateSuffix = ".ivolces.com"
+		publicSuffix  = ".volces.com"
+	)
+	if strings.HasSuffix(strings.ToLower(endpoint), privateSuffix) {
+		endpoint = endpoint[:len(endpoint)-len(privateSuffix)] + publicSuffix
+	}
+	if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
+		return endpoint
+	}
+	if useSSL {
+		return "https://" + endpoint
+	}
+	return "http://" + endpoint
+}
+
 func (r *episodeQATOSReader) GetObjectWithMetadata(ctx context.Context, bucket, objectName string, byteRange *httpRange) (episodeQATOSObject, error) {
 	headers := http.Header{}
 	if byteRange != nil {

internal/api/handlers/storage.go

@@ -64,30 +64,31 @@ func (h *StorageHandler) requireBearerToken(c *gin.Context) bool {
 
 // authorizeGetObject allows either a valid Bearer JWT (e.g. Range reads from the SPA worker)
 // or a short-lived dl_token query parameter (e.g. <a download> navigation without custom headers).
-func (h *StorageHandler) authorizeGetObject(c *gin.Context, bucket, objectName string) (usedDownloadToken bool, ok bool) {
+func (h *StorageHandler) authorizeGetObject(c *gin.Context, bucket, objectName string) (usedDownloadToken bool, downloadTTL time.Duration, ok bool) {
 	if h.authCfg == nil {
 		logger.Printf("[S3] auth config is nil; refusing request")
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "auth is not configured"})
-		return false, false
+		return false, 0, false
 	}
 
 	dl := strings.TrimSpace(c.Query("dl_token"))
 	if dl != "" {
-		if err := auth.ParseStorageDownloadToken(dl, h.authCfg, bucket, objectName); err != nil {
+		claims, err := auth.ParseStorageDownloadTokenClaims(dl, h.authCfg, bucket, objectName)
+		if err != nil {
 			if err == auth.ErrExpiredToken {
 				c.JSON(http.StatusUnauthorized, gin.H{"error": "download token has expired"})
-				return false, false
+				return false, 0, false
 			}
 			c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid download token"})
-			return false, false
+			return false, 0, false
 		}
-		return true, true
+		return true, time.Until(claims.ExpiresAt.Time), true
 	}
 
 	if !h.requireBearerToken(c) {
-		return false, false
+		return false, 0, false
 	}
-	return false, true
+	return false, 0, true
 }
 
 // RegisterRoutes registers storage-related routes on the given router group.
@@ -237,12 +238,24 @@ func (h *StorageHandler) GetObject(c *gin.Context) {
 		return
 	}
 
-	usedDownloadToken, authed := h.authorizeGetObject(c, bucket, objectName)
+	usedDownloadToken, downloadTTL, authed := h.authorizeGetObject(c, bucket, objectName)
 	if !authed {
 		return
 	}
 
 	if h.usesTOSBucket(bucket) {
+		// Browser downloads arrive without a Range header and can go directly to TOS.
+		// Keep ranged reads on the same-origin proxy so MCAP preview does not require TOS CORS.
+		if usedDownloadToken && strings.TrimSpace(c.GetHeader("Range")) == "" {
+			directURL, err := h.tos.presignGetObject(c.Request.Context(), bucket, objectName, downloadTTL)
+			if err != nil {
+				logger.Printf("[STORAGE] TOS direct download presign failed: bucket=%s, object=%s, err=%v", bucket, objectName, err)
+				c.JSON(http.StatusBadGateway, gin.H{"error": "failed to presign object download"})
+				return
+			}
+			c.Redirect(http.StatusTemporaryRedirect, directURL)
+			return
+		}
 		h.getTOSObject(c, bucket, objectName, usedDownloadToken)
 		return
 	}

internal/api/handlers/storage_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -37,16 +38,18 @@ func (fakeEpisodeQASTSClient) AssumeRoleWithContext(volcengine.Context, *sts.Ass
 			SetAccessKeyId("temp-ak").
 			SetSecretAccessKey("temp-sk").
 			SetSessionToken("temp-token").
-			SetExpiredTime("2026-07-15T08:00:00Z"),
+			SetExpiredTime(time.Now().Add(time.Hour).UTC().Format(time.RFC3339)),
 	), nil
 }
 
 func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 
-	handler := NewStorageHandler(nil, nil, &config.StorageConfig{
+	authCfg := &config.AuthConfig{JWTSecret: "test-secret"}
+	handler := NewStorageHandler(nil, authCfg, &config.StorageConfig{
 		Type:       "tos",
 		Endpoint:   "tos-cn-beijing.volces.com",
+		Bucket:     "bucket-a",
 		Region:     "cn-beijing",
 		AccessKey:  "test-ak",
 		SecretKey:  "test-sk",
@@ -72,13 +75,21 @@ func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) {
 		return resp, nil
 	})}
 
-	req := httptest.NewRequest(http.MethodGet, "/api/v1/storage/object", nil)
+	token, err := auth.SignStorageDownloadToken("bucket-a", "device-uploads/capture.mcap", time.Minute, authCfg)
+	if err != nil {
+		t.Fatalf("sign download token: %v", err)
+	}
+	req := httptest.NewRequest(
+		http.MethodGet,
+		"/api/v1/storage/object?bucket=bucket-a&object=device-uploads/capture.mcap&dl_token="+url.QueryEscape(token),
+		nil,
+	)
 	req.Header.Set("Range", "bytes=0-0")
 	w := httptest.NewRecorder()
 	ctx, _ := gin.CreateTestContext(w)
 	ctx.Request = req
 
-	handler.getTOSObject(ctx, "bucket-a", "device-uploads/capture.mcap", false)
+	handler.GetObject(ctx)
 
 	if w.Code != http.StatusPartialContent {
 		t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusPartialContent, w.Body.String())
@@ -94,13 +105,13 @@ func TestStorageHandlerProxiesTOSRangeResponse(t *testing.T) {
 	}
 }
 
-func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) {
+func TestStorageHandlerRedirectsTOSDownloadToPresignedURL(t *testing.T) {
 	gin.SetMode(gin.TestMode)
 
 	authCfg := &config.AuthConfig{JWTSecret: "test-secret"}
 	handler := NewStorageHandler(nil, authCfg, &config.StorageConfig{
 		Type:       "tos",
-		Endpoint:   "tos-cn-beijing.volces.com",
+		Endpoint:   "tos-cn-beijing.ivolces.com",
 		Bucket:     "tos-bucket",
 		Region:     "cn-beijing",
 		AccessKey:  "test-ak",
@@ -110,8 +121,8 @@ func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) {
 	})
 	handler.tos.stsClient = fakeEpisodeQASTSClient{}
 	handler.tos.client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
-		if got := req.URL.Host; got != "tos-bucket.tos-cn-beijing.volces.com" {
-			t.Fatalf("host = %q, want tos-bucket.tos-cn-beijing.volces.com", got)
+		if got := req.URL.Host; got != "tos-bucket.tos-cn-beijing.ivolces.com" {
+			t.Fatalf("host = %q, want tos-bucket.tos-cn-beijing.ivolces.com", got)
 		}
 		return &http.Response{
 			StatusCode:    http.StatusOK,
@@ -136,11 +147,34 @@ func TestStorageHandlerGetObjectAllowsTOSWithoutS3(t *testing.T) {
 
 	handler.GetObject(ctx)
 
-	if w.Code != http.StatusOK {
-		t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusOK, w.Body.String())
+	if w.Code != http.StatusTemporaryRedirect {
+		t.Fatalf("status = %d, want %d body=%s", w.Code, http.StatusTemporaryRedirect, w.Body.String())
+	}
+	location, err := url.Parse(w.Header().Get("Location"))
+	if err != nil {
+		t.Fatalf("parse redirect location: %v", err)
+	}
+	if got := location.Scheme; got != "https" {
+		t.Fatalf("redirect scheme = %q, want https", got)
+	}
+	if got := location.Host; got != "tos-bucket.tos-cn-beijing.volces.com" {
+		t.Fatalf("redirect host = %q, want public TOS host", got)
+	}
+	if got := location.Path; got != "/device-uploads/capture.mcap" {
+		t.Fatalf("redirect path = %q, want object path", got)
+	}
+	query := location.Query()
+	for _, key := range []string{"X-Tos-Algorithm", "X-Tos-Credential", "X-Tos-Date", "X-Tos-Expires", "X-Tos-Security-Token", "X-Tos-Signature"} {
+		if query.Get(key) == "" {
+			t.Fatalf("redirect query missing %s: %s", key, location.String())
+		}
+	}
+	expires, err := strconv.Atoi(query.Get("X-Tos-Expires"))
+	if err != nil || expires < 1 || expires > 60 {
+		t.Fatalf("X-Tos-Expires = %q, want 1..60 seconds", query.Get("X-Tos-Expires"))
 	}
-	if got := w.Body.String(); got != "mcap" {
-		t.Fatalf("body = %q, want mcap", got)
+	if got := query.Get("response-content-disposition"); !strings.HasPrefix(got, "attachment;") {
+		t.Fatalf("response-content-disposition = %q, want attachment", got)
 	}
 }
 

internal/auth/storage_download_token.go

@@ -53,14 +53,14 @@ func SignStorageDownloadToken(bucket, object string, ttl time.Duration, cfg *con
 	return tok.SignedString([]byte(cfg.JWTSecret))
 }
 
-// ParseStorageDownloadToken validates token signature, expiry, kind, and bucket/object binding.
-func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) error {
+// ParseStorageDownloadTokenClaims validates a token and returns its bucket/object-bound claims.
+func ParseStorageDownloadTokenClaims(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) (*StorageDownloadClaims, error) {
 	if cfg == nil || strings.TrimSpace(cfg.JWTSecret) == "" {
-		return ErrInvalidToken
+		return nil, ErrInvalidToken
 	}
 	tokenString = strings.TrimSpace(tokenString)
 	if tokenString == "" {
-		return ErrInvalidToken
+		return nil, ErrInvalidToken
 	}
 
 	var claims StorageDownloadClaims
@@ -72,18 +72,24 @@ func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantB
 	})
 	if err != nil {
 		if errors.Is(err, jwt.ErrTokenExpired) {
-			return ErrExpiredToken
+			return nil, ErrExpiredToken
 		}
-		return ErrInvalidToken
+		return nil, ErrInvalidToken
 	}
 	if !token.Valid {
-		return ErrInvalidToken
+		return nil, ErrInvalidToken
 	}
-	if claims.Kind != StorageDownloadTokenKind {
-		return ErrInvalidToken
+	if claims.Kind != StorageDownloadTokenKind || claims.ExpiresAt == nil {
+		return nil, ErrInvalidToken
 	}
 	if claims.Bucket != wantBucket || claims.Object != wantObject {
-		return ErrInvalidToken
+		return nil, ErrInvalidToken
 	}
-	return nil
+	return &claims, nil
+}
+
+// ParseStorageDownloadToken validates token signature, expiry, kind, and bucket/object binding.
+func ParseStorageDownloadToken(tokenString string, cfg *config.AuthConfig, wantBucket, wantObject string) error {
+	_, err := ParseStorageDownloadTokenClaims(tokenString, cfg, wantBucket, wantObject)
+	return err
 }