impl(v2): add client request spans to gax.Invoke

go.work.sum

@@ -982,6 +982,7 @@ github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEP
 github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w=
 github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14=
 github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
 github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=

v2/go.mod

@@ -6,7 +6,9 @@ require (
 	github.com/google/go-cmp v0.7.0
 	go.opentelemetry.io/otel v1.44.0
 	go.opentelemetry.io/otel/metric v1.44.0
+	go.opentelemetry.io/otel/sdk v1.44.0
 	go.opentelemetry.io/otel/sdk/metric v1.44.0
+	go.opentelemetry.io/otel/trace v1.44.0
 	google.golang.org/api v0.284.0
 	google.golang.org/genproto v0.0.0-20260618152121-87f3d3e198d3
 	google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3
@@ -21,8 +23,6 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
 	golang.org/x/text v0.37.0 // indirect

v2/go.sum

@@ -31,6 +31,8 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk
 go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
 go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=

v2/invoke.go

@@ -97,6 +97,14 @@ func invoke(ctx context.Context, call APICall, settings CallSettings, sp sleeper
 		}()
 	}
 
+	var endSpan func(error)
+	if IsFeatureEnabled("TRACING") {
+		ctx, endSpan = StartClientRequestSpan(ctx)
+		defer func() {
+			endSpan(err)
+		}()
+	}
+
 	retryCount := 0
 	// Feature gate: GOOGLE_SDK_GO_EXPERIMENTAL_TRACING=true
 	tracingEnabled := IsFeatureEnabled("TRACING")

v2/invoke_tracing_test.go

@@ -0,0 +1,147 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package gax
+
+import (
+	"context"
+	"testing"
+
+	"github.com/googleapis/gax-go/v2/callctx"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	grpccodes "google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestInvokeClientRequestSpan(t *testing.T) {
+	t.Setenv("GOOGLE_SDK_GO_EXPERIMENTAL_TRACING", "true")
+	TestOnlyResetIsFeatureEnabled()
+	defer TestOnlyResetIsFeatureEnabled()
+
+	tests := []struct {
+		name         string
+		setupCtx     func() context.Context
+		callFunc     func(context.Context, CallSettings) error
+		wantSpanName string
+		wantAttrs    map[string]string
+		wantStatus   codes.Code
+		wantErr      bool
+	}{
+		{
+			name: "success_with_client_span_name",
+			setupCtx: func() context.Context {
+				return callctx.WithTelemetryContext(context.Background(), "client_span_name", "MyClient.MyMethod")
+			},
+			callFunc: func(ctx context.Context, settings CallSettings) error {
+				return nil
+			},
+			wantSpanName: "MyClient.MyMethod",
+			wantAttrs:    map[string]string{},
+			wantStatus:   codes.Ok,
+			wantErr:      false,
+		},
+		{
+			name: "success_with_fallback_rpc_method",
+			setupCtx: func() context.Context {
+				return callctx.WithTelemetryContext(context.Background(), "rpc_method", "my.pkg.Service/Method")
+			},
+			callFunc: func(ctx context.Context, settings CallSettings) error {
+				return nil
+			},
+			wantSpanName: "my.pkg.Service/Method",
+			wantAttrs: map[string]string{
+				"rpc.method": "my.pkg.Service/Method",
+			},
+			wantStatus: codes.Ok,
+			wantErr:    false,
+		},
+		{
+			name: "failure_records_error_and_attributes",
+			setupCtx: func() context.Context {
+				return callctx.WithTelemetryContext(context.Background(), "client_span_name", "MyMethod")
+			},
+			callFunc: func(ctx context.Context, settings CallSettings) error {
+				return status.Error(grpccodes.Unavailable, "service unavailable")
+			},
+			wantSpanName: "MyMethod",
+			wantAttrs: map[string]string{
+				"error.type":               "UNAVAILABLE",
+				"rpc.response.status_code": "UNAVAILABLE",
+				"status.message":           "service unavailable",
+			},
+			wantStatus: codes.Error,
+			wantErr:    true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := tt.setupCtx()
+
+			sr := tracetest.NewSpanRecorder()
+			tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
+
+			oldProvider := otel.GetTracerProvider()
+			otel.SetTracerProvider(tp)
+			defer otel.SetTracerProvider(oldProvider)
+
+			err := Invoke(ctx, tt.callFunc)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("Invoke() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			spans := sr.Ended()
+			if len(spans) == 0 {
+				t.Fatalf("expected at least 1 span recorded")
+			}
+
+			span := spans[0]
+			if span.Name() != tt.wantSpanName {
+				t.Errorf("expected span name %q, got %q", tt.wantSpanName, span.Name())
+			}
+			if span.Status().Code != tt.wantStatus {
+				t.Errorf("expected span status %v, got %v", tt.wantStatus, span.Status().Code)
+			}
+
+			gotAttrs := make(map[string]string)
+			for _, a := range span.Attributes() {
+				gotAttrs[string(a.Key)] = a.Value.AsString()
+			}
+
+			for k, v := range tt.wantAttrs {
+				if gotAttrs[k] != v {
+					t.Errorf("expected attribute %q=%q, got %q", k, v, gotAttrs[k])
+				}
+			}
+		})
+	}
+}

v2/telemetry.go

@@ -42,7 +42,9 @@ import (
 	"github.com/googleapis/gax-go/v2/callctx"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
+	otelcodes "go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -433,10 +435,6 @@ func recordMetric(ctx context.Context, settings CallSettings, d time.Duration, e
 		return
 	}
 
-	// Use context.WithoutCancel to ensure metric records even if context is canceled
-	// preserving any trace context that might be required for exemplars.
-	recordCtx := context.WithoutCancel(ctx)
-
 	// Pre-allocate to avoid repeated appends (5 is the max number of dynamic attributes added here)
 	attrs := make([]attribute.KeyValue, 0, len(settings.clientMetrics.attributes())+5)
 	attrs = append(attrs, settings.clientMetrics.attributes()...)
@@ -465,5 +463,83 @@ func recordMetric(ctx context.Context, settings CallSettings, d time.Duration, e
 		attrs = append(attrs, attribute.String("url.template", urlTemplate))
 	}
 
-	settings.clientMetrics.durationHistogram().Record(recordCtx, d.Seconds(), metric.WithAttributes(attrs...))
+	settings.clientMetrics.durationHistogram().Record(ctx, d.Seconds(), metric.WithAttributes(attrs...))
+}
+
+// StartClientRequestSpan starts a client request span if tracing is enabled.
+// It reads the span name from the context (key: client_span_name) and falls back to rpc_method.
+// It returns the updated context and a closure to end the span, which avoids leaking OTel types to callers.
+func StartClientRequestSpan(ctx context.Context) (context.Context, func(error)) {
+	if !IsFeatureEnabled("TRACING") {
+		return ctx, func(error) {}
+	}
+	spanName, _ := callctx.TelemetryFromContext(ctx, "client_span_name")
+	if spanName == "" {
+		spanName, _ = callctx.TelemetryFromContext(ctx, "rpc_method")
+		if spanName == "" {
+			spanName = "ClientRequest"
+		}
+	}
+
+	tp := otel.GetTracerProvider()
+	tracer := tp.Tracer("github.com/googleapis/gax-go/v2")
+
+	ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindInternal))
+
+	// Add static attributes from context if available
+	if rpcMethod, ok := callctx.TelemetryFromContext(ctx, "rpc_method"); ok && rpcMethod != "" {
+		span.SetAttributes(attribute.String("rpc.method", rpcMethod))
+	}
+	if urlTemplate, ok := callctx.TelemetryFromContext(ctx, "url_template"); ok && urlTemplate != "" {
+		span.SetAttributes(attribute.String("url.template", urlTemplate))
+	}
+	if resName, ok := callctx.TelemetryFromContext(ctx, "resource_name"); ok && resName != "" {
+		span.SetAttributes(attribute.String("gcp.resource.destination.id", resName))
+	}
+
+	return ctx, func(err error) {
+		EndClientRequestSpan(ctx, span, err)
+	}
+}
+
+// EndClientRequestSpan ends the client request span and records error details if present.
+func EndClientRequestSpan(ctx context.Context, span trace.Span, err error) {
+	if span == nil {
+		return
+	}
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(otelcodes.Error, err.Error())
+
+		// Extract structured error info
+		errInfo := ExtractTelemetryErrorInfo(ctx, err)
+		if errInfo.ErrorType != "" {
+			span.SetAttributes(attribute.String("error.type", errInfo.ErrorType))
+		}
+		span.SetAttributes(attribute.String("rpc.response.status_code", errInfo.StatusCode))
+		if errInfo.StatusMessage != "" {
+			span.SetAttributes(attribute.String("status.message", errInfo.StatusMessage))
+		}
+		if errInfo.Domain != "" {
+			span.SetAttributes(attribute.String("gcp.errors.domain", errInfo.Domain))
+		}
+		for k, v := range errInfo.Metadata {
+			span.SetAttributes(attribute.String("gcp.errors.metadata."+k, v))
+		}
+	} else {
+		span.SetStatus(otelcodes.Ok, "OK")
+		span.SetAttributes(attribute.String("rpc.response.status_code", "OK"))
+	}
+
+	// Extract transport telemetry if available
+	if td := ExtractTransportTelemetry(ctx); td != nil {
+		if td.ServerAddress() != "" {
+			span.SetAttributes(attribute.String("server.address", td.ServerAddress()))
+		}
+		if td.ServerPort() != 0 {
+			span.SetAttributes(attribute.Int("server.port", td.ServerPort()))
+		}
+	}
+
+	span.End()
 }