Bump Microsoft.ApplicationInsights to 3.1.2 and add local telemetry export

Directory.Packages.props

@@ -59,7 +59,10 @@
          stable release on nuget.org. Promote when the internal feed ships a non-preview version.
          Tracking context: https://github.com/microsoft/testfx/issues/9091 (Task 5). -->
     <MicrosoftTestingInternalFrameworkVersion>1.5.0-preview.24577.4</MicrosoftTestingInternalFrameworkVersion>
-    <SystemThreadingTasksExtensionsVersion>4.5.4</SystemThreadingTasksExtensionsVersion>
+    <!-- Must stay >= the version required by Azure.Core (pulled in transitively on .NETFramework via
+         Microsoft.ApplicationInsights 3.x -> Azure.Monitor.OpenTelemetry.Exporter -> Azure.Core).
+         Anything lower produces an NU1605 downgrade error on the .NETFramework adapter assemblies. -->
+    <SystemThreadingTasksExtensionsVersion>4.6.3</SystemThreadingTasksExtensionsVersion>
     <!-- System.Memory flows in transitively (via System.Reflection.Metadata / System.Collections.Immutable)
          at 4.5.5, whose .NETFramework assembly version is 4.0.1.2. We pin it explicitly to 4.6.3
          (assembly 4.0.5.0) so the .NETFramework adapter assemblies reference 4.0.5.0. Hosts that unify
@@ -77,10 +80,11 @@
     <PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.556" />
   </ItemGroup>
   <ItemGroup Label="Product dependencies">
-    <!-- If this gets updated, please revise if the explicit dependency on System.Diagnostics.DiagnosticSource is still needed -->
-    <PackageVersion Include="Microsoft.ApplicationInsights" Version="2.23.0" />
-    <!-- This comes transitvely via Microsoft.ApplicationInsights, but we want to upgrade it because 5.0.0 is not maintained -->
-    <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="6.0.0" />
+    <!-- Microsoft.ApplicationInsights 3.x is a shim over OpenTelemetry + Azure Monitor Exporter.
+         Its dependency closure (via Azure.Monitor.OpenTelemetry.Exporter -> Azure.Core) already
+         brings a modern, maintained System.Diagnostics.DiagnosticSource transitively, so we no
+         longer need an explicit reference/pin here. See https://github.com/microsoft/testfx/issues/7465. -->
+    <PackageVersion Include="Microsoft.ApplicationInsights" Version="3.1.2" />
     <PackageVersion Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildVersion)" />
     <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />
     <PackageVersion Include="Microsoft.CodeAnalysis" Version="$(MicrosoftCodeAnalysisVersion)" />

src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClient.cs

@@ -24,7 +24,30 @@ public AppInsightTelemetryClient(string? currentSessionId, string osVersion)
     }
 
     public void TrackEvent(string eventName, Dictionary<string, string> properties, Dictionary<string, double> metrics)
-        => _telemetryClient.TrackEvent(eventName, properties, metrics);
+    {
+        // Microsoft.ApplicationInsights 3.x (an OpenTelemetry shim) removed the metrics parameter
+        // from TrackEvent. Tracking metrics separately via TrackMetric() would emit uncorrelated
+        // metric instruments that are neither enriched with this client's context (session/OS) nor
+        // tied back to the event, and would additionally require every metric key to satisfy the
+        // OpenTelemetry instrument-name syntax (no spaces, must start with a letter).
+        //
+        // Instead we fold the numeric measurements into the event's properties as invariant-culture
+        // strings. This keeps a single correlated, context-enriched customEvent and imposes no naming
+        // restrictions on the keys. Consumers read these values back with todouble(customDimensions[...]).
+        if (metrics.Count == 0)
+        {
+            _telemetryClient.TrackEvent(eventName, properties);
+            return;
+        }
+
+        var combinedProperties = new Dictionary<string, string>(properties);
+        foreach (KeyValuePair<string, double> metric in metrics)
+        {
+            combinedProperties[metric.Key] = metric.Value.ToString("R", CultureInfo.InvariantCulture);
+        }
+
+        _telemetryClient.TrackEvent(eventName, combinedProperties);
+    }
 
     public void Flush()
         => _telemetryClient.Flush();

src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightTelemetryClientFactory.cs

@@ -1,10 +1,19 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.Testing.Platform;
+
 namespace Microsoft.Testing.Extensions.Telemetry;
 
 internal sealed class AppInsightTelemetryClientFactory : ITelemetryClientFactory
 {
+    private readonly string? _localExportFilePath;
+
+    public AppInsightTelemetryClientFactory(string? localExportFilePath = null)
+        => _localExportFilePath = localExportFilePath;
+
     public ITelemetryClient Create(string? currentSessionId, string osVersion)
-        => new AppInsightTelemetryClient(currentSessionId, osVersion);
+        => RoslynString.IsNullOrWhiteSpace(_localExportFilePath)
+            ? new AppInsightTelemetryClient(currentSessionId, osVersion)
+            : new LocalFileTelemetryClient(_localExportFilePath, currentSessionId, osVersion);
 }

src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs

@@ -32,6 +32,11 @@ internal sealed partial class AppInsightsProvider :
     // Note: We're currently using the same environment variable as dotnet CLI.
     public static readonly string SessionIdEnvVar = "TESTINGPLATFORM_APPINSIGHTS_SESSIONID";
 
+    // When set to a file path, telemetry is written locally to that file (as JSON Lines) via
+    // LocalFileTelemetryClient instead of being shipped to Application Insights. This is the
+    // "local exporter" hook used to verify what telemetry would be collected, without the network.
+    public static readonly string LocalExportPathEnvVar = "TESTINGPLATFORM_TELEMETRY_LOCALEXPORTPATH";
+
     // Allows us to correlate events produced from the same process.
     // Not calling this ProcessId, because it has a different meaning.
     private static readonly string CurrentReporterId = Guid.NewGuid().ToString();

src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsTelemetryProviderExtensions.cs

@@ -44,6 +44,10 @@ public static void AddAppInsightsTelemetryProvider(this ITestApplicationBuilder
 
             // We want to flow down the processes the same session id for correlation purposes.
             environment.SetEnvironmentVariable(AppInsightsProvider.SessionIdEnvVar, sessionId);
+
+            // Opt-in local export: when set, telemetry is written to this file instead of AppInsights.
+            string? localExportPath = environment.GetEnvironmentVariable(AppInsightsProvider.LocalExportPathEnvVar);
+
             return new AppInsightsProvider(
                 services.GetRequiredService<IEnvironment>(),
                 services.GetTestApplicationCancellationTokenSource(),
@@ -52,7 +56,7 @@ public static void AddAppInsightsTelemetryProvider(this ITestApplicationBuilder
                 services.GetClock(),
                 services.GetConfiguration(),
                 services.GetRequiredService<ITelemetryInformation>(),
-                new AppInsightTelemetryClientFactory(),
+                new AppInsightTelemetryClientFactory(localExportPath),
                 sessionId);
         });
 #pragma warning restore IDE0022 // Use expression body for method

src/Platform/Microsoft.Testing.Extensions.Telemetry/InternalAPI/InternalAPI.Unshipped.txt

@@ -1,7 +1,14 @@
 #nullable enable
+*REMOVED*Microsoft.Testing.Extensions.Telemetry.AppInsightTelemetryClientFactory.AppInsightTelemetryClientFactory() -> void
+Microsoft.Testing.Extensions.Telemetry.AppInsightTelemetryClientFactory.AppInsightTelemetryClientFactory(string? localExportFilePath = null) -> void
+Microsoft.Testing.Extensions.Telemetry.LocalFileTelemetryClient
+Microsoft.Testing.Extensions.Telemetry.LocalFileTelemetryClient.Flush() -> void
+Microsoft.Testing.Extensions.Telemetry.LocalFileTelemetryClient.LocalFileTelemetryClient(string! filePath, string? currentSessionId, string! osVersion) -> void
+Microsoft.Testing.Extensions.Telemetry.LocalFileTelemetryClient.TrackEvent(string! eventName, System.Collections.Generic.Dictionary<string!, string!>! properties, System.Collections.Generic.Dictionary<string!, double>! metrics) -> void
 Microsoft.Testing.Platform.Helpers.RuntimeFeatureHelper
 Microsoft.Testing.Platform.Messages.SingleConsumerUnboundedChannel<T>.WaitToRead() -> bool
 Microsoft.Testing.Platform.RoslynDebug
 static Microsoft.Testing.Platform.Helpers.RuntimeFeatureHelper.IsMultiThreaded.get -> bool
 static Microsoft.Testing.Platform.RoslynDebug.Assert(bool b) -> void
 static Microsoft.Testing.Platform.RoslynDebug.Assert(bool b, string! message) -> void
+static readonly Microsoft.Testing.Extensions.Telemetry.AppInsightsProvider.LocalExportPathEnvVar -> string!

src/Platform/Microsoft.Testing.Extensions.Telemetry/LocalFileTelemetryClient.cs

@@ -0,0 +1,140 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform;
+
+namespace Microsoft.Testing.Extensions.Telemetry;
+
+/// <summary>
+/// A local, network-free <see cref="ITelemetryClient"/> that appends every telemetry event to a
+/// file as a single JSON line (JSON Lines / NDJSON). It is the "local exporter" equivalent used to
+/// verify what telemetry would be collected — without shipping anything to Application Insights.
+/// It is selected instead of <see cref="AppInsightTelemetryClient"/> when the
+/// <see cref="AppInsightsProvider.LocalExportPathEnvVar"/> environment variable points at a file.
+/// </summary>
+internal sealed class LocalFileTelemetryClient : ITelemetryClient
+{
+    private readonly string _filePath;
+    private readonly string? _sessionId;
+    private readonly string _osVersion;
+
+    public LocalFileTelemetryClient(string filePath, string? currentSessionId, string osVersion)
+    {
+        _filePath = filePath;
+        _sessionId = currentSessionId;
+        _osVersion = osVersion;
+
+        string? directory = Path.GetDirectoryName(_filePath);
+        if (!RoslynString.IsNullOrEmpty(directory))
+        {
+            Directory.CreateDirectory(directory);
+        }
+    }
+
+    public void TrackEvent(string eventName, Dictionary<string, string> properties, Dictionary<string, double> metrics)
+    {
+        var builder = new StringBuilder();
+        builder.Append('{');
+        AppendJsonString(builder, "eventName", eventName);
+        builder.Append(',');
+        AppendJsonString(builder, "sessionId", _sessionId ?? string.Empty);
+        builder.Append(',');
+        AppendJsonString(builder, "osVersion", _osVersion);
+
+        builder.Append(",\"properties\":{");
+        bool first = true;
+        foreach (KeyValuePair<string, string> property in properties)
+        {
+            if (!first)
+            {
+                builder.Append(',');
+            }
+
+            AppendJsonString(builder, property.Key, property.Value);
+            first = false;
+        }
+
+        builder.Append("},\"metrics\":{");
+        first = true;
+        foreach (KeyValuePair<string, double> metric in metrics)
+        {
+            if (!first)
+            {
+                builder.Append(',');
+            }
+
+            AppendJsonKey(builder, metric.Key);
+            builder.Append(metric.Value.ToString("R", CultureInfo.InvariantCulture));
+            first = false;
+        }
+
+        builder.Append("}}");
+
+        // TrackEvent is only ever invoked from the provider's single-consumer ingest loop, so no
+        // synchronization is needed here (mirroring AppInsightTelemetryClient).
+        File.AppendAllText(_filePath, builder.ToString() + Environment.NewLine);
+    }
+
+    // No-op: writes are flushed to disk synchronously as each event is tracked.
+    public void Flush()
+    {
+    }
+
+    private static void AppendJsonString(StringBuilder builder, string key, string value)
+    {
+        AppendJsonKey(builder, key);
+        AppendEscaped(builder, value);
+    }
+
+    private static void AppendJsonKey(StringBuilder builder, string key)
+    {
+        AppendEscaped(builder, key);
+        builder.Append(':');
+    }
+
+    private static void AppendEscaped(StringBuilder builder, string value)
+    {
+        builder.Append('"');
+        foreach (char c in value)
+        {
+            switch (c)
+            {
+                case '"':
+                    builder.Append("\\\"");
+                    break;
+                case '\\':
+                    builder.Append("\\\\");
+                    break;
+                case '\b':
+                    builder.Append("\\b");
+                    break;
+                case '\f':
+                    builder.Append("\\f");
+                    break;
+                case '\n':
+                    builder.Append("\\n");
+                    break;
+                case '\r':
+                    builder.Append("\\r");
+                    break;
+                case '\t':
+                    builder.Append("\\t");
+                    break;
+                default:
+                    if (c < ' ')
+                    {
+                        builder.Append("\\u");
+                        builder.Append(((int)c).ToString("x4", CultureInfo.InvariantCulture));
+                    }
+                    else
+                    {
+                        builder.Append(c);
+                    }
+
+                    break;
+            }
+        }
+
+        builder.Append('"');
+    }
+}

src/Platform/Microsoft.Testing.Extensions.Telemetry/Microsoft.Testing.Extensions.Telemetry.csproj

@@ -54,10 +54,6 @@ This package provides telemetry for the platform.]]>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.ApplicationInsights" />
-
-    <!-- Microsoft.ApplicationInsights already has a dependency on DiagnosticSource. But we want to update the dependency to 6.0.0 instead of 5.0.0 -->
-    <!-- This is because System.Diagnostics.DiagnosticSource is marked as not maintained -->
-    <PackageReference Include="System.Diagnostics.DiagnosticSource" />
   </ItemGroup>
 
   <ItemGroup>

src/Platform/Microsoft.Testing.Extensions.Telemetry/PACKAGE.md

@@ -16,6 +16,7 @@ This package extends Microsoft.Testing.Platform with:
 
 - **Usage telemetry**: collects usage data to help understand product usage and prioritize improvements
 - **Opt-out support**: telemetry can be disabled via the `TESTINGPLATFORM_TELEMETRY_OPTOUT` or `DOTNET_CLI_TELEMETRY_OPTOUT` environment variables
+- **Local export (verification)**: set `TESTINGPLATFORM_TELEMETRY_LOCALEXPORTPATH` to a file path to write the collected telemetry events locally (as JSON Lines) instead of sending them to Application Insights. This is useful to inspect exactly what would be collected, without any network traffic.
 - **Disclosure**: telemetry information is shown on first run, with opt-out guidance
 
 This package is an optional, opt-in extension. To enable telemetry when using Microsoft.Testing.Platform (including when running tests with [MSTest](https://www.nuget.org/packages/MSTest)), you must explicitly reference the `Microsoft.Testing.Extensions.Telemetry` package from your test project or from your own test framework or tooling package.

src/Platform/Microsoft.Testing.Platform/InternalAPI/InternalAPI.Unshipped.txt

@@ -313,3 +313,5 @@ Microsoft.Testing.Platform.OutputDevice.SlowTestThresholdState.SlowTestThreshold
 static Microsoft.Testing.Platform.OutputDevice.ProgressReportingConfiguration.GetThreshold(Microsoft.Testing.Platform.Helpers.IEnvironment! environment, string! variableName, int defaultSeconds) -> System.TimeSpan
 Microsoft.Testing.Platform.Extensions.Messages.TestNodeExecutionCompletedProperty
 static Microsoft.Testing.Platform.Extensions.Messages.TestNodeExecutionCompletedProperty.CachedInstance.get -> Microsoft.Testing.Platform.Extensions.Messages.TestNodeExecutionCompletedProperty!
+*REMOVED*const Microsoft.Testing.Platform.Telemetry.TelemetryProperties.VersionValue = "20" -> string!
+const Microsoft.Testing.Platform.Telemetry.TelemetryProperties.VersionValue = "21" -> string!

src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryProperties.cs

@@ -10,7 +10,11 @@ internal static class TelemetryProperties
     public const string ReporterIdPropertyName = "reporter id";
     public const string IsCIPropertyName = "is ci";
 
-    public const string VersionValue = "20";
+    // Bump this whenever the telemetry schema changes so dashboards can branch old vs new.
+    // 21: Microsoft.ApplicationInsights 3.x (OpenTelemetry shim) removed TrackEvent's metrics
+    //     parameter, so numeric measurements are now folded into event properties
+    //     (customDimensions) instead of customMeasurements. See #7465.
+    public const string VersionValue = "21";
 
     public const string True = "true";
     public const string False = "false";