diff --git a/.gitignore b/.gitignore
index 471480c76..32a28c1d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,9 @@ generated/
.vscode/settings.json
eng/tools/ToolDescriptionEvaluator/prompts.json
+.perf-results/
+BenchmarkDotNet.Artifacts/
+build_output.txt
eng/tools/ToolDescriptionEvaluator/results.md
eng/tools/ToolDescriptionEvaluator/tools.json
deploy-log.txt
diff --git a/.vscode/cspell.json b/.vscode/cspell.json
index 96f23a281..3d08c8546 100644
--- a/.vscode/cspell.json
+++ b/.vscode/cspell.json
@@ -63,6 +63,7 @@
"aadsts",
"accesstoken",
"addattachment",
+ "addbuildtag",
"agentpool",
"aiml",
"amazonq",
@@ -83,12 +84,15 @@
"classfilters",
"cloudevents",
"codesign",
+ "coldstart",
"confidentialledger",
"configfile",
"creds",
"credscan",
"datatable",
"devcert",
+ "diagnoser",
+ "diagnosers",
"diagnosticservices",
"doubleslash",
"dyld",
@@ -150,7 +154,9 @@
"testdb",
"testproxy",
"testrg",
+ "testruns",
"testsub",
+ "tiktoken",
"toplevel",
"uniqueprefix",
"vectorizable",
diff --git a/Directory.Packages.props b/Directory.Packages.props
index b2c47acac..ab5dd46c1 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -124,5 +124,9 @@
+
+
+
+
diff --git a/Microsoft.Mcp.slnx b/Microsoft.Mcp.slnx
index eee8c3d9e..b45b6a321 100644
--- a/Microsoft.Mcp.slnx
+++ b/Microsoft.Mcp.slnx
@@ -78,6 +78,9 @@
+
+
+
diff --git a/eng/perf-workloads.json b/eng/perf-workloads.json
new file mode 100644
index 000000000..2b44dac15
--- /dev/null
+++ b/eng/perf-workloads.json
@@ -0,0 +1,166 @@
+{
+ "catalog_version": "1.0.0",
+ "description": "Versioned, machine-readable definitions of Azure MCP Server performance workloads and the controlled-environment and statistical rules they run under (issue #3120). Bump catalog_version and record the reason whenever a workload definition changes, since a change can shift baselines.",
+ "environment_requirements": {
+ "runner": "Dedicated or sufficiently isolated runner. Baseline-quality runs must not share the host with other CI work; developer laptops are for exploration only.",
+ "runtime": "The .NET SDK pinned by global.json; record the exact runtime in each result's environment block.",
+ "build_configuration": "Release.",
+ "network": "Loopback only for server-overhead workloads. The measured path must exclude Azure and network time (deterministic no-op / tools/list).",
+ "configuration": "Server launched with the documented args per workload; no unrelated environment variables set."
+ },
+ "statistical_rules": {
+ "warmup": "Every scenario performs warm-up iterations that are discarded before measurement.",
+ "min_repetitions": 10,
+ "percentiles": ["p50", "p95", "p99"],
+ "reported_metrics": ["p50", "p95", "p99", "throughput", "cpu", "memory"],
+ "comparison": "Compare current vs approved baseline as ratios. Changes below 5% are ignored, changes above 5% warn, and budget breaches block. Prefer median/percentiles over the mean for latency.",
+ "notes": "Micro-benchmarks additionally report BenchmarkDotNet StdDev / confidence intervals; the host environment (OS, CPU, runtime) is captured in the BenchmarkDotNet report header."
+ },
+ "baseline_qualification": {
+ "min_successful_runs": 10,
+ "min_span_days": 14,
+ "controlled_runner_required": true,
+ "auto_update_after_regression": false,
+ "replacement_requires": ["linked issue", "measured impact", "owner approval"]
+ },
+ "workloads": [
+ {
+ "id": "startup-cli-cold",
+ "issue": 3118,
+ "harness": "eng/scripts/Test-StartupPerformance.ps1",
+ "transport": "cli",
+ "server_mode": null,
+ "description": "CLI cold start: 'azmcp tools list' (process spawn to exit).",
+ "warmup": "First run reported as cold; the remainder are warm.",
+ "repetitions": 10,
+ "metrics": ["p50", "p95", "p99", "cold_ms", "warm_median_ms"]
+ },
+ {
+ "id": "startup-stdio-default",
+ "issue": 3118,
+ "harness": "eng/scripts/Test-StartupPerformance.ps1",
+ "transport": "stdio",
+ "server_mode": "default",
+ "description": "MCP stdio startup to a usable tool list: 'server start' then initialize + tools/list (~71 tools).",
+ "warmup": "First run reported as cold; the remainder are warm.",
+ "repetitions": 10,
+ "metrics": ["p50", "p95", "p99", "full_schema_tokens_exact", "full_schema_serialize_ms"]
+ },
+ {
+ "id": "startup-stdio-namespace",
+ "issue": 3118,
+ "harness": "eng/scripts/Test-StartupPerformance.ps1",
+ "transport": "stdio",
+ "server_mode": "namespace",
+ "description": "MCP stdio startup with 'server start --mode namespace'.",
+ "warmup": "First run reported as cold; the remainder are warm.",
+ "repetitions": 10,
+ "metrics": ["p50", "p95", "p99"]
+ },
+ {
+ "id": "startup-stdio-all",
+ "issue": 3118,
+ "harness": "eng/scripts/Test-StartupPerformance.ps1",
+ "transport": "stdio",
+ "server_mode": "all",
+ "description": "MCP stdio startup with 'server start --mode all' (full tool surface, ~500+ tools).",
+ "warmup": "First run reported as cold; the remainder are warm.",
+ "repetitions": 10,
+ "metrics": ["p50", "p95", "p99", "full_schema_tokens_exact", "full_schema_serialize_ms"]
+ },
+ {
+ "id": "startup-http-default",
+ "issue": 3118,
+ "harness": "eng/scripts/Test-StartupPerformance.ps1",
+ "transport": "http",
+ "server_mode": "default",
+ "description": "MCP remote HTTP startup: initialize + tools/list over the HTTP transport, plus server readiness time.",
+ "warmup": "First run reported as cold; the remainder are warm.",
+ "repetitions": 10,
+ "metrics": ["p50", "p95", "p99", "readiness_ms"]
+ },
+ {
+ "id": "discovery-scaling",
+ "issue": 3118,
+ "harness": "ToolSurfaceScalingBenchmarks (BenchmarkDotNet)",
+ "transport": "in-process",
+ "server_mode": null,
+ "description": "tools/list serialization cost at current and projected tool counts.",
+ "tool_counts": [71, 250, 510, 1000],
+ "warmup": "BenchmarkDotNet warmup iterations.",
+ "repetitions": "BenchmarkDotNet default job (>=15 iterations).",
+ "metrics": ["mean", "allocated"]
+ },
+ {
+ "id": "dispatch-layers",
+ "issue": 3119,
+ "harness": "CommandDispatchBenchmarks (BenchmarkDotNet)",
+ "transport": "in-process",
+ "server_mode": null,
+ "description": "Per-layer command dispatch overhead (routing, parse, bind, validate, serialize, execute frame, full dispatch) on a deterministic no-op command; excludes Azure.",
+ "warmup": "BenchmarkDotNet warmup iterations.",
+ "repetitions": "BenchmarkDotNet default job (>=15 iterations).",
+ "metrics": ["mean", "allocated"]
+ },
+ {
+ "id": "dispatch-transport-stdio",
+ "issue": 3119,
+ "harness": "eng/scripts/Measure-DispatchOverhead.ps1",
+ "transport": "stdio",
+ "server_mode": "default",
+ "description": "tools/list request overhead over stdio.",
+ "warmup": "PERF_DISPATCH_WARMUP calls (default 20), not counted.",
+ "repetitions": 200,
+ "metrics": ["p50", "p95", "p99"]
+ },
+ {
+ "id": "dispatch-transport-http",
+ "issue": 3119,
+ "harness": "eng/scripts/Measure-DispatchOverhead.ps1",
+ "transport": "http",
+ "server_mode": "default",
+ "description": "tools/list request overhead over the HTTP transport; the http-minus-stdio p50 delta isolates transport cost.",
+ "warmup": "PERF_DISPATCH_WARMUP calls (default 20), not counted.",
+ "repetitions": 200,
+ "metrics": ["p50", "p95", "p99", "transport_overhead_tools_list_p50"]
+ },
+ {
+ "id": "resource-default",
+ "issue": 3121,
+ "harness": "eng/scripts/Measure-ResourceUsage.ps1",
+ "transport": "http",
+ "server_mode": "default",
+ "description": "CPU/memory/handle sampling across idle, startup, discovery, steady-state, and soak phases for the default tool surface.",
+ "phases": ["startup", "idle", "discovery", "steady_state", "soak"],
+ "concurrency": 4,
+ "warmup": "Soak analysis discards the first 25% of samples.",
+ "repetitions": "Continuous 100ms sampling per phase.",
+ "metrics": ["working_set_mb", "private_mb", "handles", "cpu_percent", "soak_slope"]
+ },
+ {
+ "id": "resource-all",
+ "issue": 3121,
+ "harness": "eng/scripts/Measure-ResourceUsage.ps1",
+ "transport": "http",
+ "server_mode": "all",
+ "description": "CPU/memory/handle sampling for the full tool surface.",
+ "phases": ["startup", "idle", "discovery", "steady_state", "soak"],
+ "concurrency": 4,
+ "warmup": "Soak analysis discards the first 25% of samples.",
+ "repetitions": "Continuous 100ms sampling per phase.",
+ "metrics": ["working_set_mb", "private_mb", "handles", "cpu_percent", "soak_slope"]
+ },
+ {
+ "id": "concurrency-sweep",
+ "issue": 3122,
+ "harness": "eng/scripts/Measure-ConcurrencyScaling.ps1",
+ "transport": "http",
+ "server_mode": "default",
+ "description": "Concurrency sweep over parallel MCP clients issuing tools/list; produces a saturation curve.",
+ "concurrency_levels": [1, 2, 4, 8, 16],
+ "warmup": "PERF_CONC_WARMUP_SECONDS per level (default 1s), not counted.",
+ "repetitions": "PERF_CONC_DURATION_SECONDS timed window per level (default 5s).",
+ "metrics": ["throughput_rps", "p50", "p95", "p99", "error_rate_pct", "scaling_efficiency", "tool_count_mismatches"]
+ }
+ ]
+}
diff --git a/eng/pipelines/templates/common.yml b/eng/pipelines/templates/common.yml
index 3972b2df8..85d605a70 100644
--- a/eng/pipelines/templates/common.yml
+++ b/eng/pipelines/templates/common.yml
@@ -33,6 +33,9 @@ parameters:
- name: BuildTimeoutInMinutes
type: number
default: 90
+- name: RunPerfTests
+ type: boolean
+ default: false
- name: IsTestPipeline
type: boolean
default: false
@@ -119,6 +122,30 @@ extends:
jobs:
- template: /eng/pipelines/templates/jobs/vsix/pack-vsix-pr.yml
+ - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.RunPerfTests, 'true')) }}:
+ - stage: Perf
+ displayName: 'Performance test'
+ dependsOn:
+ - Build
+ pool:
+ name: $(WINDOWSPOOL)
+ image: $(WINDOWSVMIMAGE)
+ os: windows
+ variables:
+ - template: /eng/pipelines/templates/variables/image.yml
+ - template: /eng/pipelines/templates/variables/globals.yml
+ jobs:
+ - template: /eng/pipelines/templates/jobs/perf-test.yml
+ parameters:
+ Runs: 10
+ # Uncomment to gate on regressions once a baseline is committed:
+ # BaselinePath: eng/perf-baseline.json
+ # ResourceBaselinePath: eng/resource-baseline.json
+ # DispatchBaselinePath: eng/dispatch-baseline.json
+ # ConcurrencyBaselinePath: eng/concurrency-baseline.json
+ # Uncomment to run the (~30 min) BenchmarkDotNet micro-benchmarks:
+ # RunMicroBenchmarks: true
+
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.RunLiveTests, 'true')) }}:
- stage: Test
displayName: 'Live test'
diff --git a/eng/pipelines/templates/jobs/perf-test.yml b/eng/pipelines/templates/jobs/perf-test.yml
new file mode 100644
index 000000000..e359b4da8
--- /dev/null
+++ b/eng/pipelines/templates/jobs/perf-test.yml
@@ -0,0 +1,187 @@
+parameters:
+- name: Runs
+ type: number
+ default: 10
+- name: BaselinePath
+ type: string
+ default: ''
+ # Path to a JSON baseline file. When set, a second run gates on the tiered
+ # budgets from issue #3118 (p50/p95 +10%, p99 +20%, scaling efficiency +15%).
+- name: ResourceConcurrency
+ type: number
+ default: 4
+- name: ResourceSoakSeconds
+ type: number
+ default: 45
+- name: ResourceBaselinePath
+ type: string
+ default: ''
+ # Path to a JSON resource baseline. When set, gates on the resource budgets
+ # from issue #3121 (steady CPU/memory +10%, peak memory +15%, no soak leak).
+- name: DispatchBaselinePath
+ type: string
+ default: ''
+ # Path to a JSON dispatch baseline. When set, gates on the dispatch budgets
+ # from issue #3119 (p50/p95 +10%, p99 +20%, transport change >5% reported).
+- name: ConcurrencyBaselinePath
+ type: string
+ default: ''
+ # Path to a JSON concurrency baseline. When set, gates on the concurrency budgets
+ # from issue #3122 (throughput -10%, p95/scaling -15%, p99 -20%, error +0.5pp).
+- name: RunMicroBenchmarks
+ type: boolean
+ default: false
+ # The BenchmarkDotNet micro-benchmarks are opt-in: the generated boilerplate
+ # references the whole server (60+ tool assemblies) and the full suite takes
+ # ~30 minutes, so enable this only for dedicated/nightly perf runs.
+- name: MicroBenchmarkFilter
+ type: string
+ default: '*'
+ # Glob passed to BenchmarkDotNet --filter (e.g. '*ServerStartup*' to scope a run).
+
+jobs:
+- job: PerfTest
+ displayName: "Performance test"
+ timeoutInMinutes: 60
+ pool:
+ name: $(WINDOWSPOOL)
+ image: $(WINDOWSVMIMAGE)
+ os: windows
+ templateContext:
+ outputs:
+ - output: pipelineArtifact
+ condition: always()
+ targetPath: $(Build.ArtifactStagingDirectory)/perf
+ artifactName: perf-results
+
+ steps:
+ - checkout: self
+
+ - task: UseDotNet@2
+ displayName: "Use .NET SDK from global.json"
+ retryCountOnTaskFailure: 3
+ inputs:
+ useGlobalJson: true
+
+ - task: Powershell@2
+ displayName: "Build benchmark tool (Release)"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Build-PerfBenchmarks.ps1
+
+ - task: Powershell@2
+ displayName: "Run performance measurements"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Test-StartupPerformance.ps1
+ arguments: >
+ -Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
+ -Runs ${{ parameters.Runs }}
+ -OutputPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
+
+ # Regression gate — only active when BaselinePath is provided.
+ # Reads the already-written results JSON; does NOT re-run the benchmarks.
+ - ${{ if ne(parameters.BaselinePath, '') }}:
+ - task: Powershell@2
+ displayName: "Check regression against baseline"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Check-StartupPerformanceRegression.ps1
+ arguments: >
+ -ResultsPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
+ -BaselinePath '${{ parameters.BaselinePath }}'
+
+ - task: Powershell@2
+ displayName: "Run resource-usage measurements"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-ResourceUsage.ps1
+ arguments: >
+ -Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
+ -BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
+ -Modes default,all
+ -Concurrency ${{ parameters.ResourceConcurrency }}
+ -SoakSeconds ${{ parameters.ResourceSoakSeconds }}
+ -OutputPath '$(Build.ArtifactStagingDirectory)/perf/resource-usage.json'
+
+ # Resource regression gate — only active when ResourceBaselinePath is provided.
+ - ${{ if ne(parameters.ResourceBaselinePath, '') }}:
+ - task: Powershell@2
+ displayName: "Check resource regression against baseline"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Check-ResourceUsageRegression.ps1
+ arguments: >
+ -ResultsPath '$(Build.ArtifactStagingDirectory)/perf/resource-usage.json'
+ -BaselinePath '${{ parameters.ResourceBaselinePath }}'
+
+ - task: Powershell@2
+ displayName: "Run dispatch-overhead measurements"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-DispatchOverhead.ps1
+ arguments: >
+ -Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
+ -BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
+ -OutputPath '$(Build.ArtifactStagingDirectory)/perf/dispatch-overhead.json'
+
+ # Dispatch regression gate — only active when DispatchBaselinePath is provided.
+ - ${{ if ne(parameters.DispatchBaselinePath, '') }}:
+ - task: Powershell@2
+ displayName: "Check dispatch regression against baseline"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Check-DispatchOverheadRegression.ps1
+ arguments: >
+ -ResultsPath '$(Build.ArtifactStagingDirectory)/perf/dispatch-overhead.json'
+ -BaselinePath '${{ parameters.DispatchBaselinePath }}'
+
+ - task: Powershell@2
+ displayName: "Run concurrency-scaling measurements"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-ConcurrencyScaling.ps1
+ arguments: >
+ -Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
+ -BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
+ -Levels 1,2,4,8,16
+ -DurationSeconds 10
+ -OutputPath '$(Build.ArtifactStagingDirectory)/perf/concurrency-scaling.json'
+
+ # Concurrency regression gate — only active when ConcurrencyBaselinePath is provided.
+ - ${{ if ne(parameters.ConcurrencyBaselinePath, '') }}:
+ - task: Powershell@2
+ displayName: "Check concurrency regression against baseline"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Check-ConcurrencyScalingRegression.ps1
+ arguments: >
+ -ResultsPath '$(Build.ArtifactStagingDirectory)/perf/concurrency-scaling.json'
+ -BaselinePath '${{ parameters.ConcurrencyBaselinePath }}'
+
+ # Runs the BenchmarkDotNet micro-benchmarks (per-phase startup allocations and
+ # tool-catalog serialization scaling). Opt-in via RunMicroBenchmarks because the
+ # generated boilerplate references the whole server (60+ tool assemblies) and the
+ # full suite takes ~30 minutes. --buildTimeout is raised because that boilerplate
+ # build exceeds BenchmarkDotNet's default 2-minute timeout. Each benchmark class
+ # carries its own job config, so no --job override is passed.
+ - ${{ if eq(parameters.RunMicroBenchmarks, true) }}:
+ - task: Powershell@2
+ displayName: "Run micro-benchmarks (BenchmarkDotNet)"
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Invoke-PerfMicroBenchmarks.ps1
+ arguments: >
+ -BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
+ -ArtifactsPath '$(Build.ArtifactStagingDirectory)/perf/benchmark-dotnet'
+ -Filter '${{ parameters.MicroBenchmarkFilter }}'
+
+ - task: Powershell@2
+ displayName: "Publish metrics to Azure Monitor (optional)"
+ condition: and(succeeded(), eq(variables['System.TeamProject'], 'internal'))
+ continueOnError: true
+ inputs:
+ pwsh: true
+ filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-StartupPerformanceMetrics.ps1
+ arguments: >
+ -ResultsPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
diff --git a/eng/scripts/Build-PerfBenchmarks.ps1 b/eng/scripts/Build-PerfBenchmarks.ps1
new file mode 100644
index 000000000..2f4566679
--- /dev/null
+++ b/eng/scripts/Build-PerfBenchmarks.ps1
@@ -0,0 +1,27 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Builds the Azure MCP Server performance-benchmark project in Release for the perf pipeline.
+
+.DESCRIPTION
+ Kept as a script (rather than inline pipeline YAML) so CI behavior can be reproduced locally.
+
+ NU1902 (the NuGet audit warning) is already non-fatal via WarningsNotAsErrors in
+ Directory.Build.props. A command-line -p:NoWarn is deliberately NOT passed here: a global
+ NoWarn property overrides every project's own (for example the Insights project's
+ MCP9005 sampling-deprecation suppression), which breaks a clean build.
+
+.PARAMETER Project
+ Path to the performance-benchmark csproj to build.
+#>
+param(
+ [string] $Project = 'servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj'
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+dotnet build $Project -c Release -consoleLoggerParameters:NoSummary
+if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
diff --git a/eng/scripts/Check-ConcurrencyScalingRegression.ps1 b/eng/scripts/Check-ConcurrencyScalingRegression.ps1
new file mode 100644
index 000000000..52c37d878
--- /dev/null
+++ b/eng/scripts/Check-ConcurrencyScalingRegression.ps1
@@ -0,0 +1,59 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Gates the concurrency / throughput / scalability results against a saved baseline (#3122).
+
+.DESCRIPTION
+ Reads the results JSON produced by Measure-ConcurrencyScaling.ps1 and a baseline JSON of the
+ same shape, then applies the tiered budgets from issue #3122 per concurrency level:
+ * throughput may not decrease by more than ThroughputDropPct (default 10%)
+ * concurrency scaling efficiency may not degrade by more than ScalingDropPct (default 15%)
+ * p95 latency may not degrade by more than LatencyP95RisePct (default 15%)
+ * p99 latency may not regress by more than LatencyP99RisePct (default 20%)
+ * timeout/server-error rate may not increase by more than ErrorRatePpLimit (default 0.5 points)
+ * any concurrent tool-count mismatch is a hard failure (state leakage / thread safety)
+ Fails (throws) if any budget is exceeded. Does not re-run any measurements.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Measure-ConcurrencyScaling.ps1.
+
+.PARAMETER BaselinePath
+ Path to the baseline JSON to compare against.
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath,
+ [Parameter(Mandatory)][string] $BaselinePath,
+ [double] $ThroughputDropPct = 0.10,
+ [double] $ScalingDropPct = 0.15,
+ [double] $LatencyP95RisePct = 0.15,
+ [double] $LatencyP99RisePct = 0.20,
+ [double] $ErrorRatePpLimit = 0.5,
+ [double] $WarnPct = 0.05
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot 'ConcurrencyPerformance.Common.ps1')
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$baseline = Get-Content $BaselinePath | ConvertFrom-Json
+
+$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $results.levels -BaselineLevels $baseline.levels `
+ -ThroughputDropPct $ThroughputDropPct -ScalingDropPct $ScalingDropPct `
+ -LatencyP95RisePct $LatencyP95RisePct -LatencyP99RisePct $LatencyP99RisePct `
+ -ErrorRatePpLimit $ErrorRatePpLimit -WarnPct $WarnPct
+
+if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Concurrency warnings: $($gate.Warnings -join ', ')"
+}
+
+if ($gate.Failures.Count -gt 0) {
+ Write-Error "Concurrency regression detected in: $($gate.Failures -join ', ')"
+}
+else {
+ Write-Host " All concurrency budget checks passed."
+}
diff --git a/eng/scripts/Check-DispatchOverheadRegression.ps1 b/eng/scripts/Check-DispatchOverheadRegression.ps1
new file mode 100644
index 000000000..6cbf15bb8
--- /dev/null
+++ b/eng/scripts/Check-DispatchOverheadRegression.ps1
@@ -0,0 +1,61 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Gates the command-dispatch / transport overhead results against a saved baseline (#3119).
+
+.DESCRIPTION
+ Reads the results JSON produced by Measure-DispatchOverhead.ps1 and a baseline JSON of the
+ same shape, then applies the tiered budgets from issue #3119:
+ * p50 and p95 of each transport/operation may not regress by more than FailP50P95 (+10%)
+ * p99 may not regress by more than FailP99 (+20%)
+ * any change above Warn (+5%), including transport overhead, is reported as a warning
+ Fails (throws) if any budget is exceeded. Does not re-run any measurements.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Measure-DispatchOverhead.ps1.
+
+.PARAMETER BaselinePath
+ Path to the baseline JSON to compare against.
+
+.PARAMETER FailP50P95
+ Maximum allowed p50/p95 regression ratio (default 1.10 = +10%).
+
+.PARAMETER FailP99
+ Maximum allowed p99 regression ratio (default 1.20 = +20%).
+
+.PARAMETER Warn
+ Ratio above which a change is reported as a warning (default 1.05 = +5%).
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath,
+ [Parameter(Mandatory)][string] $BaselinePath,
+ [double] $FailP50P95 = 1.10,
+ [double] $FailP99 = 1.20,
+ [double] $Warn = 1.05
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot 'DispatchPerformance.Common.ps1')
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$baseline = Get-Content $BaselinePath | ConvertFrom-Json
+
+$gate = Invoke-DispatchRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios `
+ -FailP50P95 $FailP50P95 -FailP99 $FailP99 -Warn $Warn
+
+if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Dispatch warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
+}
+
+if ($gate.Failures.Count -gt 0) {
+ Write-Error "Dispatch regression detected in: $($gate.Failures -join ', ')"
+}
+else {
+ Write-Host " All dispatch budget checks passed."
+}
diff --git a/eng/scripts/Check-ResourceUsageRegression.ps1 b/eng/scripts/Check-ResourceUsageRegression.ps1
new file mode 100644
index 000000000..2a35ff229
--- /dev/null
+++ b/eng/scripts/Check-ResourceUsageRegression.ps1
@@ -0,0 +1,62 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Gates the resource-usage results against a saved baseline (issue #3121).
+
+.DESCRIPTION
+ Reads the results JSON produced by Measure-ResourceUsage.ps1 and a baseline JSON of the
+ same shape, then applies the tiered resource budgets from issue #3121:
+ * steady-state CPU and memory may not regress by more than FailSteady (default +10%)
+ * peak memory may not regress by more than FailPeak (default +15%)
+ * a soak phase flagged with unbounded growth is a hard failure (memory/handle leak)
+ * any change above Warn (default +5%) is reported as a warning
+ Fails (throws) if any budget is exceeded. Does not re-run any measurements.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Measure-ResourceUsage.ps1.
+
+.PARAMETER BaselinePath
+ Path to the baseline JSON to compare against.
+
+.PARAMETER FailSteady
+ Maximum allowed steady-state CPU/memory regression ratio (default 1.10 = +10%).
+
+.PARAMETER FailPeak
+ Maximum allowed peak-memory regression ratio (default 1.15 = +15%).
+
+.PARAMETER Warn
+ Ratio above which a change is reported as a warning (default 1.05 = +5%).
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath,
+ [Parameter(Mandatory)][string] $BaselinePath,
+ [double] $FailSteady = 1.10,
+ [double] $FailPeak = 1.15,
+ [double] $Warn = 1.05
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot 'ResourcePerformance.Common.ps1')
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$baseline = Get-Content $BaselinePath | ConvertFrom-Json
+
+$gate = Invoke-ResourceRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios `
+ -FailSteady $FailSteady -FailPeak $FailPeak -Warn $Warn
+
+if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Resource warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
+}
+
+if ($gate.Failures.Count -gt 0) {
+ Write-Error "Resource regression detected in: $($gate.Failures -join ', ')"
+}
+else {
+ Write-Host " All resource budget checks passed."
+}
diff --git a/eng/scripts/Check-StartupPerformanceRegression.ps1 b/eng/scripts/Check-StartupPerformanceRegression.ps1
new file mode 100644
index 000000000..26ca790b9
--- /dev/null
+++ b/eng/scripts/Check-StartupPerformanceRegression.ps1
@@ -0,0 +1,67 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Gates the startup performance results against a saved baseline.
+
+.DESCRIPTION
+ Reads the results JSON produced by Test-StartupPerformance.ps1 and a baseline JSON
+ of the same shape, then applies the tiered budgets from issue #3118:
+ * p50 and p95 may not regress by more than FailP50P95 (default +10%)
+ * p99 may not regress by more than FailP99 (default +20%)
+ * discovery scaling efficiency may not degrade by more than ScalingDegrade (+15%)
+ * any change above Warn (default +5%) is reported as a warning
+ Fails (throws) if any budget is exceeded. Does not re-run any benchmarks.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Test-StartupPerformance.ps1.
+
+.PARAMETER BaselinePath
+ Path to the baseline JSON to compare against.
+
+.PARAMETER FailP50P95
+ Maximum allowed p50/p95 regression ratio (default 1.10 = +10%).
+
+.PARAMETER FailP99
+ Maximum allowed p99 regression ratio (default 1.20 = +20%).
+
+.PARAMETER ScalingDegrade
+ Maximum allowed discovery-scaling-efficiency degradation ratio (default 1.15 = +15%).
+
+.PARAMETER Warn
+ Ratio above which a change is reported as a warning (default 1.05 = +5%).
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath,
+ [Parameter(Mandatory)][string] $BaselinePath,
+ [double] $FailP50P95 = 1.10,
+ [double] $FailP99 = 1.20,
+ [double] $ScalingDegrade = 1.15,
+ [double] $Warn = 1.05
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$baseline = Get-Content $BaselinePath | ConvertFrom-Json
+
+$gate = Invoke-StartupRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios `
+ -FailP50P95 $FailP50P95 -FailP99 $FailP99 `
+ -ScalingDegrade $ScalingDegrade -Warn $Warn
+
+if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Performance warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
+}
+
+if ($gate.Failures.Count -gt 0) {
+ Write-Error "Regression detected in: $($gate.Failures -join ', ')"
+}
+else {
+ Write-Host " All budget checks passed."
+}
diff --git a/eng/scripts/ConcurrencyPerformance.Common.Tests.ps1 b/eng/scripts/ConcurrencyPerformance.Common.Tests.ps1
new file mode 100644
index 000000000..1b6fbec44
--- /dev/null
+++ b/eng/scripts/ConcurrencyPerformance.Common.Tests.ps1
@@ -0,0 +1,149 @@
+# Install-Module -Name Pester -Force -SkipPublisherCheck
+# Invoke-Pester -Path eng/scripts/ConcurrencyPerformance.Common.Tests.ps1
+#
+# Focused tests for the pure concurrency-gate helpers used by Measure-ConcurrencyScaling.ps1.
+
+BeforeAll {
+ . $PSScriptRoot/ConcurrencyPerformance.Common.ps1
+
+ function script:New-Level {
+ param($Concurrency, $Throughput, $Scaling, $P95, $P99, $ErrorRate = 0, $Mismatches = 0)
+ return [ordered]@{
+ concurrency = $Concurrency
+ throughput_rps = $Throughput
+ scaling_efficiency = $Scaling
+ error_rate_pct = $ErrorRate
+ tool_count_mismatches = $Mismatches
+ latency_ms = [ordered]@{ p50 = ($P95 * 0.6); p95 = $P95; p99 = $P99 }
+ }
+ }
+
+ function script:New-UniformLevels {
+ return @(
+ (New-Level -Concurrency 1 -Throughput 100 -Scaling 1.0 -P95 5 -P99 7),
+ (New-Level -Concurrency 4 -Throughput 350 -Scaling 0.9 -P95 8 -P99 12)
+ )
+ }
+}
+
+Describe "Test-ConcurrencyBudget" {
+ It "reports the fraction dropped for a higher-is-better metric" {
+ $r = Test-ConcurrencyBudget -Current 85 -Baseline 100 -FailFraction 0.10 -WarnFraction 0.05 -Direction 'higher'
+ $r.Status | Should -Be 'FAIL'
+ [math]::Round($r.Change, 2) | Should -Be 0.15
+ }
+
+ It "reports the fraction risen for a lower-is-better metric" {
+ # +20% rise is at the fail boundary (not > 0.20) but above the 5% warn threshold.
+ $r = Test-ConcurrencyBudget -Current 12 -Baseline 10 -FailFraction 0.20 -WarnFraction 0.05 -Direction 'lower'
+ $r.Status | Should -Be 'WARN'
+ [math]::Round($r.Change, 2) | Should -Be 0.20
+ }
+
+ It "skips when the baseline is not positive" {
+ (Test-ConcurrencyBudget -Current 5 -Baseline 0 -FailFraction 0.1 -WarnFraction 0.05 -Direction 'higher').Status | Should -Be 'SKIP'
+ }
+}
+
+Describe "Invoke-ConcurrencyRegressionGate" {
+ It "reports no failures or warnings for an identical baseline" {
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels (New-UniformLevels) -BaselineLevels (New-UniformLevels)
+ @($gate.Failures).Count | Should -Be 0
+ @($gate.Warnings).Count | Should -Be 0
+ }
+
+ It "fails when throughput drops more than 10%" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].throughput_rps = $b[1].throughput_rps * 0.85
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 throughput_rps'
+ }
+
+ It "fails when scaling efficiency degrades more than 15%" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].scaling_efficiency = $b[1].scaling_efficiency * 0.80
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 scaling_efficiency'
+ }
+
+ It "fails when p95 latency degrades more than 15%" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].latency_ms.p95 = $b[1].latency_ms.p95 * 1.20
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 latency_p95'
+ }
+
+ It "fails when p99 latency regresses more than 20%" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].latency_ms.p99 = $b[1].latency_ms.p99 * 1.25
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 latency_p99'
+ }
+
+ It "fails when the error rate increases more than 0.5 percentage points" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].error_rate_pct = $b[1].error_rate_pct + 0.6
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 error_rate_pct'
+ }
+
+ It "fails on any concurrent tool-count mismatch (state leakage)" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].tool_count_mismatches = 3
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 state_leakage'
+ }
+
+ It "warns but does not fail for a throughput drop between 5% and 10%" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].throughput_rps = $b[1].throughput_rps * 0.93
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ @($gate.Failures).Count | Should -Be 0
+ $gate.Warnings | Should -Contain 'c4 throughput_rps'
+ }
+
+ It "skips a concurrency level absent from the baseline" {
+ $s = New-UniformLevels
+ $b = @($s[0]) # only concurrency 1
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ @($gate.Failures).Count | Should -Be 0
+ }
+
+ It "flags a state-leakage mismatch even when the level is absent from the baseline (F-008)" {
+ $s = New-UniformLevels
+ $s += (New-Level -Concurrency 16 -Throughput 900 -Scaling 0.7 -P95 20 -P99 30 -Mismatches 5)
+ $b = New-UniformLevels # no concurrency 16
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c16 state_leakage'
+ }
+
+ It "fails when no level overlaps the baseline (F-007)" {
+ $s = @(New-Level -Concurrency 32 -Throughput 100 -Scaling 1.0 -P95 5 -P99 7)
+ $b = New-UniformLevels # concurrency 1 and 4 only
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'no_comparable_metrics'
+ }
+
+ It "fails when a latency metric the baseline expects is missing from results (F-007)" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].latency_ms.Remove('p95') # results dropped p95 at concurrency 4
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 latency_p95 (missing in results)'
+ }
+
+ It "fails when the error-rate metric the baseline expects is missing from results (F-007)" {
+ $s = New-UniformLevels
+ $b = New-UniformLevels
+ $s[1].Remove('error_rate_pct') # results dropped error_rate_pct at concurrency 4
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
+ $gate.Failures | Should -Contain 'c4 error_rate_pct (missing in results)'
+ }
+}
diff --git a/eng/scripts/ConcurrencyPerformance.Common.ps1 b/eng/scripts/ConcurrencyPerformance.Common.ps1
new file mode 100644
index 000000000..57ed5023f
--- /dev/null
+++ b/eng/scripts/ConcurrencyPerformance.Common.ps1
@@ -0,0 +1,167 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Side-effect-free helpers for the concurrency / throughput / scalability gate (issue #3122).
+
+.DESCRIPTION
+ Dot-source this file to reuse the pure functions in Measure-ConcurrencyScaling.ps1 and
+ Check-ConcurrencyScalingRegression.ps1 and to unit-test them without launching a server.
+ Reuses Get-MemberValue from StartupPerformance.Common.ps1.
+#>
+
+. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
+
+# ---------------------------------------------------------------------------
+# Evaluates one budget. Direction 'higher' means bigger is better (throughput,
+# scaling efficiency) and reports the fraction dropped; 'lower' means smaller is
+# better (latency) and reports the fraction risen. Returns a status + change.
+# ---------------------------------------------------------------------------
+function Test-ConcurrencyBudget {
+ [CmdletBinding()]
+ param(
+ [double] $Current,
+ [double] $Baseline,
+ [double] $FailFraction,
+ [double] $WarnFraction,
+ [ValidateSet('higher', 'lower')] [string] $Direction
+ )
+
+ if ($Baseline -le 0) {
+ return @{ Status = 'SKIP'; Change = 0 }
+ }
+
+ $change = if ($Direction -eq 'higher') { 1 - ($Current / $Baseline) } else { ($Current / $Baseline) - 1 }
+ $status = if ($change -gt $FailFraction) { 'FAIL' } elseif ($change -gt $WarnFraction) { 'WARN' } else { 'PASS' }
+ return @{ Status = $status; Change = $change }
+}
+
+# ---------------------------------------------------------------------------
+# Tiered concurrency-regression gate shared by Measure-ConcurrencyScaling.ps1 and
+# Check-ConcurrencyScalingRegression.ps1. For each concurrency level present in both
+# results and baseline it applies the acceptance-criteria budgets from issue #3122:
+# * throughput may not decrease by more than ThroughputDropPct (default 10%)
+# * concurrency scaling efficiency may not degrade by more than ScalingDropPct (default 15%)
+# * p95 latency may not degrade by more than LatencyP95RisePct (default 15%)
+# * p99 latency may not regress by more than LatencyP99RisePct (default 20%)
+# * timeout/server-error rate may not increase by more than ErrorRatePpLimit (default 0.5 points)
+# * any concurrent tool-count mismatch is a hard failure (state leakage / thread safety)
+# * any change above WarnPct (default 5%) is reported as a warning
+#
+# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a line per check.
+# ---------------------------------------------------------------------------
+function Invoke-ConcurrencyRegressionGate {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)] $ResultLevels,
+ [Parameter(Mandatory)] $BaselineLevels,
+ [double] $ThroughputDropPct = 0.10,
+ [double] $ScalingDropPct = 0.15,
+ [double] $LatencyP95RisePct = 0.15,
+ [double] $LatencyP99RisePct = 0.20,
+ [double] $ErrorRatePpLimit = 0.5,
+ [double] $WarnPct = 0.05
+ )
+
+ $failures = @()
+ $warnings = @()
+ $evaluated = 0
+
+ $baselineByConcurrency = @{}
+ foreach ($bl in $BaselineLevels) {
+ $baselineByConcurrency[[string](Get-MemberValue $bl 'concurrency')] = $bl
+ }
+
+ $seenConcurrency = @{}
+
+ foreach ($level in $ResultLevels) {
+ $concurrency = Get-MemberValue $level 'concurrency'
+ $seenConcurrency[[string]$concurrency] = $true
+
+ # State leakage / thread safety — a hard failure regardless of the baseline, so it is
+ # evaluated before the baseline-presence check below can skip this level.
+ $mismatches = Get-MemberValue $level 'tool_count_mismatches'
+ if ($mismatches -and $mismatches -gt 0) {
+ $failures += "c$concurrency state_leakage"
+ Write-Host (" [FAIL] c{0} tool_count_mismatches={1} (state leakage / thread-safety failure)" -f $concurrency, $mismatches)
+ }
+
+ $baseline = $baselineByConcurrency[[string]$concurrency]
+ if ($null -eq $baseline) {
+ Write-Host (" [SKIP] concurrency={0}: absent in baseline (relative budgets)" -f $concurrency)
+ continue
+ }
+
+ $curLatency = Get-MemberValue $level 'latency_ms'
+ $baseLatency = Get-MemberValue $baseline 'latency_ms'
+
+ $checks = @(
+ @{ Label = 'throughput_rps'; Cur = (Get-MemberValue $level 'throughput_rps'); Base = (Get-MemberValue $baseline 'throughput_rps'); Fail = $ThroughputDropPct; Dir = 'higher'; Unit = ' rps' }
+ @{ Label = 'scaling_efficiency'; Cur = (Get-MemberValue $level 'scaling_efficiency'); Base = (Get-MemberValue $baseline 'scaling_efficiency'); Fail = $ScalingDropPct; Dir = 'higher'; Unit = '' }
+ @{ Label = 'latency_p95'; Cur = (Get-MemberValue $curLatency 'p95'); Base = (Get-MemberValue $baseLatency 'p95'); Fail = $LatencyP95RisePct; Dir = 'lower'; Unit = ' ms' }
+ @{ Label = 'latency_p99'; Cur = (Get-MemberValue $curLatency 'p99'); Base = (Get-MemberValue $baseLatency 'p99'); Fail = $LatencyP99RisePct; Dir = 'lower'; Unit = ' ms' }
+ )
+
+ foreach ($check in $checks) {
+ if ($null -eq $check.Base) {
+ continue
+ }
+ if ($null -eq $check.Cur) {
+ $failures += "c$concurrency $($check.Label) (missing in results)"
+ Write-Host (" [FAIL] c{0} {1}: expected by baseline but absent in results" -f $concurrency, $check.Label)
+ continue
+ }
+ $evaluated++
+ $result = Test-ConcurrencyBudget -Current $check.Cur -Baseline $check.Base `
+ -FailFraction $check.Fail -WarnFraction $WarnPct -Direction $check.Dir
+ $label = "c$concurrency $($check.Label)"
+ switch ($result.Status) {
+ 'FAIL' { $failures += $label }
+ 'WARN' { $warnings += $label }
+ }
+ Write-Host (" [{0}] {1}: current={2}{3} baseline={4}{3} ({5}{6}%)" -f `
+ $result.Status, $label, $check.Cur, $check.Unit, $check.Base, `
+ $(if ($result.Change -ge 0) { '+' } else { '' }), [math]::Round($result.Change * 100, 1))
+ }
+
+ # Error/timeout rate — an absolute percentage-point budget.
+ $curError = Get-MemberValue $level 'error_rate_pct'
+ $baseError = Get-MemberValue $baseline 'error_rate_pct'
+ if ($null -ne $baseError) {
+ $label = "c$concurrency error_rate_pct"
+ if ($null -eq $curError) {
+ $failures += "$label (missing in results)"
+ Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $label)
+ }
+ else {
+ $evaluated++
+ $delta = $curError - $baseError
+ if ($delta -gt $ErrorRatePpLimit) {
+ $failures += $label
+ $status = 'FAIL'
+ }
+ else {
+ $status = 'PASS'
+ }
+ Write-Host (" [{0}] {1}: current={2}% baseline={3}% (+{4} pp)" -f `
+ $status, $label, $curError, $baseError, [math]::Round($delta, 3))
+ }
+ }
+ }
+
+ # Baseline levels the results no longer cover are treated as an expected-but-missing metric.
+ foreach ($blKey in $baselineByConcurrency.Keys) {
+ if (-not $seenConcurrency.ContainsKey($blKey)) {
+ $failures += "c$blKey missing_in_results"
+ Write-Host (" [FAIL] concurrency={0}: expected by baseline but absent in results" -f $blKey)
+ }
+ }
+
+ if ($evaluated -eq 0) {
+ $failures += 'no_comparable_metrics'
+ Write-Host " [FAIL] no comparable concurrency metrics between results and baseline (incompatible or empty baseline)"
+ }
+
+ return [ordered]@{ Failures = $failures; Warnings = $warnings }
+}
diff --git a/eng/scripts/DispatchPerformance.Common.Tests.ps1 b/eng/scripts/DispatchPerformance.Common.Tests.ps1
new file mode 100644
index 000000000..2eae6f023
--- /dev/null
+++ b/eng/scripts/DispatchPerformance.Common.Tests.ps1
@@ -0,0 +1,115 @@
+# Install-Module -Name Pester -Force -SkipPublisherCheck
+# Invoke-Pester -Path eng/scripts/DispatchPerformance.Common.Tests.ps1
+#
+# Focused tests for the pure dispatch-gate helpers used by Measure-DispatchOverhead.ps1.
+
+BeforeAll {
+ . $PSScriptRoot/DispatchPerformance.Common.ps1
+
+ function script:New-Stats {
+ param($P50, $P95, $P99)
+ return [ordered]@{ p50 = $P50; p95 = $P95; p99 = $P99 }
+ }
+
+ function script:New-DispatchScenarios {
+ param($StdioList, $HttpList)
+ return [ordered]@{
+ stdio = [ordered]@{ tools_list = $StdioList }
+ http = [ordered]@{ tools_list = $HttpList }
+ }
+ }
+
+ function script:New-UniformScenarios {
+ param($StdioP50 = 5.0, $HttpP50 = 6.0)
+ $stdio = New-Stats -P50 $StdioP50 -P95 ($StdioP50 * 1.5) -P99 ($StdioP50 * 2)
+ $http = New-Stats -P50 $HttpP50 -P95 ($HttpP50 * 1.5) -P99 ($HttpP50 * 2)
+ return New-DispatchScenarios -StdioList $stdio -HttpList $http
+ }
+}
+
+Describe "Get-TransportOverhead" {
+ It "returns http minus stdio tools/list p50" {
+ $s = New-DispatchScenarios `
+ -StdioList (New-Stats -P50 5 -P95 6 -P99 7) `
+ -HttpList (New-Stats -P50 6.5 -P95 8 -P99 9)
+ Get-TransportOverhead $s | Should -Be 1.5
+ }
+
+ It "returns null when a tools/list stat is absent" {
+ $s = [ordered]@{ stdio = [ordered]@{ } ; http = [ordered]@{ tools_list = (New-Stats -P50 6 -P95 8 -P99 9) } }
+ Get-TransportOverhead $s | Should -Be $null
+ }
+}
+
+Describe "Invoke-DispatchRegressionGate" {
+ It "reports no failures or warnings for an identical baseline" {
+ $s = New-UniformScenarios
+ $b = New-UniformScenarios
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ @($gate.Warnings).Count | Should -Be 0
+ }
+
+ It "fails when stdio tools/list p95 regresses beyond 10%" {
+ $b = New-UniformScenarios
+ $s = New-UniformScenarios
+ $s.stdio.tools_list.p95 = $b.stdio.tools_list.p95 * 1.15
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'stdio/tools_list (p95)'
+ }
+
+ It "fails when http tools/list p99 regresses beyond 20%" {
+ $b = New-UniformScenarios
+ $s = New-UniformScenarios
+ $s.http.tools_list.p99 = $b.http.tools_list.p99 * 1.25
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'http/tools_list (p99)'
+ }
+
+ It "warns but does not fail for a p50 change between 5% and 10%" {
+ $b = New-UniformScenarios
+ $s = New-UniformScenarios
+ $s.stdio.tools_list.p50 = $b.stdio.tools_list.p50 * 1.07
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ $gate.Warnings | Should -Contain 'stdio/tools_list (p50)'
+ }
+
+ It "warns when transport overhead grows beyond 5%" {
+ # Baseline overhead = 6 - 5 = 1. Raise http p50 to 6.4 => overhead 1.4 (+40%),
+ # while keeping the http p50 change (+6.7%) below the +10% fail threshold.
+ $b = New-UniformScenarios -StdioP50 5.0 -HttpP50 6.0
+ $s = New-UniformScenarios -StdioP50 5.0 -HttpP50 6.4
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Warnings | Should -Contain 'transport_overhead_tools_list_p50'
+ }
+
+ It "skips a transport absent from the baseline" {
+ $s = New-UniformScenarios
+ $b = [ordered]@{ stdio = $s.stdio } # no http
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ }
+
+ It "fails when no transport overlaps the baseline (F-007)" {
+ $s = New-UniformScenarios
+ $b = [ordered]@{ } # empty baseline: nothing to compare
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'no_comparable_metrics'
+ }
+
+ It "treats a positive current against a zero baseline as a failure (F-010)" {
+ $b = New-UniformScenarios
+ $b.stdio.tools_list.p95 = 0
+ $s = New-UniformScenarios
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'stdio/tools_list (p95)'
+ }
+
+ It "fails when a transport the baseline expects is missing from results (F-007)" {
+ $b = New-UniformScenarios
+ $s = [ordered]@{ stdio = $b.stdio } # results dropped http
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'http (missing in results)'
+ }
+}
diff --git a/eng/scripts/DispatchPerformance.Common.ps1 b/eng/scripts/DispatchPerformance.Common.ps1
new file mode 100644
index 000000000..6cd967820
--- /dev/null
+++ b/eng/scripts/DispatchPerformance.Common.ps1
@@ -0,0 +1,147 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Side-effect-free helpers for the command dispatch / transport overhead gate (issue #3119).
+
+.DESCRIPTION
+ Dot-source this file to reuse the pure functions in Measure-DispatchOverhead.ps1 and
+ Check-DispatchOverheadRegression.ps1 and to unit-test them without launching a server.
+ Reuses Get-MemberValue from StartupPerformance.Common.ps1.
+#>
+
+. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
+
+# ---------------------------------------------------------------------------
+# Tiered dispatch/transport regression gate shared by Measure-DispatchOverhead.ps1 and
+# Check-DispatchOverheadRegression.ps1. For each transport (stdio, http) and operation
+# (ping, tools_list) it applies the acceptance-criteria budgets from issue #3119:
+# * p50 and p95 may not regress by more than FailP50P95 (default +10%)
+# * p99 may not regress by more than FailP99 (default +20%)
+# * any change above Warn (default +5%) is reported as a warning
+# It also reports transport-specific overhead (http minus stdio tools/list p50) changes above Warn.
+#
+# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a line per check.
+# ---------------------------------------------------------------------------
+function Invoke-DispatchRegressionGate {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)] $ResultScenarios,
+ [Parameter(Mandatory)] $BaselineScenarios,
+ [double] $FailP50P95 = 1.10,
+ [double] $FailP99 = 1.20,
+ [double] $Warn = 1.05
+ )
+
+ $failures = @()
+ $warnings = @()
+ $evaluated = 0
+
+ $transports = @('stdio', 'http')
+ $operations = @('tools_list')
+ $percentileBudgets = @(
+ @{ Metric = 'p50'; Limit = $FailP50P95 }
+ @{ Metric = 'p95'; Limit = $FailP50P95 }
+ @{ Metric = 'p99'; Limit = $FailP99 }
+ )
+
+ foreach ($transport in $transports) {
+ $curTransport = Get-MemberValue $ResultScenarios $transport
+ $baseTransport = Get-MemberValue $BaselineScenarios $transport
+ if ($null -eq $baseTransport) {
+ Write-Host (" [SKIP] {0}: absent in baseline" -f $transport)
+ continue
+ }
+ if ($null -eq $curTransport) {
+ $failures += "$transport (missing in results)"
+ Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $transport)
+ continue
+ }
+
+ foreach ($operation in $operations) {
+ $curStats = Get-MemberValue $curTransport $operation
+ $baseStats = Get-MemberValue $baseTransport $operation
+ if ($null -eq $baseStats) {
+ Write-Host (" [SKIP] {0}/{1}: absent in baseline" -f $transport, $operation)
+ continue
+ }
+ if ($null -eq $curStats) {
+ $failures += "$transport/$operation (missing in results)"
+ Write-Host (" [FAIL] {0}/{1}: expected by baseline but absent in results" -f $transport, $operation)
+ continue
+ }
+
+ foreach ($budget in $percentileBudgets) {
+ $metric = $budget.Metric
+ $current = Get-MemberValue $curStats $metric
+ $baseline = Get-MemberValue $baseStats $metric
+ if ($null -eq $baseline) {
+ continue
+ }
+ $label = "$transport/$operation ($metric)"
+ if ($null -eq $current) {
+ $failures += "$label (missing in results)"
+ Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $label)
+ continue
+ }
+
+ $evaluated++
+ if ($baseline -gt 0) {
+ $ratio = $current / $baseline
+ if ($ratio -gt $budget.Limit) { $failures += $label; $status = 'FAIL' }
+ elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
+ else { $status = 'PASS' }
+ Write-Host (" [{0}] {1}: current={2}ms baseline={3}ms (+{4}%)" -f `
+ $status, $label, $current, $baseline, [math]::Round(($ratio - 1) * 100, 1))
+ }
+ elseif ($current -gt 0) {
+ $failures += $label
+ Write-Host (" [FAIL] {0}: current={1}ms vs a zero baseline (not comparable)" -f $label, $current)
+ }
+ else {
+ Write-Host (" [PASS] {0}: current=0ms baseline=0ms (unchanged)" -f $label)
+ }
+ }
+ }
+ }
+
+ # Transport-specific overhead: the extra tools/list p50 that HTTP adds over stdio.
+ $curOverhead = Get-TransportOverhead $ResultScenarios
+ $baseOverhead = Get-TransportOverhead $BaselineScenarios
+ if ($null -ne $curOverhead -and $null -ne $baseOverhead -and $baseOverhead -gt 0) {
+ $ratio = $curOverhead / $baseOverhead
+ if ($ratio -gt $Warn) {
+ $warnings += 'transport_overhead_tools_list_p50'
+ $status = 'WARN'
+ }
+ else {
+ $status = 'PASS'
+ }
+ Write-Host (" [{0}] transport_overhead (http-stdio tools/list p50): current={1}ms baseline={2}ms (+{3}%)" -f `
+ $status, [math]::Round($curOverhead, 4), [math]::Round($baseOverhead, 4), [math]::Round(($ratio - 1) * 100, 1))
+ }
+
+ if ($evaluated -eq 0) {
+ $failures += 'no_comparable_metrics'
+ Write-Host " [FAIL] no comparable dispatch metrics between results and baseline (incompatible or empty baseline)"
+ }
+
+ return [ordered]@{ Failures = $failures; Warnings = $warnings }
+}
+
+# ---------------------------------------------------------------------------
+# The HTTP-minus-stdio tools/list p50 delta — the transport-specific overhead for a real
+# request. Returns $null when either transport's tools/list p50 is absent.
+# ---------------------------------------------------------------------------
+function Get-TransportOverhead {
+ [CmdletBinding()]
+ param($Scenarios)
+
+ $stdio = Get-MemberValue (Get-MemberValue (Get-MemberValue $Scenarios 'stdio') 'tools_list') 'p50'
+ $http = Get-MemberValue (Get-MemberValue (Get-MemberValue $Scenarios 'http') 'tools_list') 'p50'
+ if ($null -eq $stdio -or $null -eq $http) {
+ return $null
+ }
+ return $http - $stdio
+}
diff --git a/eng/scripts/Invoke-PerfMicroBenchmarks.ps1 b/eng/scripts/Invoke-PerfMicroBenchmarks.ps1
new file mode 100644
index 000000000..46902bc95
--- /dev/null
+++ b/eng/scripts/Invoke-PerfMicroBenchmarks.ps1
@@ -0,0 +1,49 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Runs the BenchmarkDotNet micro-benchmarks for the perf pipeline and guards against a silent
+ no-op run.
+
+.DESCRIPTION
+ Extracted from the pipeline YAML so CI behavior can be reproduced locally. --buildTimeout is
+ raised because the generated boilerplate references the whole server (60+ tool assemblies) and
+ exceeds BenchmarkDotNet's default 2-minute build timeout. BenchmarkDotNet can exit 0 even when
+ the boilerplate build fails and nothing runs, so this fails explicitly if no benchmarks executed.
+
+.PARAMETER BenchmarkExe
+ Path to the built performance-benchmark executable.
+
+.PARAMETER ArtifactsPath
+ Directory for BenchmarkDotNet artifacts and the captured run log.
+
+.PARAMETER Filter
+ Glob passed to BenchmarkDotNet --filter (e.g. '*ServerStartup*').
+
+.PARAMETER BuildTimeoutSeconds
+ --buildTimeout for the generated boilerplate build.
+#>
+param(
+ [Parameter(Mandatory)][string] $BenchmarkExe,
+ [Parameter(Mandatory)][string] $ArtifactsPath,
+ [string] $Filter = '*',
+ [int] $BuildTimeoutSeconds = 600
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+New-Item -ItemType Directory -Force -Path $ArtifactsPath | Out-Null
+$logPath = Join-Path $ArtifactsPath 'run.log'
+
+& $BenchmarkExe --filter $Filter --artifacts $ArtifactsPath --buildTimeout $BuildTimeoutSeconds 2>&1 |
+ Tee-Object -FilePath $logPath
+$code = $LASTEXITCODE
+if ($code -ne 0) { exit $code }
+
+$log = Get-Content $logPath -Raw
+if ($log -match 'failed to build the auto-generated boilerplate' -or $log -match 'executed benchmarks: 0') {
+ Write-Error 'BenchmarkDotNet executed no benchmarks (boilerplate build failure or empty filter).'
+ exit 1
+}
diff --git a/eng/scripts/Measure-ConcurrencyScaling.ps1 b/eng/scripts/Measure-ConcurrencyScaling.ps1
new file mode 100644
index 000000000..747439409
--- /dev/null
+++ b/eng/scripts/Measure-ConcurrencyScaling.ps1
@@ -0,0 +1,145 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Measures Azure MCP Server throughput, latency, and scaling under a sweep of concurrency
+ levels (issue #3122).
+
+.DESCRIPTION
+ Runs the performance-benchmark harness in --mcp-concurrency mode, which launches the server
+ over HTTP and, at each concurrency level, drives a pool of parallel MCP clients issuing
+ tools/list for a fixed window. For each level it records throughput, latency p50/p95/p99,
+ error/timeout rate, concurrency scaling efficiency, and a state-isolation signal (responses
+ whose tool count does not match the expected count). When a baseline is supplied the results
+ are gated against the tiered budgets from issue #3122 (throughput within 10%, p95 & scaling
+ within 15%, p99 within 20%, error-rate increase within 0.5 points, no state leakage).
+
+.PARAMETER Executable
+ Path to the azmcp server executable. Defaults to the Release build.
+
+.PARAMETER BenchmarkExe
+ Path to the performance-benchmark executable. Defaults to the Release build.
+
+.PARAMETER Levels
+ Concurrency levels to sweep.
+
+.PARAMETER DurationSeconds
+ Timed load window per level.
+
+.PARAMETER WarmupSeconds
+ Warmup window per level (not counted).
+
+.PARAMETER TimeoutMs
+ Per-request timeout; a timeout counts as an error.
+
+.PARAMETER OutputPath
+ Path to write the JSON results. Defaults to .perf-results/concurrency-scaling.json.
+
+.PARAMETER BaselinePath
+ Optional baseline JSON (previously produced by this script) to gate against.
+
+.EXAMPLE
+ ./eng/scripts/Measure-ConcurrencyScaling.ps1 -Levels 1,2,4,8,16 -DurationSeconds 10
+#>
+
+param(
+ [string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
+ [string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
+ [int[]] $Levels = @(1, 2, 4, 8, 16),
+ [int] $DurationSeconds = 5,
+ [int] $WarmupSeconds = 1,
+ [int] $TimeoutMs = 10000,
+ [string] $OutputPath = '.perf-results/concurrency-scaling.json',
+ [string] $BaselinePath
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$ScriptDir = $PSScriptRoot
+$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
+. (Join-Path $ScriptDir 'ConcurrencyPerformance.Common.ps1')
+
+foreach ($path in @($Executable, $BenchmarkExe)) {
+ if (-not (Test-Path $path)) {
+ throw "Required executable not found: $path (build the solution in Release first)."
+ }
+}
+$Executable = (Resolve-Path $Executable).Path
+$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
+$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
+
+$env:PERF_CONC_LEVELS = ($Levels -join ',')
+$env:PERF_CONC_DURATION_SECONDS = $DurationSeconds
+$env:PERF_CONC_WARMUP_SECONDS = $WarmupSeconds
+$env:PERF_CONC_TIMEOUT_MS = $TimeoutMs
+
+Write-Host "=== Concurrency scaling: levels [$($Levels -join ', ')], ${DurationSeconds}s per level ==="
+
+# The harness sweeps every level in a single invocation; size the timeout to the whole sweep
+# (plus headroom) so a hung run fails fast rather than consuming the entire job.
+$concurrencyTimeoutSeconds = 180 + $Levels.Count * ($DurationSeconds + $WarmupSeconds) * 3
+$data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
+ -ArgumentList @('--mcp-concurrency', $Executable, 'server', 'start') -TimeoutSeconds $concurrencyTimeoutSeconds
+
+foreach ($level in $data.levels) {
+ Write-Host (" c{0,-3} throughput={1,8} rps p50={2,7} ms p95={3,7} ms p99={4,7} ms err={5}% scaling={6} mismatch={7}" -f `
+ $level.concurrency, $level.throughput_rps, $level.latency_ms.p50, $level.latency_ms.p95, `
+ $level.latency_ms.p99, $level.error_rate_pct, $level.scaling_efficiency, $level.tool_count_mismatches)
+}
+Write-Host ""
+
+$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
+
+$results = [ordered]@{
+ timestamp = (Get-Date).ToUniversalTime().ToString('o')
+ commit = $commit
+ environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
+ duration_seconds = $DurationSeconds
+ expected_tool_count = $data.expected_tool_count
+ levels = $data.levels
+}
+
+$results | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
+Write-Host "Results written to: $OutputPath"
+
+# ---------------------------------------------------------------------------
+# Optional: regression check against a baseline file
+# ---------------------------------------------------------------------------
+if ($BaselinePath) {
+ if (-not (Test-Path $BaselinePath)) {
+ Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
+ }
+ else {
+ Write-Host ""
+ Write-Host "=== Concurrency regression check (throughput +10%, p95/scaling +15%, p99 +20%, error +0.5pp) ==="
+
+ $baseline = Get-Content $BaselinePath | ConvertFrom-Json
+ $gate = Invoke-ConcurrencyRegressionGate -ResultLevels $results.levels -BaselineLevels $baseline.levels
+
+ if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Concurrency warnings (>5%): $($gate.Warnings -join ', ')"
+ }
+
+ if ($gate.Failures.Count -gt 0) {
+ Write-Error "Concurrency regression detected in: $($gate.Failures -join ', ')"
+ }
+ else {
+ Write-Host " All concurrency budget checks passed."
+ }
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Emit Azure DevOps pipeline variables (no-op outside AzDO)
+# ---------------------------------------------------------------------------
+$peak = $data.levels | Sort-Object throughput_rps -Descending | Select-Object -First 1
+if ($peak) {
+ Write-Host "##vso[task.setvariable variable=ConcurrencyPeakThroughputRps]$($peak.throughput_rps)"
+ Write-Host "##vso[task.setvariable variable=ConcurrencyPeakAtLevel]$($peak.concurrency)"
+}
+$maxLevel = $data.levels | Sort-Object concurrency -Descending | Select-Object -First 1
+if ($maxLevel) {
+ Write-Host "##vso[task.setvariable variable=ConcurrencyMaxLevelScalingEfficiency]$($maxLevel.scaling_efficiency)"
+}
diff --git a/eng/scripts/Measure-DispatchOverhead.ps1 b/eng/scripts/Measure-DispatchOverhead.ps1
new file mode 100644
index 000000000..6b6d42b36
--- /dev/null
+++ b/eng/scripts/Measure-DispatchOverhead.ps1
@@ -0,0 +1,166 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Measures Azure MCP Server command-dispatch and transport overhead over stdio and HTTP
+ (issue #3119) by timing many repetitions of ping and tools/list on a live server.
+
+.DESCRIPTION
+ For each transport this runs the performance-benchmark harness in --mcp-dispatch mode,
+ which times ping (the transport/protocol floor) and tools/list (a real request with
+ server-side serialization) across many iterations. Per-transport p50/p95/p99 are computed
+ with the shared percentile helpers, the HTTP-minus-stdio ping delta isolates transport
+ overhead, and (when a baseline is supplied) results are gated against the tiered budgets
+ from issue #3119 (p50/p95 within 10%, p99 within 20%, transport change above 5% reported).
+
+.PARAMETER Executable
+ Path to the azmcp server executable. Defaults to the Release build.
+
+.PARAMETER BenchmarkExe
+ Path to the performance-benchmark executable. Defaults to the Release build.
+
+.PARAMETER Iterations
+ Timed samples per operation per transport.
+
+.PARAMETER Warmup
+ Warmup calls (not timed) per operation.
+
+.PARAMETER OutputPath
+ Path to write the JSON results. Defaults to .perf-results/dispatch-overhead.json.
+
+.PARAMETER BaselinePath
+ Optional baseline JSON (previously produced by this script) to gate against.
+
+.EXAMPLE
+ ./eng/scripts/Measure-DispatchOverhead.ps1
+
+.EXAMPLE
+ ./eng/scripts/Measure-DispatchOverhead.ps1 -Iterations 500 -BaselinePath eng/dispatch-baseline.json
+#>
+
+param(
+ [string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
+ [string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
+ [int] $Iterations = 200,
+ [int] $Warmup = 20,
+ [string] $OutputPath = '.perf-results/dispatch-overhead.json',
+ [string] $BaselinePath
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$ScriptDir = $PSScriptRoot
+$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
+. (Join-Path $ScriptDir 'DispatchPerformance.Common.ps1')
+
+foreach ($path in @($Executable, $BenchmarkExe)) {
+ if (-not (Test-Path $path)) {
+ throw "Required executable not found: $path (build the solution in Release first)."
+ }
+}
+$Executable = (Resolve-Path $Executable).Path
+$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
+$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
+
+$env:PERF_DISPATCH_ITERATIONS = $Iterations
+$env:PERF_DISPATCH_WARMUP = $Warmup
+
+# Generous per-invocation timeout so a hung/crashed harness fails fast instead of consuming
+# the whole job. tools/list is sub-second, so this scales gently with the iteration count.
+$harnessTimeoutSeconds = [math]::Max(180, $Iterations)
+
+function Get-DispatchStats {
+ param([double[]] $Samples)
+ $s = Get-TimingStats -Samples $Samples
+ return [ordered]@{
+ average = $s.average
+ median = $s.median
+ p50 = $s.p50
+ p95 = $s.p95
+ p99 = $s.p99
+ count = $Samples.Count
+ }
+}
+
+$scenarios = [ordered]@{}
+
+foreach ($transport in @('stdio', 'http')) {
+ Write-Host "=== Dispatch overhead: $transport ($Iterations iterations) ==="
+ $mode = "--mcp-dispatch-$transport"
+
+ $data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
+ -ArgumentList @($mode, $Executable, 'server', 'start') -TimeoutSeconds $harnessTimeoutSeconds
+ $toolsList = Get-DispatchStats -Samples ([double[]]$data.tools_list_ms)
+
+ $scenario = [ordered]@{ tools_list = $toolsList }
+ if ($data.PSObject.Properties.Name -contains 'readiness_ms' -and $null -ne $data.readiness_ms) {
+ $scenario.readiness_ms = $data.readiness_ms
+ }
+ $scenarios[$transport] = $scenario
+
+ Write-Host (" tools/list: p50={0} ms p95={1} ms p99={2} ms" -f $toolsList.p50, $toolsList.p95, $toolsList.p99)
+ Write-Host ""
+}
+
+$overhead = Get-TransportOverhead $scenarios
+if ($null -ne $overhead) {
+ Write-Host ("=== Transport overhead (http - stdio tools/list p50): {0} ms ===" -f [math]::Round($overhead, 4))
+ Write-Host ""
+}
+
+$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
+
+$results = [ordered]@{
+ timestamp = (Get-Date).ToUniversalTime().ToString('o')
+ commit = $commit
+ environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
+ iterations = $Iterations
+ warmup = $Warmup
+ transport_overhead_tools_list_p50 = $overhead
+ scenarios = $scenarios
+}
+
+$results | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
+Write-Host "Results written to: $OutputPath"
+
+# ---------------------------------------------------------------------------
+# Optional: regression check against a baseline file
+# ---------------------------------------------------------------------------
+if ($BaselinePath) {
+ if (-not (Test-Path $BaselinePath)) {
+ Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
+ }
+ else {
+ Write-Host ""
+ Write-Host "=== Dispatch regression check (p50/p95 +10%, p99 +20%, transport warn +5%) ==="
+
+ $baseline = Get-Content $BaselinePath | ConvertFrom-Json
+ $gate = Invoke-DispatchRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios
+
+ if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Dispatch warnings (>5%): $($gate.Warnings -join ', ')"
+ }
+
+ if ($gate.Failures.Count -gt 0) {
+ Write-Error "Dispatch regression detected in: $($gate.Failures -join ', ')"
+ }
+ else {
+ Write-Host " All dispatch budget checks passed."
+ }
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Emit Azure DevOps pipeline variables (no-op outside AzDO)
+# ---------------------------------------------------------------------------
+foreach ($transport in @('stdio', 'http')) {
+ $s = $scenarios[$transport]
+ $t = (Get-Culture).TextInfo.ToTitleCase($transport)
+ Write-Host "##vso[task.setvariable variable=Dispatch${t}ToolsListP50Ms]$($s.tools_list.p50)"
+}
+if ($null -ne $overhead) {
+ Write-Host "##vso[task.setvariable variable=DispatchTransportOverheadToolsListP50Ms]$overhead"
+}
diff --git a/eng/scripts/Measure-ResourceUsage.ps1 b/eng/scripts/Measure-ResourceUsage.ps1
new file mode 100644
index 000000000..4eba52723
--- /dev/null
+++ b/eng/scripts/Measure-ResourceUsage.ps1
@@ -0,0 +1,185 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Measures the Azure MCP Server's CPU, memory, and handle baselines across server modes
+ and concurrency levels (issue #3121).
+
+.DESCRIPTION
+ For each requested server mode this script launches the performance-benchmark harness
+ in --mcp-resource mode, which starts the server over HTTP and samples process resource
+ counters across the startup, idle, discovery, steady-state, and optional soak phases.
+ Results for every mode are collected into a single JSON document and, when a baseline is
+ supplied, gated against the tiered resource budgets from issue #3121 (steady-state CPU
+ and memory within 10%, peak memory within 15%, no unbounded soak growth).
+
+.PARAMETER Executable
+ Path to the azmcp server executable. Defaults to the Release build.
+
+.PARAMETER BenchmarkExe
+ Path to the performance-benchmark executable. Defaults to the Release build.
+
+.PARAMETER Modes
+ Server modes to measure. Valid values: default, namespace, all.
+
+.PARAMETER Concurrency
+ Number of concurrent load clients driving steady-state and soak load.
+
+.PARAMETER IdleSeconds
+ Idle observation window (no client connected) in seconds.
+
+.PARAMETER SteadySeconds
+ Steady-state load duration in seconds.
+
+.PARAMETER SoakSeconds
+ Sustained soak duration in seconds. 0 disables the soak/leak-detection phase.
+
+.PARAMETER SampleMs
+ Resource sampling interval in milliseconds.
+
+.PARAMETER OutputPath
+ Path to write the JSON results. Defaults to .perf-results/resource-usage.json.
+
+.PARAMETER BaselinePath
+ Optional baseline JSON (previously produced by this script) to gate against.
+
+.EXAMPLE
+ ./eng/scripts/Measure-ResourceUsage.ps1
+
+.EXAMPLE
+ ./eng/scripts/Measure-ResourceUsage.ps1 -Modes default,all -Concurrency 4 -SoakSeconds 120
+#>
+
+param(
+ [string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
+ [string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
+ [ValidateSet('default', 'namespace', 'all')]
+ [string[]] $Modes = @('default', 'all'),
+ [int] $Concurrency = 1,
+ [int] $IdleSeconds = 3,
+ [int] $SteadySeconds = 10,
+ [int] $SoakSeconds = 0,
+ [int] $SampleMs = 100,
+ [string] $OutputPath = '.perf-results/resource-usage.json',
+ [string] $BaselinePath
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$ScriptDir = $PSScriptRoot
+$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
+. (Join-Path $ScriptDir 'ResourcePerformance.Common.ps1')
+
+foreach ($path in @($Executable, $BenchmarkExe)) {
+ if (-not (Test-Path $path)) {
+ throw "Required executable not found: $path (build the solution in Release first)."
+ }
+}
+$Executable = (Resolve-Path $Executable).Path
+$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
+$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
+
+function Get-ModeServerArgs {
+ param([string] $Mode)
+ switch ($Mode) {
+ 'default' { return @('server', 'start') }
+ 'namespace' { return @('server', 'start', '--mode', 'namespace') }
+ 'all' { return @('server', 'start', '--mode', 'all') }
+ default { throw "Unknown mode '$Mode'." }
+ }
+}
+
+# Propagate the run parameters to the harness via environment variables.
+$env:PERF_CONCURRENCY = $Concurrency
+$env:PERF_IDLE_SECONDS = $IdleSeconds
+$env:PERF_STEADY_SECONDS = $SteadySeconds
+$env:PERF_SOAK_SECONDS = $SoakSeconds
+$env:PERF_SAMPLE_MS = $SampleMs
+
+# Per-mode timeout covers idle + steady + soak (plus startup/headroom) so a hung harness
+# fails fast instead of consuming the whole job.
+$resourceTimeoutSeconds = 180 + ($IdleSeconds + $SteadySeconds + $SoakSeconds) * 3
+
+$scenarios = [ordered]@{}
+
+foreach ($mode in $Modes) {
+ $serverArgs = Get-ModeServerArgs -Mode $mode
+ Write-Host "=== Resource usage: $mode mode (concurrency=$Concurrency, steady=${SteadySeconds}s, soak=${SoakSeconds}s) ==="
+
+ $data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
+ -ArgumentList (@('--mcp-resource', $Executable) + $serverArgs) -TimeoutSeconds $resourceTimeoutSeconds
+ $scenarios[$mode] = $data
+
+ $steady = $data.phases.steady_state
+ Write-Host (" tools={0} steady: ws-mean={1} MB ws-max={2} MB cpu={3}% {4} req ({5} rps)" -f `
+ $data.tool_count, $steady.working_set_mb.mean, $steady.working_set_mb.max, `
+ $steady.cpu_percent, $steady.requests, $steady.throughput_rps)
+ Write-Host (" peak: ws={0} MB private={1} MB handles={2} growth idle->steady: ws={3} MB handles={4}" -f `
+ $data.peak.working_set_mb, $data.peak.private_mb, $data.peak.handles, `
+ $data.growth.idle_to_steady_working_set_mb, $data.growth.idle_to_steady_handles)
+ if ($null -ne $data.soak) {
+ Write-Host (" soak: {0} req over {1}s ws-slope={2} MB/min (R2={3}) handle-slope={4}/min unbounded={5}" -f `
+ $data.soak.requests, $data.soak.duration_s, $data.soak.working_set_slope_mb_per_min, `
+ $data.soak.working_set_r2, $data.soak.handle_slope_per_min, $data.soak.unbounded_growth)
+ }
+ Write-Host ""
+}
+
+$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
+
+$results = [ordered]@{
+ timestamp = (Get-Date).ToUniversalTime().ToString('o')
+ commit = $commit
+ environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
+ concurrency = $Concurrency
+ idle_seconds = $IdleSeconds
+ steady_seconds = $SteadySeconds
+ soak_seconds = $SoakSeconds
+ sample_ms = $SampleMs
+ scenarios = $scenarios
+}
+
+$results | ConvertTo-Json -Depth 12 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
+Write-Host "Results written to: $OutputPath"
+
+# ---------------------------------------------------------------------------
+# Optional: regression check against a baseline file
+# ---------------------------------------------------------------------------
+if ($BaselinePath) {
+ if (-not (Test-Path $BaselinePath)) {
+ Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
+ }
+ else {
+ Write-Host ""
+ Write-Host "=== Resource regression check (steady CPU/mem +10%, peak mem +15%, no soak leak) ==="
+
+ $baseline = Get-Content $BaselinePath | ConvertFrom-Json
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios
+
+ if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Resource warnings (>5%): $($gate.Warnings -join ', ')"
+ }
+
+ if ($gate.Failures.Count -gt 0) {
+ Write-Error "Resource regression detected in: $($gate.Failures -join ', ')"
+ }
+ else {
+ Write-Host " All resource budget checks passed."
+ }
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Emit Azure DevOps pipeline variables (no-op outside AzDO)
+# ---------------------------------------------------------------------------
+foreach ($mode in $Modes) {
+ $data = $scenarios[$mode]
+ $steady = $data.phases.steady_state
+ $modeVar = (Get-Culture).TextInfo.ToTitleCase($mode)
+ Write-Host "##vso[task.setvariable variable=Resource${modeVar}SteadyWorkingSetMb]$($steady.working_set_mb.mean)"
+ Write-Host "##vso[task.setvariable variable=Resource${modeVar}SteadyCpuPercent]$($steady.cpu_percent)"
+ Write-Host "##vso[task.setvariable variable=Resource${modeVar}PeakWorkingSetMb]$($data.peak.working_set_mb)"
+}
diff --git a/eng/scripts/Publish-StartupPerformanceMetrics.ps1 b/eng/scripts/Publish-StartupPerformanceMetrics.ps1
new file mode 100644
index 000000000..2b4ff107c
--- /dev/null
+++ b/eng/scripts/Publish-StartupPerformanceMetrics.ps1
@@ -0,0 +1,57 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Emits the startup performance metrics from a results JSON file.
+
+.DESCRIPTION
+ Reads the results JSON produced by Test-StartupPerformance.ps1 and logs each tracked
+ metric. The metrics are currently written as pipeline log warnings so they surface in
+ the build timeline; replace the Write-Host sink with a real telemetry destination
+ (Azure Monitor, Application Insights, etc.) to persist them.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Test-StartupPerformance.ps1.
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$s = $results.scenarios
+
+$metrics = @(
+ @{ name = 'startup.cli.cold.median_ms'; value = $s.cli_cold_start_ms.median }
+ @{ name = 'startup.cli.cold.p95_ms'; value = $s.cli_cold_start_ms.p95 }
+ @{ name = 'startup.mcp.default.median_ms'; value = $s.mcp_stdio_to_tools_list_ms.median }
+ @{ name = 'startup.mcp.default.p95_ms'; value = $s.mcp_stdio_to_tools_list_ms.p95 }
+ @{ name = 'startup.mcp.default.p99_ms'; value = $s.mcp_stdio_to_tools_list_ms.p99 }
+ @{ name = 'startup.mcp.namespace.median_ms'; value = $s.mcp_namespace_mode_startup_ms.median }
+ @{ name = 'startup.mcp.all.median_ms'; value = $s.mcp_all_mode_startup_ms.median }
+ @{ name = 'startup.mcp.all.p95_ms'; value = $s.mcp_all_mode_startup_ms.p95 }
+ @{ name = 'startup.mcp.all.p99_ms'; value = $s.mcp_all_mode_startup_ms.p99 }
+ @{ name = 'startup.mcp.http.median_ms'; value = $s.mcp_http_default_startup_ms.median }
+ @{ name = 'startup.mcp.http.p95_ms'; value = $s.mcp_http_default_startup_ms.p95 }
+ @{ name = 'startup.mcp.http.readiness_median_ms'; value = $s.mcp_http_readiness_ms.median }
+ @{ name = 'serialize.default.full_schema_ms'; value = $s.tools_list_payload.full_schema_serialize_ms }
+ @{ name = 'serialize.all.full_schema_ms'; value = $s.all_mode_tools_list_payload.full_schema_serialize_ms }
+ @{ name = 'serialize.http.full_schema_ms'; value = $s.http_default_tools_list_payload.full_schema_serialize_ms }
+ @{ name = 'tokens.default.full_schema_gpt4o'; value = $s.tools_list_payload.full_schema_tokens_exact }
+ @{ name = 'tokens.namespace.full_schema_gpt4o'; value = $s.namespace_mode_tools_list_payload.full_schema_tokens_exact }
+ @{ name = 'tokens.all.full_schema_gpt4o'; value = $s.all_mode_tools_list_payload.full_schema_tokens_exact }
+ @{ name = 'tools.default.count'; value = $s.tools_list_payload.tool_count }
+ @{ name = 'tools.all.count'; value = $s.all_mode_tools_list_payload.tool_count }
+)
+
+foreach ($m in $metrics) {
+ Write-Host ("##vso[task.logissue type=warning]METRIC: {0} = {1}" -f $m.name, $m.value)
+ # Replace the log statement above with your preferred telemetry sink, e.g.:
+ # az monitor metrics alert create ...
+ # Invoke-RestMethod -Uri $AppInsightsEndpoint -Body @{ name=$m.name; value=$m.value } ...
+}
+Write-Host "Metrics logged. Wire up a telemetry sink to persist them."
diff --git a/eng/scripts/ResourcePerformance.Common.Tests.ps1 b/eng/scripts/ResourcePerformance.Common.Tests.ps1
new file mode 100644
index 000000000..041458d6f
--- /dev/null
+++ b/eng/scripts/ResourcePerformance.Common.Tests.ps1
@@ -0,0 +1,161 @@
+# Install-Module -Name Pester -Force -SkipPublisherCheck
+# Invoke-Pester -Path eng/scripts/ResourcePerformance.Common.Tests.ps1
+#
+# Focused tests for the pure resource-gate helpers used by Measure-ResourceUsage.ps1.
+
+BeforeAll {
+ . $PSScriptRoot/ResourcePerformance.Common.ps1
+
+ function script:New-ResourceScenario {
+ param(
+ [double] $SteadyMem,
+ [double] $SteadyCpu,
+ [double] $PeakMem,
+ [switch] $WithSoak,
+ [bool] $Unbounded = $false
+ )
+
+ $scenario = [ordered]@{
+ phases = [ordered]@{
+ steady_state = [ordered]@{
+ working_set_mb = [ordered]@{ mean = $SteadyMem; max = $SteadyMem }
+ cpu_percent = $SteadyCpu
+ }
+ }
+ peak = [ordered]@{ working_set_mb = $PeakMem }
+ }
+
+ if ($WithSoak) {
+ $scenario.soak = [ordered]@{
+ unbounded_growth = $Unbounded
+ working_set_slope_mb_per_min = 1
+ working_set_r2 = 0.9
+ handle_slope_per_min = 1
+ handle_r2 = 0.1
+ }
+ }
+
+ return $scenario
+ }
+}
+
+Describe "Get-MemberNames" {
+ It "returns the keys of an ordered dictionary" {
+ $names = Get-MemberNames ([ordered]@{ a = 1; b = 2 })
+ $names | Should -Be @('a', 'b')
+ }
+
+ It "returns the property names of a PSCustomObject" {
+ $obj = [pscustomobject]@{ x = 1; y = 2 }
+ $names = Get-MemberNames $obj
+ $names | Should -Be @('x', 'y')
+ }
+
+ It "returns an empty set for null" {
+ @(Get-MemberNames $null).Count | Should -Be 0
+ }
+}
+
+Describe "Invoke-ResourceRegressionGate" {
+ It "reports no failures or warnings for an identical baseline" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ @($gate.Warnings).Count | Should -Be 0
+ }
+
+ It "fails when steady-state memory regresses beyond 10%" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 115 -SteadyCpu 10 -PeakMem 150) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default steady_working_set_mb'
+ }
+
+ It "warns but does not fail for a steady-state memory change between 5% and 10%" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 107 -SteadyCpu 10 -PeakMem 150) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ $gate.Warnings | Should -Contain 'default steady_working_set_mb'
+ }
+
+ It "fails when steady-state CPU regresses beyond 10%" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 12 -PeakMem 150) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default steady_cpu_percent'
+ }
+
+ It "allows peak memory growth up to 15% as a warning" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 168) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ $gate.Warnings | Should -Contain 'default peak_working_set_mb'
+ }
+
+ It "fails when peak memory regresses beyond 15%" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 180) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default peak_working_set_mb'
+ }
+
+ It "fails when a soak phase is flagged with unbounded growth" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150 -WithSoak -Unbounded $true) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default soak_unbounded_growth'
+ }
+
+ It "passes a soak phase that is not flagged" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150 -WithSoak -Unbounded $false) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ }
+
+ It "flags an incompatible (empty) baseline instead of silently passing (F-007)" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 999 -SteadyCpu 99 -PeakMem 999) }
+ $b = [ordered]@{ }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'no_comparable_metrics'
+ }
+
+ It "compares overlapping modes and skips a newly added mode without failing" {
+ $s = [ordered]@{
+ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150)
+ all = (New-ResourceScenario -SteadyMem 200 -SteadyCpu 20 -PeakMem 280)
+ }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ }
+
+ It "flags soak unbounded growth even when the mode is absent from the baseline (F-008)" {
+ $s = [ordered]@{ all = (New-ResourceScenario -SteadyMem 200 -SteadyCpu 20 -PeakMem 280 -WithSoak -Unbounded $true) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'all soak_unbounded_growth'
+ }
+
+ It "treats positive CPU against a zero baseline as a failure, not a pass (F-010)" {
+ $s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 25 -PeakMem 150) }
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 0 -PeakMem 150) }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default steady_cpu_percent'
+ }
+
+ It "fails when a steady metric the baseline expects is missing from results (F-007)" {
+ $b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
+ $s = [ordered]@{
+ default = [ordered]@{
+ phases = [ordered]@{ steady_state = [ordered]@{ working_set_mb = [ordered]@{ mean = 100; max = 100 } } }
+ peak = [ordered]@{ working_set_mb = 150 }
+ }
+ }
+ $gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'default steady_cpu_percent (missing in results)'
+ }
+}
diff --git a/eng/scripts/ResourcePerformance.Common.ps1 b/eng/scripts/ResourcePerformance.Common.ps1
new file mode 100644
index 000000000..aadd37a3c
--- /dev/null
+++ b/eng/scripts/ResourcePerformance.Common.ps1
@@ -0,0 +1,151 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Side-effect-free helpers for the resource-usage baseline gate (issue #3121).
+
+.DESCRIPTION
+ Dot-source this file to reuse the pure functions in Measure-ResourceUsage.ps1 and
+ Check-ResourceUsageRegression.ps1 and to unit-test them without launching a server.
+ Reuses the generic accessor Get-MemberValue from StartupPerformance.Common.ps1.
+#>
+
+. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
+
+# ---------------------------------------------------------------------------
+# Returns the member names of a hashtable/ordered dictionary or a PSCustomObject
+# (as produced by ConvertFrom-Json), so callers can enumerate scenario maps
+# regardless of how the results JSON was materialized.
+# ---------------------------------------------------------------------------
+function Get-MemberNames {
+ [CmdletBinding()]
+ param($Object)
+
+ if ($null -eq $Object) { return @() }
+ if ($Object -is [System.Collections.IDictionary]) { return @($Object.Keys) }
+ return @($Object.PSObject.Properties.Name)
+}
+
+# ---------------------------------------------------------------------------
+# Tiered resource-regression gate shared by Measure-ResourceUsage.ps1 and
+# Check-ResourceUsageRegression.ps1. For each server-mode scenario present in
+# both the results and the baseline it applies the acceptance-criteria budgets
+# from issue #3121:
+# * steady-state CPU and memory may not increase by more than 10% (FailSteady = 1.10)
+# * peak memory may not increase by more than 15% (FailPeak = 1.15)
+# * any change above 5% is reported as a warning (Warn = 1.05)
+# * a soak phase flagged with unbounded_growth is a hard failure (memory/handle leak)
+#
+# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a
+# human-readable line per check via Write-Host.
+# ---------------------------------------------------------------------------
+function Invoke-ResourceRegressionGate {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)] $ResultScenarios,
+ [Parameter(Mandatory)] $BaselineScenarios,
+ [double] $FailSteady = 1.10,
+ [double] $FailPeak = 1.15,
+ [double] $Warn = 1.05
+ )
+
+ $failures = @()
+ $warnings = @()
+ $evaluated = 0
+
+ $seenModes = @{}
+
+ foreach ($mode in (Get-MemberNames $ResultScenarios)) {
+ $seenModes[[string]$mode] = $true
+ $r = Get-MemberValue $ResultScenarios $mode
+ $b = Get-MemberValue $BaselineScenarios $mode
+
+ # Leak detection: a soak phase flagged unbounded is a hard failure regardless of the
+ # baseline, so it is evaluated before the baseline-presence check below can skip this mode.
+ $soak = Get-MemberValue $r 'soak'
+ if ($null -ne $soak) {
+ if (Get-MemberValue $soak 'unbounded_growth') {
+ $failures += "$mode soak_unbounded_growth"
+ Write-Host (" [FAIL] {0} soak_unbounded_growth: working_set_slope={1} MB/min (R2={2}), handle_slope={3}/min (R2={4})" -f `
+ $mode, (Get-MemberValue $soak 'working_set_slope_mb_per_min'), (Get-MemberValue $soak 'working_set_r2'), `
+ (Get-MemberValue $soak 'handle_slope_per_min'), (Get-MemberValue $soak 'handle_r2'))
+ }
+ else {
+ Write-Host (" [PASS] {0} soak: no unbounded growth" -f $mode)
+ }
+ }
+
+ if ($null -eq $b) {
+ Write-Host (" [SKIP] {0}: absent in baseline (relative budgets)" -f $mode)
+ continue
+ }
+
+ $rSteady = Get-MemberValue (Get-MemberValue $r 'phases') 'steady_state'
+ $bSteady = Get-MemberValue (Get-MemberValue $b 'phases') 'steady_state'
+
+ $checks = @(
+ @{ Label = 'steady_working_set_mb'; Unit = 'MB'
+ Cur = Get-MemberValue (Get-MemberValue $rSteady 'working_set_mb') 'mean'
+ Base = Get-MemberValue (Get-MemberValue $bSteady 'working_set_mb') 'mean'
+ Limit = $FailSteady }
+ @{ Label = 'steady_cpu_percent'; Unit = '%'
+ Cur = Get-MemberValue $rSteady 'cpu_percent'
+ Base = Get-MemberValue $bSteady 'cpu_percent'
+ Limit = $FailSteady }
+ @{ Label = 'peak_working_set_mb'; Unit = 'MB'
+ Cur = Get-MemberValue (Get-MemberValue $r 'peak') 'working_set_mb'
+ Base = Get-MemberValue (Get-MemberValue $b 'peak') 'working_set_mb'
+ Limit = $FailPeak }
+ )
+
+ foreach ($c in $checks) {
+ if ($null -eq $c.Base) {
+ Write-Host (" [SKIP] {0} {1}: absent in baseline" -f $mode, $c.Label)
+ continue
+ }
+ if ($null -eq $c.Cur) {
+ $failures += "$mode $($c.Label) (missing in results)"
+ Write-Host (" [FAIL] {0} {1}: expected by baseline but absent in results" -f $mode, $c.Label)
+ continue
+ }
+
+ $label = "$mode $($c.Label)"
+ $evaluated++
+ if ($c.Base -gt 0) {
+ $ratio = $c.Cur / $c.Base
+ if ($ratio -gt $c.Limit) { $failures += $label; $status = 'FAIL' }
+ elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
+ else { $status = 'PASS' }
+ Write-Host (" [{0}] {1}: current={2}{3} baseline={4}{3} (+{5}%)" -f `
+ $status, $label, $c.Cur, $c.Unit, $c.Base, [math]::Round(($ratio - 1) * 100, 1))
+ }
+ elseif ($c.Cur -gt 0) {
+ # Positive current against a zero baseline is not comparable — fail rather than
+ # silently pass with a fabricated 0% change.
+ $failures += $label
+ Write-Host (" [FAIL] {0}: current={1}{2} vs a zero baseline (not comparable)" -f $label, $c.Cur, $c.Unit)
+ }
+ else {
+ Write-Host (" [PASS] {0}: current=0{1} baseline=0{1} (unchanged)" -f $label, $c.Unit)
+ }
+ }
+
+ # Leak detection is evaluated near the top of the loop (before the baseline skip).
+ }
+
+ # Baseline modes the results no longer cover are treated as an expected-but-missing metric.
+ foreach ($mode in (Get-MemberNames $BaselineScenarios)) {
+ if (-not $seenModes.ContainsKey([string]$mode)) {
+ $failures += "$mode missing_in_results"
+ Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $mode)
+ }
+ }
+
+ if ($evaluated -eq 0) {
+ $failures += 'no_comparable_metrics'
+ Write-Host " [FAIL] no comparable resource metrics between results and baseline (incompatible or empty baseline)"
+ }
+
+ return [ordered]@{ Failures = $failures; Warnings = $warnings }
+}
diff --git a/eng/scripts/StartupPerformance.Common.Tests.ps1 b/eng/scripts/StartupPerformance.Common.Tests.ps1
new file mode 100644
index 000000000..7c6612dcd
--- /dev/null
+++ b/eng/scripts/StartupPerformance.Common.Tests.ps1
@@ -0,0 +1,222 @@
+# Install-Module -Name Pester -Force -SkipPublisherCheck
+# Invoke-Pester -Path eng/scripts/StartupPerformance.Common.Tests.ps1
+#
+# Focused tests for the pure helpers used by Test-StartupPerformance.ps1.
+
+BeforeAll {
+ . $PSScriptRoot/StartupPerformance.Common.ps1
+
+ $script:RunFolder = Join-Path $PSScriptRoot '.perf-common-testruns'
+ if (Test-Path $script:RunFolder) {
+ Remove-Item -Recurse -Force $script:RunFolder
+ }
+ New-Item -ItemType Directory -Path $script:RunFolder | Out-Null
+}
+
+AfterAll {
+ if (Test-Path $script:RunFolder) {
+ Remove-Item -Recurse -Force $script:RunFolder
+ }
+}
+
+Describe "Get-TimingStats" {
+ It "returns the single middle value as the median for an odd number of samples" {
+ # sorted: 3,5,9 -> middle = 5
+ $stats = Get-TimingStats -Samples @(9, 3, 5)
+ $stats.median | Should -Be 5
+ }
+
+ It "averages the two middle values as the median for an even number of samples" {
+ # sorted: 10,20,30,40 -> (20+30)/2 = 25
+ $stats = Get-TimingStats -Samples @(40, 10, 30, 20)
+ $stats.median | Should -Be 25
+ }
+
+ It "produces a fractional median when the two middle values differ by an odd amount" {
+ # sorted: 10,11 -> (10+11)/2 = 10.5
+ $stats = Get-TimingStats -Samples @(11, 10)
+ $stats.median | Should -Be 10.5
+ }
+
+ It "reports the first sample in original (unsorted) order" {
+ $stats = Get-TimingStats -Samples @(40, 10, 30, 20)
+ $stats.first | Should -Be 40
+ }
+
+ It "computes the average rounded to one decimal place" {
+ # (1+2+2)/3 = 1.666... -> 1.7
+ $stats = Get-TimingStats -Samples @(1, 2, 2)
+ $stats.average | Should -Be 1.7
+ }
+
+ It "handles a single sample" {
+ $stats = Get-TimingStats -Samples @(42)
+ $stats.median | Should -Be 42
+ $stats.first | Should -Be 42
+ $stats.average | Should -Be 42
+ }
+
+ It "throws when no samples are provided" {
+ { Get-TimingStats -Samples @() } | Should -Throw
+ }
+}
+
+Describe "Get-TimingStats percentiles" {
+ It "reports p50 equal to the median for an odd number of samples" {
+ $stats = Get-TimingStats -Samples @(9, 3, 5)
+ $stats.p50 | Should -Be 5
+ }
+
+ It "orders p50 <= p95 <= p99 and keeps them within the sample range" {
+ $stats = Get-TimingStats -Samples @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+ $stats.p50 | Should -BeLessOrEqual $stats.p95
+ $stats.p95 | Should -BeLessOrEqual $stats.p99
+ $stats.p99 | Should -BeLessOrEqual 10
+ $stats.p95 | Should -BeGreaterThan 9 # rank 8.55 lands between 9 and 10
+ }
+
+ It "returns the single value for all percentiles when one sample" {
+ $stats = Get-TimingStats -Samples @(42)
+ $stats.p50 | Should -Be 42
+ $stats.p95 | Should -Be 42
+ $stats.p99 | Should -Be 42
+ }
+}
+
+Describe "Get-Percentile" {
+ It "returns the min at p0 and max at p100" {
+ (Get-Percentile -Samples @(10, 20, 30, 40) -Percentile 0.0) | Should -Be 10
+ (Get-Percentile -Samples @(10, 20, 30, 40) -Percentile 1.0) | Should -Be 40
+ }
+
+ It "interpolates between ranks" {
+ # sorted 10,20,30,40; rank = 0.5*3 = 1.5 -> 20 + 0.5*(30-20) = 25
+ (Get-Percentile -Samples @(40, 10, 30, 20) -Percentile 0.5) | Should -Be 25
+ }
+
+ It "returns an exact sample when the rank is an integer" {
+ # sorted 1..11, rank = 0.9*(11-1) = 9 -> sorted[9] = 10
+ (Get-Percentile -Samples @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) -Percentile 0.9) | Should -Be 10
+ }
+
+ It "throws for out-of-range percentiles" {
+ { Get-Percentile -Samples @(1) -Percentile 1.5 } | Should -Throw
+ { Get-Percentile -Samples @(1) -Percentile -0.1 } | Should -Throw
+ }
+
+ It "throws when no samples are provided" {
+ { Get-Percentile -Samples @() -Percentile 0.5 } | Should -Throw
+ }
+}
+
+Describe "Split-ColdWarm" {
+ It "treats the first sample as cold and the rest as warm" {
+ $split = Split-ColdWarm -Samples @(100, 40, 50, 60)
+ $split.cold_ms | Should -Be 100
+ # warm samples 40,50,60 -> median 50
+ $split.warm.median | Should -Be 50
+ }
+
+ It "falls back to the single sample for warm when only one sample exists" {
+ $split = Split-ColdWarm -Samples @(77)
+ $split.cold_ms | Should -Be 77
+ $split.warm.median | Should -Be 77
+ }
+
+ It "throws when no samples are provided" {
+ { Split-ColdWarm -Samples @() } | Should -Throw
+ }
+}
+
+Describe "Resolve-PerfOutputPath" {
+ It "resolves a bare filename against the base directory and returns an absolute path" {
+ $result = Resolve-PerfOutputPath -Path 'startup-e2e.json' -BaseDirectory $script:RunFolder
+ [System.IO.Path]::IsPathRooted($result) | Should -Be $true
+ Split-Path -Parent $result | Should -Be ([System.IO.Path]::GetFullPath($script:RunFolder))
+ }
+
+ It "creates a missing parent directory for a relative nested path" {
+ $nested = Join-Path 'nested-a' 'nested-b/out.json'
+ $result = Resolve-PerfOutputPath -Path $nested -BaseDirectory $script:RunFolder
+ Test-Path (Split-Path -Parent $result) | Should -Be $true
+ }
+
+ It "creates a missing parent directory for an absolute path" {
+ $abs = Join-Path $script:RunFolder 'abs-dir/out.json'
+ $result = Resolve-PerfOutputPath -Path $abs
+ Test-Path (Split-Path -Parent $result) | Should -Be $true
+ $result | Should -Be ([System.IO.Path]::GetFullPath($abs))
+ }
+
+ It "does not throw for a bare filename (regression for empty Split-Path parent)" {
+ { Resolve-PerfOutputPath -Path 'only-a-filename.json' -BaseDirectory $script:RunFolder } | Should -Not -Throw
+ }
+}
+
+Describe "Get-PerfRunMetadata" {
+ It "records all controlled-environment dimensions required by issue #3120" {
+ $m = Get-PerfRunMetadata
+ foreach ($key in @('commit', 'timestamp', 'runtime', 'os', 'os_architecture', 'cpu_cores', 'machine_memory_gb', 'workload_catalog_version')) {
+ $m.Contains($key) | Should -BeTrue -Because "metadata must include '$key'"
+ }
+ }
+
+ It "reports a positive processor count" {
+ (Get-PerfRunMetadata).cpu_cores | Should -BeGreaterThan 0
+ }
+}
+
+Describe "Get-WorkloadCatalogVersion" {
+ It "returns the catalog version from the committed workload catalog" {
+ Get-WorkloadCatalogVersion | Should -Not -BeNullOrEmpty
+ }
+
+ It "returns 'unknown' when the catalog file is absent" {
+ Get-WorkloadCatalogVersion -CatalogPath (Join-Path $script:RunFolder 'no-such-catalog.json') | Should -Be 'unknown'
+ }
+}
+
+Describe "Get-TimingStats sub-millisecond precision" {
+ It "preserves fractional-millisecond percentiles instead of rounding to whole ms (F-004)" {
+ # Regression: double samples previously bound to [long[]] and were rounded to 0,0,1,1.
+ $stats = Get-TimingStats -Samples @(0.41, 0.49, 0.51, 0.59)
+ $stats.median | Should -Be 0.5
+ $stats.p95 | Should -BeGreaterThan 0.5
+ $stats.p95 | Should -BeLessThan 0.6
+ }
+}
+
+Describe "Invoke-StartupRegressionGate" {
+ It "reports no failures for an identical baseline" {
+ $s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
+ $b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
+ $gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -Be 0
+ }
+
+ It "fails when nothing overlaps results and baseline (F-007)" {
+ $s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
+ $b = [ordered]@{ }
+ $gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'no_comparable_metrics'
+ }
+
+ It "fails when a metric the baseline expects is missing from results (F-007)" {
+ $s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120 } }
+ $b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
+ $gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ $gate.Failures | Should -Contain 'mcp_stdio_default (p99, missing in results)'
+ }
+
+ It "treats a positive current against a zero baseline as a failure (F-010)" {
+ $s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
+ $b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 0; p95 = 0; p99 = 0 } }
+ $gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
+ @($gate.Failures).Count | Should -BeGreaterThan 0
+ }
+
+ It "accepts a -RepoRoot argument when stamping run metadata (F-009)" {
+ $m = Get-PerfRunMetadata -RepoRoot $PSScriptRoot
+ $m.Contains('commit') | Should -BeTrue
+ }
+}
diff --git a/eng/scripts/StartupPerformance.Common.ps1 b/eng/scripts/StartupPerformance.Common.ps1
new file mode 100644
index 000000000..39884247f
--- /dev/null
+++ b/eng/scripts/StartupPerformance.Common.ps1
@@ -0,0 +1,430 @@
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Shared, side-effect-free helpers for the startup performance measurement scripts.
+
+.DESCRIPTION
+ These functions are extracted into their own file so they can be dot-sourced by
+ Test-StartupPerformance.ps1 and covered by focused Pester tests without spawning
+ the server or running the full measurement pipeline.
+#>
+
+Set-StrictMode -Version Latest
+
+# ---------------------------------------------------------------------------
+# Compute the p-th percentile of a sample array using linear interpolation
+# between closest ranks (the inclusive method, equivalent to Excel's
+# PERCENTILE.INC). p is expressed as a fraction in [0, 1].
+# A single sample returns that sample for any percentile.
+# ---------------------------------------------------------------------------
+function Get-Percentile {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)][double[]] $Samples,
+ [Parameter(Mandatory)][double] $Percentile
+ )
+
+ if ($null -eq $Samples -or $Samples.Count -eq 0) {
+ throw "Get-Percentile requires at least one sample."
+ }
+ if ($Percentile -lt 0 -or $Percentile -gt 1) {
+ throw "Get-Percentile requires Percentile in the range [0, 1]."
+ }
+
+ $sorted = @($Samples | Sort-Object)
+ $count = $sorted.Count
+ if ($count -eq 1) { return [double]$sorted[0] }
+
+ $rank = $Percentile * ($count - 1)
+ $lower = [int][math]::Floor($rank)
+ $upper = [int][math]::Ceiling($rank)
+ if ($lower -eq $upper) { return [double]$sorted[$lower] }
+
+ $frac = $rank - $lower
+ return [double]$sorted[$lower] + ($sorted[$upper] - $sorted[$lower]) * $frac
+}
+
+# ---------------------------------------------------------------------------
+# Compute first/average/median/p50/p95/p99 timing stats from a sample array.
+# For an even number of samples the median is the average of the two middle
+# values (the statistically correct definition); for an odd count it is the
+# single middle value. p50/p95/p99 are computed via Get-Percentile (linear
+# interpolation) so they align with the budgets in the acceptance criteria.
+# ---------------------------------------------------------------------------
+function Get-TimingStats {
+ [CmdletBinding()]
+ param([Parameter(Mandatory)][double[]] $Samples)
+
+ if ($null -eq $Samples -or $Samples.Count -eq 0) {
+ throw "Get-TimingStats requires at least one sample."
+ }
+
+ $sorted = @($Samples | Sort-Object)
+ $count = $sorted.Count
+
+ if ($count % 2 -eq 1) {
+ $median = $sorted[[math]::Floor($count / 2)]
+ }
+ else {
+ $upper = $count / 2
+ $lower = $upper - 1
+ $median = ($sorted[$lower] + $sorted[$upper]) / 2
+ }
+
+ return [ordered]@{
+ first = $Samples[0]
+ average = [math]::Round(($Samples | Measure-Object -Average).Average, 1)
+ median = $median
+ p50 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.50), 4)
+ p95 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.95), 4)
+ p99 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.99), 4)
+ all = $Samples
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Split a sample array into cold (first invocation) and warm (all subsequent
+# invocations) timing stats. Process-level cold vs warm startup is captured by
+# treating the first sample as cold (OS/file-cache/JIT cold) and the remaining
+# samples as warm. When only one sample exists, warm falls back to that sample.
+# ---------------------------------------------------------------------------
+function Split-ColdWarm {
+ [CmdletBinding()]
+ param([Parameter(Mandatory)][long[]] $Samples)
+
+ if ($null -eq $Samples -or $Samples.Count -eq 0) {
+ throw "Split-ColdWarm requires at least one sample."
+ }
+
+ $cold = [long]$Samples[0]
+ $warmSamples = if ($Samples.Count -gt 1) { @($Samples[1..($Samples.Count - 1)]) } else { @($Samples[0]) }
+
+ return [ordered]@{
+ cold_ms = $cold
+ warm = Get-TimingStats -Samples $warmSamples
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Reads a named member from either an ordered hashtable (results built in-process)
+# or a PSCustomObject (parsed from JSON). Returns $null when the member is absent
+# so callers can gracefully skip metrics missing from an older baseline.
+# ---------------------------------------------------------------------------
+function Get-MemberValue {
+ [CmdletBinding()]
+ param($InputObject, [Parameter(Mandatory)][string] $Name)
+
+ if ($null -eq $InputObject) { return $null }
+ if ($InputObject -is [System.Collections.IDictionary]) {
+ if ($InputObject.Contains($Name)) { return $InputObject[$Name] }
+ return $null
+ }
+ $prop = $InputObject.PSObject.Properties[$Name]
+ if ($prop) { return $prop.Value }
+ return $null
+}
+
+# ---------------------------------------------------------------------------
+# Runs an external perf-harness process with a hard timeout, capturing stdout and
+# stderr and rejecting a non-zero exit code. Prevents a hung or crashed harness
+# from stalling the whole job or being mistaken for a successful measurement.
+# Returns the captured stdout string.
+# ---------------------------------------------------------------------------
+function Invoke-PerfHarnessProcess {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)][string] $FilePath,
+ [string[]] $ArgumentList = @(),
+ [int] $TimeoutSeconds = 120
+ )
+
+ $psi = [System.Diagnostics.ProcessStartInfo]::new()
+ $psi.FileName = $FilePath
+ $psi.RedirectStandardOutput = $true
+ $psi.RedirectStandardError = $true
+ $psi.UseShellExecute = $false
+ $psi.CreateNoWindow = $true
+ foreach ($arg in $ArgumentList) { $null = $psi.ArgumentList.Add($arg) }
+
+ $proc = [System.Diagnostics.Process]::new()
+ $proc.StartInfo = $psi
+ try {
+ if (-not $proc.Start()) { throw "Failed to start process '$FilePath'." }
+
+ $stdoutTask = $proc.StandardOutput.ReadToEndAsync()
+ $stderrTask = $proc.StandardError.ReadToEndAsync()
+
+ if (-not $proc.WaitForExit($TimeoutSeconds * 1000)) {
+ try { $proc.Kill($true) }
+ catch { if (-not $proc.HasExited) { throw } }
+ $null = $proc.WaitForExit(5000)
+ $errOut = $stderrTask.GetAwaiter().GetResult()
+ $errMsg = if ([string]::IsNullOrWhiteSpace($errOut)) { '' } else { $errOut.Trim() }
+ throw "Process '$FilePath' timed out after $TimeoutSeconds seconds. Stderr: $errMsg"
+ }
+
+ $outText = $stdoutTask.GetAwaiter().GetResult()
+ $errOut = $stderrTask.GetAwaiter().GetResult()
+ if ($proc.ExitCode -ne 0) {
+ $errMsg = if ([string]::IsNullOrWhiteSpace($errOut)) { '' } else { $errOut.Trim() }
+ throw "Process '$FilePath' exited with code $($proc.ExitCode). Stderr: $errMsg"
+ }
+
+ return $outText
+ }
+ finally {
+ $proc.Dispose()
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Runs a perf-harness process (see Invoke-PerfHarnessProcess) and returns the last
+# JSON object it emitted on stdout, parsed. Throws if the harness produced no JSON
+# line (after already rejecting non-zero exits), so a silent/garbled run fails loudly.
+# ---------------------------------------------------------------------------
+function Invoke-PerfHarnessJson {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)][string] $FilePath,
+ [string[]] $ArgumentList = @(),
+ [int] $TimeoutSeconds = 120
+ )
+
+ $stdout = Invoke-PerfHarnessProcess -FilePath $FilePath -ArgumentList $ArgumentList -TimeoutSeconds $TimeoutSeconds
+ $jsonLine = @($stdout -split "`r?`n") | Where-Object { $_ -match '^\s*\{' } | Select-Object -Last 1
+ if ([string]::IsNullOrWhiteSpace($jsonLine)) {
+ throw "Harness '$FilePath' did not emit a JSON result line."
+ }
+ return $jsonLine | ConvertFrom-Json
+}
+
+# ---------------------------------------------------------------------------
+# Tiered startup-regression gate shared by Test-StartupPerformance.ps1 and
+# Check-StartupPerformanceRegression.ps1. Compares p50/p95/p99 of each tracked
+# latency scenario against the baseline using the acceptance-criteria budgets
+# from issue #3118:
+# * p50 and p95 may not regress by more than 10% (FailP50P95 = 1.10)
+# * p99 may not regress by more than 20% (FailP99 = 1.20)
+# * any change above 5% is reported as a warning (Warn = 1.05)
+# Discovery scaling efficiency (per-tool startup cost for the all-tools surface)
+# may not degrade by more than 15% (ScalingDegrade = 1.15).
+#
+# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a
+# human-readable line per check via Write-Host.
+# ---------------------------------------------------------------------------
+function Invoke-StartupRegressionGate {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)] $ResultScenarios,
+ [Parameter(Mandatory)] $BaselineScenarios,
+ [double] $FailP50P95 = 1.10,
+ [double] $FailP99 = 1.20,
+ [double] $Warn = 1.05,
+ [double] $ScalingDegrade = 1.15
+ )
+
+ $failures = @()
+ $warnings = @()
+
+ # Latency scenarios: display name -> scenario property holding a stats object.
+ $latencyScenarios = [ordered]@{
+ 'cli_cold_start' = 'cli_cold_start_ms'
+ 'mcp_stdio_default' = 'mcp_stdio_to_tools_list_ms'
+ 'mcp_namespace' = 'mcp_namespace_mode_startup_ms'
+ 'mcp_all' = 'mcp_all_mode_startup_ms'
+ 'mcp_http_default' = 'mcp_http_default_startup_ms'
+ }
+
+ $percentileBudgets = @(
+ @{ Metric = 'p50'; Limit = $FailP50P95 }
+ @{ Metric = 'p95'; Limit = $FailP50P95 }
+ @{ Metric = 'p99'; Limit = $FailP99 }
+ )
+
+ $evaluated = 0
+
+ foreach ($name in $latencyScenarios.Keys) {
+ $key = $latencyScenarios[$name]
+ $curStats = Get-MemberValue $ResultScenarios $key
+ $baseStats = Get-MemberValue $BaselineScenarios $key
+ if ($null -eq $baseStats) {
+ Write-Host (" [SKIP] {0}: absent in baseline" -f $name)
+ continue
+ }
+ if ($null -eq $curStats) {
+ $failures += "$name (missing in results)"
+ Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $name)
+ continue
+ }
+
+ foreach ($budget in $percentileBudgets) {
+ $metric = $budget.Metric
+ $current = Get-MemberValue $curStats $metric
+ $baseline = Get-MemberValue $baseStats $metric
+ if ($null -eq $baseline) {
+ Write-Host (" [SKIP] {0} ({1}): absent in baseline" -f $name, $metric)
+ continue
+ }
+ if ($null -eq $current) {
+ $failures += "$name ($metric, missing in results)"
+ Write-Host (" [FAIL] {0} ({1}): expected by baseline but absent in results" -f $name, $metric)
+ continue
+ }
+
+ $label = "$name ($metric)"
+ $evaluated++
+ if ($baseline -gt 0) {
+ $ratio = $current / $baseline
+ $limit = [math]::Round($baseline * $budget.Limit, 1)
+ if ($ratio -gt $budget.Limit) { $failures += $label; $status = 'FAIL' }
+ elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
+ else { $status = 'PASS' }
+ Write-Host (" [{0}] {1}: current={2}ms baseline={3}ms limit={4}ms (+{5}%)" -f `
+ $status, $label, $current, $baseline, $limit, [math]::Round(($ratio - 1) * 100, 1))
+ }
+ elseif ($current -gt 0) {
+ # Positive current against a zero baseline is not comparable — fail rather than
+ # silently pass with a fabricated 0% change.
+ $failures += $label
+ Write-Host (" [FAIL] {0}: current={1}ms vs a zero baseline (not comparable)" -f $label, $current)
+ }
+ else {
+ Write-Host (" [PASS] {0}: current=0ms baseline=0ms (unchanged)" -f $label)
+ }
+ }
+ }
+
+ # Discovery scaling efficiency: per-tool startup cost for the all-tools surface.
+ $curAll = Get-MemberValue $ResultScenarios 'mcp_all_mode_startup_ms'
+ $curAllPl = Get-MemberValue $ResultScenarios 'all_mode_tools_list_payload'
+ $baseAll = Get-MemberValue $BaselineScenarios 'mcp_all_mode_startup_ms'
+ $baseAllPl = Get-MemberValue $BaselineScenarios 'all_mode_tools_list_payload'
+ $curP50 = Get-MemberValue $curAll 'p50'
+ $curCount = Get-MemberValue $curAllPl 'tool_count'
+ $baseP50 = Get-MemberValue $baseAll 'p50'
+ $baseCount = Get-MemberValue $baseAllPl 'tool_count'
+ if ($curP50 -and $curCount -and $baseP50 -and $baseCount -and $curCount -gt 0 -and $baseCount -gt 0) {
+ $curPerTool = $curP50 / $curCount
+ $basePerTool = $baseP50 / $baseCount
+ $ratio = if ($basePerTool -gt 0) { $curPerTool / $basePerTool } else { 1 }
+ $evaluated++
+ if ($ratio -gt $ScalingDegrade) {
+ $failures += 'discovery_scaling_efficiency'
+ $status = 'FAIL'
+ }
+ elseif ($ratio -gt $Warn) {
+ $warnings += 'discovery_scaling_efficiency'
+ $status = 'WARN'
+ }
+ else {
+ $status = 'PASS'
+ }
+ Write-Host (" [{0}] discovery_scaling_efficiency: current={1}ms/tool baseline={2}ms/tool (+{3}%)" -f `
+ $status, [math]::Round($curPerTool, 3), [math]::Round($basePerTool, 3), [math]::Round(($ratio - 1) * 100, 1))
+ }
+ else {
+ Write-Host " [SKIP] discovery_scaling_efficiency: all-mode metrics absent"
+ }
+
+ if ($evaluated -eq 0) {
+ # Nothing overlapped between results and baseline — an incompatible or empty baseline
+ # must not be reported as a pass.
+ $failures += 'no_comparable_metrics'
+ Write-Host " [FAIL] no comparable latency metrics between results and baseline (incompatible or empty baseline)"
+ }
+
+ return [ordered]@{ Failures = $failures; Warnings = $warnings }
+}
+
+# ---------------------------------------------------------------------------
+# Normalize an output file path so its parent directory always exists.
+# A bare filename (no directory component) otherwise makes Split-Path return an
+# empty string, which causes New-Item -Path '' to fail. Relative paths are
+# resolved against BaseDirectory so the parent directory is unambiguous.
+# Returns the absolute file path.
+# ---------------------------------------------------------------------------
+function Resolve-PerfOutputPath {
+ [CmdletBinding()]
+ param(
+ [Parameter(Mandatory)][string] $Path,
+ [string] $BaseDirectory = (Get-Location).Path
+ )
+
+ if (-not [System.IO.Path]::IsPathRooted($Path)) {
+ $Path = Join-Path $BaseDirectory $Path
+ }
+ $Path = [System.IO.Path]::GetFullPath($Path)
+
+ $parent = Split-Path -Parent $Path
+ if ($parent -and -not (Test-Path $parent)) {
+ $null = New-Item -ItemType Directory -Force -Path $parent
+ }
+
+ return $Path
+}
+
+# ---------------------------------------------------------------------------
+# Reads the catalog_version from the versioned workload catalog (eng/perf-workloads.json),
+# used to stamp every result so a number can be tied to the workload definition that produced
+# it (issue #3120). Returns 'unknown' if the catalog is missing or unreadable.
+# ---------------------------------------------------------------------------
+function Get-WorkloadCatalogVersion {
+ [CmdletBinding()]
+ param(
+ [string] $CatalogPath = (Join-Path $PSScriptRoot '../perf-workloads.json')
+ )
+
+ try {
+ if (Test-Path $CatalogPath) {
+ return (Get-Content $CatalogPath -Raw | ConvertFrom-Json).catalog_version
+ }
+ }
+ catch {
+ # Fall through to the default below.
+ }
+ return 'unknown'
+}
+
+# ---------------------------------------------------------------------------
+# Captures the controlled-environment metadata every perf result must record (issue #3120):
+# commit, runtime, OS, CPU, machine memory, plus the workload catalog version. Harnesses embed
+# this under an "environment" key so results are self-describing and comparable across runs.
+# Memory is best-effort and cross-platform; unresolved values are null rather than fatal.
+# ---------------------------------------------------------------------------
+function Get-PerfRunMetadata {
+ [CmdletBinding()]
+ param([string] $RepoRoot)
+
+ # Resolve the commit against the repository that holds the script, not the caller's
+ # working directory, so invoking a harness by absolute path from another checkout
+ # (or from outside any repo) still stamps the correct commit.
+ $gitLocation = if ($RepoRoot) { @('-C', $RepoRoot) } else { @() }
+ $commit = try { (& git @gitLocation rev-parse --short HEAD 2>$null).Trim() } catch { '' }
+
+ $memoryGb = $null
+ try {
+ if ($IsWindows) {
+ $bytes = (Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction Stop).TotalPhysicalMemory
+ $memoryGb = [math]::Round($bytes / 1GB, 1)
+ }
+ elseif ($IsLinux -and (Test-Path '/proc/meminfo')) {
+ $kb = ((Get-Content '/proc/meminfo' | Where-Object { $_ -match '^MemTotal:' }) -replace '[^\d]', '')
+ if ($kb) { $memoryGb = [math]::Round([double]$kb / 1048576, 1) }
+ }
+ }
+ catch {
+ $memoryGb = $null
+ }
+
+ return [ordered]@{
+ commit = $commit
+ timestamp = (Get-Date).ToUniversalTime().ToString('o')
+ runtime = [System.Runtime.InteropServices.RuntimeInformation]::FrameworkDescription
+ os = [System.Runtime.InteropServices.RuntimeInformation]::OSDescription
+ os_architecture = "$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)"
+ cpu_cores = [Environment]::ProcessorCount
+ machine_memory_gb = $memoryGb
+ workload_catalog_version = Get-WorkloadCatalogVersion
+ }
+}
diff --git a/eng/scripts/Test-StartupPerformance.ps1 b/eng/scripts/Test-StartupPerformance.ps1
new file mode 100644
index 000000000..6f565fc58
--- /dev/null
+++ b/eng/scripts/Test-StartupPerformance.ps1
@@ -0,0 +1,381 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Measures end-to-end startup performance of the Azure MCP Server across four scenarios:
+ 1. CLI cold start : azmcp tools list (process spawn → exit)
+ 2. MCP stdio default : azmcp server start → initialize → tools/list response
+ 3. MCP stdio namespace : azmcp server start --mode namespace → tools/list response
+ 4. MCP stdio all-tools : azmcp server start --mode all → tools/list response
+
+ For each MCP scenario the script also tokenizes the tools/list response using the
+ GPT-4o o200k_base encoding (via Microsoft.ML.Tokenizers) to give exact token budgets.
+
+.PARAMETER Executable
+ Path to the azmcp executable. Defaults to the Debug build in the repo.
+
+.PARAMETER Runs
+ Number of timing samples per scenario. Default: 5.
+
+.PARAMETER OutputPath
+ Optional path to write the JSON results file.
+ If omitted, results are written to .perf-results/startup-e2e.json under the repo root.
+
+.PARAMETER BaselinePath
+ Optional path to a JSON baseline file (previously produced by this script).
+ When provided, the script compares results against the baseline and fails if any
+ scenario exceeds the tiered budgets from issue #3118 (p50/p95 +10%, p99 +20%,
+ discovery scaling efficiency +15%; changes above +5% are reported as warnings).
+
+.EXAMPLE
+ # Default run against the Debug build
+ ./eng/scripts/Test-StartupPerformance.ps1
+
+.EXAMPLE
+ # Run against a Release build with more samples
+ ./eng/scripts/Test-StartupPerformance.ps1 -Executable ./servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe -Runs 10
+
+.EXAMPLE
+ # Compare against a saved baseline and fail on regression
+ ./eng/scripts/Test-StartupPerformance.ps1 -BaselinePath eng/perf-baseline.json
+#>
+
+param(
+ [string] $Executable,
+ [int] $Runs = 10,
+ [string] $OutputPath,
+ [string] $BaselinePath
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+# ---------------------------------------------------------------------------
+# Resolve repo root and executable path
+# ---------------------------------------------------------------------------
+$ScriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { $PWD.Path }
+$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
+
+# Shared, unit-tested helpers (Get-TimingStats, Resolve-PerfOutputPath).
+. (Join-Path $ScriptDir 'StartupPerformance.Common.ps1')
+
+if (-not $Executable) {
+ $Executable = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/src/bin/Debug/net10.0/azmcp.exe'
+ if ($IsLinux -or $IsMacOS) {
+ $Executable = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/src/bin/Debug/net10.0/azmcp'
+ }
+}
+
+if (-not (Test-Path $Executable)) {
+ Write-Error "azmcp executable not found at: $Executable`nBuild first: dotnet build servers/Azure.Mcp.Server/src"
+}
+
+# Resolve to a full path so child transports (e.g. the stdio SDK client) don't
+# fail on a leading './' that Windows cmd cannot parse.
+$Executable = (Resolve-Path $Executable).Path
+
+$OutputPath = if ($OutputPath) { $OutputPath } else { Join-Path $RepoRoot '.perf-results/startup-e2e.json' }
+$OutputPath = Resolve-PerfOutputPath -Path $OutputPath -BaseDirectory $RepoRoot
+
+# ---------------------------------------------------------------------------
+# Resolve benchmark exe path (Release build preferred, Debug as fallback)
+# ---------------------------------------------------------------------------
+$BenchmarkExeSuffix = if ($IsWindows) { '.exe' } else { '' }
+$BenchmarkExeBase = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin'
+$BenchmarkExe = Join-Path $BenchmarkExeBase "Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks$BenchmarkExeSuffix"
+if (-not (Test-Path $BenchmarkExe)) {
+ $BenchmarkExe = Join-Path $BenchmarkExeBase "Debug/net10.0/Azure.Mcp.Server.PerformanceBenchmarks$BenchmarkExeSuffix"
+}
+if (-not (Test-Path $BenchmarkExe)) {
+ Write-Error "Benchmark exe not found. Build first: dotnet build -c Release servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks"
+}
+
+Write-Host "azmcp executable : $Executable"
+Write-Host "Benchmark exe : $BenchmarkExe"
+Write-Host "Samples per scenario : $Runs"
+Write-Host ""
+
+# ---------------------------------------------------------------------------
+# Helper: measure-cli-coldstart
+# Time from OS process creation to process exit for a single CLI invocation.
+# ---------------------------------------------------------------------------
+function Measure-CliColdStart {
+ param([string] $ExePath)
+
+ $psi = [System.Diagnostics.ProcessStartInfo]@{
+ FileName = $ExePath
+ Arguments = 'tools list'
+ RedirectStandardOutput = $true
+ RedirectStandardError = $true
+ UseShellExecute = $false
+ CreateNoWindow = $true
+ }
+
+ $timeoutMs = 30000
+ $sw = [System.Diagnostics.Stopwatch]::StartNew()
+ $proc = [System.Diagnostics.Process]::Start($psi)
+ try {
+ # Drain both streams asynchronously to prevent buffer deadlocks
+ $stdoutTask = $proc.StandardOutput.ReadToEndAsync()
+ $stderrTask = $proc.StandardError.ReadToEndAsync()
+
+ if (-not $proc.WaitForExit($timeoutMs)) {
+ $sw.Stop()
+ try { $proc.Kill($true) }
+ catch { if (-not $proc.HasExited) { throw } }
+ $null = $proc.WaitForExit(5000)
+ $stderr = $stderrTask.GetAwaiter().GetResult()
+ $stderrMessage = if ([string]::IsNullOrWhiteSpace($stderr)) { '' } else { $stderr.Trim() }
+ throw "CLI cold start timed out after $timeoutMs ms for '$ExePath tools list'. stderr: $stderrMessage"
+ }
+
+ $sw.Stop()
+ $null = $stdoutTask.GetAwaiter().GetResult()
+ $stderr = $stderrTask.GetAwaiter().GetResult()
+
+ if ($proc.ExitCode -ne 0) {
+ $stderrMessage = if ([string]::IsNullOrWhiteSpace($stderr)) { '' } else { $stderr.Trim() }
+ throw "CLI cold start failed with exit code $($proc.ExitCode) for '$ExePath tools list'. stderr: $stderrMessage"
+ }
+
+ return $sw.ElapsedMilliseconds
+ }
+ finally {
+ if ($null -ne $proc) { $proc.Dispose() }
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Helper: invoke-benchmark-mcp-startup
+# Calls the benchmark binary with --mcp-startup so the official MCP C# SDK
+# performs the initialize + tools/list handshake and reports timing.
+# The stopwatch inside the binary starts before McpClient.CreateAsync (which
+# spawns the server process) and stops after ListToolsAsync completes.
+# Returns the parsed JSON result object:
+# { elapsed_ms, tool_count,
+# name_description: { bytes, exact_tokens_gpt4o_o200k, approx_tokens_bytes_div_4 },
+# full_schema: { bytes, exact_tokens_gpt4o_o200k, approx_tokens_bytes_div_4 } }
+# ---------------------------------------------------------------------------
+function Invoke-BenchmarkMcpStartup {
+ param(
+ [string] $BenchmarkExe,
+ [string] $ExePath,
+ [string[]] $ServerArgTokens,
+ [int] $TimeoutSeconds = 60,
+ # '--mcp-startup' (stdio) or '--mcp-startup-http' (HTTP transport).
+ [string] $BenchmarkMode = '--mcp-startup'
+ )
+
+ # Bounded execution (hard timeout, stderr capture, non-zero exit rejection) and JSON parsing
+ # are shared with the other perf runners via Invoke-PerfHarnessJson in StartupPerformance.Common.ps1.
+ return Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
+ -ArgumentList (@($BenchmarkMode, $ExePath) + $ServerArgTokens) `
+ -TimeoutSeconds $TimeoutSeconds
+}
+
+# ---------------------------------------------------------------------------
+# Helper: print a tools/list payload summary to the console
+# ---------------------------------------------------------------------------
+function Write-PayloadStats {
+ param([string] $Label, $Payload)
+ Write-Host "=== tools/list payload ($Label) ==="
+ Write-Host (" Tool count : {0}" -f $Payload.tool_count)
+ Write-Host (" Name+description bytes : {0:N0}" -f $Payload.name_description_bytes)
+ Write-Host (" Name+description exact tokens (GPT-4o) : {0:N0}" -f $Payload.name_description_tokens_exact)
+ Write-Host (" Full schema bytes (incl. inputSchema) : {0:N0}" -f $Payload.full_schema_bytes)
+ Write-Host (" Full schema exact tokens (GPT-4o) [LLM cost]: {0:N0}" -f $Payload.full_schema_tokens_exact)
+ Write-Host (" Full schema serialization time (ms) : {0}" -f $Payload.full_schema_serialize_ms)
+ Write-Host ""
+}
+
+# ---------------------------------------------------------------------------
+# Helper: run one MCP scenario (timing loop + cold/warm split + percentiles +
+# token/serialization measurement). Works for both stdio and HTTP transports
+# via the $BenchmarkMode switch. Uses the MCP C# SDK (via the benchmark binary)
+# for protocol-correct timing.
+# Returns [ordered]@{ stats = ...; coldWarm = ...; payload = ... }
+# ---------------------------------------------------------------------------
+function Invoke-McpScenario {
+ param(
+ [string] $Label,
+ [string[]] $ServerArgs,
+ [string] $ExePath,
+ [int] $Runs,
+ [string] $BenchmarkExe,
+ [string] $BenchmarkMode = '--mcp-startup',
+ [int] $TimeoutSeconds = 60
+ )
+ Write-Host "=== $Label ==="
+ $ms = @()
+ $readinessMs = @()
+ $lastBenchResult = $null
+ for ($i = 1; $i -le $Runs; $i++) {
+ $benchResult = Invoke-BenchmarkMcpStartup -BenchmarkExe $BenchmarkExe `
+ -ExePath $ExePath -ServerArgTokens $ServerArgs `
+ -BenchmarkMode $BenchmarkMode -TimeoutSeconds $TimeoutSeconds
+ $ms += [long]$benchResult.elapsed_ms
+ if ($benchResult.PSObject.Properties.Name -contains 'readiness_ms') {
+ $readinessMs += [long]$benchResult.readiness_ms
+ }
+ $lastBenchResult = $benchResult
+ Write-Host (" Run {0}: {1} ms" -f $i, $benchResult.elapsed_ms)
+ }
+ $stats = Get-TimingStats -Samples $ms
+ $coldWarm = Split-ColdWarm -Samples $ms
+ Write-Host (" → cold={0} ms warm-median={1} ms p95={2} ms p99={3} ms" -f `
+ $coldWarm.cold_ms, $coldWarm.warm.median, $stats.p95, $stats.p99)
+ Write-Host ""
+ $payload = [ordered]@{
+ tool_count = [int]$lastBenchResult.tool_count
+ name_description_bytes = [int]$lastBenchResult.name_description.bytes
+ name_description_tokens_exact = [int]$lastBenchResult.name_description.exact_tokens_gpt4o_o200k
+ full_schema_bytes = [int]$lastBenchResult.full_schema.bytes
+ full_schema_tokens_exact = [int]$lastBenchResult.full_schema.exact_tokens_gpt4o_o200k
+ full_schema_serialize_ms = [double]$lastBenchResult.full_schema.serialize_ms
+ }
+ Write-PayloadStats -Label $Label -Payload $payload
+ $result = [ordered]@{ stats = $stats; coldWarm = $coldWarm; payload = $payload }
+ if ($readinessMs.Count -gt 0) {
+ $result.readiness = Get-TimingStats -Samples $readinessMs
+ }
+ return $result
+}
+
+# ---------------------------------------------------------------------------
+# Scenario 1 – CLI cold start
+# ---------------------------------------------------------------------------
+Write-Host "=== Scenario 1: CLI cold start (azmcp tools list) ==="
+$cliMs = @()
+for ($i = 1; $i -le $Runs; $i++) {
+ $ms = Measure-CliColdStart -ExePath $Executable
+ $cliMs += $ms
+ Write-Host (" Run {0}: {1} ms" -f $i, $ms)
+}
+
+$cli = Get-TimingStats -Samples $cliMs
+$cliColdWarm = Split-ColdWarm -Samples $cliMs
+Write-Host (" → cold={0} ms warm-median={1} ms p95={2} ms p99={3} ms" -f `
+ $cliColdWarm.cold_ms, $cliColdWarm.warm.median, $cli.p95, $cli.p99)
+Write-Host ""
+
+# ---------------------------------------------------------------------------
+# Scenarios 2-4 – MCP stdio startup across three server modes
+# Timing is performed by the benchmark binary using the official MCP C# SDK
+# (McpClient.CreateAsync + ListToolsAsync via StdioClientTransport).
+# ---------------------------------------------------------------------------
+$s2 = Invoke-McpScenario -Label 'Scenario 2: MCP stdio startup (default mode)' `
+ -ServerArgs @('server', 'start') -ExePath $Executable -Runs $Runs `
+ -BenchmarkExe $BenchmarkExe
+
+$s3 = Invoke-McpScenario -Label 'Scenario 3: MCP stdio startup (--mode namespace)' `
+ -ServerArgs @('server', 'start', '--mode', 'namespace') -ExePath $Executable -Runs $Runs `
+ -BenchmarkExe $BenchmarkExe
+
+$s4 = Invoke-McpScenario -Label 'Scenario 4: MCP stdio startup (--mode all)' `
+ -ServerArgs @('server', 'start', '--mode', 'all') -ExePath $Executable -Runs $Runs `
+ -BenchmarkExe $BenchmarkExe
+
+# ---------------------------------------------------------------------------
+# Scenario 5 – MCP remote HTTP startup (default mode)
+# The benchmark binary spawns the server with --transport http
+# --dangerously-disable-http-incoming-auth on a free loopback port, polls
+# readiness, then times the initialize + tools/list handshake over
+# HttpClientTransport. readiness_ms captures server-ready time separately.
+# HTTP startup includes web-host build + Kestrel bind, so it needs a longer
+# per-run timeout than stdio.
+# ---------------------------------------------------------------------------
+$s5 = Invoke-McpScenario -Label 'Scenario 5: MCP remote HTTP startup (default mode)' `
+ -ServerArgs @('server', 'start') -ExePath $Executable -Runs $Runs `
+ -BenchmarkExe $BenchmarkExe -BenchmarkMode '--mcp-startup-http' `
+ -TimeoutSeconds 90
+
+# ---------------------------------------------------------------------------
+# Write results JSON
+# ---------------------------------------------------------------------------
+$gitCommit = (& git -C $RepoRoot rev-parse --short HEAD 2>$null) ?? 'unknown'
+
+$results = [ordered]@{
+ timestamp = (Get-Date -Format 'o')
+ commit = $gitCommit
+ executable = $Executable
+ runs = $Runs
+ environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
+ scenarios = [ordered]@{
+ cli_cold_start_ms = $cli
+ cli_cold_warm = $cliColdWarm
+ mcp_stdio_to_tools_list_ms = $s2.stats
+ mcp_stdio_cold_warm = $s2.coldWarm
+ tools_list_payload = $s2.payload
+ mcp_namespace_mode_startup_ms = $s3.stats
+ mcp_namespace_cold_warm = $s3.coldWarm
+ namespace_mode_tools_list_payload = $s3.payload
+ mcp_all_mode_startup_ms = $s4.stats
+ mcp_all_cold_warm = $s4.coldWarm
+ all_mode_tools_list_payload = $s4.payload
+ mcp_http_default_startup_ms = $s5.stats
+ mcp_http_cold_warm = $s5.coldWarm
+ mcp_http_readiness_ms = $s5.readiness
+ http_default_tools_list_payload = $s5.payload
+ }
+}
+
+$resultsJson = $results | ConvertTo-Json -Depth 10
+$resultsJson | Out-File -FilePath $OutputPath -Encoding utf8 -Force
+Write-Host "Results written to: $OutputPath"
+
+# ---------------------------------------------------------------------------
+# Optional: regression check against a baseline file
+# ---------------------------------------------------------------------------
+if ($BaselinePath) {
+ if (-not (Test-Path $BaselinePath)) {
+ Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
+ }
+ else {
+ Write-Host ""
+ Write-Host "=== Regression check (p50/p95 +10%, p99 +20%, scaling +15%, warn +5%) ==="
+
+ $baseline = Get-Content $BaselinePath | ConvertFrom-Json
+ $gate = Invoke-StartupRegressionGate -ResultScenarios $results.scenarios `
+ -BaselineScenarios $baseline.scenarios
+
+ if ($gate.Warnings.Count -gt 0) {
+ Write-Warning "Startup performance warnings (>5%): $($gate.Warnings -join ', ')"
+ }
+
+ if ($gate.Failures.Count -gt 0) {
+ Write-Error "Startup regression detected in: $($gate.Failures -join ', ')"
+ }
+ else {
+ Write-Host " All budget checks passed."
+ }
+ }
+}
+
+# ---------------------------------------------------------------------------
+# Emit Azure DevOps pipeline variables (no-op outside AzDO)
+# ---------------------------------------------------------------------------
+Write-Host "##vso[task.setvariable variable=CliColdStartMedianMs]$($cli.median)"
+Write-Host "##vso[task.setvariable variable=CliColdStartColdMs]$($cliColdWarm.cold_ms)"
+Write-Host "##vso[task.setvariable variable=CliColdStartWarmMedianMs]$($cliColdWarm.warm.median)"
+Write-Host "##vso[task.setvariable variable=CliColdStartP95Ms]$($cli.p95)"
+Write-Host "##vso[task.setvariable variable=CliColdStartP99Ms]$($cli.p99)"
+Write-Host "##vso[task.setvariable variable=McpStdioStartupMedianMs]$($s2.stats.median)"
+Write-Host "##vso[task.setvariable variable=McpStdioStartupP95Ms]$($s2.stats.p95)"
+Write-Host "##vso[task.setvariable variable=McpStdioStartupP99Ms]$($s2.stats.p99)"
+Write-Host "##vso[task.setvariable variable=ToolsListToolCount]$($s2.payload.tool_count)"
+Write-Host "##vso[task.setvariable variable=ToolsListNameDescTokensGpt4o]$($s2.payload.name_description_tokens_exact)"
+Write-Host "##vso[task.setvariable variable=ToolsListFullSchemaTokensGpt4o]$($s2.payload.full_schema_tokens_exact)"
+Write-Host "##vso[task.setvariable variable=ToolsListFullSchemaSerializeMs]$($s2.payload.full_schema_serialize_ms)"
+Write-Host "##vso[task.setvariable variable=McpNamespaceModeStartupMedianMs]$($s3.stats.median)"
+Write-Host "##vso[task.setvariable variable=NamespaceModeToolsListToolCount]$($s3.payload.tool_count)"
+Write-Host "##vso[task.setvariable variable=NamespaceModeToolsListFullSchemaTokensGpt4o]$($s3.payload.full_schema_tokens_exact)"
+Write-Host "##vso[task.setvariable variable=AllModeStartupMedianMs]$($s4.stats.median)"
+Write-Host "##vso[task.setvariable variable=AllModeStartupP95Ms]$($s4.stats.p95)"
+Write-Host "##vso[task.setvariable variable=AllModeStartupP99Ms]$($s4.stats.p99)"
+Write-Host "##vso[task.setvariable variable=AllModeToolsListToolCount]$($s4.payload.tool_count)"
+Write-Host "##vso[task.setvariable variable=AllModeToolsListFullSchemaTokensGpt4o]$($s4.payload.full_schema_tokens_exact)"
+Write-Host "##vso[task.setvariable variable=McpHttpDefaultStartupMedianMs]$($s5.stats.median)"
+Write-Host "##vso[task.setvariable variable=McpHttpDefaultStartupP95Ms]$($s5.stats.p95)"
+Write-Host "##vso[task.setvariable variable=McpHttpReadinessMedianMs]$(if ($s5.Contains('readiness')) { $s5.readiness.median } else { 'n/a' })"
+Write-Host "##vso[build.addbuildtag]perf-tracked"
diff --git a/eng/scripts/Update-StartupPerformanceTrend.ps1 b/eng/scripts/Update-StartupPerformanceTrend.ps1
new file mode 100644
index 000000000..cba966c09
--- /dev/null
+++ b/eng/scripts/Update-StartupPerformanceTrend.ps1
@@ -0,0 +1,106 @@
+#!/bin/env pwsh
+#Requires -Version 7
+
+<#
+.SYNOPSIS
+ Appends a startup-performance run to a persistent trend file and regenerates a
+ Markdown trend report.
+
+.DESCRIPTION
+ Reads the results JSON produced by Test-StartupPerformance.ps1 and appends a compact,
+ one-line-per-run record to a JSON Lines (.jsonl) trend file. It then regenerates a
+ Markdown table summarizing the most recent runs so reviewers can eyeball the trend
+ (issue #3118: "Baseline results and trend report").
+
+ The JSONL file is the durable, machine-readable history; the Markdown file is a
+ human-readable rendering of the tail. Both are safe to commit as build artifacts.
+
+.PARAMETER ResultsPath
+ Path to the results JSON written by Test-StartupPerformance.ps1.
+
+.PARAMETER TrendPath
+ Path to the JSONL trend file to append to. Created if it does not exist.
+ Defaults to .perf-results/startup-trend.jsonl under the repo root.
+
+.PARAMETER MarkdownPath
+ Path to the Markdown report to (re)write. Defaults to .perf-results/startup-trend.md.
+
+.PARAMETER MaxRows
+ Maximum number of most-recent rows to render in the Markdown table (default 20).
+#>
+
+param(
+ [Parameter(Mandatory)][string] $ResultsPath,
+ [string] $TrendPath,
+ [string] $MarkdownPath,
+ [int] $MaxRows = 20
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
+
+$ScriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { $PWD.Path }
+$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
+
+if (-not $TrendPath) { $TrendPath = Join-Path $RepoRoot '.perf-results/startup-trend.jsonl' }
+if (-not $MarkdownPath) { $MarkdownPath = Join-Path $RepoRoot '.perf-results/startup-trend.md' }
+
+$TrendPath = Resolve-PerfOutputPath -Path $TrendPath -BaseDirectory $RepoRoot
+$MarkdownPath = Resolve-PerfOutputPath -Path $MarkdownPath -BaseDirectory $RepoRoot
+
+$results = Get-Content $ResultsPath | ConvertFrom-Json
+$s = $results.scenarios
+
+# Compact record: one flat object per run so the JSONL stays diff-friendly.
+$record = [ordered]@{
+ timestamp = $results.timestamp
+ commit = $results.commit
+ runs = $results.runs
+ cli_cold_ms = (Get-MemberValue (Get-MemberValue $s 'cli_cold_start_ms') 'median')
+ cli_p95_ms = (Get-MemberValue (Get-MemberValue $s 'cli_cold_start_ms') 'p95')
+ stdio_default_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_stdio_to_tools_list_ms') 'median')
+ stdio_default_p95_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_stdio_to_tools_list_ms') 'p95')
+ all_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_all_mode_startup_ms') 'median')
+ all_p95_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_all_mode_startup_ms') 'p95')
+ http_default_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_http_default_startup_ms') 'median')
+ http_readiness_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_http_readiness_ms') 'median')
+ default_tool_count = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'tool_count')
+ default_full_schema_tokens = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'full_schema_tokens_exact')
+ all_tool_count = (Get-MemberValue (Get-MemberValue $s 'all_mode_tools_list_payload') 'tool_count')
+ all_full_schema_tokens = (Get-MemberValue (Get-MemberValue $s 'all_mode_tools_list_payload') 'full_schema_tokens_exact')
+ default_full_schema_serialize_ms = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'full_schema_serialize_ms')
+}
+
+# Append as a single compact JSON line.
+$recordJson = $record | ConvertTo-Json -Depth 5 -Compress
+Add-Content -Path $TrendPath -Value $recordJson -Encoding utf8
+Write-Host "Appended trend record to: $TrendPath"
+
+# Re-render the Markdown tail from the full JSONL history.
+$allRecords = Get-Content $TrendPath | Where-Object { $_.Trim() } | ForEach-Object { $_ | ConvertFrom-Json }
+$recent = @($allRecords | Select-Object -Last $MaxRows)
+
+$sb = [System.Text.StringBuilder]::new()
+$null = $sb.AppendLine('# Azure MCP Server startup performance trend')
+$null = $sb.AppendLine()
+$null = $sb.AppendLine("_Last updated: $(Get-Date -Format 'o'). Showing the $($recent.Count) most recent of $($allRecords.Count) runs._")
+$null = $sb.AppendLine()
+$null = $sb.AppendLine('| Commit | CLI cold (ms) | stdio p50/p95 (ms) | all p50/p95 (ms) | HTTP p50 (ms) | HTTP ready (ms) | default tools | default tokens | all tokens |')
+$null = $sb.AppendLine('|--------|--------------:|-------------------:|-----------------:|--------------:|----------------:|--------------:|---------------:|-----------:|')
+foreach ($r in $recent) {
+ $null = $sb.AppendLine(("| {0} | {1} | {2}/{3} | {4}/{5} | {6} | {7} | {8} | {9} | {10} |" -f `
+ (Get-MemberValue $r 'commit'),
+ (Get-MemberValue $r 'cli_cold_ms'),
+ (Get-MemberValue $r 'stdio_default_median_ms'), (Get-MemberValue $r 'stdio_default_p95_ms'),
+ (Get-MemberValue $r 'all_median_ms'), (Get-MemberValue $r 'all_p95_ms'),
+ (Get-MemberValue $r 'http_default_median_ms'),
+ (Get-MemberValue $r 'http_readiness_median_ms'),
+ (Get-MemberValue $r 'default_tool_count'),
+ (Get-MemberValue $r 'default_full_schema_tokens'),
+ (Get-MemberValue $r 'all_full_schema_tokens')))
+}
+
+$sb.ToString() | Out-File -FilePath $MarkdownPath -Encoding utf8 -Force
+Write-Host "Wrote Markdown trend report to: $MarkdownPath"
diff --git a/servers/Azure.Mcp.Server/build.yml b/servers/Azure.Mcp.Server/build.yml
index b4e9cf975..1fce35d5a 100644
--- a/servers/Azure.Mcp.Server/build.yml
+++ b/servers/Azure.Mcp.Server/build.yml
@@ -29,6 +29,7 @@ extends:
PublishTarget: ${{ parameters.PublishTarget }}
RunLiveTests: true
RunVSCodeIntegrationTests: ${{ parameters.RunVSCodeIntegrationTests }}
+ RunPerfTests: true
PackageDocker: true
PackageVSIX: true
PackagePyPI: true
diff --git a/servers/Azure.Mcp.Server/changelog-entries/1777318267439.yaml b/servers/Azure.Mcp.Server/changelog-entries/1777318267439.yaml
new file mode 100644
index 000000000..f521e8cfa
--- /dev/null
+++ b/servers/Azure.Mcp.Server/changelog-entries/1777318267439.yaml
@@ -0,0 +1,3 @@
+changes:
+ - section: "Other Changes"
+ description: "Added startup performance benchmarks and end-to-end measurement script (eng/scripts/Test-StartupPerformance.ps1) that measures CLI cold-start time and MCP startup time across stdio (default, namespace, all-tools) and remote HTTP transports, reporting cold vs. warm latency, p50/p95/p99 percentiles, server readiness time, tools/list JSON serialization time, and exact GPT-4o token budgets for each payload. A tool-catalog scaling benchmark measures discovery cost at current and projected tool counts. Results are gated against a baseline with tiered budgets (p50/p95 within 10%, p99 within 20%, scaling efficiency within 15%), summarized in a running trend report (eng/scripts/Update-StartupPerformanceTrend.ps1), and wired into the nightly AzDO pipeline via a new `Perf` stage in `common.yml`."
diff --git a/servers/Azure.Mcp.Server/changelog-entries/concurrency-scaling-benchmarks.yaml b/servers/Azure.Mcp.Server/changelog-entries/concurrency-scaling-benchmarks.yaml
new file mode 100644
index 000000000..de9cbf556
--- /dev/null
+++ b/servers/Azure.Mcp.Server/changelog-entries/concurrency-scaling-benchmarks.yaml
@@ -0,0 +1,3 @@
+changes:
+ - section: "Other Changes"
+ description: "Added concurrency, throughput, and scalability validation for the Azure MCP Server (issue #3122). A new end-to-end harness (eng/scripts/Measure-ConcurrencyScaling.ps1 with a --mcp-concurrency mode) launches the server over HTTP and sweeps a ladder of concurrency levels, driving parallel MCP clients at each level and recording throughput, latency p50/p95/p99, error/timeout rate, concurrency scaling efficiency, and a state-isolation signal (concurrent responses whose tool count does not match the expected count, indicating cross-request leakage or a thread-safety failure). The resulting saturation curve shows where throughput plateaus. Results are gated against a baseline with the tiered budgets from issue #3122 (throughput within 10%, p95 latency and scaling efficiency within 15%, p99 latency within 20%, error-rate increase within 0.5 percentage points, and no state leakage), and wired into the AzDO perf pipeline."
diff --git a/servers/Azure.Mcp.Server/changelog-entries/dispatch-overhead-benchmarks.yaml b/servers/Azure.Mcp.Server/changelog-entries/dispatch-overhead-benchmarks.yaml
new file mode 100644
index 000000000..7eb01d3d5
--- /dev/null
+++ b/servers/Azure.Mcp.Server/changelog-entries/dispatch-overhead-benchmarks.yaml
@@ -0,0 +1,3 @@
+changes:
+ - section: "Other Changes"
+ description: "Added command-dispatch and transport overhead benchmarks for the Azure MCP Server (issue #3119). New BenchmarkDotNet micro-benchmarks (CommandDispatchBenchmarks) measure each server-only dispatch layer in isolation — routing, argument parsing, option binding, validation, the execution frame, and response serialization — using a deterministic no-op command so no Azure work is included, which pinpoints the responsible layer for a regression. A new end-to-end harness (eng/scripts/Measure-DispatchOverhead.ps1 with --mcp-dispatch-stdio/--mcp-dispatch-http modes) times tools/list over stdio and HTTP and subtracts to isolate transport-specific overhead. Results are gated against a baseline with the tiered budgets from issue #3119 (p50/p95 within 10%, p99 within 20%, transport-overhead changes above 5% reported), and wired into the AzDO perf pipeline."
diff --git a/servers/Azure.Mcp.Server/changelog-entries/resource-usage-baselines.yaml b/servers/Azure.Mcp.Server/changelog-entries/resource-usage-baselines.yaml
new file mode 100644
index 000000000..85d3a20a5
--- /dev/null
+++ b/servers/Azure.Mcp.Server/changelog-entries/resource-usage-baselines.yaml
@@ -0,0 +1,3 @@
+changes:
+ - section: "Other Changes"
+ description: "Added CPU, memory, and handle resource baselines for the Azure MCP Server (eng/scripts/Measure-ResourceUsage.ps1 and a new --mcp-resource harness) that sample process resource counters across the idle, startup, discovery, steady-state, and sustained soak phases under configurable concurrency. Soak leak detection fits a least-squares trend (with a warmup exclusion) to flag unbounded memory or handle growth. Results are gated against a baseline with the tiered budgets from issue #3121 (steady-state CPU and memory within 10%, peak memory within 15%, no unbounded soak growth), and wired into the AzDO perf pipeline."
diff --git a/servers/Azure.Mcp.Server/changelog-entries/workload-catalog-metadata.yaml b/servers/Azure.Mcp.Server/changelog-entries/workload-catalog-metadata.yaml
new file mode 100644
index 000000000..601b3617c
--- /dev/null
+++ b/servers/Azure.Mcp.Server/changelog-entries/workload-catalog-metadata.yaml
@@ -0,0 +1,3 @@
+changes:
+ - section: "Other Changes"
+ description: "Added a versioned, machine-readable performance workload catalog (eng/perf-workloads.json) and controlled-environment definition for the Azure MCP Server (issue #3120). The catalog defines every perf workload (startup, discovery scaling, dispatch, resource, concurrency) with its harness, transport, server mode, warm-up, repetitions, and metrics, plus shared environment requirements, statistical rules, and baseline-qualification governance. Every measurement harness now embeds a self-describing environment block (commit, runtime, OS, CPU, machine memory, and workload catalog version) in its results, and the default measured repetitions were raised to at least 10."
diff --git a/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj
new file mode 100644
index 000000000..aa46ce12b
--- /dev/null
+++ b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj
@@ -0,0 +1,31 @@
+
+
+ Exe
+ false
+
+ false
+ false
+ false
+ false
+ true
+ $(NoWarn);IL2026;IL3050;IL2075;IL2072
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ PreserveNewest
+
+
+
diff --git a/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/CommandDispatchBenchmarks.cs b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/CommandDispatchBenchmarks.cs
new file mode 100644
index 000000000..be727e8ee
--- /dev/null
+++ b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/CommandDispatchBenchmarks.cs
@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.CommandLine;
+using System.Text.Json;
+using Azure.Mcp.Server;
+using Azure.Mcp.Server.Perf.Dispatch;
+using BenchmarkDotNet.Attributes;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Mcp.Core.Commands;
+using Microsoft.Mcp.Core.Models;
+using Microsoft.Mcp.Core.Models.Command;
+
+namespace Azure.Mcp.Server.Perf.Benchmarks;
+
+///
+/// Measures the Azure MCP Server's own per-call dispatch overhead in isolation (issue #3119):
+/// the cost of routing a tool name to a command, parsing MCP arguments, binding and validating
+/// options, running the execution frame, and serializing the response — all with a deterministic
+/// so no Azure or network latency is included.
+///
+/// Each layer is a separate benchmark so a regression can be attributed to the responsible layer.
+/// [MemoryDiagnoser] also records allocations per layer.
+///
+/// Run with:
+/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks `
+/// -- --filter '*CommandDispatch*' --buildTimeout 600
+///
+[MemoryDiagnoser]
+public class CommandDispatchBenchmarks
+{
+ // A real, registered, Azure-free-to-look-up command name used to measure routing cost.
+ private const string RealToolName = "subscription_list";
+
+ private ServiceProvider _serviceProvider = null!;
+ private ICommandFactory _commandFactory = null!;
+ private NoOpCommand _noop = null!;
+ private IBaseCommand _noopBase = null!;
+ private Command _noopSysCommand = null!;
+ private Dictionary _args = null!;
+ private ParseResult _parseResult = null!;
+ private NoOpCommand.NoOpOptions _boundOptions = null!;
+ private CommandResponse _response = null!;
+
+ [GlobalSetup]
+ public async Task Setup()
+ {
+ // Build the full server so routing measures a realistic command map (~500+ commands).
+ var services = new ServiceCollection();
+ Program.ConfigureServices(services);
+ services.AddLogging(builder => builder.SetMinimumLevel(LogLevel.None));
+ _serviceProvider = services.BuildServiceProvider();
+ await Program.InitializeServicesAsync(_serviceProvider);
+ _commandFactory = _serviceProvider.GetRequiredService();
+
+ _noop = new NoOpCommand();
+ _noopBase = _noop;
+ _noopSysCommand = _noop.GetCommand();
+
+ _args = new Dictionary
+ {
+ ["value"] = JsonSerializer.SerializeToElement("hello"),
+ ["count"] = JsonSerializer.SerializeToElement(3),
+ };
+
+ _noopSysCommand.TryParseFromDictionary(_args, out var parseResult, out _);
+ _parseResult = parseResult!;
+ _boundOptions = _noop.BindOptions(_parseResult);
+
+ // A populated response for the serialization benchmark.
+ _response = await _noopBase.ExecuteAsync(new CommandContext(), _parseResult, CancellationToken.None);
+ }
+
+ [GlobalCleanup]
+ public async Task Cleanup()
+ {
+ // The full DI container holds services that only implement IAsyncDisposable
+ // (e.g. CosmosService), so it must be disposed asynchronously.
+ if (_serviceProvider is not null)
+ {
+ await _serviceProvider.DisposeAsync();
+ }
+ }
+
+ [Benchmark(Description = "Routing – CommandFactory tool-name lookup")]
+ public IBaseCommand? Routing() => _commandFactory.FindCommandByName(RealToolName);
+
+ [Benchmark(Description = "Parse – MCP arguments dictionary to ParseResult")]
+ public bool Parse() => _noopSysCommand.TryParseFromDictionary(_args, out _, out _);
+
+ [Benchmark(Description = "Bind – ParseResult to options object")]
+ public NoOpCommand.NoOpOptions Bind() => _noop.BindOptions(_parseResult);
+
+ [Benchmark(Description = "Validate – option validation")]
+ public bool Validate()
+ {
+ var validationResult = new ValidationResult();
+ _noop.ValidateOptions(_boundOptions, validationResult);
+ return validationResult.IsValid;
+ }
+
+ [Benchmark(Description = "SerializeResponse – CommandResponse to MCP JSON")]
+ public string SerializeResponse()
+ => JsonSerializer.Serialize(_response, ModelsJsonContext.Default.CommandResponse);
+
+ [Benchmark(Description = "ExecuteFrame – bind + validate + execute + capture")]
+ public async Task ExecuteFrame()
+ => await _noopBase.ExecuteAsync(new CommandContext(), _parseResult, CancellationToken.None);
+
+ [Benchmark(Description = "FullDispatch – parse + bind + validate + execute + serialize")]
+ public async Task FullDispatch()
+ {
+ _noopSysCommand.TryParseFromDictionary(_args, out var parseResult, out _);
+ var response = await _noopBase.ExecuteAsync(new CommandContext(), parseResult!, CancellationToken.None);
+ return JsonSerializer.Serialize(response, ModelsJsonContext.Default.CommandResponse);
+ }
+}
diff --git a/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ServerStartupBenchmarks.cs b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ServerStartupBenchmarks.cs
new file mode 100644
index 000000000..a7d8bccc2
--- /dev/null
+++ b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ServerStartupBenchmarks.cs
@@ -0,0 +1,156 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Azure.Mcp.Server;
+using Azure.Mcp.Server.Perf;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Mcp.Core.Commands;
+
+namespace Azure.Mcp.Server.Perf.Benchmarks;
+
+///
+/// Measures each phase of the Azure MCP Server startup pipeline in isolation so regressions
+/// can be pinpointed to the specific phase that changed.
+///
+/// Each phase's prerequisites are built in a per-target
+/// hook so they are excluded from the timed body — e.g. the BuildServiceProvider benchmark
+/// times only BuildServiceProvider(), not the ConfigureServices work that precedes it.
+/// To keep setup/measurement one-to-one, the job runs a single invocation per iteration
+/// (InvocationCount = 1, UnrollFactor = 1), which is the pattern BenchmarkDotNet requires
+/// when IterationSetup establishes non-idempotent state.
+///
+/// Run with:
+/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks -- --filter *
+///
+[Config(typeof(IsolatedPhaseConfig))]
+public class ServerStartupBenchmarks
+{
+ // Per-phase prerequisites, prepared in IterationSetup so they are not part of the
+ // timed body. Each is disposed in DisposeProviders after the iteration is recorded.
+ private IServiceCollection _servicesForConfigure = null!;
+ private IServiceCollection _servicesForBuild = null!;
+ private ServiceProvider? _builtProvider;
+ private ServiceProvider? _providerForInitialize;
+ private ServiceProvider? _providerForCommandTree;
+
+ private sealed class IsolatedPhaseConfig : ManualConfig
+ {
+ public IsolatedPhaseConfig()
+ {
+ AddJob(Job.Default
+ .WithLaunchCount(1)
+ .WithWarmupCount(2)
+ .WithIterationCount(10)
+ .WithInvocationCount(1)
+ .WithUnrollFactor(1));
+ AddDiagnoser(MemoryDiagnoser.Default);
+ }
+ }
+
+ [IterationCleanup]
+ public async Task DisposeProviders()
+ {
+ // Providers may resolve async-only-disposable services (e.g. CosmosService implements
+ // IAsyncDisposable but not IDisposable), for which a synchronous Dispose() throws.
+ // Dispose asynchronously in the cleanup hook, outside the timed benchmark body.
+ await (_builtProvider?.DisposeAsync() ?? ValueTask.CompletedTask);
+ await (_providerForInitialize?.DisposeAsync() ?? ValueTask.CompletedTask);
+ await (_providerForCommandTree?.DisposeAsync() ?? ValueTask.CompletedTask);
+ _builtProvider = null;
+ _providerForInitialize = null;
+ _providerForCommandTree = null;
+ }
+
+ // -------------------------------------------------------------------------
+ // Phase 1 – instantiate all IAreaSetup objects (the 60+ toolset registrations)
+ // -------------------------------------------------------------------------
+
+ [Benchmark(Description = "RegisterAreas – instantiate all IAreaSetup objects")]
+ public object RegisterAreas() => Program.RegisterAreas();
+
+ // -------------------------------------------------------------------------
+ // Phase 2 – wire the DI container (call all area.ConfigureServices)
+ // -------------------------------------------------------------------------
+
+ [IterationSetup(Target = nameof(ConfigureServices))]
+ public void SetupConfigureServices() => _servicesForConfigure = new ServiceCollection();
+
+ [Benchmark(Description = "ConfigureServices – register services into the DI container")]
+ public IServiceCollection ConfigureServices()
+ {
+ Program.ConfigureServices(_servicesForConfigure);
+ return _servicesForConfigure;
+ }
+
+ // -------------------------------------------------------------------------
+ // Phase 3 – compile the DI container (BuildServiceProvider)
+ // -------------------------------------------------------------------------
+
+ [IterationSetup(Target = nameof(BuildServiceProvider))]
+ public void SetupBuildServiceProvider() => _servicesForBuild = CreateSilentServices();
+
+ [Benchmark(Description = "BuildServiceProvider – compile the DI container")]
+ public ServiceProvider BuildServiceProvider()
+ {
+ _builtProvider = _servicesForBuild.BuildServiceProvider();
+ return _builtProvider;
+ }
+
+ // -------------------------------------------------------------------------
+ // Phase 4 – async service initialization (telemetry, user-agent policy, etc.)
+ // -------------------------------------------------------------------------
+
+ [IterationSetup(Target = nameof(InitializeServicesAsync))]
+ public void SetupInitializeServices() => _providerForInitialize = CreateSilentServices().BuildServiceProvider();
+
+ [Benchmark(Description = "InitializeServicesAsync – async service warm-up")]
+ public async Task InitializeServicesAsync() => await Program.InitializeServicesAsync(_providerForInitialize!);
+
+ // -------------------------------------------------------------------------
+ // Phase 5 – build the full System.CommandLine command tree
+ // -------------------------------------------------------------------------
+
+ [IterationSetup(Target = nameof(BuildCommandTree))]
+ public void SetupBuildCommandTree()
+ {
+ _providerForCommandTree = CreateSilentServices().BuildServiceProvider();
+ Program.InitializeServicesAsync(_providerForCommandTree).GetAwaiter().GetResult();
+ }
+
+ [Benchmark(Description = "CommandFactory.RootCommand – build the full command tree")]
+ public System.CommandLine.RootCommand BuildCommandTree() =>
+ _providerForCommandTree!.GetRequiredService().RootCommand;
+
+ // -------------------------------------------------------------------------
+ // Full pipeline – DI build + service init + command tree.
+ // This is the cumulative end-to-end measurement, so ConfigureServices/BuildServiceProvider
+ // remain inside the timed body by design. RegisterAreas runs once as a static initializer
+ // and is NOT included here (it has its own dedicated benchmark above).
+ // -------------------------------------------------------------------------
+
+ [Benchmark(Description = "FullStartup – DI build + service init + command tree (excludes RegisterAreas static init)")]
+ public async Task FullStartup()
+ {
+ await using var sp = CreateSilentServices().BuildServiceProvider();
+ await Program.InitializeServicesAsync(sp);
+ return sp.GetRequiredService().RootCommand;
+ }
+
+ // -------------------------------------------------------------------------
+ // Shared setup – registers all services with logging suppressed
+ // -------------------------------------------------------------------------
+
+ private static IServiceCollection CreateSilentServices()
+ {
+ var services = new ServiceCollection();
+ Program.ConfigureServices(services);
+ services.AddLogging(b => b.SetMinimumLevel(LogLevel.None));
+ return services;
+ }
+}
+
diff --git a/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ToolSurfaceScalingBenchmarks.cs b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ToolSurfaceScalingBenchmarks.cs
new file mode 100644
index 000000000..99295d445
--- /dev/null
+++ b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Benchmarks/ToolSurfaceScalingBenchmarks.cs
@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Text.Json;
+using BenchmarkDotNet.Attributes;
+using ModelContextProtocol.Protocol;
+
+namespace Azure.Mcp.Server.Perf.Benchmarks;
+
+///
+/// Measures how tool-discovery cost scales as the exposed tool catalog grows, at current
+/// and projected tool counts (issue #3118: "Measure tool and namespace discovery at
+/// current and projected tool counts" / "Track scaling behavior as the tool catalog grows").
+///
+/// For each the benchmark builds a synthetic tool catalog whose
+/// per-tool shape (name + description + a small typed JSON input schema) approximates a real
+/// Azure MCP tool, then times the JSON serialization of the whole catalog — the dominant,
+/// transport-independent cost of answering a tools/list request. Comparing the mean time per
+/// tool across counts yields the discovery scaling-efficiency metric: if serialization is
+/// linear, mean/ToolCount stays roughly constant as the catalog grows.
+///
+/// Run with:
+/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks -- --filter *ToolSurfaceScaling*
+///
+[MemoryDiagnoser]
+public class ToolSurfaceScalingBenchmarks
+{
+ // 71 ≈ current default surface, 510 ≈ current all-tools surface; the rest bracket
+ // projected growth so the scaling curve is visible on both sides of today's counts.
+ [Params(71, 250, 510, 1000)]
+ public int ToolCount { get; set; }
+
+ private Tool[] _tools = [];
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ _tools = new Tool[ToolCount];
+ for (var i = 0; i < ToolCount; i++)
+ {
+ _tools[i] = CreateSyntheticTool(i);
+ }
+ }
+
+ [Benchmark(Description = "SerializeToolCatalog – serialize the full tools/list payload")]
+ public string SerializeToolCatalog() => JsonSerializer.Serialize(_tools);
+
+ ///
+ /// Builds a single synthetic tool whose name, description, and input schema are sized to
+ /// resemble a typical Azure MCP tool so per-tool serialization cost is representative.
+ ///
+ private static Tool CreateSyntheticTool(int index)
+ {
+ var inputSchema = JsonDocument.Parse($$"""
+ {
+ "type": "object",
+ "properties": {
+ "subscription": { "type": "string", "description": "The Azure subscription id or name for tool {{index}}." },
+ "resourceGroup": { "type": "string", "description": "The resource group name." },
+ "name": { "type": "string", "description": "The target resource name." },
+ "limit": { "type": "integer", "description": "Maximum number of results to return." }
+ },
+ "required": ["subscription"]
+ }
+ """).RootElement;
+
+ return new Tool
+ {
+ Name = $"azmcp_service{index % 60}_resource_operation{index}",
+ Description = $"Performs operation {index} against an Azure resource, returning a structured result. " +
+ "Use this tool when the user asks to inspect or manage the corresponding Azure resource.",
+ InputSchema = inputSchema,
+ };
+ }
+}
diff --git a/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/ConcurrencyScalingMeasurement.cs b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/ConcurrencyScalingMeasurement.cs
new file mode 100644
index 000000000..12973d31f
--- /dev/null
+++ b/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/ConcurrencyScalingMeasurement.cs
@@ -0,0 +1,224 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.Json;
+using ModelContextProtocol.Client;
+
+namespace Azure.Mcp.Server.Perf;
+
+///
+/// Measures the Azure MCP Server's behavior under concurrent load (issue #3122). It launches
+/// the server over HTTP and drives a sweep of concurrency levels; at each level a pool of
+/// parallel MCP clients issues tools/list for a fixed window, and the harness records the
+/// throughput, latency distribution (p50/p95/p99), error/timeout rate, and a state-isolation
+/// signal (responses whose tool count does not match the expected count — a sign of a
+/// thread-safety or request-state-leakage failure). Per-level scaling efficiency is the
+/// throughput relative to perfect linear scaling from the first level, so the emitted "saturation
+/// curve" shows where throughput plateaus or collapses.
+///
+/// Mode:
+/// --mcp-concurrency <exe> [serverArgs...]
+///
+/// Tunable via environment variables:
+/// PERF_CONC_LEVELS comma-separated concurrency levels (default "1,2,4,8,16")
+/// PERF_CONC_DURATION_SECONDS timed window per level (default 5)
+/// PERF_CONC_WARMUP_SECONDS warmup window per level, not counted (default 1)
+/// PERF_CONC_TIMEOUT_MS per-request timeout; a timeout counts as an error (default 10000)
+///
+/// Emits a single compact JSON line for the PowerShell harness to parse.
+///
+internal static class ConcurrencyScalingMeasurement
+{
+ private static readonly TimeSpan s_readinessTimeout = TimeSpan.FromSeconds(60);
+
+ internal static async Task RunAsync(string exePath, string[] serverArgs)
+ {
+ var levels = GetEnvIntArray("PERF_CONC_LEVELS", [1, 2, 4, 8, 16]);
+ var durationSeconds = GetEnvInt("PERF_CONC_DURATION_SECONDS", 5);
+ var warmupSeconds = GetEnvInt("PERF_CONC_WARMUP_SECONDS", 1);
+ var timeoutMs = GetEnvInt("PERF_CONC_TIMEOUT_MS", 10000);
+
+ var port = PerfServerProcess.GetFreeLoopbackPort();
+ var endpoint = new Uri($"http://127.0.0.1:{port}");
+ using var server = PerfServerProcess.StartHttpServer(exePath, serverArgs, endpoint);
+
+ try
+ {
+ var readinessMs = await PerfServerProcess.WaitForPortAsync(port, s_readinessTimeout, server);
+
+ // Establish the expected tool count once; concurrent responses that differ from it
+ // indicate cross-request state leakage or a thread-safety failure.
+ int expectedToolCount;
+ {
+ await using var probe = await McpClient.CreateAsync(NewHttpTransport(endpoint, "perf-conc-probe"));
+ expectedToolCount = (await probe.ListToolsAsync()).Count;
+ }
+
+ var levelResults = new List