mirror of
https://github.com/microsoft/mcp.git
synced 2026-11-04 12:01:31 +00:00
Add Performance Scenarios + Pipeline Infra (#2510)
This commit is contained in:
Vinay Gera authored and 
GitHub committed
1 parent
8002bb1613
commit
5b563effc6
50 files changed
+5103
-2
No files matched your search
@@ -29,6 +29,9 @@ generated/
|
||||
.vscode/settings.json
|
||||
|
||||
eng/tools/ToolDescriptionEvaluator/prompts.json
|
||||
.perf-results/
|
||||
BenchmarkDotNet.Artifacts/
|
||||
build_output.txt
|
||||
eng/tools/ToolDescriptionEvaluator/results.md
|
||||
eng/tools/ToolDescriptionEvaluator/tools.json
|
||||
deploy-log.txt
|
||||
|
||||
Vendored
+6
@@ -63,6 +63,7 @@
|
||||
"aadsts",
|
||||
"accesstoken",
|
||||
"addattachment",
|
||||
"addbuildtag",
|
||||
"agentpool",
|
||||
"aiml",
|
||||
"amazonq",
|
||||
@@ -83,12 +84,15 @@
|
||||
"classfilters",
|
||||
"cloudevents",
|
||||
"codesign",
|
||||
"coldstart",
|
||||
"confidentialledger",
|
||||
"configfile",
|
||||
"creds",
|
||||
"credscan",
|
||||
"datatable",
|
||||
"devcert",
|
||||
"diagnoser",
|
||||
"diagnosers",
|
||||
"diagnosticservices",
|
||||
"doubleslash",
|
||||
"dyld",
|
||||
@@ -150,7 +154,9 @@
|
||||
"testdb",
|
||||
"testproxy",
|
||||
"testrg",
|
||||
"testruns",
|
||||
"testsub",
|
||||
"tiktoken",
|
||||
"toplevel",
|
||||
"uniqueprefix",
|
||||
"vectorizable",
|
||||
|
||||
@@ -124,5 +124,9 @@
|
||||
<PackageVersion Include="xunit.v3.extensibility.core" Version="3.2.2" />
|
||||
<PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
|
||||
<PackageVersion Include="coverlet.MTP" Version="10.0.1" />
|
||||
<!-- Performance benchmark dependencies -->
|
||||
<PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
|
||||
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0" />
|
||||
<PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="0.22.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -78,6 +78,9 @@
|
||||
</Folder>
|
||||
<Folder Name="/servers/" />
|
||||
<Folder Name="/servers/Azure.Mcp.Server/" />
|
||||
<Folder Name="/servers/Azure.Mcp.Server/perf/">
|
||||
<Project Path="servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/servers/Azure.Mcp.Server/src/">
|
||||
<Project Path="servers/Azure.Mcp.Server/src/Azure.Mcp.Server.csproj" />
|
||||
</Folder>
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
{
|
||||
"catalog_version": "1.0.0",
|
||||
"description": "Versioned, machine-readable definitions of Azure MCP Server performance workloads and the controlled-environment and statistical rules they run under (issue #3120). Bump catalog_version and record the reason whenever a workload definition changes, since a change can shift baselines.",
|
||||
"environment_requirements": {
|
||||
"runner": "Dedicated or sufficiently isolated runner. Baseline-quality runs must not share the host with other CI work; developer laptops are for exploration only.",
|
||||
"runtime": "The .NET SDK pinned by global.json; record the exact runtime in each result's environment block.",
|
||||
"build_configuration": "Release.",
|
||||
"network": "Loopback only for server-overhead workloads. The measured path must exclude Azure and network time (deterministic no-op / tools/list).",
|
||||
"configuration": "Server launched with the documented args per workload; no unrelated environment variables set."
|
||||
},
|
||||
"statistical_rules": {
|
||||
"warmup": "Every scenario performs warm-up iterations that are discarded before measurement.",
|
||||
"min_repetitions": 10,
|
||||
"percentiles": ["p50", "p95", "p99"],
|
||||
"reported_metrics": ["p50", "p95", "p99", "throughput", "cpu", "memory"],
|
||||
"comparison": "Compare current vs approved baseline as ratios. Changes below 5% are ignored, changes above 5% warn, and budget breaches block. Prefer median/percentiles over the mean for latency.",
|
||||
"notes": "Micro-benchmarks additionally report BenchmarkDotNet StdDev / confidence intervals; the host environment (OS, CPU, runtime) is captured in the BenchmarkDotNet report header."
|
||||
},
|
||||
"baseline_qualification": {
|
||||
"min_successful_runs": 10,
|
||||
"min_span_days": 14,
|
||||
"controlled_runner_required": true,
|
||||
"auto_update_after_regression": false,
|
||||
"replacement_requires": ["linked issue", "measured impact", "owner approval"]
|
||||
},
|
||||
"workloads": [
|
||||
{
|
||||
"id": "startup-cli-cold",
|
||||
"issue": 3118,
|
||||
"harness": "eng/scripts/Test-StartupPerformance.ps1",
|
||||
"transport": "cli",
|
||||
"server_mode": null,
|
||||
"description": "CLI cold start: 'azmcp tools list' (process spawn to exit).",
|
||||
"warmup": "First run reported as cold; the remainder are warm.",
|
||||
"repetitions": 10,
|
||||
"metrics": ["p50", "p95", "p99", "cold_ms", "warm_median_ms"]
|
||||
},
|
||||
{
|
||||
"id": "startup-stdio-default",
|
||||
"issue": 3118,
|
||||
"harness": "eng/scripts/Test-StartupPerformance.ps1",
|
||||
"transport": "stdio",
|
||||
"server_mode": "default",
|
||||
"description": "MCP stdio startup to a usable tool list: 'server start' then initialize + tools/list (~71 tools).",
|
||||
"warmup": "First run reported as cold; the remainder are warm.",
|
||||
"repetitions": 10,
|
||||
"metrics": ["p50", "p95", "p99", "full_schema_tokens_exact", "full_schema_serialize_ms"]
|
||||
},
|
||||
{
|
||||
"id": "startup-stdio-namespace",
|
||||
"issue": 3118,
|
||||
"harness": "eng/scripts/Test-StartupPerformance.ps1",
|
||||
"transport": "stdio",
|
||||
"server_mode": "namespace",
|
||||
"description": "MCP stdio startup with 'server start --mode namespace'.",
|
||||
"warmup": "First run reported as cold; the remainder are warm.",
|
||||
"repetitions": 10,
|
||||
"metrics": ["p50", "p95", "p99"]
|
||||
},
|
||||
{
|
||||
"id": "startup-stdio-all",
|
||||
"issue": 3118,
|
||||
"harness": "eng/scripts/Test-StartupPerformance.ps1",
|
||||
"transport": "stdio",
|
||||
"server_mode": "all",
|
||||
"description": "MCP stdio startup with 'server start --mode all' (full tool surface, ~500+ tools).",
|
||||
"warmup": "First run reported as cold; the remainder are warm.",
|
||||
"repetitions": 10,
|
||||
"metrics": ["p50", "p95", "p99", "full_schema_tokens_exact", "full_schema_serialize_ms"]
|
||||
},
|
||||
{
|
||||
"id": "startup-http-default",
|
||||
"issue": 3118,
|
||||
"harness": "eng/scripts/Test-StartupPerformance.ps1",
|
||||
"transport": "http",
|
||||
"server_mode": "default",
|
||||
"description": "MCP remote HTTP startup: initialize + tools/list over the HTTP transport, plus server readiness time.",
|
||||
"warmup": "First run reported as cold; the remainder are warm.",
|
||||
"repetitions": 10,
|
||||
"metrics": ["p50", "p95", "p99", "readiness_ms"]
|
||||
},
|
||||
{
|
||||
"id": "discovery-scaling",
|
||||
"issue": 3118,
|
||||
"harness": "ToolSurfaceScalingBenchmarks (BenchmarkDotNet)",
|
||||
"transport": "in-process",
|
||||
"server_mode": null,
|
||||
"description": "tools/list serialization cost at current and projected tool counts.",
|
||||
"tool_counts": [71, 250, 510, 1000],
|
||||
"warmup": "BenchmarkDotNet warmup iterations.",
|
||||
"repetitions": "BenchmarkDotNet default job (>=15 iterations).",
|
||||
"metrics": ["mean", "allocated"]
|
||||
},
|
||||
{
|
||||
"id": "dispatch-layers",
|
||||
"issue": 3119,
|
||||
"harness": "CommandDispatchBenchmarks (BenchmarkDotNet)",
|
||||
"transport": "in-process",
|
||||
"server_mode": null,
|
||||
"description": "Per-layer command dispatch overhead (routing, parse, bind, validate, serialize, execute frame, full dispatch) on a deterministic no-op command; excludes Azure.",
|
||||
"warmup": "BenchmarkDotNet warmup iterations.",
|
||||
"repetitions": "BenchmarkDotNet default job (>=15 iterations).",
|
||||
"metrics": ["mean", "allocated"]
|
||||
},
|
||||
{
|
||||
"id": "dispatch-transport-stdio",
|
||||
"issue": 3119,
|
||||
"harness": "eng/scripts/Measure-DispatchOverhead.ps1",
|
||||
"transport": "stdio",
|
||||
"server_mode": "default",
|
||||
"description": "tools/list request overhead over stdio.",
|
||||
"warmup": "PERF_DISPATCH_WARMUP calls (default 20), not counted.",
|
||||
"repetitions": 200,
|
||||
"metrics": ["p50", "p95", "p99"]
|
||||
},
|
||||
{
|
||||
"id": "dispatch-transport-http",
|
||||
"issue": 3119,
|
||||
"harness": "eng/scripts/Measure-DispatchOverhead.ps1",
|
||||
"transport": "http",
|
||||
"server_mode": "default",
|
||||
"description": "tools/list request overhead over the HTTP transport; the http-minus-stdio p50 delta isolates transport cost.",
|
||||
"warmup": "PERF_DISPATCH_WARMUP calls (default 20), not counted.",
|
||||
"repetitions": 200,
|
||||
"metrics": ["p50", "p95", "p99", "transport_overhead_tools_list_p50"]
|
||||
},
|
||||
{
|
||||
"id": "resource-default",
|
||||
"issue": 3121,
|
||||
"harness": "eng/scripts/Measure-ResourceUsage.ps1",
|
||||
"transport": "http",
|
||||
"server_mode": "default",
|
||||
"description": "CPU/memory/handle sampling across idle, startup, discovery, steady-state, and soak phases for the default tool surface.",
|
||||
"phases": ["startup", "idle", "discovery", "steady_state", "soak"],
|
||||
"concurrency": 4,
|
||||
"warmup": "Soak analysis discards the first 25% of samples.",
|
||||
"repetitions": "Continuous 100ms sampling per phase.",
|
||||
"metrics": ["working_set_mb", "private_mb", "handles", "cpu_percent", "soak_slope"]
|
||||
},
|
||||
{
|
||||
"id": "resource-all",
|
||||
"issue": 3121,
|
||||
"harness": "eng/scripts/Measure-ResourceUsage.ps1",
|
||||
"transport": "http",
|
||||
"server_mode": "all",
|
||||
"description": "CPU/memory/handle sampling for the full tool surface.",
|
||||
"phases": ["startup", "idle", "discovery", "steady_state", "soak"],
|
||||
"concurrency": 4,
|
||||
"warmup": "Soak analysis discards the first 25% of samples.",
|
||||
"repetitions": "Continuous 100ms sampling per phase.",
|
||||
"metrics": ["working_set_mb", "private_mb", "handles", "cpu_percent", "soak_slope"]
|
||||
},
|
||||
{
|
||||
"id": "concurrency-sweep",
|
||||
"issue": 3122,
|
||||
"harness": "eng/scripts/Measure-ConcurrencyScaling.ps1",
|
||||
"transport": "http",
|
||||
"server_mode": "default",
|
||||
"description": "Concurrency sweep over parallel MCP clients issuing tools/list; produces a saturation curve.",
|
||||
"concurrency_levels": [1, 2, 4, 8, 16],
|
||||
"warmup": "PERF_CONC_WARMUP_SECONDS per level (default 1s), not counted.",
|
||||
"repetitions": "PERF_CONC_DURATION_SECONDS timed window per level (default 5s).",
|
||||
"metrics": ["throughput_rps", "p50", "p95", "p99", "error_rate_pct", "scaling_efficiency", "tool_count_mismatches"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -33,6 +33,9 @@ parameters:
|
||||
- name: BuildTimeoutInMinutes
|
||||
type: number
|
||||
default: 90
|
||||
- name: RunPerfTests
|
||||
type: boolean
|
||||
default: false
|
||||
- name: IsTestPipeline
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -119,6 +122,30 @@ extends:
|
||||
jobs:
|
||||
- template: /eng/pipelines/templates/jobs/vsix/pack-vsix-pr.yml
|
||||
|
||||
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.RunPerfTests, 'true')) }}:
|
||||
- stage: Perf
|
||||
displayName: 'Performance test'
|
||||
dependsOn:
|
||||
- Build
|
||||
pool:
|
||||
name: $(WINDOWSPOOL)
|
||||
image: $(WINDOWSVMIMAGE)
|
||||
os: windows
|
||||
variables:
|
||||
- template: /eng/pipelines/templates/variables/image.yml
|
||||
- template: /eng/pipelines/templates/variables/globals.yml
|
||||
jobs:
|
||||
- template: /eng/pipelines/templates/jobs/perf-test.yml
|
||||
parameters:
|
||||
Runs: 10
|
||||
# Uncomment to gate on regressions once a baseline is committed:
|
||||
# BaselinePath: eng/perf-baseline.json
|
||||
# ResourceBaselinePath: eng/resource-baseline.json
|
||||
# DispatchBaselinePath: eng/dispatch-baseline.json
|
||||
# ConcurrencyBaselinePath: eng/concurrency-baseline.json
|
||||
# Uncomment to run the (~30 min) BenchmarkDotNet micro-benchmarks:
|
||||
# RunMicroBenchmarks: true
|
||||
|
||||
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.RunLiveTests, 'true')) }}:
|
||||
- stage: Test
|
||||
displayName: 'Live test'
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
parameters:
|
||||
- name: Runs
|
||||
type: number
|
||||
default: 10
|
||||
- name: BaselinePath
|
||||
type: string
|
||||
default: ''
|
||||
# Path to a JSON baseline file. When set, a second run gates on the tiered
|
||||
# budgets from issue #3118 (p50/p95 +10%, p99 +20%, scaling efficiency +15%).
|
||||
- name: ResourceConcurrency
|
||||
type: number
|
||||
default: 4
|
||||
- name: ResourceSoakSeconds
|
||||
type: number
|
||||
default: 45
|
||||
- name: ResourceBaselinePath
|
||||
type: string
|
||||
default: ''
|
||||
# Path to a JSON resource baseline. When set, gates on the resource budgets
|
||||
# from issue #3121 (steady CPU/memory +10%, peak memory +15%, no soak leak).
|
||||
- name: DispatchBaselinePath
|
||||
type: string
|
||||
default: ''
|
||||
# Path to a JSON dispatch baseline. When set, gates on the dispatch budgets
|
||||
# from issue #3119 (p50/p95 +10%, p99 +20%, transport change >5% reported).
|
||||
- name: ConcurrencyBaselinePath
|
||||
type: string
|
||||
default: ''
|
||||
# Path to a JSON concurrency baseline. When set, gates on the concurrency budgets
|
||||
# from issue #3122 (throughput -10%, p95/scaling -15%, p99 -20%, error +0.5pp).
|
||||
- name: RunMicroBenchmarks
|
||||
type: boolean
|
||||
default: false
|
||||
# The BenchmarkDotNet micro-benchmarks are opt-in: the generated boilerplate
|
||||
# references the whole server (60+ tool assemblies) and the full suite takes
|
||||
# ~30 minutes, so enable this only for dedicated/nightly perf runs.
|
||||
- name: MicroBenchmarkFilter
|
||||
type: string
|
||||
default: '*'
|
||||
# Glob passed to BenchmarkDotNet --filter (e.g. '*ServerStartup*' to scope a run).
|
||||
|
||||
jobs:
|
||||
- job: PerfTest
|
||||
displayName: "Performance test"
|
||||
timeoutInMinutes: 60
|
||||
pool:
|
||||
name: $(WINDOWSPOOL)
|
||||
image: $(WINDOWSVMIMAGE)
|
||||
os: windows
|
||||
templateContext:
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
condition: always()
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/perf
|
||||
artifactName: perf-results
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
|
||||
- task: UseDotNet@2
|
||||
displayName: "Use .NET SDK from global.json"
|
||||
retryCountOnTaskFailure: 3
|
||||
inputs:
|
||||
useGlobalJson: true
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Build benchmark tool (Release)"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Build-PerfBenchmarks.ps1
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Run performance measurements"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Test-StartupPerformance.ps1
|
||||
arguments: >
|
||||
-Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
|
||||
-Runs ${{ parameters.Runs }}
|
||||
-OutputPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
|
||||
|
||||
# Regression gate — only active when BaselinePath is provided.
|
||||
# Reads the already-written results JSON; does NOT re-run the benchmarks.
|
||||
- ${{ if ne(parameters.BaselinePath, '') }}:
|
||||
- task: Powershell@2
|
||||
displayName: "Check regression against baseline"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Check-StartupPerformanceRegression.ps1
|
||||
arguments: >
|
||||
-ResultsPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
|
||||
-BaselinePath '${{ parameters.BaselinePath }}'
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Run resource-usage measurements"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-ResourceUsage.ps1
|
||||
arguments: >
|
||||
-Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
|
||||
-BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
|
||||
-Modes default,all
|
||||
-Concurrency ${{ parameters.ResourceConcurrency }}
|
||||
-SoakSeconds ${{ parameters.ResourceSoakSeconds }}
|
||||
-OutputPath '$(Build.ArtifactStagingDirectory)/perf/resource-usage.json'
|
||||
|
||||
# Resource regression gate — only active when ResourceBaselinePath is provided.
|
||||
- ${{ if ne(parameters.ResourceBaselinePath, '') }}:
|
||||
- task: Powershell@2
|
||||
displayName: "Check resource regression against baseline"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Check-ResourceUsageRegression.ps1
|
||||
arguments: >
|
||||
-ResultsPath '$(Build.ArtifactStagingDirectory)/perf/resource-usage.json'
|
||||
-BaselinePath '${{ parameters.ResourceBaselinePath }}'
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Run dispatch-overhead measurements"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-DispatchOverhead.ps1
|
||||
arguments: >
|
||||
-Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
|
||||
-BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
|
||||
-OutputPath '$(Build.ArtifactStagingDirectory)/perf/dispatch-overhead.json'
|
||||
|
||||
# Dispatch regression gate — only active when DispatchBaselinePath is provided.
|
||||
- ${{ if ne(parameters.DispatchBaselinePath, '') }}:
|
||||
- task: Powershell@2
|
||||
displayName: "Check dispatch regression against baseline"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Check-DispatchOverheadRegression.ps1
|
||||
arguments: >
|
||||
-ResultsPath '$(Build.ArtifactStagingDirectory)/perf/dispatch-overhead.json'
|
||||
-BaselinePath '${{ parameters.DispatchBaselinePath }}'
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Run concurrency-scaling measurements"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Measure-ConcurrencyScaling.ps1
|
||||
arguments: >
|
||||
-Executable '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe'
|
||||
-BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
|
||||
-Levels 1,2,4,8,16
|
||||
-DurationSeconds 10
|
||||
-OutputPath '$(Build.ArtifactStagingDirectory)/perf/concurrency-scaling.json'
|
||||
|
||||
# Concurrency regression gate — only active when ConcurrencyBaselinePath is provided.
|
||||
- ${{ if ne(parameters.ConcurrencyBaselinePath, '') }}:
|
||||
- task: Powershell@2
|
||||
displayName: "Check concurrency regression against baseline"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Check-ConcurrencyScalingRegression.ps1
|
||||
arguments: >
|
||||
-ResultsPath '$(Build.ArtifactStagingDirectory)/perf/concurrency-scaling.json'
|
||||
-BaselinePath '${{ parameters.ConcurrencyBaselinePath }}'
|
||||
|
||||
# Runs the BenchmarkDotNet micro-benchmarks (per-phase startup allocations and
|
||||
# tool-catalog serialization scaling). Opt-in via RunMicroBenchmarks because the
|
||||
# generated boilerplate references the whole server (60+ tool assemblies) and the
|
||||
# full suite takes ~30 minutes. --buildTimeout is raised because that boilerplate
|
||||
# build exceeds BenchmarkDotNet's default 2-minute timeout. Each benchmark class
|
||||
# carries its own job config, so no --job override is passed.
|
||||
- ${{ if eq(parameters.RunMicroBenchmarks, true) }}:
|
||||
- task: Powershell@2
|
||||
displayName: "Run micro-benchmarks (BenchmarkDotNet)"
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Invoke-PerfMicroBenchmarks.ps1
|
||||
arguments: >
|
||||
-BenchmarkExe '$(Build.SourcesDirectory)/servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe'
|
||||
-ArtifactsPath '$(Build.ArtifactStagingDirectory)/perf/benchmark-dotnet'
|
||||
-Filter '${{ parameters.MicroBenchmarkFilter }}'
|
||||
|
||||
- task: Powershell@2
|
||||
displayName: "Publish metrics to Azure Monitor (optional)"
|
||||
condition: and(succeeded(), eq(variables['System.TeamProject'], 'internal'))
|
||||
continueOnError: true
|
||||
inputs:
|
||||
pwsh: true
|
||||
filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-StartupPerformanceMetrics.ps1
|
||||
arguments: >
|
||||
-ResultsPath '$(Build.ArtifactStagingDirectory)/perf/startup-e2e.json'
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Builds the Azure MCP Server performance-benchmark project in Release for the perf pipeline.
|
||||
|
||||
.DESCRIPTION
|
||||
Kept as a script (rather than inline pipeline YAML) so CI behavior can be reproduced locally.
|
||||
|
||||
NU1902 (the NuGet audit warning) is already non-fatal via WarningsNotAsErrors in
|
||||
Directory.Build.props. A command-line -p:NoWarn is deliberately NOT passed here: a global
|
||||
NoWarn property overrides every project's own <NoWarn> (for example the Insights project's
|
||||
MCP9005 sampling-deprecation suppression), which breaks a clean build.
|
||||
|
||||
.PARAMETER Project
|
||||
Path to the performance-benchmark csproj to build.
|
||||
#>
|
||||
param(
|
||||
[string] $Project = 'servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/Azure.Mcp.Server.PerformanceBenchmarks.csproj'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
dotnet build $Project -c Release -consoleLoggerParameters:NoSummary
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Gates the concurrency / throughput / scalability results against a saved baseline (#3122).
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Measure-ConcurrencyScaling.ps1 and a baseline JSON of the
|
||||
same shape, then applies the tiered budgets from issue #3122 per concurrency level:
|
||||
* throughput may not decrease by more than ThroughputDropPct (default 10%)
|
||||
* concurrency scaling efficiency may not degrade by more than ScalingDropPct (default 15%)
|
||||
* p95 latency may not degrade by more than LatencyP95RisePct (default 15%)
|
||||
* p99 latency may not regress by more than LatencyP99RisePct (default 20%)
|
||||
* timeout/server-error rate may not increase by more than ErrorRatePpLimit (default 0.5 points)
|
||||
* any concurrent tool-count mismatch is a hard failure (state leakage / thread safety)
|
||||
Fails (throws) if any budget is exceeded. Does not re-run any measurements.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Measure-ConcurrencyScaling.ps1.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Path to the baseline JSON to compare against.
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath,
|
||||
[Parameter(Mandatory)][string] $BaselinePath,
|
||||
[double] $ThroughputDropPct = 0.10,
|
||||
[double] $ScalingDropPct = 0.15,
|
||||
[double] $LatencyP95RisePct = 0.15,
|
||||
[double] $LatencyP99RisePct = 0.20,
|
||||
[double] $ErrorRatePpLimit = 0.5,
|
||||
[double] $WarnPct = 0.05
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
. (Join-Path $PSScriptRoot 'ConcurrencyPerformance.Common.ps1')
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $results.levels -BaselineLevels $baseline.levels `
|
||||
-ThroughputDropPct $ThroughputDropPct -ScalingDropPct $ScalingDropPct `
|
||||
-LatencyP95RisePct $LatencyP95RisePct -LatencyP99RisePct $LatencyP99RisePct `
|
||||
-ErrorRatePpLimit $ErrorRatePpLimit -WarnPct $WarnPct
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Concurrency warnings: $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Concurrency regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All concurrency budget checks passed."
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Gates the command-dispatch / transport overhead results against a saved baseline (#3119).
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Measure-DispatchOverhead.ps1 and a baseline JSON of the
|
||||
same shape, then applies the tiered budgets from issue #3119:
|
||||
* p50 and p95 of each transport/operation may not regress by more than FailP50P95 (+10%)
|
||||
* p99 may not regress by more than FailP99 (+20%)
|
||||
* any change above Warn (+5%), including transport overhead, is reported as a warning
|
||||
Fails (throws) if any budget is exceeded. Does not re-run any measurements.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Measure-DispatchOverhead.ps1.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Path to the baseline JSON to compare against.
|
||||
|
||||
.PARAMETER FailP50P95
|
||||
Maximum allowed p50/p95 regression ratio (default 1.10 = +10%).
|
||||
|
||||
.PARAMETER FailP99
|
||||
Maximum allowed p99 regression ratio (default 1.20 = +20%).
|
||||
|
||||
.PARAMETER Warn
|
||||
Ratio above which a change is reported as a warning (default 1.05 = +5%).
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath,
|
||||
[Parameter(Mandatory)][string] $BaselinePath,
|
||||
[double] $FailP50P95 = 1.10,
|
||||
[double] $FailP99 = 1.20,
|
||||
[double] $Warn = 1.05
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
. (Join-Path $PSScriptRoot 'DispatchPerformance.Common.ps1')
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios `
|
||||
-FailP50P95 $FailP50P95 -FailP99 $FailP99 -Warn $Warn
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Dispatch warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Dispatch regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All dispatch budget checks passed."
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Gates the resource-usage results against a saved baseline (issue #3121).
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Measure-ResourceUsage.ps1 and a baseline JSON of the
|
||||
same shape, then applies the tiered resource budgets from issue #3121:
|
||||
* steady-state CPU and memory may not regress by more than FailSteady (default +10%)
|
||||
* peak memory may not regress by more than FailPeak (default +15%)
|
||||
* a soak phase flagged with unbounded growth is a hard failure (memory/handle leak)
|
||||
* any change above Warn (default +5%) is reported as a warning
|
||||
Fails (throws) if any budget is exceeded. Does not re-run any measurements.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Measure-ResourceUsage.ps1.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Path to the baseline JSON to compare against.
|
||||
|
||||
.PARAMETER FailSteady
|
||||
Maximum allowed steady-state CPU/memory regression ratio (default 1.10 = +10%).
|
||||
|
||||
.PARAMETER FailPeak
|
||||
Maximum allowed peak-memory regression ratio (default 1.15 = +15%).
|
||||
|
||||
.PARAMETER Warn
|
||||
Ratio above which a change is reported as a warning (default 1.05 = +5%).
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath,
|
||||
[Parameter(Mandatory)][string] $BaselinePath,
|
||||
[double] $FailSteady = 1.10,
|
||||
[double] $FailPeak = 1.15,
|
||||
[double] $Warn = 1.05
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
. (Join-Path $PSScriptRoot 'ResourcePerformance.Common.ps1')
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios `
|
||||
-FailSteady $FailSteady -FailPeak $FailPeak -Warn $Warn
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Resource warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Resource regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All resource budget checks passed."
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Gates the startup performance results against a saved baseline.
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Test-StartupPerformance.ps1 and a baseline JSON
|
||||
of the same shape, then applies the tiered budgets from issue #3118:
|
||||
* p50 and p95 may not regress by more than FailP50P95 (default +10%)
|
||||
* p99 may not regress by more than FailP99 (default +20%)
|
||||
* discovery scaling efficiency may not degrade by more than ScalingDegrade (+15%)
|
||||
* any change above Warn (default +5%) is reported as a warning
|
||||
Fails (throws) if any budget is exceeded. Does not re-run any benchmarks.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Test-StartupPerformance.ps1.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Path to the baseline JSON to compare against.
|
||||
|
||||
.PARAMETER FailP50P95
|
||||
Maximum allowed p50/p95 regression ratio (default 1.10 = +10%).
|
||||
|
||||
.PARAMETER FailP99
|
||||
Maximum allowed p99 regression ratio (default 1.20 = +20%).
|
||||
|
||||
.PARAMETER ScalingDegrade
|
||||
Maximum allowed discovery-scaling-efficiency degradation ratio (default 1.15 = +15%).
|
||||
|
||||
.PARAMETER Warn
|
||||
Ratio above which a change is reported as a warning (default 1.05 = +5%).
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath,
|
||||
[Parameter(Mandatory)][string] $BaselinePath,
|
||||
[double] $FailP50P95 = 1.10,
|
||||
[double] $FailP99 = 1.20,
|
||||
[double] $ScalingDegrade = 1.15,
|
||||
[double] $Warn = 1.05
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios `
|
||||
-FailP50P95 $FailP50P95 -FailP99 $FailP99 `
|
||||
-ScalingDegrade $ScalingDegrade -Warn $Warn
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Performance warnings (>$([math]::Round(($Warn - 1) * 100))%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All budget checks passed."
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
# Install-Module -Name Pester -Force -SkipPublisherCheck
|
||||
# Invoke-Pester -Path eng/scripts/ConcurrencyPerformance.Common.Tests.ps1
|
||||
#
|
||||
# Focused tests for the pure concurrency-gate helpers used by Measure-ConcurrencyScaling.ps1.
|
||||
|
||||
BeforeAll {
|
||||
. $PSScriptRoot/ConcurrencyPerformance.Common.ps1
|
||||
|
||||
function script:New-Level {
|
||||
param($Concurrency, $Throughput, $Scaling, $P95, $P99, $ErrorRate = 0, $Mismatches = 0)
|
||||
return [ordered]@{
|
||||
concurrency = $Concurrency
|
||||
throughput_rps = $Throughput
|
||||
scaling_efficiency = $Scaling
|
||||
error_rate_pct = $ErrorRate
|
||||
tool_count_mismatches = $Mismatches
|
||||
latency_ms = [ordered]@{ p50 = ($P95 * 0.6); p95 = $P95; p99 = $P99 }
|
||||
}
|
||||
}
|
||||
|
||||
function script:New-UniformLevels {
|
||||
return @(
|
||||
(New-Level -Concurrency 1 -Throughput 100 -Scaling 1.0 -P95 5 -P99 7),
|
||||
(New-Level -Concurrency 4 -Throughput 350 -Scaling 0.9 -P95 8 -P99 12)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Test-ConcurrencyBudget" {
|
||||
It "reports the fraction dropped for a higher-is-better metric" {
|
||||
$r = Test-ConcurrencyBudget -Current 85 -Baseline 100 -FailFraction 0.10 -WarnFraction 0.05 -Direction 'higher'
|
||||
$r.Status | Should -Be 'FAIL'
|
||||
[math]::Round($r.Change, 2) | Should -Be 0.15
|
||||
}
|
||||
|
||||
It "reports the fraction risen for a lower-is-better metric" {
|
||||
# +20% rise is at the fail boundary (not > 0.20) but above the 5% warn threshold.
|
||||
$r = Test-ConcurrencyBudget -Current 12 -Baseline 10 -FailFraction 0.20 -WarnFraction 0.05 -Direction 'lower'
|
||||
$r.Status | Should -Be 'WARN'
|
||||
[math]::Round($r.Change, 2) | Should -Be 0.20
|
||||
}
|
||||
|
||||
It "skips when the baseline is not positive" {
|
||||
(Test-ConcurrencyBudget -Current 5 -Baseline 0 -FailFraction 0.1 -WarnFraction 0.05 -Direction 'higher').Status | Should -Be 'SKIP'
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Invoke-ConcurrencyRegressionGate" {
|
||||
It "reports no failures or warnings for an identical baseline" {
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels (New-UniformLevels) -BaselineLevels (New-UniformLevels)
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
@($gate.Warnings).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "fails when throughput drops more than 10%" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].throughput_rps = $b[1].throughput_rps * 0.85
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 throughput_rps'
|
||||
}
|
||||
|
||||
It "fails when scaling efficiency degrades more than 15%" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].scaling_efficiency = $b[1].scaling_efficiency * 0.80
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 scaling_efficiency'
|
||||
}
|
||||
|
||||
It "fails when p95 latency degrades more than 15%" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].latency_ms.p95 = $b[1].latency_ms.p95 * 1.20
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 latency_p95'
|
||||
}
|
||||
|
||||
It "fails when p99 latency regresses more than 20%" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].latency_ms.p99 = $b[1].latency_ms.p99 * 1.25
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 latency_p99'
|
||||
}
|
||||
|
||||
It "fails when the error rate increases more than 0.5 percentage points" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].error_rate_pct = $b[1].error_rate_pct + 0.6
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 error_rate_pct'
|
||||
}
|
||||
|
||||
It "fails on any concurrent tool-count mismatch (state leakage)" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].tool_count_mismatches = 3
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 state_leakage'
|
||||
}
|
||||
|
||||
It "warns but does not fail for a throughput drop between 5% and 10%" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].throughput_rps = $b[1].throughput_rps * 0.93
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
$gate.Warnings | Should -Contain 'c4 throughput_rps'
|
||||
}
|
||||
|
||||
It "skips a concurrency level absent from the baseline" {
|
||||
$s = New-UniformLevels
|
||||
$b = @($s[0]) # only concurrency 1
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "flags a state-leakage mismatch even when the level is absent from the baseline (F-008)" {
|
||||
$s = New-UniformLevels
|
||||
$s += (New-Level -Concurrency 16 -Throughput 900 -Scaling 0.7 -P95 20 -P99 30 -Mismatches 5)
|
||||
$b = New-UniformLevels # no concurrency 16
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c16 state_leakage'
|
||||
}
|
||||
|
||||
It "fails when no level overlaps the baseline (F-007)" {
|
||||
$s = @(New-Level -Concurrency 32 -Throughput 100 -Scaling 1.0 -P95 5 -P99 7)
|
||||
$b = New-UniformLevels # concurrency 1 and 4 only
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'no_comparable_metrics'
|
||||
}
|
||||
|
||||
It "fails when a latency metric the baseline expects is missing from results (F-007)" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].latency_ms.Remove('p95') # results dropped p95 at concurrency 4
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 latency_p95 (missing in results)'
|
||||
}
|
||||
|
||||
It "fails when the error-rate metric the baseline expects is missing from results (F-007)" {
|
||||
$s = New-UniformLevels
|
||||
$b = New-UniformLevels
|
||||
$s[1].Remove('error_rate_pct') # results dropped error_rate_pct at concurrency 4
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $s -BaselineLevels $b
|
||||
$gate.Failures | Should -Contain 'c4 error_rate_pct (missing in results)'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Side-effect-free helpers for the concurrency / throughput / scalability gate (issue #3122).
|
||||
|
||||
.DESCRIPTION
|
||||
Dot-source this file to reuse the pure functions in Measure-ConcurrencyScaling.ps1 and
|
||||
Check-ConcurrencyScalingRegression.ps1 and to unit-test them without launching a server.
|
||||
Reuses Get-MemberValue from StartupPerformance.Common.ps1.
|
||||
#>
|
||||
|
||||
. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Evaluates one budget. Direction 'higher' means bigger is better (throughput,
|
||||
# scaling efficiency) and reports the fraction dropped; 'lower' means smaller is
|
||||
# better (latency) and reports the fraction risen. Returns a status + change.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Test-ConcurrencyBudget {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[double] $Current,
|
||||
[double] $Baseline,
|
||||
[double] $FailFraction,
|
||||
[double] $WarnFraction,
|
||||
[ValidateSet('higher', 'lower')] [string] $Direction
|
||||
)
|
||||
|
||||
if ($Baseline -le 0) {
|
||||
return @{ Status = 'SKIP'; Change = 0 }
|
||||
}
|
||||
|
||||
$change = if ($Direction -eq 'higher') { 1 - ($Current / $Baseline) } else { ($Current / $Baseline) - 1 }
|
||||
$status = if ($change -gt $FailFraction) { 'FAIL' } elseif ($change -gt $WarnFraction) { 'WARN' } else { 'PASS' }
|
||||
return @{ Status = $status; Change = $change }
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tiered concurrency-regression gate shared by Measure-ConcurrencyScaling.ps1 and
|
||||
# Check-ConcurrencyScalingRegression.ps1. For each concurrency level present in both
|
||||
# results and baseline it applies the acceptance-criteria budgets from issue #3122:
|
||||
# * throughput may not decrease by more than ThroughputDropPct (default 10%)
|
||||
# * concurrency scaling efficiency may not degrade by more than ScalingDropPct (default 15%)
|
||||
# * p95 latency may not degrade by more than LatencyP95RisePct (default 15%)
|
||||
# * p99 latency may not regress by more than LatencyP99RisePct (default 20%)
|
||||
# * timeout/server-error rate may not increase by more than ErrorRatePpLimit (default 0.5 points)
|
||||
# * any concurrent tool-count mismatch is a hard failure (state leakage / thread safety)
|
||||
# * any change above WarnPct (default 5%) is reported as a warning
|
||||
#
|
||||
# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a line per check.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-ConcurrencyRegressionGate {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] $ResultLevels,
|
||||
[Parameter(Mandatory)] $BaselineLevels,
|
||||
[double] $ThroughputDropPct = 0.10,
|
||||
[double] $ScalingDropPct = 0.15,
|
||||
[double] $LatencyP95RisePct = 0.15,
|
||||
[double] $LatencyP99RisePct = 0.20,
|
||||
[double] $ErrorRatePpLimit = 0.5,
|
||||
[double] $WarnPct = 0.05
|
||||
)
|
||||
|
||||
$failures = @()
|
||||
$warnings = @()
|
||||
$evaluated = 0
|
||||
|
||||
$baselineByConcurrency = @{}
|
||||
foreach ($bl in $BaselineLevels) {
|
||||
$baselineByConcurrency[[string](Get-MemberValue $bl 'concurrency')] = $bl
|
||||
}
|
||||
|
||||
$seenConcurrency = @{}
|
||||
|
||||
foreach ($level in $ResultLevels) {
|
||||
$concurrency = Get-MemberValue $level 'concurrency'
|
||||
$seenConcurrency[[string]$concurrency] = $true
|
||||
|
||||
# State leakage / thread safety — a hard failure regardless of the baseline, so it is
|
||||
# evaluated before the baseline-presence check below can skip this level.
|
||||
$mismatches = Get-MemberValue $level 'tool_count_mismatches'
|
||||
if ($mismatches -and $mismatches -gt 0) {
|
||||
$failures += "c$concurrency state_leakage"
|
||||
Write-Host (" [FAIL] c{0} tool_count_mismatches={1} (state leakage / thread-safety failure)" -f $concurrency, $mismatches)
|
||||
}
|
||||
|
||||
$baseline = $baselineByConcurrency[[string]$concurrency]
|
||||
if ($null -eq $baseline) {
|
||||
Write-Host (" [SKIP] concurrency={0}: absent in baseline (relative budgets)" -f $concurrency)
|
||||
continue
|
||||
}
|
||||
|
||||
$curLatency = Get-MemberValue $level 'latency_ms'
|
||||
$baseLatency = Get-MemberValue $baseline 'latency_ms'
|
||||
|
||||
$checks = @(
|
||||
@{ Label = 'throughput_rps'; Cur = (Get-MemberValue $level 'throughput_rps'); Base = (Get-MemberValue $baseline 'throughput_rps'); Fail = $ThroughputDropPct; Dir = 'higher'; Unit = ' rps' }
|
||||
@{ Label = 'scaling_efficiency'; Cur = (Get-MemberValue $level 'scaling_efficiency'); Base = (Get-MemberValue $baseline 'scaling_efficiency'); Fail = $ScalingDropPct; Dir = 'higher'; Unit = '' }
|
||||
@{ Label = 'latency_p95'; Cur = (Get-MemberValue $curLatency 'p95'); Base = (Get-MemberValue $baseLatency 'p95'); Fail = $LatencyP95RisePct; Dir = 'lower'; Unit = ' ms' }
|
||||
@{ Label = 'latency_p99'; Cur = (Get-MemberValue $curLatency 'p99'); Base = (Get-MemberValue $baseLatency 'p99'); Fail = $LatencyP99RisePct; Dir = 'lower'; Unit = ' ms' }
|
||||
)
|
||||
|
||||
foreach ($check in $checks) {
|
||||
if ($null -eq $check.Base) {
|
||||
continue
|
||||
}
|
||||
if ($null -eq $check.Cur) {
|
||||
$failures += "c$concurrency $($check.Label) (missing in results)"
|
||||
Write-Host (" [FAIL] c{0} {1}: expected by baseline but absent in results" -f $concurrency, $check.Label)
|
||||
continue
|
||||
}
|
||||
$evaluated++
|
||||
$result = Test-ConcurrencyBudget -Current $check.Cur -Baseline $check.Base `
|
||||
-FailFraction $check.Fail -WarnFraction $WarnPct -Direction $check.Dir
|
||||
$label = "c$concurrency $($check.Label)"
|
||||
switch ($result.Status) {
|
||||
'FAIL' { $failures += $label }
|
||||
'WARN' { $warnings += $label }
|
||||
}
|
||||
Write-Host (" [{0}] {1}: current={2}{3} baseline={4}{3} ({5}{6}%)" -f `
|
||||
$result.Status, $label, $check.Cur, $check.Unit, $check.Base, `
|
||||
$(if ($result.Change -ge 0) { '+' } else { '' }), [math]::Round($result.Change * 100, 1))
|
||||
}
|
||||
|
||||
# Error/timeout rate — an absolute percentage-point budget.
|
||||
$curError = Get-MemberValue $level 'error_rate_pct'
|
||||
$baseError = Get-MemberValue $baseline 'error_rate_pct'
|
||||
if ($null -ne $baseError) {
|
||||
$label = "c$concurrency error_rate_pct"
|
||||
if ($null -eq $curError) {
|
||||
$failures += "$label (missing in results)"
|
||||
Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $label)
|
||||
}
|
||||
else {
|
||||
$evaluated++
|
||||
$delta = $curError - $baseError
|
||||
if ($delta -gt $ErrorRatePpLimit) {
|
||||
$failures += $label
|
||||
$status = 'FAIL'
|
||||
}
|
||||
else {
|
||||
$status = 'PASS'
|
||||
}
|
||||
Write-Host (" [{0}] {1}: current={2}% baseline={3}% (+{4} pp)" -f `
|
||||
$status, $label, $curError, $baseError, [math]::Round($delta, 3))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Baseline levels the results no longer cover are treated as an expected-but-missing metric.
|
||||
foreach ($blKey in $baselineByConcurrency.Keys) {
|
||||
if (-not $seenConcurrency.ContainsKey($blKey)) {
|
||||
$failures += "c$blKey missing_in_results"
|
||||
Write-Host (" [FAIL] concurrency={0}: expected by baseline but absent in results" -f $blKey)
|
||||
}
|
||||
}
|
||||
|
||||
if ($evaluated -eq 0) {
|
||||
$failures += 'no_comparable_metrics'
|
||||
Write-Host " [FAIL] no comparable concurrency metrics between results and baseline (incompatible or empty baseline)"
|
||||
}
|
||||
|
||||
return [ordered]@{ Failures = $failures; Warnings = $warnings }
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# Install-Module -Name Pester -Force -SkipPublisherCheck
|
||||
# Invoke-Pester -Path eng/scripts/DispatchPerformance.Common.Tests.ps1
|
||||
#
|
||||
# Focused tests for the pure dispatch-gate helpers used by Measure-DispatchOverhead.ps1.
|
||||
|
||||
BeforeAll {
|
||||
. $PSScriptRoot/DispatchPerformance.Common.ps1
|
||||
|
||||
function script:New-Stats {
|
||||
param($P50, $P95, $P99)
|
||||
return [ordered]@{ p50 = $P50; p95 = $P95; p99 = $P99 }
|
||||
}
|
||||
|
||||
function script:New-DispatchScenarios {
|
||||
param($StdioList, $HttpList)
|
||||
return [ordered]@{
|
||||
stdio = [ordered]@{ tools_list = $StdioList }
|
||||
http = [ordered]@{ tools_list = $HttpList }
|
||||
}
|
||||
}
|
||||
|
||||
function script:New-UniformScenarios {
|
||||
param($StdioP50 = 5.0, $HttpP50 = 6.0)
|
||||
$stdio = New-Stats -P50 $StdioP50 -P95 ($StdioP50 * 1.5) -P99 ($StdioP50 * 2)
|
||||
$http = New-Stats -P50 $HttpP50 -P95 ($HttpP50 * 1.5) -P99 ($HttpP50 * 2)
|
||||
return New-DispatchScenarios -StdioList $stdio -HttpList $http
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-TransportOverhead" {
|
||||
It "returns http minus stdio tools/list p50" {
|
||||
$s = New-DispatchScenarios `
|
||||
-StdioList (New-Stats -P50 5 -P95 6 -P99 7) `
|
||||
-HttpList (New-Stats -P50 6.5 -P95 8 -P99 9)
|
||||
Get-TransportOverhead $s | Should -Be 1.5
|
||||
}
|
||||
|
||||
It "returns null when a tools/list stat is absent" {
|
||||
$s = [ordered]@{ stdio = [ordered]@{ } ; http = [ordered]@{ tools_list = (New-Stats -P50 6 -P95 8 -P99 9) } }
|
||||
Get-TransportOverhead $s | Should -Be $null
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Invoke-DispatchRegressionGate" {
|
||||
It "reports no failures or warnings for an identical baseline" {
|
||||
$s = New-UniformScenarios
|
||||
$b = New-UniformScenarios
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
@($gate.Warnings).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "fails when stdio tools/list p95 regresses beyond 10%" {
|
||||
$b = New-UniformScenarios
|
||||
$s = New-UniformScenarios
|
||||
$s.stdio.tools_list.p95 = $b.stdio.tools_list.p95 * 1.15
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'stdio/tools_list (p95)'
|
||||
}
|
||||
|
||||
It "fails when http tools/list p99 regresses beyond 20%" {
|
||||
$b = New-UniformScenarios
|
||||
$s = New-UniformScenarios
|
||||
$s.http.tools_list.p99 = $b.http.tools_list.p99 * 1.25
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'http/tools_list (p99)'
|
||||
}
|
||||
|
||||
It "warns but does not fail for a p50 change between 5% and 10%" {
|
||||
$b = New-UniformScenarios
|
||||
$s = New-UniformScenarios
|
||||
$s.stdio.tools_list.p50 = $b.stdio.tools_list.p50 * 1.07
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
$gate.Warnings | Should -Contain 'stdio/tools_list (p50)'
|
||||
}
|
||||
|
||||
It "warns when transport overhead grows beyond 5%" {
|
||||
# Baseline overhead = 6 - 5 = 1. Raise http p50 to 6.4 => overhead 1.4 (+40%),
|
||||
# while keeping the http p50 change (+6.7%) below the +10% fail threshold.
|
||||
$b = New-UniformScenarios -StdioP50 5.0 -HttpP50 6.0
|
||||
$s = New-UniformScenarios -StdioP50 5.0 -HttpP50 6.4
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Warnings | Should -Contain 'transport_overhead_tools_list_p50'
|
||||
}
|
||||
|
||||
It "skips a transport absent from the baseline" {
|
||||
$s = New-UniformScenarios
|
||||
$b = [ordered]@{ stdio = $s.stdio } # no http
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "fails when no transport overlaps the baseline (F-007)" {
|
||||
$s = New-UniformScenarios
|
||||
$b = [ordered]@{ } # empty baseline: nothing to compare
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'no_comparable_metrics'
|
||||
}
|
||||
|
||||
It "treats a positive current against a zero baseline as a failure (F-010)" {
|
||||
$b = New-UniformScenarios
|
||||
$b.stdio.tools_list.p95 = 0
|
||||
$s = New-UniformScenarios
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'stdio/tools_list (p95)'
|
||||
}
|
||||
|
||||
It "fails when a transport the baseline expects is missing from results (F-007)" {
|
||||
$b = New-UniformScenarios
|
||||
$s = [ordered]@{ stdio = $b.stdio } # results dropped http
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'http (missing in results)'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Side-effect-free helpers for the command dispatch / transport overhead gate (issue #3119).
|
||||
|
||||
.DESCRIPTION
|
||||
Dot-source this file to reuse the pure functions in Measure-DispatchOverhead.ps1 and
|
||||
Check-DispatchOverheadRegression.ps1 and to unit-test them without launching a server.
|
||||
Reuses Get-MemberValue from StartupPerformance.Common.ps1.
|
||||
#>
|
||||
|
||||
. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tiered dispatch/transport regression gate shared by Measure-DispatchOverhead.ps1 and
|
||||
# Check-DispatchOverheadRegression.ps1. For each transport (stdio, http) and operation
|
||||
# (ping, tools_list) it applies the acceptance-criteria budgets from issue #3119:
|
||||
# * p50 and p95 may not regress by more than FailP50P95 (default +10%)
|
||||
# * p99 may not regress by more than FailP99 (default +20%)
|
||||
# * any change above Warn (default +5%) is reported as a warning
|
||||
# It also reports transport-specific overhead (http minus stdio tools/list p50) changes above Warn.
|
||||
#
|
||||
# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a line per check.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-DispatchRegressionGate {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] $ResultScenarios,
|
||||
[Parameter(Mandatory)] $BaselineScenarios,
|
||||
[double] $FailP50P95 = 1.10,
|
||||
[double] $FailP99 = 1.20,
|
||||
[double] $Warn = 1.05
|
||||
)
|
||||
|
||||
$failures = @()
|
||||
$warnings = @()
|
||||
$evaluated = 0
|
||||
|
||||
$transports = @('stdio', 'http')
|
||||
$operations = @('tools_list')
|
||||
$percentileBudgets = @(
|
||||
@{ Metric = 'p50'; Limit = $FailP50P95 }
|
||||
@{ Metric = 'p95'; Limit = $FailP50P95 }
|
||||
@{ Metric = 'p99'; Limit = $FailP99 }
|
||||
)
|
||||
|
||||
foreach ($transport in $transports) {
|
||||
$curTransport = Get-MemberValue $ResultScenarios $transport
|
||||
$baseTransport = Get-MemberValue $BaselineScenarios $transport
|
||||
if ($null -eq $baseTransport) {
|
||||
Write-Host (" [SKIP] {0}: absent in baseline" -f $transport)
|
||||
continue
|
||||
}
|
||||
if ($null -eq $curTransport) {
|
||||
$failures += "$transport (missing in results)"
|
||||
Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $transport)
|
||||
continue
|
||||
}
|
||||
|
||||
foreach ($operation in $operations) {
|
||||
$curStats = Get-MemberValue $curTransport $operation
|
||||
$baseStats = Get-MemberValue $baseTransport $operation
|
||||
if ($null -eq $baseStats) {
|
||||
Write-Host (" [SKIP] {0}/{1}: absent in baseline" -f $transport, $operation)
|
||||
continue
|
||||
}
|
||||
if ($null -eq $curStats) {
|
||||
$failures += "$transport/$operation (missing in results)"
|
||||
Write-Host (" [FAIL] {0}/{1}: expected by baseline but absent in results" -f $transport, $operation)
|
||||
continue
|
||||
}
|
||||
|
||||
foreach ($budget in $percentileBudgets) {
|
||||
$metric = $budget.Metric
|
||||
$current = Get-MemberValue $curStats $metric
|
||||
$baseline = Get-MemberValue $baseStats $metric
|
||||
if ($null -eq $baseline) {
|
||||
continue
|
||||
}
|
||||
$label = "$transport/$operation ($metric)"
|
||||
if ($null -eq $current) {
|
||||
$failures += "$label (missing in results)"
|
||||
Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $label)
|
||||
continue
|
||||
}
|
||||
|
||||
$evaluated++
|
||||
if ($baseline -gt 0) {
|
||||
$ratio = $current / $baseline
|
||||
if ($ratio -gt $budget.Limit) { $failures += $label; $status = 'FAIL' }
|
||||
elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
|
||||
else { $status = 'PASS' }
|
||||
Write-Host (" [{0}] {1}: current={2}ms baseline={3}ms (+{4}%)" -f `
|
||||
$status, $label, $current, $baseline, [math]::Round(($ratio - 1) * 100, 1))
|
||||
}
|
||||
elseif ($current -gt 0) {
|
||||
$failures += $label
|
||||
Write-Host (" [FAIL] {0}: current={1}ms vs a zero baseline (not comparable)" -f $label, $current)
|
||||
}
|
||||
else {
|
||||
Write-Host (" [PASS] {0}: current=0ms baseline=0ms (unchanged)" -f $label)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Transport-specific overhead: the extra tools/list p50 that HTTP adds over stdio.
|
||||
$curOverhead = Get-TransportOverhead $ResultScenarios
|
||||
$baseOverhead = Get-TransportOverhead $BaselineScenarios
|
||||
if ($null -ne $curOverhead -and $null -ne $baseOverhead -and $baseOverhead -gt 0) {
|
||||
$ratio = $curOverhead / $baseOverhead
|
||||
if ($ratio -gt $Warn) {
|
||||
$warnings += 'transport_overhead_tools_list_p50'
|
||||
$status = 'WARN'
|
||||
}
|
||||
else {
|
||||
$status = 'PASS'
|
||||
}
|
||||
Write-Host (" [{0}] transport_overhead (http-stdio tools/list p50): current={1}ms baseline={2}ms (+{3}%)" -f `
|
||||
$status, [math]::Round($curOverhead, 4), [math]::Round($baseOverhead, 4), [math]::Round(($ratio - 1) * 100, 1))
|
||||
}
|
||||
|
||||
if ($evaluated -eq 0) {
|
||||
$failures += 'no_comparable_metrics'
|
||||
Write-Host " [FAIL] no comparable dispatch metrics between results and baseline (incompatible or empty baseline)"
|
||||
}
|
||||
|
||||
return [ordered]@{ Failures = $failures; Warnings = $warnings }
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The HTTP-minus-stdio tools/list p50 delta — the transport-specific overhead for a real
|
||||
# request. Returns $null when either transport's tools/list p50 is absent.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-TransportOverhead {
|
||||
[CmdletBinding()]
|
||||
param($Scenarios)
|
||||
|
||||
$stdio = Get-MemberValue (Get-MemberValue (Get-MemberValue $Scenarios 'stdio') 'tools_list') 'p50'
|
||||
$http = Get-MemberValue (Get-MemberValue (Get-MemberValue $Scenarios 'http') 'tools_list') 'p50'
|
||||
if ($null -eq $stdio -or $null -eq $http) {
|
||||
return $null
|
||||
}
|
||||
return $http - $stdio
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs the BenchmarkDotNet micro-benchmarks for the perf pipeline and guards against a silent
|
||||
no-op run.
|
||||
|
||||
.DESCRIPTION
|
||||
Extracted from the pipeline YAML so CI behavior can be reproduced locally. --buildTimeout is
|
||||
raised because the generated boilerplate references the whole server (60+ tool assemblies) and
|
||||
exceeds BenchmarkDotNet's default 2-minute build timeout. BenchmarkDotNet can exit 0 even when
|
||||
the boilerplate build fails and nothing runs, so this fails explicitly if no benchmarks executed.
|
||||
|
||||
.PARAMETER BenchmarkExe
|
||||
Path to the built performance-benchmark executable.
|
||||
|
||||
.PARAMETER ArtifactsPath
|
||||
Directory for BenchmarkDotNet artifacts and the captured run log.
|
||||
|
||||
.PARAMETER Filter
|
||||
Glob passed to BenchmarkDotNet --filter (e.g. '*ServerStartup*').
|
||||
|
||||
.PARAMETER BuildTimeoutSeconds
|
||||
--buildTimeout for the generated boilerplate build.
|
||||
#>
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $BenchmarkExe,
|
||||
[Parameter(Mandatory)][string] $ArtifactsPath,
|
||||
[string] $Filter = '*',
|
||||
[int] $BuildTimeoutSeconds = 600
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $ArtifactsPath | Out-Null
|
||||
$logPath = Join-Path $ArtifactsPath 'run.log'
|
||||
|
||||
& $BenchmarkExe --filter $Filter --artifacts $ArtifactsPath --buildTimeout $BuildTimeoutSeconds 2>&1 |
|
||||
Tee-Object -FilePath $logPath
|
||||
$code = $LASTEXITCODE
|
||||
if ($code -ne 0) { exit $code }
|
||||
|
||||
$log = Get-Content $logPath -Raw
|
||||
if ($log -match 'failed to build the auto-generated boilerplate' -or $log -match 'executed benchmarks: 0') {
|
||||
Write-Error 'BenchmarkDotNet executed no benchmarks (boilerplate build failure or empty filter).'
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Measures Azure MCP Server throughput, latency, and scaling under a sweep of concurrency
|
||||
levels (issue #3122).
|
||||
|
||||
.DESCRIPTION
|
||||
Runs the performance-benchmark harness in --mcp-concurrency mode, which launches the server
|
||||
over HTTP and, at each concurrency level, drives a pool of parallel MCP clients issuing
|
||||
tools/list for a fixed window. For each level it records throughput, latency p50/p95/p99,
|
||||
error/timeout rate, concurrency scaling efficiency, and a state-isolation signal (responses
|
||||
whose tool count does not match the expected count). When a baseline is supplied the results
|
||||
are gated against the tiered budgets from issue #3122 (throughput within 10%, p95 & scaling
|
||||
within 15%, p99 within 20%, error-rate increase within 0.5 points, no state leakage).
|
||||
|
||||
.PARAMETER Executable
|
||||
Path to the azmcp server executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER BenchmarkExe
|
||||
Path to the performance-benchmark executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER Levels
|
||||
Concurrency levels to sweep.
|
||||
|
||||
.PARAMETER DurationSeconds
|
||||
Timed load window per level.
|
||||
|
||||
.PARAMETER WarmupSeconds
|
||||
Warmup window per level (not counted).
|
||||
|
||||
.PARAMETER TimeoutMs
|
||||
Per-request timeout; a timeout counts as an error.
|
||||
|
||||
.PARAMETER OutputPath
|
||||
Path to write the JSON results. Defaults to .perf-results/concurrency-scaling.json.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Optional baseline JSON (previously produced by this script) to gate against.
|
||||
|
||||
.EXAMPLE
|
||||
./eng/scripts/Measure-ConcurrencyScaling.ps1 -Levels 1,2,4,8,16 -DurationSeconds 10
|
||||
#>
|
||||
|
||||
param(
|
||||
[string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
|
||||
[string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
|
||||
[int[]] $Levels = @(1, 2, 4, 8, 16),
|
||||
[int] $DurationSeconds = 5,
|
||||
[int] $WarmupSeconds = 1,
|
||||
[int] $TimeoutMs = 10000,
|
||||
[string] $OutputPath = '.perf-results/concurrency-scaling.json',
|
||||
[string] $BaselinePath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$ScriptDir = $PSScriptRoot
|
||||
$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
|
||||
. (Join-Path $ScriptDir 'ConcurrencyPerformance.Common.ps1')
|
||||
|
||||
foreach ($path in @($Executable, $BenchmarkExe)) {
|
||||
if (-not (Test-Path $path)) {
|
||||
throw "Required executable not found: $path (build the solution in Release first)."
|
||||
}
|
||||
}
|
||||
$Executable = (Resolve-Path $Executable).Path
|
||||
$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
|
||||
$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
|
||||
|
||||
$env:PERF_CONC_LEVELS = ($Levels -join ',')
|
||||
$env:PERF_CONC_DURATION_SECONDS = $DurationSeconds
|
||||
$env:PERF_CONC_WARMUP_SECONDS = $WarmupSeconds
|
||||
$env:PERF_CONC_TIMEOUT_MS = $TimeoutMs
|
||||
|
||||
Write-Host "=== Concurrency scaling: levels [$($Levels -join ', ')], ${DurationSeconds}s per level ==="
|
||||
|
||||
# The harness sweeps every level in a single invocation; size the timeout to the whole sweep
|
||||
# (plus headroom) so a hung run fails fast rather than consuming the entire job.
|
||||
$concurrencyTimeoutSeconds = 180 + $Levels.Count * ($DurationSeconds + $WarmupSeconds) * 3
|
||||
$data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
|
||||
-ArgumentList @('--mcp-concurrency', $Executable, 'server', 'start') -TimeoutSeconds $concurrencyTimeoutSeconds
|
||||
|
||||
foreach ($level in $data.levels) {
|
||||
Write-Host (" c{0,-3} throughput={1,8} rps p50={2,7} ms p95={3,7} ms p99={4,7} ms err={5}% scaling={6} mismatch={7}" -f `
|
||||
$level.concurrency, $level.throughput_rps, $level.latency_ms.p50, $level.latency_ms.p95, `
|
||||
$level.latency_ms.p99, $level.error_rate_pct, $level.scaling_efficiency, $level.tool_count_mismatches)
|
||||
}
|
||||
Write-Host ""
|
||||
|
||||
$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
|
||||
|
||||
$results = [ordered]@{
|
||||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||||
commit = $commit
|
||||
environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
|
||||
duration_seconds = $DurationSeconds
|
||||
expected_tool_count = $data.expected_tool_count
|
||||
levels = $data.levels
|
||||
}
|
||||
|
||||
$results | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
|
||||
Write-Host "Results written to: $OutputPath"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: regression check against a baseline file
|
||||
# ---------------------------------------------------------------------------
|
||||
if ($BaselinePath) {
|
||||
if (-not (Test-Path $BaselinePath)) {
|
||||
Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
|
||||
}
|
||||
else {
|
||||
Write-Host ""
|
||||
Write-Host "=== Concurrency regression check (throughput +10%, p95/scaling +15%, p99 +20%, error +0.5pp) ==="
|
||||
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
$gate = Invoke-ConcurrencyRegressionGate -ResultLevels $results.levels -BaselineLevels $baseline.levels
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Concurrency warnings (>5%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Concurrency regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All concurrency budget checks passed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit Azure DevOps pipeline variables (no-op outside AzDO)
|
||||
# ---------------------------------------------------------------------------
|
||||
$peak = $data.levels | Sort-Object throughput_rps -Descending | Select-Object -First 1
|
||||
if ($peak) {
|
||||
Write-Host "##vso[task.setvariable variable=ConcurrencyPeakThroughputRps]$($peak.throughput_rps)"
|
||||
Write-Host "##vso[task.setvariable variable=ConcurrencyPeakAtLevel]$($peak.concurrency)"
|
||||
}
|
||||
$maxLevel = $data.levels | Sort-Object concurrency -Descending | Select-Object -First 1
|
||||
if ($maxLevel) {
|
||||
Write-Host "##vso[task.setvariable variable=ConcurrencyMaxLevelScalingEfficiency]$($maxLevel.scaling_efficiency)"
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Measures Azure MCP Server command-dispatch and transport overhead over stdio and HTTP
|
||||
(issue #3119) by timing many repetitions of ping and tools/list on a live server.
|
||||
|
||||
.DESCRIPTION
|
||||
For each transport this runs the performance-benchmark harness in --mcp-dispatch mode,
|
||||
which times ping (the transport/protocol floor) and tools/list (a real request with
|
||||
server-side serialization) across many iterations. Per-transport p50/p95/p99 are computed
|
||||
with the shared percentile helpers, the HTTP-minus-stdio ping delta isolates transport
|
||||
overhead, and (when a baseline is supplied) results are gated against the tiered budgets
|
||||
from issue #3119 (p50/p95 within 10%, p99 within 20%, transport change above 5% reported).
|
||||
|
||||
.PARAMETER Executable
|
||||
Path to the azmcp server executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER BenchmarkExe
|
||||
Path to the performance-benchmark executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER Iterations
|
||||
Timed samples per operation per transport.
|
||||
|
||||
.PARAMETER Warmup
|
||||
Warmup calls (not timed) per operation.
|
||||
|
||||
.PARAMETER OutputPath
|
||||
Path to write the JSON results. Defaults to .perf-results/dispatch-overhead.json.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Optional baseline JSON (previously produced by this script) to gate against.
|
||||
|
||||
.EXAMPLE
|
||||
./eng/scripts/Measure-DispatchOverhead.ps1
|
||||
|
||||
.EXAMPLE
|
||||
./eng/scripts/Measure-DispatchOverhead.ps1 -Iterations 500 -BaselinePath eng/dispatch-baseline.json
|
||||
#>
|
||||
|
||||
param(
|
||||
[string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
|
||||
[string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
|
||||
[int] $Iterations = 200,
|
||||
[int] $Warmup = 20,
|
||||
[string] $OutputPath = '.perf-results/dispatch-overhead.json',
|
||||
[string] $BaselinePath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$ScriptDir = $PSScriptRoot
|
||||
$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
|
||||
. (Join-Path $ScriptDir 'DispatchPerformance.Common.ps1')
|
||||
|
||||
foreach ($path in @($Executable, $BenchmarkExe)) {
|
||||
if (-not (Test-Path $path)) {
|
||||
throw "Required executable not found: $path (build the solution in Release first)."
|
||||
}
|
||||
}
|
||||
$Executable = (Resolve-Path $Executable).Path
|
||||
$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
|
||||
$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
|
||||
|
||||
$env:PERF_DISPATCH_ITERATIONS = $Iterations
|
||||
$env:PERF_DISPATCH_WARMUP = $Warmup
|
||||
|
||||
# Generous per-invocation timeout so a hung/crashed harness fails fast instead of consuming
|
||||
# the whole job. tools/list is sub-second, so this scales gently with the iteration count.
|
||||
$harnessTimeoutSeconds = [math]::Max(180, $Iterations)
|
||||
|
||||
function Get-DispatchStats {
|
||||
param([double[]] $Samples)
|
||||
$s = Get-TimingStats -Samples $Samples
|
||||
return [ordered]@{
|
||||
average = $s.average
|
||||
median = $s.median
|
||||
p50 = $s.p50
|
||||
p95 = $s.p95
|
||||
p99 = $s.p99
|
||||
count = $Samples.Count
|
||||
}
|
||||
}
|
||||
|
||||
$scenarios = [ordered]@{}
|
||||
|
||||
foreach ($transport in @('stdio', 'http')) {
|
||||
Write-Host "=== Dispatch overhead: $transport ($Iterations iterations) ==="
|
||||
$mode = "--mcp-dispatch-$transport"
|
||||
|
||||
$data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
|
||||
-ArgumentList @($mode, $Executable, 'server', 'start') -TimeoutSeconds $harnessTimeoutSeconds
|
||||
$toolsList = Get-DispatchStats -Samples ([double[]]$data.tools_list_ms)
|
||||
|
||||
$scenario = [ordered]@{ tools_list = $toolsList }
|
||||
if ($data.PSObject.Properties.Name -contains 'readiness_ms' -and $null -ne $data.readiness_ms) {
|
||||
$scenario.readiness_ms = $data.readiness_ms
|
||||
}
|
||||
$scenarios[$transport] = $scenario
|
||||
|
||||
Write-Host (" tools/list: p50={0} ms p95={1} ms p99={2} ms" -f $toolsList.p50, $toolsList.p95, $toolsList.p99)
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
$overhead = Get-TransportOverhead $scenarios
|
||||
if ($null -ne $overhead) {
|
||||
Write-Host ("=== Transport overhead (http - stdio tools/list p50): {0} ms ===" -f [math]::Round($overhead, 4))
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
|
||||
|
||||
$results = [ordered]@{
|
||||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||||
commit = $commit
|
||||
environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
|
||||
iterations = $Iterations
|
||||
warmup = $Warmup
|
||||
transport_overhead_tools_list_p50 = $overhead
|
||||
scenarios = $scenarios
|
||||
}
|
||||
|
||||
$results | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
|
||||
Write-Host "Results written to: $OutputPath"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: regression check against a baseline file
|
||||
# ---------------------------------------------------------------------------
|
||||
if ($BaselinePath) {
|
||||
if (-not (Test-Path $BaselinePath)) {
|
||||
Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
|
||||
}
|
||||
else {
|
||||
Write-Host ""
|
||||
Write-Host "=== Dispatch regression check (p50/p95 +10%, p99 +20%, transport warn +5%) ==="
|
||||
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
$gate = Invoke-DispatchRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Dispatch warnings (>5%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Dispatch regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All dispatch budget checks passed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit Azure DevOps pipeline variables (no-op outside AzDO)
|
||||
# ---------------------------------------------------------------------------
|
||||
foreach ($transport in @('stdio', 'http')) {
|
||||
$s = $scenarios[$transport]
|
||||
$t = (Get-Culture).TextInfo.ToTitleCase($transport)
|
||||
Write-Host "##vso[task.setvariable variable=Dispatch${t}ToolsListP50Ms]$($s.tools_list.p50)"
|
||||
}
|
||||
if ($null -ne $overhead) {
|
||||
Write-Host "##vso[task.setvariable variable=DispatchTransportOverheadToolsListP50Ms]$overhead"
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Measures the Azure MCP Server's CPU, memory, and handle baselines across server modes
|
||||
and concurrency levels (issue #3121).
|
||||
|
||||
.DESCRIPTION
|
||||
For each requested server mode this script launches the performance-benchmark harness
|
||||
in --mcp-resource mode, which starts the server over HTTP and samples process resource
|
||||
counters across the startup, idle, discovery, steady-state, and optional soak phases.
|
||||
Results for every mode are collected into a single JSON document and, when a baseline is
|
||||
supplied, gated against the tiered resource budgets from issue #3121 (steady-state CPU
|
||||
and memory within 10%, peak memory within 15%, no unbounded soak growth).
|
||||
|
||||
.PARAMETER Executable
|
||||
Path to the azmcp server executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER BenchmarkExe
|
||||
Path to the performance-benchmark executable. Defaults to the Release build.
|
||||
|
||||
.PARAMETER Modes
|
||||
Server modes to measure. Valid values: default, namespace, all.
|
||||
|
||||
.PARAMETER Concurrency
|
||||
Number of concurrent load clients driving steady-state and soak load.
|
||||
|
||||
.PARAMETER IdleSeconds
|
||||
Idle observation window (no client connected) in seconds.
|
||||
|
||||
.PARAMETER SteadySeconds
|
||||
Steady-state load duration in seconds.
|
||||
|
||||
.PARAMETER SoakSeconds
|
||||
Sustained soak duration in seconds. 0 disables the soak/leak-detection phase.
|
||||
|
||||
.PARAMETER SampleMs
|
||||
Resource sampling interval in milliseconds.
|
||||
|
||||
.PARAMETER OutputPath
|
||||
Path to write the JSON results. Defaults to .perf-results/resource-usage.json.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Optional baseline JSON (previously produced by this script) to gate against.
|
||||
|
||||
.EXAMPLE
|
||||
./eng/scripts/Measure-ResourceUsage.ps1
|
||||
|
||||
.EXAMPLE
|
||||
./eng/scripts/Measure-ResourceUsage.ps1 -Modes default,all -Concurrency 4 -SoakSeconds 120
|
||||
#>
|
||||
|
||||
param(
|
||||
[string] $Executable = './servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe',
|
||||
[string] $BenchmarkExe = './servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin/Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks.exe',
|
||||
[ValidateSet('default', 'namespace', 'all')]
|
||||
[string[]] $Modes = @('default', 'all'),
|
||||
[int] $Concurrency = 1,
|
||||
[int] $IdleSeconds = 3,
|
||||
[int] $SteadySeconds = 10,
|
||||
[int] $SoakSeconds = 0,
|
||||
[int] $SampleMs = 100,
|
||||
[string] $OutputPath = '.perf-results/resource-usage.json',
|
||||
[string] $BaselinePath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$ScriptDir = $PSScriptRoot
|
||||
$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
|
||||
. (Join-Path $ScriptDir 'ResourcePerformance.Common.ps1')
|
||||
|
||||
foreach ($path in @($Executable, $BenchmarkExe)) {
|
||||
if (-not (Test-Path $path)) {
|
||||
throw "Required executable not found: $path (build the solution in Release first)."
|
||||
}
|
||||
}
|
||||
$Executable = (Resolve-Path $Executable).Path
|
||||
$BenchmarkExe = (Resolve-Path $BenchmarkExe).Path
|
||||
$OutputPath = Resolve-PerfOutputPath -Path $OutputPath
|
||||
|
||||
function Get-ModeServerArgs {
|
||||
param([string] $Mode)
|
||||
switch ($Mode) {
|
||||
'default' { return @('server', 'start') }
|
||||
'namespace' { return @('server', 'start', '--mode', 'namespace') }
|
||||
'all' { return @('server', 'start', '--mode', 'all') }
|
||||
default { throw "Unknown mode '$Mode'." }
|
||||
}
|
||||
}
|
||||
|
||||
# Propagate the run parameters to the harness via environment variables.
|
||||
$env:PERF_CONCURRENCY = $Concurrency
|
||||
$env:PERF_IDLE_SECONDS = $IdleSeconds
|
||||
$env:PERF_STEADY_SECONDS = $SteadySeconds
|
||||
$env:PERF_SOAK_SECONDS = $SoakSeconds
|
||||
$env:PERF_SAMPLE_MS = $SampleMs
|
||||
|
||||
# Per-mode timeout covers idle + steady + soak (plus startup/headroom) so a hung harness
|
||||
# fails fast instead of consuming the whole job.
|
||||
$resourceTimeoutSeconds = 180 + ($IdleSeconds + $SteadySeconds + $SoakSeconds) * 3
|
||||
|
||||
$scenarios = [ordered]@{}
|
||||
|
||||
foreach ($mode in $Modes) {
|
||||
$serverArgs = Get-ModeServerArgs -Mode $mode
|
||||
Write-Host "=== Resource usage: $mode mode (concurrency=$Concurrency, steady=${SteadySeconds}s, soak=${SoakSeconds}s) ==="
|
||||
|
||||
$data = Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
|
||||
-ArgumentList (@('--mcp-resource', $Executable) + $serverArgs) -TimeoutSeconds $resourceTimeoutSeconds
|
||||
$scenarios[$mode] = $data
|
||||
|
||||
$steady = $data.phases.steady_state
|
||||
Write-Host (" tools={0} steady: ws-mean={1} MB ws-max={2} MB cpu={3}% {4} req ({5} rps)" -f `
|
||||
$data.tool_count, $steady.working_set_mb.mean, $steady.working_set_mb.max, `
|
||||
$steady.cpu_percent, $steady.requests, $steady.throughput_rps)
|
||||
Write-Host (" peak: ws={0} MB private={1} MB handles={2} growth idle->steady: ws={3} MB handles={4}" -f `
|
||||
$data.peak.working_set_mb, $data.peak.private_mb, $data.peak.handles, `
|
||||
$data.growth.idle_to_steady_working_set_mb, $data.growth.idle_to_steady_handles)
|
||||
if ($null -ne $data.soak) {
|
||||
Write-Host (" soak: {0} req over {1}s ws-slope={2} MB/min (R2={3}) handle-slope={4}/min unbounded={5}" -f `
|
||||
$data.soak.requests, $data.soak.duration_s, $data.soak.working_set_slope_mb_per_min, `
|
||||
$data.soak.working_set_r2, $data.soak.handle_slope_per_min, $data.soak.unbounded_growth)
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
$commit = try { (& git -C $RepoRoot rev-parse --short HEAD 2>$null).Trim() } catch { '' }
|
||||
|
||||
$results = [ordered]@{
|
||||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||||
commit = $commit
|
||||
environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
|
||||
concurrency = $Concurrency
|
||||
idle_seconds = $IdleSeconds
|
||||
steady_seconds = $SteadySeconds
|
||||
soak_seconds = $SoakSeconds
|
||||
sample_ms = $SampleMs
|
||||
scenarios = $scenarios
|
||||
}
|
||||
|
||||
$results | ConvertTo-Json -Depth 12 | Out-File -FilePath $OutputPath -Encoding utf8 -Force
|
||||
Write-Host "Results written to: $OutputPath"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: regression check against a baseline file
|
||||
# ---------------------------------------------------------------------------
|
||||
if ($BaselinePath) {
|
||||
if (-not (Test-Path $BaselinePath)) {
|
||||
Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
|
||||
}
|
||||
else {
|
||||
Write-Host ""
|
||||
Write-Host "=== Resource regression check (steady CPU/mem +10%, peak mem +15%, no soak leak) ==="
|
||||
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Resource warnings (>5%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Resource regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All resource budget checks passed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit Azure DevOps pipeline variables (no-op outside AzDO)
|
||||
# ---------------------------------------------------------------------------
|
||||
foreach ($mode in $Modes) {
|
||||
$data = $scenarios[$mode]
|
||||
$steady = $data.phases.steady_state
|
||||
$modeVar = (Get-Culture).TextInfo.ToTitleCase($mode)
|
||||
Write-Host "##vso[task.setvariable variable=Resource${modeVar}SteadyWorkingSetMb]$($steady.working_set_mb.mean)"
|
||||
Write-Host "##vso[task.setvariable variable=Resource${modeVar}SteadyCpuPercent]$($steady.cpu_percent)"
|
||||
Write-Host "##vso[task.setvariable variable=Resource${modeVar}PeakWorkingSetMb]$($data.peak.working_set_mb)"
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Emits the startup performance metrics from a results JSON file.
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Test-StartupPerformance.ps1 and logs each tracked
|
||||
metric. The metrics are currently written as pipeline log warnings so they surface in
|
||||
the build timeline; replace the Write-Host sink with a real telemetry destination
|
||||
(Azure Monitor, Application Insights, etc.) to persist them.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Test-StartupPerformance.ps1.
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$s = $results.scenarios
|
||||
|
||||
$metrics = @(
|
||||
@{ name = 'startup.cli.cold.median_ms'; value = $s.cli_cold_start_ms.median }
|
||||
@{ name = 'startup.cli.cold.p95_ms'; value = $s.cli_cold_start_ms.p95 }
|
||||
@{ name = 'startup.mcp.default.median_ms'; value = $s.mcp_stdio_to_tools_list_ms.median }
|
||||
@{ name = 'startup.mcp.default.p95_ms'; value = $s.mcp_stdio_to_tools_list_ms.p95 }
|
||||
@{ name = 'startup.mcp.default.p99_ms'; value = $s.mcp_stdio_to_tools_list_ms.p99 }
|
||||
@{ name = 'startup.mcp.namespace.median_ms'; value = $s.mcp_namespace_mode_startup_ms.median }
|
||||
@{ name = 'startup.mcp.all.median_ms'; value = $s.mcp_all_mode_startup_ms.median }
|
||||
@{ name = 'startup.mcp.all.p95_ms'; value = $s.mcp_all_mode_startup_ms.p95 }
|
||||
@{ name = 'startup.mcp.all.p99_ms'; value = $s.mcp_all_mode_startup_ms.p99 }
|
||||
@{ name = 'startup.mcp.http.median_ms'; value = $s.mcp_http_default_startup_ms.median }
|
||||
@{ name = 'startup.mcp.http.p95_ms'; value = $s.mcp_http_default_startup_ms.p95 }
|
||||
@{ name = 'startup.mcp.http.readiness_median_ms'; value = $s.mcp_http_readiness_ms.median }
|
||||
@{ name = 'serialize.default.full_schema_ms'; value = $s.tools_list_payload.full_schema_serialize_ms }
|
||||
@{ name = 'serialize.all.full_schema_ms'; value = $s.all_mode_tools_list_payload.full_schema_serialize_ms }
|
||||
@{ name = 'serialize.http.full_schema_ms'; value = $s.http_default_tools_list_payload.full_schema_serialize_ms }
|
||||
@{ name = 'tokens.default.full_schema_gpt4o'; value = $s.tools_list_payload.full_schema_tokens_exact }
|
||||
@{ name = 'tokens.namespace.full_schema_gpt4o'; value = $s.namespace_mode_tools_list_payload.full_schema_tokens_exact }
|
||||
@{ name = 'tokens.all.full_schema_gpt4o'; value = $s.all_mode_tools_list_payload.full_schema_tokens_exact }
|
||||
@{ name = 'tools.default.count'; value = $s.tools_list_payload.tool_count }
|
||||
@{ name = 'tools.all.count'; value = $s.all_mode_tools_list_payload.tool_count }
|
||||
)
|
||||
|
||||
foreach ($m in $metrics) {
|
||||
Write-Host ("##vso[task.logissue type=warning]METRIC: {0} = {1}" -f $m.name, $m.value)
|
||||
# Replace the log statement above with your preferred telemetry sink, e.g.:
|
||||
# az monitor metrics alert create ...
|
||||
# Invoke-RestMethod -Uri $AppInsightsEndpoint -Body @{ name=$m.name; value=$m.value } ...
|
||||
}
|
||||
Write-Host "Metrics logged. Wire up a telemetry sink to persist them."
|
||||
@@ -0,0 +1,161 @@
|
||||
# Install-Module -Name Pester -Force -SkipPublisherCheck
|
||||
# Invoke-Pester -Path eng/scripts/ResourcePerformance.Common.Tests.ps1
|
||||
#
|
||||
# Focused tests for the pure resource-gate helpers used by Measure-ResourceUsage.ps1.
|
||||
|
||||
BeforeAll {
|
||||
. $PSScriptRoot/ResourcePerformance.Common.ps1
|
||||
|
||||
function script:New-ResourceScenario {
|
||||
param(
|
||||
[double] $SteadyMem,
|
||||
[double] $SteadyCpu,
|
||||
[double] $PeakMem,
|
||||
[switch] $WithSoak,
|
||||
[bool] $Unbounded = $false
|
||||
)
|
||||
|
||||
$scenario = [ordered]@{
|
||||
phases = [ordered]@{
|
||||
steady_state = [ordered]@{
|
||||
working_set_mb = [ordered]@{ mean = $SteadyMem; max = $SteadyMem }
|
||||
cpu_percent = $SteadyCpu
|
||||
}
|
||||
}
|
||||
peak = [ordered]@{ working_set_mb = $PeakMem }
|
||||
}
|
||||
|
||||
if ($WithSoak) {
|
||||
$scenario.soak = [ordered]@{
|
||||
unbounded_growth = $Unbounded
|
||||
working_set_slope_mb_per_min = 1
|
||||
working_set_r2 = 0.9
|
||||
handle_slope_per_min = 1
|
||||
handle_r2 = 0.1
|
||||
}
|
||||
}
|
||||
|
||||
return $scenario
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-MemberNames" {
|
||||
It "returns the keys of an ordered dictionary" {
|
||||
$names = Get-MemberNames ([ordered]@{ a = 1; b = 2 })
|
||||
$names | Should -Be @('a', 'b')
|
||||
}
|
||||
|
||||
It "returns the property names of a PSCustomObject" {
|
||||
$obj = [pscustomobject]@{ x = 1; y = 2 }
|
||||
$names = Get-MemberNames $obj
|
||||
$names | Should -Be @('x', 'y')
|
||||
}
|
||||
|
||||
It "returns an empty set for null" {
|
||||
@(Get-MemberNames $null).Count | Should -Be 0
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Invoke-ResourceRegressionGate" {
|
||||
It "reports no failures or warnings for an identical baseline" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
@($gate.Warnings).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "fails when steady-state memory regresses beyond 10%" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 115 -SteadyCpu 10 -PeakMem 150) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default steady_working_set_mb'
|
||||
}
|
||||
|
||||
It "warns but does not fail for a steady-state memory change between 5% and 10%" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 107 -SteadyCpu 10 -PeakMem 150) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
$gate.Warnings | Should -Contain 'default steady_working_set_mb'
|
||||
}
|
||||
|
||||
It "fails when steady-state CPU regresses beyond 10%" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 12 -PeakMem 150) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default steady_cpu_percent'
|
||||
}
|
||||
|
||||
It "allows peak memory growth up to 15% as a warning" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 168) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
$gate.Warnings | Should -Contain 'default peak_working_set_mb'
|
||||
}
|
||||
|
||||
It "fails when peak memory regresses beyond 15%" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 180) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default peak_working_set_mb'
|
||||
}
|
||||
|
||||
It "fails when a soak phase is flagged with unbounded growth" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150 -WithSoak -Unbounded $true) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default soak_unbounded_growth'
|
||||
}
|
||||
|
||||
It "passes a soak phase that is not flagged" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150 -WithSoak -Unbounded $false) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "flags an incompatible (empty) baseline instead of silently passing (F-007)" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 999 -SteadyCpu 99 -PeakMem 999) }
|
||||
$b = [ordered]@{ }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'no_comparable_metrics'
|
||||
}
|
||||
|
||||
It "compares overlapping modes and skips a newly added mode without failing" {
|
||||
$s = [ordered]@{
|
||||
default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150)
|
||||
all = (New-ResourceScenario -SteadyMem 200 -SteadyCpu 20 -PeakMem 280)
|
||||
}
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "flags soak unbounded growth even when the mode is absent from the baseline (F-008)" {
|
||||
$s = [ordered]@{ all = (New-ResourceScenario -SteadyMem 200 -SteadyCpu 20 -PeakMem 280 -WithSoak -Unbounded $true) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'all soak_unbounded_growth'
|
||||
}
|
||||
|
||||
It "treats positive CPU against a zero baseline as a failure, not a pass (F-010)" {
|
||||
$s = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 25 -PeakMem 150) }
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 0 -PeakMem 150) }
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default steady_cpu_percent'
|
||||
}
|
||||
|
||||
It "fails when a steady metric the baseline expects is missing from results (F-007)" {
|
||||
$b = [ordered]@{ default = (New-ResourceScenario -SteadyMem 100 -SteadyCpu 10 -PeakMem 150) }
|
||||
$s = [ordered]@{
|
||||
default = [ordered]@{
|
||||
phases = [ordered]@{ steady_state = [ordered]@{ working_set_mb = [ordered]@{ mean = 100; max = 100 } } }
|
||||
peak = [ordered]@{ working_set_mb = 150 }
|
||||
}
|
||||
}
|
||||
$gate = Invoke-ResourceRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'default steady_cpu_percent (missing in results)'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Side-effect-free helpers for the resource-usage baseline gate (issue #3121).
|
||||
|
||||
.DESCRIPTION
|
||||
Dot-source this file to reuse the pure functions in Measure-ResourceUsage.ps1 and
|
||||
Check-ResourceUsageRegression.ps1 and to unit-test them without launching a server.
|
||||
Reuses the generic accessor Get-MemberValue from StartupPerformance.Common.ps1.
|
||||
#>
|
||||
|
||||
. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Returns the member names of a hashtable/ordered dictionary or a PSCustomObject
|
||||
# (as produced by ConvertFrom-Json), so callers can enumerate scenario maps
|
||||
# regardless of how the results JSON was materialized.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-MemberNames {
|
||||
[CmdletBinding()]
|
||||
param($Object)
|
||||
|
||||
if ($null -eq $Object) { return @() }
|
||||
if ($Object -is [System.Collections.IDictionary]) { return @($Object.Keys) }
|
||||
return @($Object.PSObject.Properties.Name)
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tiered resource-regression gate shared by Measure-ResourceUsage.ps1 and
|
||||
# Check-ResourceUsageRegression.ps1. For each server-mode scenario present in
|
||||
# both the results and the baseline it applies the acceptance-criteria budgets
|
||||
# from issue #3121:
|
||||
# * steady-state CPU and memory may not increase by more than 10% (FailSteady = 1.10)
|
||||
# * peak memory may not increase by more than 15% (FailPeak = 1.15)
|
||||
# * any change above 5% is reported as a warning (Warn = 1.05)
|
||||
# * a soak phase flagged with unbounded_growth is a hard failure (memory/handle leak)
|
||||
#
|
||||
# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a
|
||||
# human-readable line per check via Write-Host.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-ResourceRegressionGate {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] $ResultScenarios,
|
||||
[Parameter(Mandatory)] $BaselineScenarios,
|
||||
[double] $FailSteady = 1.10,
|
||||
[double] $FailPeak = 1.15,
|
||||
[double] $Warn = 1.05
|
||||
)
|
||||
|
||||
$failures = @()
|
||||
$warnings = @()
|
||||
$evaluated = 0
|
||||
|
||||
$seenModes = @{}
|
||||
|
||||
foreach ($mode in (Get-MemberNames $ResultScenarios)) {
|
||||
$seenModes[[string]$mode] = $true
|
||||
$r = Get-MemberValue $ResultScenarios $mode
|
||||
$b = Get-MemberValue $BaselineScenarios $mode
|
||||
|
||||
# Leak detection: a soak phase flagged unbounded is a hard failure regardless of the
|
||||
# baseline, so it is evaluated before the baseline-presence check below can skip this mode.
|
||||
$soak = Get-MemberValue $r 'soak'
|
||||
if ($null -ne $soak) {
|
||||
if (Get-MemberValue $soak 'unbounded_growth') {
|
||||
$failures += "$mode soak_unbounded_growth"
|
||||
Write-Host (" [FAIL] {0} soak_unbounded_growth: working_set_slope={1} MB/min (R2={2}), handle_slope={3}/min (R2={4})" -f `
|
||||
$mode, (Get-MemberValue $soak 'working_set_slope_mb_per_min'), (Get-MemberValue $soak 'working_set_r2'), `
|
||||
(Get-MemberValue $soak 'handle_slope_per_min'), (Get-MemberValue $soak 'handle_r2'))
|
||||
}
|
||||
else {
|
||||
Write-Host (" [PASS] {0} soak: no unbounded growth" -f $mode)
|
||||
}
|
||||
}
|
||||
|
||||
if ($null -eq $b) {
|
||||
Write-Host (" [SKIP] {0}: absent in baseline (relative budgets)" -f $mode)
|
||||
continue
|
||||
}
|
||||
|
||||
$rSteady = Get-MemberValue (Get-MemberValue $r 'phases') 'steady_state'
|
||||
$bSteady = Get-MemberValue (Get-MemberValue $b 'phases') 'steady_state'
|
||||
|
||||
$checks = @(
|
||||
@{ Label = 'steady_working_set_mb'; Unit = 'MB'
|
||||
Cur = Get-MemberValue (Get-MemberValue $rSteady 'working_set_mb') 'mean'
|
||||
Base = Get-MemberValue (Get-MemberValue $bSteady 'working_set_mb') 'mean'
|
||||
Limit = $FailSteady }
|
||||
@{ Label = 'steady_cpu_percent'; Unit = '%'
|
||||
Cur = Get-MemberValue $rSteady 'cpu_percent'
|
||||
Base = Get-MemberValue $bSteady 'cpu_percent'
|
||||
Limit = $FailSteady }
|
||||
@{ Label = 'peak_working_set_mb'; Unit = 'MB'
|
||||
Cur = Get-MemberValue (Get-MemberValue $r 'peak') 'working_set_mb'
|
||||
Base = Get-MemberValue (Get-MemberValue $b 'peak') 'working_set_mb'
|
||||
Limit = $FailPeak }
|
||||
)
|
||||
|
||||
foreach ($c in $checks) {
|
||||
if ($null -eq $c.Base) {
|
||||
Write-Host (" [SKIP] {0} {1}: absent in baseline" -f $mode, $c.Label)
|
||||
continue
|
||||
}
|
||||
if ($null -eq $c.Cur) {
|
||||
$failures += "$mode $($c.Label) (missing in results)"
|
||||
Write-Host (" [FAIL] {0} {1}: expected by baseline but absent in results" -f $mode, $c.Label)
|
||||
continue
|
||||
}
|
||||
|
||||
$label = "$mode $($c.Label)"
|
||||
$evaluated++
|
||||
if ($c.Base -gt 0) {
|
||||
$ratio = $c.Cur / $c.Base
|
||||
if ($ratio -gt $c.Limit) { $failures += $label; $status = 'FAIL' }
|
||||
elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
|
||||
else { $status = 'PASS' }
|
||||
Write-Host (" [{0}] {1}: current={2}{3} baseline={4}{3} (+{5}%)" -f `
|
||||
$status, $label, $c.Cur, $c.Unit, $c.Base, [math]::Round(($ratio - 1) * 100, 1))
|
||||
}
|
||||
elseif ($c.Cur -gt 0) {
|
||||
# Positive current against a zero baseline is not comparable — fail rather than
|
||||
# silently pass with a fabricated 0% change.
|
||||
$failures += $label
|
||||
Write-Host (" [FAIL] {0}: current={1}{2} vs a zero baseline (not comparable)" -f $label, $c.Cur, $c.Unit)
|
||||
}
|
||||
else {
|
||||
Write-Host (" [PASS] {0}: current=0{1} baseline=0{1} (unchanged)" -f $label, $c.Unit)
|
||||
}
|
||||
}
|
||||
|
||||
# Leak detection is evaluated near the top of the loop (before the baseline skip).
|
||||
}
|
||||
|
||||
# Baseline modes the results no longer cover are treated as an expected-but-missing metric.
|
||||
foreach ($mode in (Get-MemberNames $BaselineScenarios)) {
|
||||
if (-not $seenModes.ContainsKey([string]$mode)) {
|
||||
$failures += "$mode missing_in_results"
|
||||
Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $mode)
|
||||
}
|
||||
}
|
||||
|
||||
if ($evaluated -eq 0) {
|
||||
$failures += 'no_comparable_metrics'
|
||||
Write-Host " [FAIL] no comparable resource metrics between results and baseline (incompatible or empty baseline)"
|
||||
}
|
||||
|
||||
return [ordered]@{ Failures = $failures; Warnings = $warnings }
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
# Install-Module -Name Pester -Force -SkipPublisherCheck
|
||||
# Invoke-Pester -Path eng/scripts/StartupPerformance.Common.Tests.ps1
|
||||
#
|
||||
# Focused tests for the pure helpers used by Test-StartupPerformance.ps1.
|
||||
|
||||
BeforeAll {
|
||||
. $PSScriptRoot/StartupPerformance.Common.ps1
|
||||
|
||||
$script:RunFolder = Join-Path $PSScriptRoot '.perf-common-testruns'
|
||||
if (Test-Path $script:RunFolder) {
|
||||
Remove-Item -Recurse -Force $script:RunFolder
|
||||
}
|
||||
New-Item -ItemType Directory -Path $script:RunFolder | Out-Null
|
||||
}
|
||||
|
||||
AfterAll {
|
||||
if (Test-Path $script:RunFolder) {
|
||||
Remove-Item -Recurse -Force $script:RunFolder
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-TimingStats" {
|
||||
It "returns the single middle value as the median for an odd number of samples" {
|
||||
# sorted: 3,5,9 -> middle = 5
|
||||
$stats = Get-TimingStats -Samples @(9, 3, 5)
|
||||
$stats.median | Should -Be 5
|
||||
}
|
||||
|
||||
It "averages the two middle values as the median for an even number of samples" {
|
||||
# sorted: 10,20,30,40 -> (20+30)/2 = 25
|
||||
$stats = Get-TimingStats -Samples @(40, 10, 30, 20)
|
||||
$stats.median | Should -Be 25
|
||||
}
|
||||
|
||||
It "produces a fractional median when the two middle values differ by an odd amount" {
|
||||
# sorted: 10,11 -> (10+11)/2 = 10.5
|
||||
$stats = Get-TimingStats -Samples @(11, 10)
|
||||
$stats.median | Should -Be 10.5
|
||||
}
|
||||
|
||||
It "reports the first sample in original (unsorted) order" {
|
||||
$stats = Get-TimingStats -Samples @(40, 10, 30, 20)
|
||||
$stats.first | Should -Be 40
|
||||
}
|
||||
|
||||
It "computes the average rounded to one decimal place" {
|
||||
# (1+2+2)/3 = 1.666... -> 1.7
|
||||
$stats = Get-TimingStats -Samples @(1, 2, 2)
|
||||
$stats.average | Should -Be 1.7
|
||||
}
|
||||
|
||||
It "handles a single sample" {
|
||||
$stats = Get-TimingStats -Samples @(42)
|
||||
$stats.median | Should -Be 42
|
||||
$stats.first | Should -Be 42
|
||||
$stats.average | Should -Be 42
|
||||
}
|
||||
|
||||
It "throws when no samples are provided" {
|
||||
{ Get-TimingStats -Samples @() } | Should -Throw
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-TimingStats percentiles" {
|
||||
It "reports p50 equal to the median for an odd number of samples" {
|
||||
$stats = Get-TimingStats -Samples @(9, 3, 5)
|
||||
$stats.p50 | Should -Be 5
|
||||
}
|
||||
|
||||
It "orders p50 <= p95 <= p99 and keeps them within the sample range" {
|
||||
$stats = Get-TimingStats -Samples @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||
$stats.p50 | Should -BeLessOrEqual $stats.p95
|
||||
$stats.p95 | Should -BeLessOrEqual $stats.p99
|
||||
$stats.p99 | Should -BeLessOrEqual 10
|
||||
$stats.p95 | Should -BeGreaterThan 9 # rank 8.55 lands between 9 and 10
|
||||
}
|
||||
|
||||
It "returns the single value for all percentiles when one sample" {
|
||||
$stats = Get-TimingStats -Samples @(42)
|
||||
$stats.p50 | Should -Be 42
|
||||
$stats.p95 | Should -Be 42
|
||||
$stats.p99 | Should -Be 42
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-Percentile" {
|
||||
It "returns the min at p0 and max at p100" {
|
||||
(Get-Percentile -Samples @(10, 20, 30, 40) -Percentile 0.0) | Should -Be 10
|
||||
(Get-Percentile -Samples @(10, 20, 30, 40) -Percentile 1.0) | Should -Be 40
|
||||
}
|
||||
|
||||
It "interpolates between ranks" {
|
||||
# sorted 10,20,30,40; rank = 0.5*3 = 1.5 -> 20 + 0.5*(30-20) = 25
|
||||
(Get-Percentile -Samples @(40, 10, 30, 20) -Percentile 0.5) | Should -Be 25
|
||||
}
|
||||
|
||||
It "returns an exact sample when the rank is an integer" {
|
||||
# sorted 1..11, rank = 0.9*(11-1) = 9 -> sorted[9] = 10
|
||||
(Get-Percentile -Samples @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) -Percentile 0.9) | Should -Be 10
|
||||
}
|
||||
|
||||
It "throws for out-of-range percentiles" {
|
||||
{ Get-Percentile -Samples @(1) -Percentile 1.5 } | Should -Throw
|
||||
{ Get-Percentile -Samples @(1) -Percentile -0.1 } | Should -Throw
|
||||
}
|
||||
|
||||
It "throws when no samples are provided" {
|
||||
{ Get-Percentile -Samples @() -Percentile 0.5 } | Should -Throw
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Split-ColdWarm" {
|
||||
It "treats the first sample as cold and the rest as warm" {
|
||||
$split = Split-ColdWarm -Samples @(100, 40, 50, 60)
|
||||
$split.cold_ms | Should -Be 100
|
||||
# warm samples 40,50,60 -> median 50
|
||||
$split.warm.median | Should -Be 50
|
||||
}
|
||||
|
||||
It "falls back to the single sample for warm when only one sample exists" {
|
||||
$split = Split-ColdWarm -Samples @(77)
|
||||
$split.cold_ms | Should -Be 77
|
||||
$split.warm.median | Should -Be 77
|
||||
}
|
||||
|
||||
It "throws when no samples are provided" {
|
||||
{ Split-ColdWarm -Samples @() } | Should -Throw
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Resolve-PerfOutputPath" {
|
||||
It "resolves a bare filename against the base directory and returns an absolute path" {
|
||||
$result = Resolve-PerfOutputPath -Path 'startup-e2e.json' -BaseDirectory $script:RunFolder
|
||||
[System.IO.Path]::IsPathRooted($result) | Should -Be $true
|
||||
Split-Path -Parent $result | Should -Be ([System.IO.Path]::GetFullPath($script:RunFolder))
|
||||
}
|
||||
|
||||
It "creates a missing parent directory for a relative nested path" {
|
||||
$nested = Join-Path 'nested-a' 'nested-b/out.json'
|
||||
$result = Resolve-PerfOutputPath -Path $nested -BaseDirectory $script:RunFolder
|
||||
Test-Path (Split-Path -Parent $result) | Should -Be $true
|
||||
}
|
||||
|
||||
It "creates a missing parent directory for an absolute path" {
|
||||
$abs = Join-Path $script:RunFolder 'abs-dir/out.json'
|
||||
$result = Resolve-PerfOutputPath -Path $abs
|
||||
Test-Path (Split-Path -Parent $result) | Should -Be $true
|
||||
$result | Should -Be ([System.IO.Path]::GetFullPath($abs))
|
||||
}
|
||||
|
||||
It "does not throw for a bare filename (regression for empty Split-Path parent)" {
|
||||
{ Resolve-PerfOutputPath -Path 'only-a-filename.json' -BaseDirectory $script:RunFolder } | Should -Not -Throw
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-PerfRunMetadata" {
|
||||
It "records all controlled-environment dimensions required by issue #3120" {
|
||||
$m = Get-PerfRunMetadata
|
||||
foreach ($key in @('commit', 'timestamp', 'runtime', 'os', 'os_architecture', 'cpu_cores', 'machine_memory_gb', 'workload_catalog_version')) {
|
||||
$m.Contains($key) | Should -BeTrue -Because "metadata must include '$key'"
|
||||
}
|
||||
}
|
||||
|
||||
It "reports a positive processor count" {
|
||||
(Get-PerfRunMetadata).cpu_cores | Should -BeGreaterThan 0
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-WorkloadCatalogVersion" {
|
||||
It "returns the catalog version from the committed workload catalog" {
|
||||
Get-WorkloadCatalogVersion | Should -Not -BeNullOrEmpty
|
||||
}
|
||||
|
||||
It "returns 'unknown' when the catalog file is absent" {
|
||||
Get-WorkloadCatalogVersion -CatalogPath (Join-Path $script:RunFolder 'no-such-catalog.json') | Should -Be 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Get-TimingStats sub-millisecond precision" {
|
||||
It "preserves fractional-millisecond percentiles instead of rounding to whole ms (F-004)" {
|
||||
# Regression: double samples previously bound to [long[]] and were rounded to 0,0,1,1.
|
||||
$stats = Get-TimingStats -Samples @(0.41, 0.49, 0.51, 0.59)
|
||||
$stats.median | Should -Be 0.5
|
||||
$stats.p95 | Should -BeGreaterThan 0.5
|
||||
$stats.p95 | Should -BeLessThan 0.6
|
||||
}
|
||||
}
|
||||
|
||||
Describe "Invoke-StartupRegressionGate" {
|
||||
It "reports no failures for an identical baseline" {
|
||||
$s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
|
||||
$b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -Be 0
|
||||
}
|
||||
|
||||
It "fails when nothing overlaps results and baseline (F-007)" {
|
||||
$s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
|
||||
$b = [ordered]@{ }
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'no_comparable_metrics'
|
||||
}
|
||||
|
||||
It "fails when a metric the baseline expects is missing from results (F-007)" {
|
||||
$s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120 } }
|
||||
$b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
$gate.Failures | Should -Contain 'mcp_stdio_default (p99, missing in results)'
|
||||
}
|
||||
|
||||
It "treats a positive current against a zero baseline as a failure (F-010)" {
|
||||
$s = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 100; p95 = 120; p99 = 140 } }
|
||||
$b = [ordered]@{ mcp_stdio_to_tools_list_ms = [ordered]@{ p50 = 0; p95 = 0; p99 = 0 } }
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $s -BaselineScenarios $b
|
||||
@($gate.Failures).Count | Should -BeGreaterThan 0
|
||||
}
|
||||
|
||||
It "accepts a -RepoRoot argument when stamping run metadata (F-009)" {
|
||||
$m = Get-PerfRunMetadata -RepoRoot $PSScriptRoot
|
||||
$m.Contains('commit') | Should -BeTrue
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Shared, side-effect-free helpers for the startup performance measurement scripts.
|
||||
|
||||
.DESCRIPTION
|
||||
These functions are extracted into their own file so they can be dot-sourced by
|
||||
Test-StartupPerformance.ps1 and covered by focused Pester tests without spawning
|
||||
the server or running the full measurement pipeline.
|
||||
#>
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compute the p-th percentile of a sample array using linear interpolation
|
||||
# between closest ranks (the inclusive method, equivalent to Excel's
|
||||
# PERCENTILE.INC). p is expressed as a fraction in [0, 1].
|
||||
# A single sample returns that sample for any percentile.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-Percentile {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][double[]] $Samples,
|
||||
[Parameter(Mandatory)][double] $Percentile
|
||||
)
|
||||
|
||||
if ($null -eq $Samples -or $Samples.Count -eq 0) {
|
||||
throw "Get-Percentile requires at least one sample."
|
||||
}
|
||||
if ($Percentile -lt 0 -or $Percentile -gt 1) {
|
||||
throw "Get-Percentile requires Percentile in the range [0, 1]."
|
||||
}
|
||||
|
||||
$sorted = @($Samples | Sort-Object)
|
||||
$count = $sorted.Count
|
||||
if ($count -eq 1) { return [double]$sorted[0] }
|
||||
|
||||
$rank = $Percentile * ($count - 1)
|
||||
$lower = [int][math]::Floor($rank)
|
||||
$upper = [int][math]::Ceiling($rank)
|
||||
if ($lower -eq $upper) { return [double]$sorted[$lower] }
|
||||
|
||||
$frac = $rank - $lower
|
||||
return [double]$sorted[$lower] + ($sorted[$upper] - $sorted[$lower]) * $frac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compute first/average/median/p50/p95/p99 timing stats from a sample array.
|
||||
# For an even number of samples the median is the average of the two middle
|
||||
# values (the statistically correct definition); for an odd count it is the
|
||||
# single middle value. p50/p95/p99 are computed via Get-Percentile (linear
|
||||
# interpolation) so they align with the budgets in the acceptance criteria.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-TimingStats {
|
||||
[CmdletBinding()]
|
||||
param([Parameter(Mandatory)][double[]] $Samples)
|
||||
|
||||
if ($null -eq $Samples -or $Samples.Count -eq 0) {
|
||||
throw "Get-TimingStats requires at least one sample."
|
||||
}
|
||||
|
||||
$sorted = @($Samples | Sort-Object)
|
||||
$count = $sorted.Count
|
||||
|
||||
if ($count % 2 -eq 1) {
|
||||
$median = $sorted[[math]::Floor($count / 2)]
|
||||
}
|
||||
else {
|
||||
$upper = $count / 2
|
||||
$lower = $upper - 1
|
||||
$median = ($sorted[$lower] + $sorted[$upper]) / 2
|
||||
}
|
||||
|
||||
return [ordered]@{
|
||||
first = $Samples[0]
|
||||
average = [math]::Round(($Samples | Measure-Object -Average).Average, 1)
|
||||
median = $median
|
||||
p50 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.50), 4)
|
||||
p95 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.95), 4)
|
||||
p99 = [math]::Round((Get-Percentile -Samples $Samples -Percentile 0.99), 4)
|
||||
all = $Samples
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Split a sample array into cold (first invocation) and warm (all subsequent
|
||||
# invocations) timing stats. Process-level cold vs warm startup is captured by
|
||||
# treating the first sample as cold (OS/file-cache/JIT cold) and the remaining
|
||||
# samples as warm. When only one sample exists, warm falls back to that sample.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Split-ColdWarm {
|
||||
[CmdletBinding()]
|
||||
param([Parameter(Mandatory)][long[]] $Samples)
|
||||
|
||||
if ($null -eq $Samples -or $Samples.Count -eq 0) {
|
||||
throw "Split-ColdWarm requires at least one sample."
|
||||
}
|
||||
|
||||
$cold = [long]$Samples[0]
|
||||
$warmSamples = if ($Samples.Count -gt 1) { @($Samples[1..($Samples.Count - 1)]) } else { @($Samples[0]) }
|
||||
|
||||
return [ordered]@{
|
||||
cold_ms = $cold
|
||||
warm = Get-TimingStats -Samples $warmSamples
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads a named member from either an ordered hashtable (results built in-process)
|
||||
# or a PSCustomObject (parsed from JSON). Returns $null when the member is absent
|
||||
# so callers can gracefully skip metrics missing from an older baseline.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-MemberValue {
|
||||
[CmdletBinding()]
|
||||
param($InputObject, [Parameter(Mandatory)][string] $Name)
|
||||
|
||||
if ($null -eq $InputObject) { return $null }
|
||||
if ($InputObject -is [System.Collections.IDictionary]) {
|
||||
if ($InputObject.Contains($Name)) { return $InputObject[$Name] }
|
||||
return $null
|
||||
}
|
||||
$prop = $InputObject.PSObject.Properties[$Name]
|
||||
if ($prop) { return $prop.Value }
|
||||
return $null
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runs an external perf-harness process with a hard timeout, capturing stdout and
|
||||
# stderr and rejecting a non-zero exit code. Prevents a hung or crashed harness
|
||||
# from stalling the whole job or being mistaken for a successful measurement.
|
||||
# Returns the captured stdout string.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-PerfHarnessProcess {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $FilePath,
|
||||
[string[]] $ArgumentList = @(),
|
||||
[int] $TimeoutSeconds = 120
|
||||
)
|
||||
|
||||
$psi = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$psi.FileName = $FilePath
|
||||
$psi.RedirectStandardOutput = $true
|
||||
$psi.RedirectStandardError = $true
|
||||
$psi.UseShellExecute = $false
|
||||
$psi.CreateNoWindow = $true
|
||||
foreach ($arg in $ArgumentList) { $null = $psi.ArgumentList.Add($arg) }
|
||||
|
||||
$proc = [System.Diagnostics.Process]::new()
|
||||
$proc.StartInfo = $psi
|
||||
try {
|
||||
if (-not $proc.Start()) { throw "Failed to start process '$FilePath'." }
|
||||
|
||||
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
||||
|
||||
if (-not $proc.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
try { $proc.Kill($true) }
|
||||
catch { if (-not $proc.HasExited) { throw } }
|
||||
$null = $proc.WaitForExit(5000)
|
||||
$errOut = $stderrTask.GetAwaiter().GetResult()
|
||||
$errMsg = if ([string]::IsNullOrWhiteSpace($errOut)) { '<no stderr>' } else { $errOut.Trim() }
|
||||
throw "Process '$FilePath' timed out after $TimeoutSeconds seconds. Stderr: $errMsg"
|
||||
}
|
||||
|
||||
$outText = $stdoutTask.GetAwaiter().GetResult()
|
||||
$errOut = $stderrTask.GetAwaiter().GetResult()
|
||||
if ($proc.ExitCode -ne 0) {
|
||||
$errMsg = if ([string]::IsNullOrWhiteSpace($errOut)) { '<no stderr>' } else { $errOut.Trim() }
|
||||
throw "Process '$FilePath' exited with code $($proc.ExitCode). Stderr: $errMsg"
|
||||
}
|
||||
|
||||
return $outText
|
||||
}
|
||||
finally {
|
||||
$proc.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runs a perf-harness process (see Invoke-PerfHarnessProcess) and returns the last
|
||||
# JSON object it emitted on stdout, parsed. Throws if the harness produced no JSON
|
||||
# line (after already rejecting non-zero exits), so a silent/garbled run fails loudly.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-PerfHarnessJson {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $FilePath,
|
||||
[string[]] $ArgumentList = @(),
|
||||
[int] $TimeoutSeconds = 120
|
||||
)
|
||||
|
||||
$stdout = Invoke-PerfHarnessProcess -FilePath $FilePath -ArgumentList $ArgumentList -TimeoutSeconds $TimeoutSeconds
|
||||
$jsonLine = @($stdout -split "`r?`n") | Where-Object { $_ -match '^\s*\{' } | Select-Object -Last 1
|
||||
if ([string]::IsNullOrWhiteSpace($jsonLine)) {
|
||||
throw "Harness '$FilePath' did not emit a JSON result line."
|
||||
}
|
||||
return $jsonLine | ConvertFrom-Json
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tiered startup-regression gate shared by Test-StartupPerformance.ps1 and
|
||||
# Check-StartupPerformanceRegression.ps1. Compares p50/p95/p99 of each tracked
|
||||
# latency scenario against the baseline using the acceptance-criteria budgets
|
||||
# from issue #3118:
|
||||
# * p50 and p95 may not regress by more than 10% (FailP50P95 = 1.10)
|
||||
# * p99 may not regress by more than 20% (FailP99 = 1.20)
|
||||
# * any change above 5% is reported as a warning (Warn = 1.05)
|
||||
# Discovery scaling efficiency (per-tool startup cost for the all-tools surface)
|
||||
# may not degrade by more than 15% (ScalingDegrade = 1.15).
|
||||
#
|
||||
# Returns [ordered]@{ Failures = @(...); Warnings = @(...) } and writes a
|
||||
# human-readable line per check via Write-Host.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-StartupRegressionGate {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] $ResultScenarios,
|
||||
[Parameter(Mandatory)] $BaselineScenarios,
|
||||
[double] $FailP50P95 = 1.10,
|
||||
[double] $FailP99 = 1.20,
|
||||
[double] $Warn = 1.05,
|
||||
[double] $ScalingDegrade = 1.15
|
||||
)
|
||||
|
||||
$failures = @()
|
||||
$warnings = @()
|
||||
|
||||
# Latency scenarios: display name -> scenario property holding a stats object.
|
||||
$latencyScenarios = [ordered]@{
|
||||
'cli_cold_start' = 'cli_cold_start_ms'
|
||||
'mcp_stdio_default' = 'mcp_stdio_to_tools_list_ms'
|
||||
'mcp_namespace' = 'mcp_namespace_mode_startup_ms'
|
||||
'mcp_all' = 'mcp_all_mode_startup_ms'
|
||||
'mcp_http_default' = 'mcp_http_default_startup_ms'
|
||||
}
|
||||
|
||||
$percentileBudgets = @(
|
||||
@{ Metric = 'p50'; Limit = $FailP50P95 }
|
||||
@{ Metric = 'p95'; Limit = $FailP50P95 }
|
||||
@{ Metric = 'p99'; Limit = $FailP99 }
|
||||
)
|
||||
|
||||
$evaluated = 0
|
||||
|
||||
foreach ($name in $latencyScenarios.Keys) {
|
||||
$key = $latencyScenarios[$name]
|
||||
$curStats = Get-MemberValue $ResultScenarios $key
|
||||
$baseStats = Get-MemberValue $BaselineScenarios $key
|
||||
if ($null -eq $baseStats) {
|
||||
Write-Host (" [SKIP] {0}: absent in baseline" -f $name)
|
||||
continue
|
||||
}
|
||||
if ($null -eq $curStats) {
|
||||
$failures += "$name (missing in results)"
|
||||
Write-Host (" [FAIL] {0}: expected by baseline but absent in results" -f $name)
|
||||
continue
|
||||
}
|
||||
|
||||
foreach ($budget in $percentileBudgets) {
|
||||
$metric = $budget.Metric
|
||||
$current = Get-MemberValue $curStats $metric
|
||||
$baseline = Get-MemberValue $baseStats $metric
|
||||
if ($null -eq $baseline) {
|
||||
Write-Host (" [SKIP] {0} ({1}): absent in baseline" -f $name, $metric)
|
||||
continue
|
||||
}
|
||||
if ($null -eq $current) {
|
||||
$failures += "$name ($metric, missing in results)"
|
||||
Write-Host (" [FAIL] {0} ({1}): expected by baseline but absent in results" -f $name, $metric)
|
||||
continue
|
||||
}
|
||||
|
||||
$label = "$name ($metric)"
|
||||
$evaluated++
|
||||
if ($baseline -gt 0) {
|
||||
$ratio = $current / $baseline
|
||||
$limit = [math]::Round($baseline * $budget.Limit, 1)
|
||||
if ($ratio -gt $budget.Limit) { $failures += $label; $status = 'FAIL' }
|
||||
elseif ($ratio -gt $Warn) { $warnings += $label; $status = 'WARN' }
|
||||
else { $status = 'PASS' }
|
||||
Write-Host (" [{0}] {1}: current={2}ms baseline={3}ms limit={4}ms (+{5}%)" -f `
|
||||
$status, $label, $current, $baseline, $limit, [math]::Round(($ratio - 1) * 100, 1))
|
||||
}
|
||||
elseif ($current -gt 0) {
|
||||
# Positive current against a zero baseline is not comparable — fail rather than
|
||||
# silently pass with a fabricated 0% change.
|
||||
$failures += $label
|
||||
Write-Host (" [FAIL] {0}: current={1}ms vs a zero baseline (not comparable)" -f $label, $current)
|
||||
}
|
||||
else {
|
||||
Write-Host (" [PASS] {0}: current=0ms baseline=0ms (unchanged)" -f $label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Discovery scaling efficiency: per-tool startup cost for the all-tools surface.
|
||||
$curAll = Get-MemberValue $ResultScenarios 'mcp_all_mode_startup_ms'
|
||||
$curAllPl = Get-MemberValue $ResultScenarios 'all_mode_tools_list_payload'
|
||||
$baseAll = Get-MemberValue $BaselineScenarios 'mcp_all_mode_startup_ms'
|
||||
$baseAllPl = Get-MemberValue $BaselineScenarios 'all_mode_tools_list_payload'
|
||||
$curP50 = Get-MemberValue $curAll 'p50'
|
||||
$curCount = Get-MemberValue $curAllPl 'tool_count'
|
||||
$baseP50 = Get-MemberValue $baseAll 'p50'
|
||||
$baseCount = Get-MemberValue $baseAllPl 'tool_count'
|
||||
if ($curP50 -and $curCount -and $baseP50 -and $baseCount -and $curCount -gt 0 -and $baseCount -gt 0) {
|
||||
$curPerTool = $curP50 / $curCount
|
||||
$basePerTool = $baseP50 / $baseCount
|
||||
$ratio = if ($basePerTool -gt 0) { $curPerTool / $basePerTool } else { 1 }
|
||||
$evaluated++
|
||||
if ($ratio -gt $ScalingDegrade) {
|
||||
$failures += 'discovery_scaling_efficiency'
|
||||
$status = 'FAIL'
|
||||
}
|
||||
elseif ($ratio -gt $Warn) {
|
||||
$warnings += 'discovery_scaling_efficiency'
|
||||
$status = 'WARN'
|
||||
}
|
||||
else {
|
||||
$status = 'PASS'
|
||||
}
|
||||
Write-Host (" [{0}] discovery_scaling_efficiency: current={1}ms/tool baseline={2}ms/tool (+{3}%)" -f `
|
||||
$status, [math]::Round($curPerTool, 3), [math]::Round($basePerTool, 3), [math]::Round(($ratio - 1) * 100, 1))
|
||||
}
|
||||
else {
|
||||
Write-Host " [SKIP] discovery_scaling_efficiency: all-mode metrics absent"
|
||||
}
|
||||
|
||||
if ($evaluated -eq 0) {
|
||||
# Nothing overlapped between results and baseline — an incompatible or empty baseline
|
||||
# must not be reported as a pass.
|
||||
$failures += 'no_comparable_metrics'
|
||||
Write-Host " [FAIL] no comparable latency metrics between results and baseline (incompatible or empty baseline)"
|
||||
}
|
||||
|
||||
return [ordered]@{ Failures = $failures; Warnings = $warnings }
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalize an output file path so its parent directory always exists.
|
||||
# A bare filename (no directory component) otherwise makes Split-Path return an
|
||||
# empty string, which causes New-Item -Path '' to fail. Relative paths are
|
||||
# resolved against BaseDirectory so the parent directory is unambiguous.
|
||||
# Returns the absolute file path.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Resolve-PerfOutputPath {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $Path,
|
||||
[string] $BaseDirectory = (Get-Location).Path
|
||||
)
|
||||
|
||||
if (-not [System.IO.Path]::IsPathRooted($Path)) {
|
||||
$Path = Join-Path $BaseDirectory $Path
|
||||
}
|
||||
$Path = [System.IO.Path]::GetFullPath($Path)
|
||||
|
||||
$parent = Split-Path -Parent $Path
|
||||
if ($parent -and -not (Test-Path $parent)) {
|
||||
$null = New-Item -ItemType Directory -Force -Path $parent
|
||||
}
|
||||
|
||||
return $Path
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads the catalog_version from the versioned workload catalog (eng/perf-workloads.json),
|
||||
# used to stamp every result so a number can be tied to the workload definition that produced
|
||||
# it (issue #3120). Returns 'unknown' if the catalog is missing or unreadable.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-WorkloadCatalogVersion {
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $CatalogPath = (Join-Path $PSScriptRoot '../perf-workloads.json')
|
||||
)
|
||||
|
||||
try {
|
||||
if (Test-Path $CatalogPath) {
|
||||
return (Get-Content $CatalogPath -Raw | ConvertFrom-Json).catalog_version
|
||||
}
|
||||
}
|
||||
catch {
|
||||
# Fall through to the default below.
|
||||
}
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Captures the controlled-environment metadata every perf result must record (issue #3120):
|
||||
# commit, runtime, OS, CPU, machine memory, plus the workload catalog version. Harnesses embed
|
||||
# this under an "environment" key so results are self-describing and comparable across runs.
|
||||
# Memory is best-effort and cross-platform; unresolved values are null rather than fatal.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Get-PerfRunMetadata {
|
||||
[CmdletBinding()]
|
||||
param([string] $RepoRoot)
|
||||
|
||||
# Resolve the commit against the repository that holds the script, not the caller's
|
||||
# working directory, so invoking a harness by absolute path from another checkout
|
||||
# (or from outside any repo) still stamps the correct commit.
|
||||
$gitLocation = if ($RepoRoot) { @('-C', $RepoRoot) } else { @() }
|
||||
$commit = try { (& git @gitLocation rev-parse --short HEAD 2>$null).Trim() } catch { '' }
|
||||
|
||||
$memoryGb = $null
|
||||
try {
|
||||
if ($IsWindows) {
|
||||
$bytes = (Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction Stop).TotalPhysicalMemory
|
||||
$memoryGb = [math]::Round($bytes / 1GB, 1)
|
||||
}
|
||||
elseif ($IsLinux -and (Test-Path '/proc/meminfo')) {
|
||||
$kb = ((Get-Content '/proc/meminfo' | Where-Object { $_ -match '^MemTotal:' }) -replace '[^\d]', '')
|
||||
if ($kb) { $memoryGb = [math]::Round([double]$kb / 1048576, 1) }
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$memoryGb = $null
|
||||
}
|
||||
|
||||
return [ordered]@{
|
||||
commit = $commit
|
||||
timestamp = (Get-Date).ToUniversalTime().ToString('o')
|
||||
runtime = [System.Runtime.InteropServices.RuntimeInformation]::FrameworkDescription
|
||||
os = [System.Runtime.InteropServices.RuntimeInformation]::OSDescription
|
||||
os_architecture = "$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)"
|
||||
cpu_cores = [Environment]::ProcessorCount
|
||||
machine_memory_gb = $memoryGb
|
||||
workload_catalog_version = Get-WorkloadCatalogVersion
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Measures end-to-end startup performance of the Azure MCP Server across four scenarios:
|
||||
1. CLI cold start : azmcp tools list (process spawn → exit)
|
||||
2. MCP stdio default : azmcp server start → initialize → tools/list response
|
||||
3. MCP stdio namespace : azmcp server start --mode namespace → tools/list response
|
||||
4. MCP stdio all-tools : azmcp server start --mode all → tools/list response
|
||||
|
||||
For each MCP scenario the script also tokenizes the tools/list response using the
|
||||
GPT-4o o200k_base encoding (via Microsoft.ML.Tokenizers) to give exact token budgets.
|
||||
|
||||
.PARAMETER Executable
|
||||
Path to the azmcp executable. Defaults to the Debug build in the repo.
|
||||
|
||||
.PARAMETER Runs
|
||||
Number of timing samples per scenario. Default: 5.
|
||||
|
||||
.PARAMETER OutputPath
|
||||
Optional path to write the JSON results file.
|
||||
If omitted, results are written to .perf-results/startup-e2e.json under the repo root.
|
||||
|
||||
.PARAMETER BaselinePath
|
||||
Optional path to a JSON baseline file (previously produced by this script).
|
||||
When provided, the script compares results against the baseline and fails if any
|
||||
scenario exceeds the tiered budgets from issue #3118 (p50/p95 +10%, p99 +20%,
|
||||
discovery scaling efficiency +15%; changes above +5% are reported as warnings).
|
||||
|
||||
.EXAMPLE
|
||||
# Default run against the Debug build
|
||||
./eng/scripts/Test-StartupPerformance.ps1
|
||||
|
||||
.EXAMPLE
|
||||
# Run against a Release build with more samples
|
||||
./eng/scripts/Test-StartupPerformance.ps1 -Executable ./servers/Azure.Mcp.Server/src/bin/Release/net10.0/azmcp.exe -Runs 10
|
||||
|
||||
.EXAMPLE
|
||||
# Compare against a saved baseline and fail on regression
|
||||
./eng/scripts/Test-StartupPerformance.ps1 -BaselinePath eng/perf-baseline.json
|
||||
#>
|
||||
|
||||
param(
|
||||
[string] $Executable,
|
||||
[int] $Runs = 10,
|
||||
[string] $OutputPath,
|
||||
[string] $BaselinePath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolve repo root and executable path
|
||||
# ---------------------------------------------------------------------------
|
||||
$ScriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { $PWD.Path }
|
||||
$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
|
||||
|
||||
# Shared, unit-tested helpers (Get-TimingStats, Resolve-PerfOutputPath).
|
||||
. (Join-Path $ScriptDir 'StartupPerformance.Common.ps1')
|
||||
|
||||
if (-not $Executable) {
|
||||
$Executable = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/src/bin/Debug/net10.0/azmcp.exe'
|
||||
if ($IsLinux -or $IsMacOS) {
|
||||
$Executable = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/src/bin/Debug/net10.0/azmcp'
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path $Executable)) {
|
||||
Write-Error "azmcp executable not found at: $Executable`nBuild first: dotnet build servers/Azure.Mcp.Server/src"
|
||||
}
|
||||
|
||||
# Resolve to a full path so child transports (e.g. the stdio SDK client) don't
|
||||
# fail on a leading './' that Windows cmd cannot parse.
|
||||
$Executable = (Resolve-Path $Executable).Path
|
||||
|
||||
$OutputPath = if ($OutputPath) { $OutputPath } else { Join-Path $RepoRoot '.perf-results/startup-e2e.json' }
|
||||
$OutputPath = Resolve-PerfOutputPath -Path $OutputPath -BaseDirectory $RepoRoot
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolve benchmark exe path (Release build preferred, Debug as fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
$BenchmarkExeSuffix = if ($IsWindows) { '.exe' } else { '' }
|
||||
$BenchmarkExeBase = Join-Path $RepoRoot 'servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks/bin'
|
||||
$BenchmarkExe = Join-Path $BenchmarkExeBase "Release/net10.0/Azure.Mcp.Server.PerformanceBenchmarks$BenchmarkExeSuffix"
|
||||
if (-not (Test-Path $BenchmarkExe)) {
|
||||
$BenchmarkExe = Join-Path $BenchmarkExeBase "Debug/net10.0/Azure.Mcp.Server.PerformanceBenchmarks$BenchmarkExeSuffix"
|
||||
}
|
||||
if (-not (Test-Path $BenchmarkExe)) {
|
||||
Write-Error "Benchmark exe not found. Build first: dotnet build -c Release servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks"
|
||||
}
|
||||
|
||||
Write-Host "azmcp executable : $Executable"
|
||||
Write-Host "Benchmark exe : $BenchmarkExe"
|
||||
Write-Host "Samples per scenario : $Runs"
|
||||
Write-Host ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: measure-cli-coldstart
|
||||
# Time from OS process creation to process exit for a single CLI invocation.
|
||||
# ---------------------------------------------------------------------------
|
||||
function Measure-CliColdStart {
|
||||
param([string] $ExePath)
|
||||
|
||||
$psi = [System.Diagnostics.ProcessStartInfo]@{
|
||||
FileName = $ExePath
|
||||
Arguments = 'tools list'
|
||||
RedirectStandardOutput = $true
|
||||
RedirectStandardError = $true
|
||||
UseShellExecute = $false
|
||||
CreateNoWindow = $true
|
||||
}
|
||||
|
||||
$timeoutMs = 30000
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$proc = [System.Diagnostics.Process]::Start($psi)
|
||||
try {
|
||||
# Drain both streams asynchronously to prevent buffer deadlocks
|
||||
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
||||
|
||||
if (-not $proc.WaitForExit($timeoutMs)) {
|
||||
$sw.Stop()
|
||||
try { $proc.Kill($true) }
|
||||
catch { if (-not $proc.HasExited) { throw } }
|
||||
$null = $proc.WaitForExit(5000)
|
||||
$stderr = $stderrTask.GetAwaiter().GetResult()
|
||||
$stderrMessage = if ([string]::IsNullOrWhiteSpace($stderr)) { '<no stderr>' } else { $stderr.Trim() }
|
||||
throw "CLI cold start timed out after $timeoutMs ms for '$ExePath tools list'. stderr: $stderrMessage"
|
||||
}
|
||||
|
||||
$sw.Stop()
|
||||
$null = $stdoutTask.GetAwaiter().GetResult()
|
||||
$stderr = $stderrTask.GetAwaiter().GetResult()
|
||||
|
||||
if ($proc.ExitCode -ne 0) {
|
||||
$stderrMessage = if ([string]::IsNullOrWhiteSpace($stderr)) { '<no stderr>' } else { $stderr.Trim() }
|
||||
throw "CLI cold start failed with exit code $($proc.ExitCode) for '$ExePath tools list'. stderr: $stderrMessage"
|
||||
}
|
||||
|
||||
return $sw.ElapsedMilliseconds
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $proc) { $proc.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: invoke-benchmark-mcp-startup
|
||||
# Calls the benchmark binary with --mcp-startup so the official MCP C# SDK
|
||||
# performs the initialize + tools/list handshake and reports timing.
|
||||
# The stopwatch inside the binary starts before McpClient.CreateAsync (which
|
||||
# spawns the server process) and stops after ListToolsAsync completes.
|
||||
# Returns the parsed JSON result object:
|
||||
# { elapsed_ms, tool_count,
|
||||
# name_description: { bytes, exact_tokens_gpt4o_o200k, approx_tokens_bytes_div_4 },
|
||||
# full_schema: { bytes, exact_tokens_gpt4o_o200k, approx_tokens_bytes_div_4 } }
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-BenchmarkMcpStartup {
|
||||
param(
|
||||
[string] $BenchmarkExe,
|
||||
[string] $ExePath,
|
||||
[string[]] $ServerArgTokens,
|
||||
[int] $TimeoutSeconds = 60,
|
||||
# '--mcp-startup' (stdio) or '--mcp-startup-http' (HTTP transport).
|
||||
[string] $BenchmarkMode = '--mcp-startup'
|
||||
)
|
||||
|
||||
# Bounded execution (hard timeout, stderr capture, non-zero exit rejection) and JSON parsing
|
||||
# are shared with the other perf runners via Invoke-PerfHarnessJson in StartupPerformance.Common.ps1.
|
||||
return Invoke-PerfHarnessJson -FilePath $BenchmarkExe `
|
||||
-ArgumentList (@($BenchmarkMode, $ExePath) + $ServerArgTokens) `
|
||||
-TimeoutSeconds $TimeoutSeconds
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: print a tools/list payload summary to the console
|
||||
# ---------------------------------------------------------------------------
|
||||
function Write-PayloadStats {
|
||||
param([string] $Label, $Payload)
|
||||
Write-Host "=== tools/list payload ($Label) ==="
|
||||
Write-Host (" Tool count : {0}" -f $Payload.tool_count)
|
||||
Write-Host (" Name+description bytes : {0:N0}" -f $Payload.name_description_bytes)
|
||||
Write-Host (" Name+description exact tokens (GPT-4o) : {0:N0}" -f $Payload.name_description_tokens_exact)
|
||||
Write-Host (" Full schema bytes (incl. inputSchema) : {0:N0}" -f $Payload.full_schema_bytes)
|
||||
Write-Host (" Full schema exact tokens (GPT-4o) [LLM cost]: {0:N0}" -f $Payload.full_schema_tokens_exact)
|
||||
Write-Host (" Full schema serialization time (ms) : {0}" -f $Payload.full_schema_serialize_ms)
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: run one MCP scenario (timing loop + cold/warm split + percentiles +
|
||||
# token/serialization measurement). Works for both stdio and HTTP transports
|
||||
# via the $BenchmarkMode switch. Uses the MCP C# SDK (via the benchmark binary)
|
||||
# for protocol-correct timing.
|
||||
# Returns [ordered]@{ stats = ...; coldWarm = ...; payload = ... }
|
||||
# ---------------------------------------------------------------------------
|
||||
function Invoke-McpScenario {
|
||||
param(
|
||||
[string] $Label,
|
||||
[string[]] $ServerArgs,
|
||||
[string] $ExePath,
|
||||
[int] $Runs,
|
||||
[string] $BenchmarkExe,
|
||||
[string] $BenchmarkMode = '--mcp-startup',
|
||||
[int] $TimeoutSeconds = 60
|
||||
)
|
||||
Write-Host "=== $Label ==="
|
||||
$ms = @()
|
||||
$readinessMs = @()
|
||||
$lastBenchResult = $null
|
||||
for ($i = 1; $i -le $Runs; $i++) {
|
||||
$benchResult = Invoke-BenchmarkMcpStartup -BenchmarkExe $BenchmarkExe `
|
||||
-ExePath $ExePath -ServerArgTokens $ServerArgs `
|
||||
-BenchmarkMode $BenchmarkMode -TimeoutSeconds $TimeoutSeconds
|
||||
$ms += [long]$benchResult.elapsed_ms
|
||||
if ($benchResult.PSObject.Properties.Name -contains 'readiness_ms') {
|
||||
$readinessMs += [long]$benchResult.readiness_ms
|
||||
}
|
||||
$lastBenchResult = $benchResult
|
||||
Write-Host (" Run {0}: {1} ms" -f $i, $benchResult.elapsed_ms)
|
||||
}
|
||||
$stats = Get-TimingStats -Samples $ms
|
||||
$coldWarm = Split-ColdWarm -Samples $ms
|
||||
Write-Host (" → cold={0} ms warm-median={1} ms p95={2} ms p99={3} ms" -f `
|
||||
$coldWarm.cold_ms, $coldWarm.warm.median, $stats.p95, $stats.p99)
|
||||
Write-Host ""
|
||||
$payload = [ordered]@{
|
||||
tool_count = [int]$lastBenchResult.tool_count
|
||||
name_description_bytes = [int]$lastBenchResult.name_description.bytes
|
||||
name_description_tokens_exact = [int]$lastBenchResult.name_description.exact_tokens_gpt4o_o200k
|
||||
full_schema_bytes = [int]$lastBenchResult.full_schema.bytes
|
||||
full_schema_tokens_exact = [int]$lastBenchResult.full_schema.exact_tokens_gpt4o_o200k
|
||||
full_schema_serialize_ms = [double]$lastBenchResult.full_schema.serialize_ms
|
||||
}
|
||||
Write-PayloadStats -Label $Label -Payload $payload
|
||||
$result = [ordered]@{ stats = $stats; coldWarm = $coldWarm; payload = $payload }
|
||||
if ($readinessMs.Count -gt 0) {
|
||||
$result.readiness = Get-TimingStats -Samples $readinessMs
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 1 – CLI cold start
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "=== Scenario 1: CLI cold start (azmcp tools list) ==="
|
||||
$cliMs = @()
|
||||
for ($i = 1; $i -le $Runs; $i++) {
|
||||
$ms = Measure-CliColdStart -ExePath $Executable
|
||||
$cliMs += $ms
|
||||
Write-Host (" Run {0}: {1} ms" -f $i, $ms)
|
||||
}
|
||||
|
||||
$cli = Get-TimingStats -Samples $cliMs
|
||||
$cliColdWarm = Split-ColdWarm -Samples $cliMs
|
||||
Write-Host (" → cold={0} ms warm-median={1} ms p95={2} ms p99={3} ms" -f `
|
||||
$cliColdWarm.cold_ms, $cliColdWarm.warm.median, $cli.p95, $cli.p99)
|
||||
Write-Host ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenarios 2-4 – MCP stdio startup across three server modes
|
||||
# Timing is performed by the benchmark binary using the official MCP C# SDK
|
||||
# (McpClient.CreateAsync + ListToolsAsync via StdioClientTransport).
|
||||
# ---------------------------------------------------------------------------
|
||||
$s2 = Invoke-McpScenario -Label 'Scenario 2: MCP stdio startup (default mode)' `
|
||||
-ServerArgs @('server', 'start') -ExePath $Executable -Runs $Runs `
|
||||
-BenchmarkExe $BenchmarkExe
|
||||
|
||||
$s3 = Invoke-McpScenario -Label 'Scenario 3: MCP stdio startup (--mode namespace)' `
|
||||
-ServerArgs @('server', 'start', '--mode', 'namespace') -ExePath $Executable -Runs $Runs `
|
||||
-BenchmarkExe $BenchmarkExe
|
||||
|
||||
$s4 = Invoke-McpScenario -Label 'Scenario 4: MCP stdio startup (--mode all)' `
|
||||
-ServerArgs @('server', 'start', '--mode', 'all') -ExePath $Executable -Runs $Runs `
|
||||
-BenchmarkExe $BenchmarkExe
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario 5 – MCP remote HTTP startup (default mode)
|
||||
# The benchmark binary spawns the server with --transport http
|
||||
# --dangerously-disable-http-incoming-auth on a free loopback port, polls
|
||||
# readiness, then times the initialize + tools/list handshake over
|
||||
# HttpClientTransport. readiness_ms captures server-ready time separately.
|
||||
# HTTP startup includes web-host build + Kestrel bind, so it needs a longer
|
||||
# per-run timeout than stdio.
|
||||
# ---------------------------------------------------------------------------
|
||||
$s5 = Invoke-McpScenario -Label 'Scenario 5: MCP remote HTTP startup (default mode)' `
|
||||
-ServerArgs @('server', 'start') -ExePath $Executable -Runs $Runs `
|
||||
-BenchmarkExe $BenchmarkExe -BenchmarkMode '--mcp-startup-http' `
|
||||
-TimeoutSeconds 90
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write results JSON
|
||||
# ---------------------------------------------------------------------------
|
||||
$gitCommit = (& git -C $RepoRoot rev-parse --short HEAD 2>$null) ?? 'unknown'
|
||||
|
||||
$results = [ordered]@{
|
||||
timestamp = (Get-Date -Format 'o')
|
||||
commit = $gitCommit
|
||||
executable = $Executable
|
||||
runs = $Runs
|
||||
environment = (Get-PerfRunMetadata -RepoRoot $RepoRoot)
|
||||
scenarios = [ordered]@{
|
||||
cli_cold_start_ms = $cli
|
||||
cli_cold_warm = $cliColdWarm
|
||||
mcp_stdio_to_tools_list_ms = $s2.stats
|
||||
mcp_stdio_cold_warm = $s2.coldWarm
|
||||
tools_list_payload = $s2.payload
|
||||
mcp_namespace_mode_startup_ms = $s3.stats
|
||||
mcp_namespace_cold_warm = $s3.coldWarm
|
||||
namespace_mode_tools_list_payload = $s3.payload
|
||||
mcp_all_mode_startup_ms = $s4.stats
|
||||
mcp_all_cold_warm = $s4.coldWarm
|
||||
all_mode_tools_list_payload = $s4.payload
|
||||
mcp_http_default_startup_ms = $s5.stats
|
||||
mcp_http_cold_warm = $s5.coldWarm
|
||||
mcp_http_readiness_ms = $s5.readiness
|
||||
http_default_tools_list_payload = $s5.payload
|
||||
}
|
||||
}
|
||||
|
||||
$resultsJson = $results | ConvertTo-Json -Depth 10
|
||||
$resultsJson | Out-File -FilePath $OutputPath -Encoding utf8 -Force
|
||||
Write-Host "Results written to: $OutputPath"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: regression check against a baseline file
|
||||
# ---------------------------------------------------------------------------
|
||||
if ($BaselinePath) {
|
||||
if (-not (Test-Path $BaselinePath)) {
|
||||
Write-Warning "Baseline file not found: $BaselinePath (skipping regression check)"
|
||||
}
|
||||
else {
|
||||
Write-Host ""
|
||||
Write-Host "=== Regression check (p50/p95 +10%, p99 +20%, scaling +15%, warn +5%) ==="
|
||||
|
||||
$baseline = Get-Content $BaselinePath | ConvertFrom-Json
|
||||
$gate = Invoke-StartupRegressionGate -ResultScenarios $results.scenarios `
|
||||
-BaselineScenarios $baseline.scenarios
|
||||
|
||||
if ($gate.Warnings.Count -gt 0) {
|
||||
Write-Warning "Startup performance warnings (>5%): $($gate.Warnings -join ', ')"
|
||||
}
|
||||
|
||||
if ($gate.Failures.Count -gt 0) {
|
||||
Write-Error "Startup regression detected in: $($gate.Failures -join ', ')"
|
||||
}
|
||||
else {
|
||||
Write-Host " All budget checks passed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit Azure DevOps pipeline variables (no-op outside AzDO)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "##vso[task.setvariable variable=CliColdStartMedianMs]$($cli.median)"
|
||||
Write-Host "##vso[task.setvariable variable=CliColdStartColdMs]$($cliColdWarm.cold_ms)"
|
||||
Write-Host "##vso[task.setvariable variable=CliColdStartWarmMedianMs]$($cliColdWarm.warm.median)"
|
||||
Write-Host "##vso[task.setvariable variable=CliColdStartP95Ms]$($cli.p95)"
|
||||
Write-Host "##vso[task.setvariable variable=CliColdStartP99Ms]$($cli.p99)"
|
||||
Write-Host "##vso[task.setvariable variable=McpStdioStartupMedianMs]$($s2.stats.median)"
|
||||
Write-Host "##vso[task.setvariable variable=McpStdioStartupP95Ms]$($s2.stats.p95)"
|
||||
Write-Host "##vso[task.setvariable variable=McpStdioStartupP99Ms]$($s2.stats.p99)"
|
||||
Write-Host "##vso[task.setvariable variable=ToolsListToolCount]$($s2.payload.tool_count)"
|
||||
Write-Host "##vso[task.setvariable variable=ToolsListNameDescTokensGpt4o]$($s2.payload.name_description_tokens_exact)"
|
||||
Write-Host "##vso[task.setvariable variable=ToolsListFullSchemaTokensGpt4o]$($s2.payload.full_schema_tokens_exact)"
|
||||
Write-Host "##vso[task.setvariable variable=ToolsListFullSchemaSerializeMs]$($s2.payload.full_schema_serialize_ms)"
|
||||
Write-Host "##vso[task.setvariable variable=McpNamespaceModeStartupMedianMs]$($s3.stats.median)"
|
||||
Write-Host "##vso[task.setvariable variable=NamespaceModeToolsListToolCount]$($s3.payload.tool_count)"
|
||||
Write-Host "##vso[task.setvariable variable=NamespaceModeToolsListFullSchemaTokensGpt4o]$($s3.payload.full_schema_tokens_exact)"
|
||||
Write-Host "##vso[task.setvariable variable=AllModeStartupMedianMs]$($s4.stats.median)"
|
||||
Write-Host "##vso[task.setvariable variable=AllModeStartupP95Ms]$($s4.stats.p95)"
|
||||
Write-Host "##vso[task.setvariable variable=AllModeStartupP99Ms]$($s4.stats.p99)"
|
||||
Write-Host "##vso[task.setvariable variable=AllModeToolsListToolCount]$($s4.payload.tool_count)"
|
||||
Write-Host "##vso[task.setvariable variable=AllModeToolsListFullSchemaTokensGpt4o]$($s4.payload.full_schema_tokens_exact)"
|
||||
Write-Host "##vso[task.setvariable variable=McpHttpDefaultStartupMedianMs]$($s5.stats.median)"
|
||||
Write-Host "##vso[task.setvariable variable=McpHttpDefaultStartupP95Ms]$($s5.stats.p95)"
|
||||
Write-Host "##vso[task.setvariable variable=McpHttpReadinessMedianMs]$(if ($s5.Contains('readiness')) { $s5.readiness.median } else { 'n/a' })"
|
||||
Write-Host "##vso[build.addbuildtag]perf-tracked"
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/bin/env pwsh
|
||||
#Requires -Version 7
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Appends a startup-performance run to a persistent trend file and regenerates a
|
||||
Markdown trend report.
|
||||
|
||||
.DESCRIPTION
|
||||
Reads the results JSON produced by Test-StartupPerformance.ps1 and appends a compact,
|
||||
one-line-per-run record to a JSON Lines (.jsonl) trend file. It then regenerates a
|
||||
Markdown table summarizing the most recent runs so reviewers can eyeball the trend
|
||||
(issue #3118: "Baseline results and trend report").
|
||||
|
||||
The JSONL file is the durable, machine-readable history; the Markdown file is a
|
||||
human-readable rendering of the tail. Both are safe to commit as build artifacts.
|
||||
|
||||
.PARAMETER ResultsPath
|
||||
Path to the results JSON written by Test-StartupPerformance.ps1.
|
||||
|
||||
.PARAMETER TrendPath
|
||||
Path to the JSONL trend file to append to. Created if it does not exist.
|
||||
Defaults to .perf-results/startup-trend.jsonl under the repo root.
|
||||
|
||||
.PARAMETER MarkdownPath
|
||||
Path to the Markdown report to (re)write. Defaults to .perf-results/startup-trend.md.
|
||||
|
||||
.PARAMETER MaxRows
|
||||
Maximum number of most-recent rows to render in the Markdown table (default 20).
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory)][string] $ResultsPath,
|
||||
[string] $TrendPath,
|
||||
[string] $MarkdownPath,
|
||||
[int] $MaxRows = 20
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
. (Join-Path $PSScriptRoot 'StartupPerformance.Common.ps1')
|
||||
|
||||
$ScriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { $PWD.Path }
|
||||
$RepoRoot = (Get-Item (Join-Path $ScriptDir '../..')).FullName
|
||||
|
||||
if (-not $TrendPath) { $TrendPath = Join-Path $RepoRoot '.perf-results/startup-trend.jsonl' }
|
||||
if (-not $MarkdownPath) { $MarkdownPath = Join-Path $RepoRoot '.perf-results/startup-trend.md' }
|
||||
|
||||
$TrendPath = Resolve-PerfOutputPath -Path $TrendPath -BaseDirectory $RepoRoot
|
||||
$MarkdownPath = Resolve-PerfOutputPath -Path $MarkdownPath -BaseDirectory $RepoRoot
|
||||
|
||||
$results = Get-Content $ResultsPath | ConvertFrom-Json
|
||||
$s = $results.scenarios
|
||||
|
||||
# Compact record: one flat object per run so the JSONL stays diff-friendly.
|
||||
$record = [ordered]@{
|
||||
timestamp = $results.timestamp
|
||||
commit = $results.commit
|
||||
runs = $results.runs
|
||||
cli_cold_ms = (Get-MemberValue (Get-MemberValue $s 'cli_cold_start_ms') 'median')
|
||||
cli_p95_ms = (Get-MemberValue (Get-MemberValue $s 'cli_cold_start_ms') 'p95')
|
||||
stdio_default_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_stdio_to_tools_list_ms') 'median')
|
||||
stdio_default_p95_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_stdio_to_tools_list_ms') 'p95')
|
||||
all_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_all_mode_startup_ms') 'median')
|
||||
all_p95_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_all_mode_startup_ms') 'p95')
|
||||
http_default_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_http_default_startup_ms') 'median')
|
||||
http_readiness_median_ms = (Get-MemberValue (Get-MemberValue $s 'mcp_http_readiness_ms') 'median')
|
||||
default_tool_count = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'tool_count')
|
||||
default_full_schema_tokens = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'full_schema_tokens_exact')
|
||||
all_tool_count = (Get-MemberValue (Get-MemberValue $s 'all_mode_tools_list_payload') 'tool_count')
|
||||
all_full_schema_tokens = (Get-MemberValue (Get-MemberValue $s 'all_mode_tools_list_payload') 'full_schema_tokens_exact')
|
||||
default_full_schema_serialize_ms = (Get-MemberValue (Get-MemberValue $s 'tools_list_payload') 'full_schema_serialize_ms')
|
||||
}
|
||||
|
||||
# Append as a single compact JSON line.
|
||||
$recordJson = $record | ConvertTo-Json -Depth 5 -Compress
|
||||
Add-Content -Path $TrendPath -Value $recordJson -Encoding utf8
|
||||
Write-Host "Appended trend record to: $TrendPath"
|
||||
|
||||
# Re-render the Markdown tail from the full JSONL history.
|
||||
$allRecords = Get-Content $TrendPath | Where-Object { $_.Trim() } | ForEach-Object { $_ | ConvertFrom-Json }
|
||||
$recent = @($allRecords | Select-Object -Last $MaxRows)
|
||||
|
||||
$sb = [System.Text.StringBuilder]::new()
|
||||
$null = $sb.AppendLine('# Azure MCP Server startup performance trend')
|
||||
$null = $sb.AppendLine()
|
||||
$null = $sb.AppendLine("_Last updated: $(Get-Date -Format 'o'). Showing the $($recent.Count) most recent of $($allRecords.Count) runs._")
|
||||
$null = $sb.AppendLine()
|
||||
$null = $sb.AppendLine('| Commit | CLI cold (ms) | stdio p50/p95 (ms) | all p50/p95 (ms) | HTTP p50 (ms) | HTTP ready (ms) | default tools | default tokens | all tokens |')
|
||||
$null = $sb.AppendLine('|--------|--------------:|-------------------:|-----------------:|--------------:|----------------:|--------------:|---------------:|-----------:|')
|
||||
foreach ($r in $recent) {
|
||||
$null = $sb.AppendLine(("| {0} | {1} | {2}/{3} | {4}/{5} | {6} | {7} | {8} | {9} | {10} |" -f `
|
||||
(Get-MemberValue $r 'commit'),
|
||||
(Get-MemberValue $r 'cli_cold_ms'),
|
||||
(Get-MemberValue $r 'stdio_default_median_ms'), (Get-MemberValue $r 'stdio_default_p95_ms'),
|
||||
(Get-MemberValue $r 'all_median_ms'), (Get-MemberValue $r 'all_p95_ms'),
|
||||
(Get-MemberValue $r 'http_default_median_ms'),
|
||||
(Get-MemberValue $r 'http_readiness_median_ms'),
|
||||
(Get-MemberValue $r 'default_tool_count'),
|
||||
(Get-MemberValue $r 'default_full_schema_tokens'),
|
||||
(Get-MemberValue $r 'all_full_schema_tokens')))
|
||||
}
|
||||
|
||||
$sb.ToString() | Out-File -FilePath $MarkdownPath -Encoding utf8 -Force
|
||||
Write-Host "Wrote Markdown trend report to: $MarkdownPath"
|
||||
@@ -29,6 +29,7 @@ extends:
|
||||
PublishTarget: ${{ parameters.PublishTarget }}
|
||||
RunLiveTests: true
|
||||
RunVSCodeIntegrationTests: ${{ parameters.RunVSCodeIntegrationTests }}
|
||||
RunPerfTests: true
|
||||
PackageDocker: true
|
||||
PackageVSIX: true
|
||||
PackagePyPI: true
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
changes:
|
||||
- section: "Other Changes"
|
||||
description: "Added startup performance benchmarks and end-to-end measurement script (eng/scripts/Test-StartupPerformance.ps1) that measures CLI cold-start time and MCP startup time across stdio (default, namespace, all-tools) and remote HTTP transports, reporting cold vs. warm latency, p50/p95/p99 percentiles, server readiness time, tools/list JSON serialization time, and exact GPT-4o token budgets for each payload. A tool-catalog scaling benchmark measures discovery cost at current and projected tool counts. Results are gated against a baseline with tiered budgets (p50/p95 within 10%, p99 within 20%, scaling efficiency within 15%), summarized in a running trend report (eng/scripts/Update-StartupPerformanceTrend.ps1), and wired into the nightly AzDO pipeline via a new `Perf` stage in `common.yml`."
|
||||
@@ -0,0 +1,3 @@
|
||||
changes:
|
||||
- section: "Other Changes"
|
||||
description: "Added concurrency, throughput, and scalability validation for the Azure MCP Server (issue #3122). A new end-to-end harness (eng/scripts/Measure-ConcurrencyScaling.ps1 with a --mcp-concurrency mode) launches the server over HTTP and sweeps a ladder of concurrency levels, driving parallel MCP clients at each level and recording throughput, latency p50/p95/p99, error/timeout rate, concurrency scaling efficiency, and a state-isolation signal (concurrent responses whose tool count does not match the expected count, indicating cross-request leakage or a thread-safety failure). The resulting saturation curve shows where throughput plateaus. Results are gated against a baseline with the tiered budgets from issue #3122 (throughput within 10%, p95 latency and scaling efficiency within 15%, p99 latency within 20%, error-rate increase within 0.5 percentage points, and no state leakage), and wired into the AzDO perf pipeline."
|
||||
@@ -0,0 +1,3 @@
|
||||
changes:
|
||||
- section: "Other Changes"
|
||||
description: "Added command-dispatch and transport overhead benchmarks for the Azure MCP Server (issue #3119). New BenchmarkDotNet micro-benchmarks (CommandDispatchBenchmarks) measure each server-only dispatch layer in isolation — routing, argument parsing, option binding, validation, the execution frame, and response serialization — using a deterministic no-op command so no Azure work is included, which pinpoints the responsible layer for a regression. A new end-to-end harness (eng/scripts/Measure-DispatchOverhead.ps1 with --mcp-dispatch-stdio/--mcp-dispatch-http modes) times tools/list over stdio and HTTP and subtracts to isolate transport-specific overhead. Results are gated against a baseline with the tiered budgets from issue #3119 (p50/p95 within 10%, p99 within 20%, transport-overhead changes above 5% reported), and wired into the AzDO perf pipeline."
|
||||
@@ -0,0 +1,3 @@
|
||||
changes:
|
||||
- section: "Other Changes"
|
||||
description: "Added CPU, memory, and handle resource baselines for the Azure MCP Server (eng/scripts/Measure-ResourceUsage.ps1 and a new --mcp-resource harness) that sample process resource counters across the idle, startup, discovery, steady-state, and sustained soak phases under configurable concurrency. Soak leak detection fits a least-squares trend (with a warmup exclusion) to flag unbounded memory or handle growth. Results are gated against a baseline with the tiered budgets from issue #3121 (steady-state CPU and memory within 10%, peak memory within 15%, no unbounded soak growth), and wired into the AzDO perf pipeline."
|
||||
@@ -0,0 +1,3 @@
|
||||
changes:
|
||||
- section: "Other Changes"
|
||||
description: "Added a versioned, machine-readable performance workload catalog (eng/perf-workloads.json) and controlled-environment definition for the Azure MCP Server (issue #3120). The catalog defines every perf workload (startup, discovery scaling, dispatch, resource, concurrency) with its harness, transport, server mode, warm-up, repetitions, and metrics, plus shared environment requirements, statistical rules, and baseline-qualification governance. Every measurement harness now embeds a self-describing environment block (commit, runtime, OS, CPU, machine memory, and workload catalog version) in its results, and the default measured repetitions were raised to at least 10."
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<IsPackable>false</IsPackable>
|
||||
<!-- This is a dev-time perf tool; AOT/trim analyzers are not needed -->
|
||||
<IsAotCompatible>false</IsAotCompatible>
|
||||
<EnableAotAnalyzer>false</EnableAotAnalyzer>
|
||||
<EnableTrimAnalyzer>false</EnableTrimAnalyzer>
|
||||
<PublishTrimmed>false</PublishTrimmed>
|
||||
<SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings>
|
||||
<NoWarn>$(NoWarn);IL2026;IL3050;IL2075;IL2072</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Azure.Mcp.Server.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="BenchmarkDotNet" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers" />
|
||||
<PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" />
|
||||
<PackageReference Include="ModelContextProtocol" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Copy the server's appsettings so ConfigureServices can initialize correctly -->
|
||||
<ItemGroup>
|
||||
<Content Include="..\..\src\appsettings.json" Link="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.CommandLine;
|
||||
using System.Text.Json;
|
||||
using Azure.Mcp.Server;
|
||||
using Azure.Mcp.Server.Perf.Dispatch;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Mcp.Core.Commands;
|
||||
using Microsoft.Mcp.Core.Models;
|
||||
using Microsoft.Mcp.Core.Models.Command;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Measures the Azure MCP Server's own per-call dispatch overhead in isolation (issue #3119):
|
||||
/// the cost of routing a tool name to a command, parsing MCP arguments, binding and validating
|
||||
/// options, running the execution frame, and serializing the response — all with a deterministic
|
||||
/// <see cref="NoOpCommand"/> so no Azure or network latency is included.
|
||||
///
|
||||
/// Each layer is a separate benchmark so a regression can be attributed to the responsible layer.
|
||||
/// <c>[MemoryDiagnoser]</c> also records allocations per layer.
|
||||
///
|
||||
/// Run with:
|
||||
/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks `
|
||||
/// -- --filter '*CommandDispatch*' --buildTimeout 600
|
||||
/// </summary>
|
||||
[MemoryDiagnoser]
|
||||
public class CommandDispatchBenchmarks
|
||||
{
|
||||
// A real, registered, Azure-free-to-look-up command name used to measure routing cost.
|
||||
private const string RealToolName = "subscription_list";
|
||||
|
||||
private ServiceProvider _serviceProvider = null!;
|
||||
private ICommandFactory _commandFactory = null!;
|
||||
private NoOpCommand _noop = null!;
|
||||
private IBaseCommand _noopBase = null!;
|
||||
private Command _noopSysCommand = null!;
|
||||
private Dictionary<string, JsonElement> _args = null!;
|
||||
private ParseResult _parseResult = null!;
|
||||
private NoOpCommand.NoOpOptions _boundOptions = null!;
|
||||
private CommandResponse _response = null!;
|
||||
|
||||
[GlobalSetup]
|
||||
public async Task Setup()
|
||||
{
|
||||
// Build the full server so routing measures a realistic command map (~500+ commands).
|
||||
var services = new ServiceCollection();
|
||||
Program.ConfigureServices(services);
|
||||
services.AddLogging(builder => builder.SetMinimumLevel(LogLevel.None));
|
||||
_serviceProvider = services.BuildServiceProvider();
|
||||
await Program.InitializeServicesAsync(_serviceProvider);
|
||||
_commandFactory = _serviceProvider.GetRequiredService<ICommandFactory>();
|
||||
|
||||
_noop = new NoOpCommand();
|
||||
_noopBase = _noop;
|
||||
_noopSysCommand = _noop.GetCommand();
|
||||
|
||||
_args = new Dictionary<string, JsonElement>
|
||||
{
|
||||
["value"] = JsonSerializer.SerializeToElement("hello"),
|
||||
["count"] = JsonSerializer.SerializeToElement(3),
|
||||
};
|
||||
|
||||
_noopSysCommand.TryParseFromDictionary(_args, out var parseResult, out _);
|
||||
_parseResult = parseResult!;
|
||||
_boundOptions = _noop.BindOptions(_parseResult);
|
||||
|
||||
// A populated response for the serialization benchmark.
|
||||
_response = await _noopBase.ExecuteAsync(new CommandContext(), _parseResult, CancellationToken.None);
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public async Task Cleanup()
|
||||
{
|
||||
// The full DI container holds services that only implement IAsyncDisposable
|
||||
// (e.g. CosmosService), so it must be disposed asynchronously.
|
||||
if (_serviceProvider is not null)
|
||||
{
|
||||
await _serviceProvider.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark(Description = "Routing – CommandFactory tool-name lookup")]
|
||||
public IBaseCommand? Routing() => _commandFactory.FindCommandByName(RealToolName);
|
||||
|
||||
[Benchmark(Description = "Parse – MCP arguments dictionary to ParseResult")]
|
||||
public bool Parse() => _noopSysCommand.TryParseFromDictionary(_args, out _, out _);
|
||||
|
||||
[Benchmark(Description = "Bind – ParseResult to options object")]
|
||||
public NoOpCommand.NoOpOptions Bind() => _noop.BindOptions(_parseResult);
|
||||
|
||||
[Benchmark(Description = "Validate – option validation")]
|
||||
public bool Validate()
|
||||
{
|
||||
var validationResult = new ValidationResult();
|
||||
_noop.ValidateOptions(_boundOptions, validationResult);
|
||||
return validationResult.IsValid;
|
||||
}
|
||||
|
||||
[Benchmark(Description = "SerializeResponse – CommandResponse to MCP JSON")]
|
||||
public string SerializeResponse()
|
||||
=> JsonSerializer.Serialize(_response, ModelsJsonContext.Default.CommandResponse);
|
||||
|
||||
[Benchmark(Description = "ExecuteFrame – bind + validate + execute + capture")]
|
||||
public async Task<CommandResponse> ExecuteFrame()
|
||||
=> await _noopBase.ExecuteAsync(new CommandContext(), _parseResult, CancellationToken.None);
|
||||
|
||||
[Benchmark(Description = "FullDispatch – parse + bind + validate + execute + serialize")]
|
||||
public async Task<string> FullDispatch()
|
||||
{
|
||||
_noopSysCommand.TryParseFromDictionary(_args, out var parseResult, out _);
|
||||
var response = await _noopBase.ExecuteAsync(new CommandContext(), parseResult!, CancellationToken.None);
|
||||
return JsonSerializer.Serialize(response, ModelsJsonContext.Default.CommandResponse);
|
||||
}
|
||||
}
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using Azure.Mcp.Server;
|
||||
using Azure.Mcp.Server.Perf;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Configs;
|
||||
using BenchmarkDotNet.Diagnosers;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Mcp.Core.Commands;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Measures each phase of the Azure MCP Server startup pipeline in isolation so regressions
|
||||
/// can be pinpointed to the specific phase that changed.
|
||||
///
|
||||
/// Each phase's prerequisites are built in a per-target <see cref="IterationSetupAttribute"/>
|
||||
/// hook so they are excluded from the timed body — e.g. the BuildServiceProvider benchmark
|
||||
/// times only <c>BuildServiceProvider()</c>, not the ConfigureServices work that precedes it.
|
||||
/// To keep setup/measurement one-to-one, the job runs a single invocation per iteration
|
||||
/// (InvocationCount = 1, UnrollFactor = 1), which is the pattern BenchmarkDotNet requires
|
||||
/// when IterationSetup establishes non-idempotent state.
|
||||
///
|
||||
/// Run with:
|
||||
/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks -- --filter *
|
||||
/// </summary>
|
||||
[Config(typeof(IsolatedPhaseConfig))]
|
||||
public class ServerStartupBenchmarks
|
||||
{
|
||||
// Per-phase prerequisites, prepared in IterationSetup so they are not part of the
|
||||
// timed body. Each is disposed in DisposeProviders after the iteration is recorded.
|
||||
private IServiceCollection _servicesForConfigure = null!;
|
||||
private IServiceCollection _servicesForBuild = null!;
|
||||
private ServiceProvider? _builtProvider;
|
||||
private ServiceProvider? _providerForInitialize;
|
||||
private ServiceProvider? _providerForCommandTree;
|
||||
|
||||
private sealed class IsolatedPhaseConfig : ManualConfig
|
||||
{
|
||||
public IsolatedPhaseConfig()
|
||||
{
|
||||
AddJob(Job.Default
|
||||
.WithLaunchCount(1)
|
||||
.WithWarmupCount(2)
|
||||
.WithIterationCount(10)
|
||||
.WithInvocationCount(1)
|
||||
.WithUnrollFactor(1));
|
||||
AddDiagnoser(MemoryDiagnoser.Default);
|
||||
}
|
||||
}
|
||||
|
||||
[IterationCleanup]
|
||||
public async Task DisposeProviders()
|
||||
{
|
||||
// Providers may resolve async-only-disposable services (e.g. CosmosService implements
|
||||
// IAsyncDisposable but not IDisposable), for which a synchronous Dispose() throws.
|
||||
// Dispose asynchronously in the cleanup hook, outside the timed benchmark body.
|
||||
await (_builtProvider?.DisposeAsync() ?? ValueTask.CompletedTask);
|
||||
await (_providerForInitialize?.DisposeAsync() ?? ValueTask.CompletedTask);
|
||||
await (_providerForCommandTree?.DisposeAsync() ?? ValueTask.CompletedTask);
|
||||
_builtProvider = null;
|
||||
_providerForInitialize = null;
|
||||
_providerForCommandTree = null;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Phase 1 – instantiate all IAreaSetup objects (the 60+ toolset registrations)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Benchmark(Description = "RegisterAreas – instantiate all IAreaSetup objects")]
|
||||
public object RegisterAreas() => Program.RegisterAreas();
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Phase 2 – wire the DI container (call all area.ConfigureServices)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[IterationSetup(Target = nameof(ConfigureServices))]
|
||||
public void SetupConfigureServices() => _servicesForConfigure = new ServiceCollection();
|
||||
|
||||
[Benchmark(Description = "ConfigureServices – register services into the DI container")]
|
||||
public IServiceCollection ConfigureServices()
|
||||
{
|
||||
Program.ConfigureServices(_servicesForConfigure);
|
||||
return _servicesForConfigure;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Phase 3 – compile the DI container (BuildServiceProvider)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[IterationSetup(Target = nameof(BuildServiceProvider))]
|
||||
public void SetupBuildServiceProvider() => _servicesForBuild = CreateSilentServices();
|
||||
|
||||
[Benchmark(Description = "BuildServiceProvider – compile the DI container")]
|
||||
public ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
_builtProvider = _servicesForBuild.BuildServiceProvider();
|
||||
return _builtProvider;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Phase 4 – async service initialization (telemetry, user-agent policy, etc.)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[IterationSetup(Target = nameof(InitializeServicesAsync))]
|
||||
public void SetupInitializeServices() => _providerForInitialize = CreateSilentServices().BuildServiceProvider();
|
||||
|
||||
[Benchmark(Description = "InitializeServicesAsync – async service warm-up")]
|
||||
public async Task InitializeServicesAsync() => await Program.InitializeServicesAsync(_providerForInitialize!);
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Phase 5 – build the full System.CommandLine command tree
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[IterationSetup(Target = nameof(BuildCommandTree))]
|
||||
public void SetupBuildCommandTree()
|
||||
{
|
||||
_providerForCommandTree = CreateSilentServices().BuildServiceProvider();
|
||||
Program.InitializeServicesAsync(_providerForCommandTree).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
[Benchmark(Description = "CommandFactory.RootCommand – build the full command tree")]
|
||||
public System.CommandLine.RootCommand BuildCommandTree() =>
|
||||
_providerForCommandTree!.GetRequiredService<ICommandFactory>().RootCommand;
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Full pipeline – DI build + service init + command tree.
|
||||
// This is the cumulative end-to-end measurement, so ConfigureServices/BuildServiceProvider
|
||||
// remain inside the timed body by design. RegisterAreas runs once as a static initializer
|
||||
// and is NOT included here (it has its own dedicated benchmark above).
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
[Benchmark(Description = "FullStartup – DI build + service init + command tree (excludes RegisterAreas static init)")]
|
||||
public async Task<System.CommandLine.RootCommand> FullStartup()
|
||||
{
|
||||
await using var sp = CreateSilentServices().BuildServiceProvider();
|
||||
await Program.InitializeServicesAsync(sp);
|
||||
return sp.GetRequiredService<ICommandFactory>().RootCommand;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Shared setup – registers all services with logging suppressed
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private static IServiceCollection CreateSilentServices()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
Program.ConfigureServices(services);
|
||||
services.AddLogging(b => b.SetMinimumLevel(LogLevel.None));
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Text.Json;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using ModelContextProtocol.Protocol;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Measures how tool-discovery cost scales as the exposed tool catalog grows, at current
|
||||
/// and projected tool counts (issue #3118: "Measure tool and namespace discovery at
|
||||
/// current and projected tool counts" / "Track scaling behavior as the tool catalog grows").
|
||||
///
|
||||
/// For each <see cref="ToolCount"/> the benchmark builds a synthetic tool catalog whose
|
||||
/// per-tool shape (name + description + a small typed JSON input schema) approximates a real
|
||||
/// Azure MCP tool, then times the JSON serialization of the whole catalog — the dominant,
|
||||
/// transport-independent cost of answering a tools/list request. Comparing the mean time per
|
||||
/// tool across counts yields the discovery scaling-efficiency metric: if serialization is
|
||||
/// linear, mean/ToolCount stays roughly constant as the catalog grows.
|
||||
///
|
||||
/// Run with:
|
||||
/// dotnet run -c Release --project servers/Azure.Mcp.Server/perf/Azure.Mcp.Server.PerformanceBenchmarks -- --filter *ToolSurfaceScaling*
|
||||
/// </summary>
|
||||
[MemoryDiagnoser]
|
||||
public class ToolSurfaceScalingBenchmarks
|
||||
{
|
||||
// 71 ≈ current default surface, 510 ≈ current all-tools surface; the rest bracket
|
||||
// projected growth so the scaling curve is visible on both sides of today's counts.
|
||||
[Params(71, 250, 510, 1000)]
|
||||
public int ToolCount { get; set; }
|
||||
|
||||
private Tool[] _tools = [];
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
_tools = new Tool[ToolCount];
|
||||
for (var i = 0; i < ToolCount; i++)
|
||||
{
|
||||
_tools[i] = CreateSyntheticTool(i);
|
||||
}
|
||||
}
|
||||
|
||||
[Benchmark(Description = "SerializeToolCatalog – serialize the full tools/list payload")]
|
||||
public string SerializeToolCatalog() => JsonSerializer.Serialize(_tools);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a single synthetic tool whose name, description, and input schema are sized to
|
||||
/// resemble a typical Azure MCP tool so per-tool serialization cost is representative.
|
||||
/// </summary>
|
||||
private static Tool CreateSyntheticTool(int index)
|
||||
{
|
||||
var inputSchema = JsonDocument.Parse($$"""
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"subscription": { "type": "string", "description": "The Azure subscription id or name for tool {{index}}." },
|
||||
"resourceGroup": { "type": "string", "description": "The resource group name." },
|
||||
"name": { "type": "string", "description": "The target resource name." },
|
||||
"limit": { "type": "integer", "description": "Maximum number of results to return." }
|
||||
},
|
||||
"required": ["subscription"]
|
||||
}
|
||||
""").RootElement;
|
||||
|
||||
return new Tool
|
||||
{
|
||||
Name = $"azmcp_service{index % 60}_resource_operation{index}",
|
||||
Description = $"Performs operation {index} against an Azure resource, returning a structured result. " +
|
||||
"Use this tool when the user asks to inspect or manage the corresponding Azure resource.",
|
||||
InputSchema = inputSchema,
|
||||
};
|
||||
}
|
||||
}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ModelContextProtocol.Client;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Measures the Azure MCP Server's behavior under concurrent load (issue #3122). It launches
|
||||
/// the server over HTTP and drives a <b>sweep of concurrency levels</b>; at each level a pool of
|
||||
/// parallel MCP clients issues <c>tools/list</c> for a fixed window, and the harness records the
|
||||
/// throughput, latency distribution (p50/p95/p99), error/timeout rate, and a state-isolation
|
||||
/// signal (responses whose tool count does not match the expected count — a sign of a
|
||||
/// thread-safety or request-state-leakage failure). Per-level <b>scaling efficiency</b> is the
|
||||
/// throughput relative to perfect linear scaling from the first level, so the emitted "saturation
|
||||
/// curve" shows where throughput plateaus or collapses.
|
||||
///
|
||||
/// Mode:
|
||||
/// --mcp-concurrency <exe> [serverArgs...]
|
||||
///
|
||||
/// Tunable via environment variables:
|
||||
/// PERF_CONC_LEVELS comma-separated concurrency levels (default "1,2,4,8,16")
|
||||
/// PERF_CONC_DURATION_SECONDS timed window per level (default 5)
|
||||
/// PERF_CONC_WARMUP_SECONDS warmup window per level, not counted (default 1)
|
||||
/// PERF_CONC_TIMEOUT_MS per-request timeout; a timeout counts as an error (default 10000)
|
||||
///
|
||||
/// Emits a single compact JSON line for the PowerShell harness to parse.
|
||||
/// </summary>
|
||||
internal static class ConcurrencyScalingMeasurement
|
||||
{
|
||||
private static readonly TimeSpan s_readinessTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
internal static async Task RunAsync(string exePath, string[] serverArgs)
|
||||
{
|
||||
var levels = GetEnvIntArray("PERF_CONC_LEVELS", [1, 2, 4, 8, 16]);
|
||||
var durationSeconds = GetEnvInt("PERF_CONC_DURATION_SECONDS", 5);
|
||||
var warmupSeconds = GetEnvInt("PERF_CONC_WARMUP_SECONDS", 1);
|
||||
var timeoutMs = GetEnvInt("PERF_CONC_TIMEOUT_MS", 10000);
|
||||
|
||||
var port = PerfServerProcess.GetFreeLoopbackPort();
|
||||
var endpoint = new Uri($"http://127.0.0.1:{port}");
|
||||
using var server = PerfServerProcess.StartHttpServer(exePath, serverArgs, endpoint);
|
||||
|
||||
try
|
||||
{
|
||||
var readinessMs = await PerfServerProcess.WaitForPortAsync(port, s_readinessTimeout, server);
|
||||
|
||||
// Establish the expected tool count once; concurrent responses that differ from it
|
||||
// indicate cross-request state leakage or a thread-safety failure.
|
||||
int expectedToolCount;
|
||||
{
|
||||
await using var probe = await McpClient.CreateAsync(NewHttpTransport(endpoint, "perf-conc-probe"));
|
||||
expectedToolCount = (await probe.ListToolsAsync()).Count;
|
||||
}
|
||||
|
||||
var levelResults = new List<object>();
|
||||
double? baselineThroughput = null;
|
||||
|
||||
foreach (var level in levels)
|
||||
{
|
||||
var r = await RunLevelAsync(endpoint, level, durationSeconds, warmupSeconds, timeoutMs, expectedToolCount);
|
||||
baselineThroughput ??= r.ThroughputRps;
|
||||
|
||||
var scalingEfficiency = baselineThroughput.Value > 0 && level > 0
|
||||
? Math.Round(r.ThroughputRps / baselineThroughput.Value / level, 3)
|
||||
: 0d;
|
||||
|
||||
levelResults.Add(new
|
||||
{
|
||||
concurrency = level,
|
||||
requests = r.Requests,
|
||||
errors = r.Errors,
|
||||
error_rate_pct = r.ErrorRatePct,
|
||||
tool_count_mismatches = r.ToolCountMismatches,
|
||||
throughput_rps = r.ThroughputRps,
|
||||
scaling_efficiency = scalingEfficiency,
|
||||
latency_ms = new { p50 = r.P50, p95 = r.P95, p99 = r.P99 },
|
||||
});
|
||||
}
|
||||
|
||||
var payload = new
|
||||
{
|
||||
server_args = string.Join(' ', serverArgs),
|
||||
readiness_ms = readinessMs,
|
||||
expected_tool_count = expectedToolCount,
|
||||
duration_seconds = durationSeconds,
|
||||
levels = levelResults,
|
||||
};
|
||||
|
||||
Console.WriteLine(JsonSerializer.Serialize(payload));
|
||||
}
|
||||
finally
|
||||
{
|
||||
PerfServerProcess.TryKillProcessTree(server);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs a single concurrency level: <paramref name="concurrency"/> parallel clients issue
|
||||
/// <c>tools/list</c> until the window closes, after a warmup window that is not counted.
|
||||
/// Returns throughput, error rate, tool-count mismatches, and latency percentiles for the
|
||||
/// timed window.
|
||||
/// </summary>
|
||||
private static async Task<LevelResult> RunLevelAsync(
|
||||
Uri endpoint, int concurrency, int durationSeconds, int warmupSeconds, int timeoutMs, int expectedToolCount)
|
||||
{
|
||||
var latencies = new ConcurrentBag<double>();
|
||||
long requests = 0;
|
||||
long errors = 0;
|
||||
long mismatches = 0;
|
||||
|
||||
var warmupDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(warmupSeconds);
|
||||
var endDeadline = warmupDeadline + TimeSpan.FromSeconds(durationSeconds);
|
||||
|
||||
var workers = new Task[concurrency];
|
||||
for (var w = 0; w < concurrency; w++)
|
||||
{
|
||||
var index = w;
|
||||
workers[w] = Task.Run(async () =>
|
||||
{
|
||||
await using var client = await McpClient.CreateAsync(NewHttpTransport(endpoint, $"perf-conc-{concurrency}-{index}"));
|
||||
while (DateTime.UtcNow < endDeadline)
|
||||
{
|
||||
var timed = DateTime.UtcNow >= warmupDeadline;
|
||||
using var cts = new CancellationTokenSource(timeoutMs);
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
try
|
||||
{
|
||||
var tools = await client.ListToolsAsync(cancellationToken: cts.Token);
|
||||
if (timed)
|
||||
{
|
||||
latencies.Add(Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
Interlocked.Increment(ref requests);
|
||||
if (tools.Count != expectedToolCount)
|
||||
{
|
||||
Interlocked.Increment(ref mismatches);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (timed)
|
||||
{
|
||||
Interlocked.Increment(ref requests);
|
||||
Interlocked.Increment(ref errors);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await Task.WhenAll(workers);
|
||||
|
||||
var sorted = latencies.ToArray();
|
||||
Array.Sort(sorted);
|
||||
var throughput = durationSeconds > 0 ? Math.Round((double)requests / durationSeconds, 1) : 0d;
|
||||
var errorRate = requests > 0 ? Math.Round((double)errors / requests * 100, 3) : 0d;
|
||||
|
||||
return new LevelResult(
|
||||
requests, errors, mismatches, errorRate, throughput,
|
||||
Percentile(sorted, 50), Percentile(sorted, 95), Percentile(sorted, 99));
|
||||
}
|
||||
|
||||
private readonly record struct LevelResult(
|
||||
long Requests, long Errors, long ToolCountMismatches, double ErrorRatePct,
|
||||
double ThroughputRps, double P50, double P95, double P99);
|
||||
|
||||
/// <summary>Linear-interpolated percentile of a pre-sorted array (0 when empty).</summary>
|
||||
private static double Percentile(double[] sorted, double p)
|
||||
{
|
||||
if (sorted.Length == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
if (sorted.Length == 1)
|
||||
{
|
||||
return Math.Round(sorted[0], 3);
|
||||
}
|
||||
|
||||
var rank = p / 100.0 * (sorted.Length - 1);
|
||||
var low = (int)Math.Floor(rank);
|
||||
var high = (int)Math.Ceiling(rank);
|
||||
var frac = rank - low;
|
||||
return Math.Round(sorted[low] + (sorted[high] - sorted[low]) * frac, 3);
|
||||
}
|
||||
|
||||
private static IClientTransport NewHttpTransport(Uri endpoint, string name)
|
||||
=> new HttpClientTransport(new HttpClientTransportOptions
|
||||
{
|
||||
Name = name,
|
||||
Endpoint = endpoint,
|
||||
TransportMode = HttpTransportMode.AutoDetect,
|
||||
});
|
||||
|
||||
private static int GetEnvInt(string name, int fallback)
|
||||
{
|
||||
var raw = Environment.GetEnvironmentVariable(name);
|
||||
return int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) ? value : fallback;
|
||||
}
|
||||
|
||||
private static int[] GetEnvIntArray(string name, int[] fallback)
|
||||
{
|
||||
var raw = Environment.GetEnvironmentVariable(name);
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var values = new List<int>();
|
||||
foreach (var part in raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
|
||||
{
|
||||
if (int.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) && value > 0)
|
||||
{
|
||||
values.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
return values.Count > 0 ? [.. values] : fallback;
|
||||
}
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ModelContextProtocol.Client;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Measures per-transport request overhead (issue #3119) by timing many repetitions of a
|
||||
/// deterministic, Azure-free MCP <c>tools/list</c> request against a live server. It runs over
|
||||
/// either stdio or the remote HTTP transport; the PowerShell harness runs both and subtracts
|
||||
/// to isolate transport-specific overhead independently of Azure latency.
|
||||
///
|
||||
/// (A <c>ping</c> floor was considered but the stateless HTTP transport does not implement the
|
||||
/// ping method, so <c>tools/list</c> — supported identically over both transports — is used as
|
||||
/// the common comparison operation.)
|
||||
///
|
||||
/// Modes:
|
||||
/// --mcp-dispatch-stdio <exe> [serverArgs...]
|
||||
/// --mcp-dispatch-http <exe> [serverArgs...]
|
||||
///
|
||||
/// Tunable via environment variables:
|
||||
/// PERF_DISPATCH_ITERATIONS timed samples per operation (default 200)
|
||||
/// PERF_DISPATCH_WARMUP warmup calls (not timed) per operation (default 20)
|
||||
///
|
||||
/// Emits a single compact JSON line with the raw per-call millisecond samples so the harness
|
||||
/// can compute p50/p95/p99 with the shared PowerShell percentile helpers.
|
||||
/// </summary>
|
||||
internal static class DispatchOverheadMeasurement
|
||||
{
|
||||
private static readonly TimeSpan s_readinessTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
internal static Task RunStdioAsync(string exePath, string[] serverArgs)
|
||||
=> RunAsync(exePath, serverArgs, useHttp: false);
|
||||
|
||||
internal static Task RunHttpAsync(string exePath, string[] serverArgs)
|
||||
=> RunAsync(exePath, serverArgs, useHttp: true);
|
||||
|
||||
private static async Task RunAsync(string exePath, string[] serverArgs, bool useHttp)
|
||||
{
|
||||
var iterations = GetEnvInt("PERF_DISPATCH_ITERATIONS", 200);
|
||||
var warmup = GetEnvInt("PERF_DISPATCH_WARMUP", 20);
|
||||
|
||||
Process? server = null;
|
||||
long? readinessMs = null;
|
||||
IClientTransport transport;
|
||||
|
||||
if (useHttp)
|
||||
{
|
||||
var port = PerfServerProcess.GetFreeLoopbackPort();
|
||||
var endpoint = new Uri($"http://127.0.0.1:{port}");
|
||||
server = PerfServerProcess.StartHttpServer(exePath, serverArgs, endpoint);
|
||||
readinessMs = await PerfServerProcess.WaitForPortAsync(port, s_readinessTimeout, server);
|
||||
transport = new HttpClientTransport(new HttpClientTransportOptions
|
||||
{
|
||||
Name = "perf-dispatch-http",
|
||||
Endpoint = endpoint,
|
||||
TransportMode = HttpTransportMode.AutoDetect,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
transport = new StdioClientTransport(new StdioClientTransportOptions
|
||||
{
|
||||
Name = "perf-dispatch-stdio",
|
||||
Command = exePath,
|
||||
Arguments = serverArgs,
|
||||
});
|
||||
}
|
||||
|
||||
var client = await McpClient.CreateAsync(transport);
|
||||
try
|
||||
{
|
||||
var toolsListMs = await MeasureAsync(iterations, warmup, async () => _ = await client.ListToolsAsync());
|
||||
EmitResult(useHttp ? "http" : "stdio", readinessMs, iterations, warmup, toolsListMs);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await client.DisposeAsync();
|
||||
if (server is not null)
|
||||
{
|
||||
PerfServerProcess.TryKillProcessTree(server);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs <paramref name="warmup"/> warmup calls (not timed), then times
|
||||
/// <paramref name="iterations"/> calls of <paramref name="action"/>, returning per-call
|
||||
/// durations in milliseconds.
|
||||
/// </summary>
|
||||
private static async Task<double[]> MeasureAsync(int iterations, int warmup, Func<Task> action)
|
||||
{
|
||||
for (var i = 0; i < warmup; i++)
|
||||
{
|
||||
await action();
|
||||
}
|
||||
|
||||
var samples = new double[iterations];
|
||||
for (var i = 0; i < iterations; i++)
|
||||
{
|
||||
var start = Stopwatch.GetTimestamp();
|
||||
await action();
|
||||
samples[i] = Stopwatch.GetElapsedTime(start).TotalMilliseconds;
|
||||
}
|
||||
|
||||
return samples;
|
||||
}
|
||||
|
||||
private static void EmitResult(
|
||||
string transport, long? readinessMs, int iterations, int warmup, double[] toolsListMs)
|
||||
{
|
||||
var result = new
|
||||
{
|
||||
transport,
|
||||
iterations,
|
||||
warmup,
|
||||
readiness_ms = readinessMs,
|
||||
tools_list_ms = Array.ConvertAll(toolsListMs, x => Math.Round(x, 4)),
|
||||
};
|
||||
|
||||
Console.WriteLine(JsonSerializer.Serialize(result));
|
||||
}
|
||||
|
||||
private static int GetEnvInt(string name, int fallback)
|
||||
{
|
||||
var raw = Environment.GetEnvironmentVariable(name);
|
||||
return int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) ? value : fallback;
|
||||
}
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using ModelContextProtocol.Client;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Measures end-to-end MCP client startup time using the official C# MCP SDK.
|
||||
/// This provides a protocol-correct timing of the full initialize → tools/list
|
||||
/// handshake as a real client would experience it, plus both token-budget metrics
|
||||
/// (name+description only, comparable across modes; and full inputSchema, the actual
|
||||
/// LLM context-window cost) and the JSON serialization time of each payload.
|
||||
///
|
||||
/// Modes:
|
||||
/// --mcp-startup <exe> [serverArgs...]
|
||||
/// Spawns the server via StdioClientTransport, performs the initialize
|
||||
/// handshake + tools/list request over stdio, and emits a compact JSON line.
|
||||
/// --mcp-startup-http <exe> [serverArgs...]
|
||||
/// Spawns the server with HTTP transport (--transport http
|
||||
/// --dangerously-disable-http-incoming-auth) on a free loopback port, polls
|
||||
/// readiness, then times the initialize + tools/list handshake over
|
||||
/// HttpClientTransport. The emitted JSON additionally includes readiness_ms
|
||||
/// (process start → server accepting connections).
|
||||
/// </summary>
|
||||
internal static class McpClientStartupMeasurement
|
||||
{
|
||||
// Number of repetitions used to average JSON serialization time so the metric is
|
||||
// stable despite sub-millisecond per-call durations.
|
||||
private const int SerializeSampleCount = 50;
|
||||
|
||||
// How long to wait for the HTTP server to start accepting connections.
|
||||
private static readonly TimeSpan s_httpReadinessTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// Spawns the MCP server at <paramref name="exePath"/> using
|
||||
/// <paramref name="serverArgs"/> over stdio, connects via the MCP SDK, and times
|
||||
/// the full initialize + tools/list round-trip.
|
||||
///
|
||||
/// The stopwatch starts just before <see cref="McpClient.CreateAsync"/> (which
|
||||
/// spawns the process and performs the initialize handshake) and stops after
|
||||
/// <c>ListToolsAsync</c> completes.
|
||||
/// </summary>
|
||||
internal static async Task RunAsync(string exePath, string[] serverArgs)
|
||||
{
|
||||
var transportOptions = new StdioClientTransportOptions
|
||||
{
|
||||
Name = "perf-harness",
|
||||
Command = exePath,
|
||||
Arguments = serverArgs,
|
||||
};
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var transport = new StdioClientTransport(transportOptions);
|
||||
await using var client = await McpClient.CreateAsync(transport);
|
||||
var tools = (await client.ListToolsAsync()).ToList();
|
||||
sw.Stop();
|
||||
|
||||
EmitResult(sw.ElapsedMilliseconds, readinessMs: null, tools);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Spawns the MCP server at <paramref name="exePath"/> in HTTP transport mode on a
|
||||
/// free loopback port, waits for it to accept connections, then times the full
|
||||
/// initialize + tools/list round-trip over <see cref="HttpClientTransport"/>.
|
||||
///
|
||||
/// The stopwatch starts before the server process is launched so <c>elapsed_ms</c>
|
||||
/// captures the complete cold path (process spawn → ready → initialize → tools/list),
|
||||
/// comparable to the stdio measurement. <c>readiness_ms</c> records the sub-interval
|
||||
/// from process start to the server accepting its first TCP connection.
|
||||
/// </summary>
|
||||
internal static async Task RunHttpAsync(string exePath, string[] serverArgs)
|
||||
{
|
||||
var port = PerfServerProcess.GetFreeLoopbackPort();
|
||||
var endpoint = new Uri($"http://127.0.0.1:{port}");
|
||||
|
||||
// Start the stopwatch before launching so elapsed_ms captures the complete cold
|
||||
// path (process spawn -> ready -> initialize -> tools/list), comparable to stdio.
|
||||
var sw = Stopwatch.StartNew();
|
||||
using var server = PerfServerProcess.StartHttpServer(exePath, serverArgs, endpoint);
|
||||
|
||||
try
|
||||
{
|
||||
var readinessMs = await PerfServerProcess.WaitForPortAsync(port, s_httpReadinessTimeout, server);
|
||||
|
||||
var transportOptions = new HttpClientTransportOptions
|
||||
{
|
||||
Name = "perf-harness-http",
|
||||
Endpoint = endpoint,
|
||||
TransportMode = HttpTransportMode.AutoDetect,
|
||||
};
|
||||
var transport = new HttpClientTransport(transportOptions);
|
||||
await using (var client = await McpClient.CreateAsync(transport))
|
||||
{
|
||||
var tools = (await client.ListToolsAsync()).ToList();
|
||||
sw.Stop();
|
||||
EmitResult(sw.ElapsedMilliseconds, readinessMs, tools);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
PerfServerProcess.TryKillProcessTree(server);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serializes the tool metrics and writes a single compact JSON line to stdout so
|
||||
/// the PowerShell harness can parse it with ConvertFrom-Json.
|
||||
/// </summary>
|
||||
private static void EmitResult(long elapsedMs, long? readinessMs, IReadOnlyList<McpClientTool> tools)
|
||||
{
|
||||
// --- name+description only (matches --count-tools; comparable across modes) ---
|
||||
var nameDescPayload = tools.Select(t => new { name = t.Name, description = t.Description }).ToArray();
|
||||
var nameDescJson = JsonSerializer.Serialize(nameDescPayload);
|
||||
var (nameDescBytes, nameDescTokens, nameDescApprox) = PerfTokenizer.Measure(nameDescJson);
|
||||
var nameDescSerializeMs = MeasureSerializeMs(nameDescPayload);
|
||||
|
||||
// --- full schema (name + description + inputSchema) ---
|
||||
// This is the actual LLM context-window cost: what a client sends to the model
|
||||
// as the tools array. ProtocolTool includes the full JSON Schema per tool.
|
||||
var fullSchemaPayload = tools.Select(t => t.ProtocolTool).ToArray();
|
||||
var fullSchemaJson = JsonSerializer.Serialize(fullSchemaPayload);
|
||||
var (fullSchemaBytes, fullSchemaTokens, fullSchemaApprox) = PerfTokenizer.Measure(fullSchemaJson);
|
||||
var fullSchemaSerializeMs = MeasureSerializeMs(fullSchemaPayload);
|
||||
|
||||
var result = new
|
||||
{
|
||||
elapsed_ms = elapsedMs,
|
||||
readiness_ms = readinessMs,
|
||||
tool_count = tools.Count,
|
||||
name_description = new
|
||||
{
|
||||
bytes = nameDescBytes,
|
||||
exact_tokens_gpt4o_o200k = nameDescTokens,
|
||||
approx_tokens_bytes_div_4 = nameDescApprox,
|
||||
serialize_ms = nameDescSerializeMs,
|
||||
},
|
||||
full_schema = new
|
||||
{
|
||||
bytes = fullSchemaBytes,
|
||||
exact_tokens_gpt4o_o200k = fullSchemaTokens,
|
||||
approx_tokens_bytes_div_4 = fullSchemaApprox,
|
||||
serialize_ms = fullSchemaSerializeMs,
|
||||
},
|
||||
};
|
||||
|
||||
// Single compact JSON line so PowerShell can parse it with ConvertFrom-Json.
|
||||
Console.WriteLine(JsonSerializer.Serialize(result));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Averages the wall-clock time to serialize <paramref name="payload"/> to JSON over
|
||||
/// <see cref="SerializeSampleCount"/> repetitions, isolating the serialization phase
|
||||
/// of tool discovery. Returns milliseconds rounded to three decimals.
|
||||
/// </summary>
|
||||
private static double MeasureSerializeMs<T>(T payload)
|
||||
{
|
||||
// Warm up once so JIT and any first-call setup are excluded from the timing.
|
||||
_ = JsonSerializer.Serialize(payload);
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
for (var i = 0; i < SerializeSampleCount; i++)
|
||||
{
|
||||
_ = JsonSerializer.Serialize(payload);
|
||||
}
|
||||
sw.Stop();
|
||||
|
||||
return Math.Round(sw.Elapsed.TotalMilliseconds / SerializeSampleCount, 3);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
using Microsoft.Mcp.Core.Commands;
|
||||
using Microsoft.Mcp.Core.Models.Command;
|
||||
using Microsoft.Mcp.Core.Options;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf.Dispatch;
|
||||
|
||||
/// <summary>
|
||||
/// A deterministic no-op command used to benchmark the server's own dispatch overhead
|
||||
/// (issue #3119) — argument binding, validation, execution frame, and response
|
||||
/// serialization — with zero Azure or network work. It is instantiated directly by the
|
||||
/// dispatch benchmarks and is <b>not</b> registered with the production server, so it never
|
||||
/// appears in the real tool surface.
|
||||
/// </summary>
|
||||
[CommandMetadata(
|
||||
Id = "0a5f5f5f-0000-0000-0000-0000000000ff",
|
||||
Name = "noop",
|
||||
Title = "No-op benchmark command",
|
||||
Description = "Deterministic no-op used to measure server dispatch overhead. Not registered in production.",
|
||||
OperationPlane = ToolOperationPlane.Control,
|
||||
Destructive = false,
|
||||
Idempotent = true,
|
||||
OpenWorld = false,
|
||||
ReadOnly = true,
|
||||
Secret = false,
|
||||
LocalRequired = false)]
|
||||
public sealed class NoOpCommand : BaseCommand<NoOpCommand.NoOpOptions, NoOpCommand.NoOpResult>
|
||||
{
|
||||
public override JsonTypeInfo<NoOpResult>? ResultTypeInfo => PerfDispatchJsonContext.Default.NoOpResult;
|
||||
|
||||
public override Task<CommandResponse> ExecuteAsync(
|
||||
CommandContext context, NoOpOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
SetResult(context, new NoOpResult(options.Value ?? "ok", options.Count ?? 0));
|
||||
return Task.FromResult(context.Response);
|
||||
}
|
||||
|
||||
public sealed class NoOpOptions
|
||||
{
|
||||
[Option(Name = "value", Description = "An echoed string value.")]
|
||||
public string? Value { get; set; }
|
||||
|
||||
[Option(Name = "count", Description = "An echoed integer value.")]
|
||||
public int? Count { get; set; }
|
||||
}
|
||||
|
||||
public sealed record NoOpResult(string Echo, int Count);
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf.Dispatch;
|
||||
|
||||
/// <summary>
|
||||
/// Source-generated JSON context for the no-op benchmark command's result type, so the
|
||||
/// dispatch benchmarks can serialize it AOT-safely with the same pattern real commands use.
|
||||
/// </summary>
|
||||
[JsonSerializable(typeof(NoOpCommand.NoOpResult))]
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
|
||||
internal partial class PerfDispatchJsonContext : JsonSerializerContext;
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Shared helpers for launching the Azure MCP Server as an external process in HTTP
|
||||
/// transport mode and probing its readiness. Used by both the startup-timing harness
|
||||
/// (<see cref="McpClientStartupMeasurement"/>) and the resource-usage harness
|
||||
/// (<see cref="ResourceUsageMeasurement"/>) so the launch, readiness-poll, and teardown
|
||||
/// behavior stay identical across measurements.
|
||||
/// </summary>
|
||||
internal static class PerfServerProcess
|
||||
{
|
||||
/// <summary>
|
||||
/// Reserves and immediately releases a free loopback TCP port so the server can bind it.
|
||||
/// </summary>
|
||||
internal static int GetFreeLoopbackPort()
|
||||
{
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
try
|
||||
{
|
||||
return ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
}
|
||||
finally
|
||||
{
|
||||
listener.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the server at <paramref name="exePath"/> in HTTP transport mode bound to
|
||||
/// <paramref name="endpoint"/>, appending the HTTP transport flags to
|
||||
/// <paramref name="serverArgs"/> and draining stdout/stderr so the process pipes never
|
||||
/// fill and block it. The caller owns the returned process and must terminate it with
|
||||
/// <see cref="TryKillProcessTree"/>.
|
||||
/// </summary>
|
||||
internal static Process StartHttpServer(string exePath, string[] serverArgs, Uri endpoint)
|
||||
{
|
||||
var httpArgs = new List<string>(serverArgs)
|
||||
{
|
||||
"--transport", "http", "--dangerously-disable-http-incoming-auth",
|
||||
};
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = exePath,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
foreach (var arg in httpArgs)
|
||||
{
|
||||
startInfo.ArgumentList.Add(arg);
|
||||
}
|
||||
startInfo.Environment["ASPNETCORE_URLS"] = endpoint.ToString();
|
||||
|
||||
var server = new Process { StartInfo = startInfo };
|
||||
if (!server.Start())
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start HTTP server process '{exePath}'.");
|
||||
}
|
||||
|
||||
// Drain the server's stdout/stderr so its pipes never fill and block it.
|
||||
_ = server.StandardOutput.ReadToEndAsync();
|
||||
_ = server.StandardError.ReadToEndAsync();
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Polls <paramref name="port"/> on the loopback interface until a TCP connection
|
||||
/// succeeds, returning the elapsed milliseconds from now. Throws if the server exits
|
||||
/// early or the timeout elapses.
|
||||
/// </summary>
|
||||
internal static async Task<long> WaitForPortAsync(int port, TimeSpan timeout, Process server)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (server.HasExited)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"HTTP server exited early with code {server.ExitCode} before accepting connections.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var probe = new TcpClient();
|
||||
var connectTask = probe.ConnectAsync(IPAddress.Loopback, port);
|
||||
var completed = await Task.WhenAny(connectTask, Task.Delay(250));
|
||||
if (completed == connectTask && probe.Connected)
|
||||
{
|
||||
sw.Stop();
|
||||
return sw.ElapsedMilliseconds;
|
||||
}
|
||||
}
|
||||
catch (SocketException)
|
||||
{
|
||||
// Not listening yet; keep polling.
|
||||
}
|
||||
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
throw new TimeoutException($"HTTP server did not accept connections on port {port} within {timeout.TotalSeconds:N0}s.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort termination of the server process and its children.
|
||||
/// </summary>
|
||||
internal static void TryKillProcessTree(Process process)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
process.WaitForExit(5000);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort during teardown; nothing actionable if the process is already gone.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Text;
|
||||
using Microsoft.ML.Tokenizers;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Shared GPT-4o tokenizer instance and byte/token measurement helper used by all
|
||||
/// measurement classes in this project.
|
||||
/// </summary>
|
||||
internal static class PerfTokenizer
|
||||
{
|
||||
// GPT-4o uses o200k_base encoding. GPT-4/GPT-3.5-turbo use cl100k_base.
|
||||
//
|
||||
// GPT-4o (o200k_base) is used as the reference model because it is the encoding shared
|
||||
// by the current generation of OpenAI models most commonly driving MCP clients (GPT-4o,
|
||||
// GPT-4o-mini, o1/o3 families), so its token counts best approximate the context-window
|
||||
// cost real agents pay for the tool surface. o200k_base is also more token-efficient than
|
||||
// the older cl100k_base, giving a conservative (lower-bound) estimate; exact counts still
|
||||
// vary per model, so these numbers are a comparable proxy rather than an absolute budget.
|
||||
internal static readonly TiktokenTokenizer Gpt4o =
|
||||
TiktokenTokenizer.CreateForModel("gpt-4o");
|
||||
|
||||
/// <summary>
|
||||
/// Returns <c>(Bytes, ExactTokens, ApproxTokens)</c> for <paramref name="text"/>
|
||||
/// using the GPT-4o o200k_base encoding.
|
||||
/// <c>ApproxTokens</c> is a quick bytes÷4 estimate useful for sanity-checking.
|
||||
/// </summary>
|
||||
internal static (int Bytes, int ExactTokens, int ApproxTokens) Measure(string text)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetByteCount(text);
|
||||
return (bytes, Gpt4o.CountTokens(text), bytes / 4);
|
||||
}
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Periodically samples an external process's resource counters (working set, private
|
||||
/// bytes, handle count, and cumulative CPU time) on a background loop, tagging every
|
||||
/// sample with the caller-supplied phase label so the samples can be aggregated per
|
||||
/// phase after the run completes.
|
||||
///
|
||||
/// The sampler collects into a private list from its own loop; callers must
|
||||
/// <see cref="Stop"/> the sampler and await the <see cref="RunAsync"/> task before
|
||||
/// reading <see cref="Samples"/> so there is no concurrent access to the backing list.
|
||||
/// </summary>
|
||||
internal sealed class ProcessResourceSampler
|
||||
{
|
||||
private readonly Process _process;
|
||||
private readonly TimeSpan _interval;
|
||||
private readonly List<ResourceSample> _samples = [];
|
||||
private readonly Stopwatch _clock = new();
|
||||
private volatile string _phase = "init";
|
||||
private volatile bool _running;
|
||||
|
||||
public ProcessResourceSampler(Process process, TimeSpan interval)
|
||||
{
|
||||
_process = process;
|
||||
_interval = interval;
|
||||
}
|
||||
|
||||
/// <summary>A single point-in-time reading of the process resource counters.</summary>
|
||||
internal readonly record struct ResourceSample(
|
||||
double ElapsedSeconds,
|
||||
string Phase,
|
||||
long WorkingSetBytes,
|
||||
long PrivateBytes,
|
||||
int HandleCount,
|
||||
double CpuSeconds);
|
||||
|
||||
/// <summary>The samples collected so far. Only read after <see cref="Stop"/> + awaiting <see cref="RunAsync"/>.</summary>
|
||||
public IReadOnlyList<ResourceSample> Samples => _samples;
|
||||
|
||||
/// <summary>Tags all subsequent samples with <paramref name="phase"/>.</summary>
|
||||
public void BeginPhase(string phase) => _phase = phase;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the sampling loop until <see cref="Stop"/> is called, taking one reading every
|
||||
/// interval plus a final reading on exit.
|
||||
/// </summary>
|
||||
public async Task RunAsync()
|
||||
{
|
||||
_running = true;
|
||||
_clock.Start();
|
||||
while (_running)
|
||||
{
|
||||
TrySample();
|
||||
await Task.Delay(_interval);
|
||||
}
|
||||
|
||||
TrySample();
|
||||
}
|
||||
|
||||
/// <summary>Signals the sampling loop to stop after its current iteration.</summary>
|
||||
public void Stop() => _running = false;
|
||||
|
||||
private void TrySample()
|
||||
{
|
||||
try
|
||||
{
|
||||
_process.Refresh();
|
||||
if (_process.HasExited)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_samples.Add(new ResourceSample(
|
||||
_clock.Elapsed.TotalSeconds,
|
||||
_phase,
|
||||
_process.WorkingSet64,
|
||||
_process.PrivateMemorySize64,
|
||||
SafeHandleCount(_process),
|
||||
_process.TotalProcessorTime.TotalSeconds));
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The process can exit between the HasExited check and a counter read; a
|
||||
// dropped sample is acceptable for a statistical baseline.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the handle count, returning 0 on platforms where it is not supported so the
|
||||
/// sampler degrades gracefully instead of throwing.
|
||||
/// </summary>
|
||||
private static int SafeHandleCount(Process process)
|
||||
{
|
||||
try
|
||||
{
|
||||
return process.HandleCount;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using Azure.Mcp.Server.Perf;
|
||||
using BenchmarkDotNet.Running;
|
||||
|
||||
// --count-tools : emit tool count + exact token count (names+descriptions)
|
||||
// --mcp-startup <exe> [serverArgs...] : time full MCP client initialize+tools/list via SDK (stdio)
|
||||
// --mcp-startup-http <exe> [serverArgs...] : time full MCP client initialize+tools/list via SDK (HTTP transport)
|
||||
// --mcp-resource <exe> [serverArgs...] : sample CPU/memory/handles across idle/startup/discovery/steady/soak phases (HTTP transport)
|
||||
// --mcp-dispatch-stdio <exe> [serverArgs...] : time ping + tools/list request overhead over stdio
|
||||
// --mcp-dispatch-http <exe> [serverArgs...] : time ping + tools/list request overhead over HTTP transport
|
||||
// --mcp-concurrency <exe> [serverArgs...] : sweep concurrency levels; throughput, latency, error rate, scaling efficiency (HTTP transport)
|
||||
// (all other args) : run BenchmarkDotNet benchmarks
|
||||
if (args.Length > 0 && args[0] == "--count-tools")
|
||||
{
|
||||
await ToolCountMeasurement.RunCountToolsAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-startup")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments (e.g. "server" "start" "--mode" "namespace")
|
||||
await McpClientStartupMeasurement.RunAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-startup-http")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments. The harness appends the
|
||||
// HTTP transport flags and measures startup over HttpClientTransport.
|
||||
await McpClientStartupMeasurement.RunHttpAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-resource")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments. The harness launches the
|
||||
// server over HTTP and samples process resource counters across each phase. Duration
|
||||
// and concurrency are controlled via PERF_* environment variables.
|
||||
await ResourceUsageMeasurement.RunAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-dispatch-stdio")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments. Times ping + tools/list
|
||||
// request overhead over stdio (issue #3119).
|
||||
await DispatchOverheadMeasurement.RunStdioAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-dispatch-http")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments. Times ping + tools/list
|
||||
// request overhead over the HTTP transport (issue #3119).
|
||||
await DispatchOverheadMeasurement.RunHttpAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.Length >= 2 && args[0] == "--mcp-concurrency")
|
||||
{
|
||||
// args[1] = path to azmcp exe; args[2..] = server arguments. Sweeps concurrency levels
|
||||
// over HTTP and reports throughput, latency, error rate, and scaling efficiency (issue #3122).
|
||||
// Levels and durations are controlled via PERF_CONC_* environment variables.
|
||||
await ConcurrencyScalingMeasurement.RunAsync(args[1], args.Length > 2 ? args[2..] : []);
|
||||
return;
|
||||
}
|
||||
|
||||
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
|
||||
+395
@@ -0,0 +1,395 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ModelContextProtocol.Client;
|
||||
using static Azure.Mcp.Server.Perf.ProcessResourceSampler;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Measures the Azure MCP Server's process-level resource consumption (CPU, memory, and
|
||||
/// handles) across the startup, idle, discovery, and steady-state phases, plus an
|
||||
/// optional sustained "soak" phase used to detect unbounded memory or handle growth.
|
||||
///
|
||||
/// The server is launched in HTTP transport mode on a free loopback port; a background
|
||||
/// <see cref="ProcessResourceSampler"/> reads the process counters throughout the run,
|
||||
/// tagging every reading with the current phase. Steady-state and soak load are driven by
|
||||
/// <c>PERF_CONCURRENCY</c> parallel MCP clients issuing repeated <c>tools/list</c>
|
||||
/// requests (an authentication-free, representative discovery workload).
|
||||
///
|
||||
/// Tunable via environment variables (all optional):
|
||||
/// PERF_SAMPLE_MS sampling interval in ms (default 100)
|
||||
/// PERF_IDLE_SECONDS idle observation window with no client (default 3)
|
||||
/// PERF_STEADY_SECONDS steady-state load duration (default 10)
|
||||
/// PERF_SOAK_SECONDS sustained soak duration; 0 disables the soak phase (default 0)
|
||||
/// PERF_CONCURRENCY number of concurrent load clients (default 1)
|
||||
/// PERF_LEAK_MB_PER_MIN working-set slope above which soak is flagged unbounded (default 5)
|
||||
/// PERF_LEAK_HANDLES_PER_MIN handle slope above which soak is flagged unbounded (default 20)
|
||||
///
|
||||
/// Emits a single compact JSON line to stdout for the PowerShell harness to parse.
|
||||
/// </summary>
|
||||
internal static class ResourceUsageMeasurement
|
||||
{
|
||||
private const double BytesPerMb = 1024.0 * 1024.0;
|
||||
|
||||
private static readonly TimeSpan s_readinessTimeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// Runs the full resource-usage measurement against the server at
|
||||
/// <paramref name="exePath"/> launched with <paramref name="serverArgs"/>.
|
||||
/// </summary>
|
||||
internal static async Task RunAsync(string exePath, string[] serverArgs)
|
||||
{
|
||||
var sampleMs = GetEnvInt("PERF_SAMPLE_MS", 100);
|
||||
var idleSeconds = GetEnvInt("PERF_IDLE_SECONDS", 3);
|
||||
var steadySeconds = GetEnvInt("PERF_STEADY_SECONDS", 10);
|
||||
var soakSeconds = GetEnvInt("PERF_SOAK_SECONDS", 0);
|
||||
var concurrency = Math.Max(1, GetEnvInt("PERF_CONCURRENCY", 1));
|
||||
|
||||
var port = PerfServerProcess.GetFreeLoopbackPort();
|
||||
var endpoint = new Uri($"http://127.0.0.1:{port}");
|
||||
|
||||
using var server = PerfServerProcess.StartHttpServer(exePath, serverArgs, endpoint);
|
||||
var sampler = new ProcessResourceSampler(server, TimeSpan.FromMilliseconds(sampleMs));
|
||||
sampler.BeginPhase("startup");
|
||||
var samplerTask = sampler.RunAsync();
|
||||
|
||||
try
|
||||
{
|
||||
// Phase: startup — process launch until it accepts its first connection.
|
||||
var readinessMs = await PerfServerProcess.WaitForPortAsync(port, s_readinessTimeout, server);
|
||||
|
||||
// Phase: idle — server ready, no client connected.
|
||||
sampler.BeginPhase("idle");
|
||||
await Task.Delay(TimeSpan.FromSeconds(idleSeconds));
|
||||
|
||||
// Phase: discovery — a single initialize + tools/list handshake.
|
||||
sampler.BeginPhase("discovery");
|
||||
int toolCount;
|
||||
{
|
||||
var transport = new HttpClientTransport(new HttpClientTransportOptions
|
||||
{
|
||||
Name = "perf-resource-discovery",
|
||||
Endpoint = endpoint,
|
||||
TransportMode = HttpTransportMode.AutoDetect,
|
||||
});
|
||||
await using var client = await McpClient.CreateAsync(transport);
|
||||
toolCount = (await client.ListToolsAsync()).Count;
|
||||
}
|
||||
|
||||
// Phase: steady-state — sustained load under PERF_CONCURRENCY clients.
|
||||
sampler.BeginPhase("steady_state");
|
||||
var steadyRequests = await DriveLoadAsync(endpoint, concurrency, TimeSpan.FromSeconds(steadySeconds));
|
||||
|
||||
// Phase: soak (optional) — longer sustained load for leak detection.
|
||||
long soakRequests = 0;
|
||||
if (soakSeconds > 0)
|
||||
{
|
||||
sampler.BeginPhase("soak");
|
||||
soakRequests = await DriveLoadAsync(endpoint, concurrency, TimeSpan.FromSeconds(soakSeconds));
|
||||
}
|
||||
|
||||
sampler.Stop();
|
||||
await samplerTask;
|
||||
|
||||
EmitResult(
|
||||
sampler.Samples, readinessMs, toolCount, concurrency, sampleMs,
|
||||
steadyRequests, steadySeconds, soakRequests, soakSeconds, serverArgs);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sampler.Stop();
|
||||
try
|
||||
{
|
||||
await samplerTask;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Sampler teardown is best-effort.
|
||||
}
|
||||
|
||||
PerfServerProcess.TryKillProcessTree(server);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drives <paramref name="concurrency"/> parallel MCP clients, each issuing
|
||||
/// <c>tools/list</c> in a tight loop until <paramref name="duration"/> elapses, and
|
||||
/// returns the total number of completed requests across all workers.
|
||||
/// </summary>
|
||||
private static async Task<long> DriveLoadAsync(Uri endpoint, int concurrency, TimeSpan duration)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + duration;
|
||||
var counts = new long[concurrency];
|
||||
var workers = new Task[concurrency];
|
||||
|
||||
for (var w = 0; w < concurrency; w++)
|
||||
{
|
||||
var index = w;
|
||||
workers[w] = Task.Run(async () =>
|
||||
{
|
||||
var transport = new HttpClientTransport(new HttpClientTransportOptions
|
||||
{
|
||||
Name = $"perf-load-{index}",
|
||||
Endpoint = endpoint,
|
||||
TransportMode = HttpTransportMode.AutoDetect,
|
||||
});
|
||||
await using var client = await McpClient.CreateAsync(transport);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
_ = await client.ListToolsAsync();
|
||||
counts[index]++;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await Task.WhenAll(workers);
|
||||
|
||||
long total = 0;
|
||||
foreach (var count in counts)
|
||||
{
|
||||
total += count;
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Aggregates the collected samples into per-phase, peak, growth, and soak statistics
|
||||
/// and writes a single compact JSON line to stdout.
|
||||
/// </summary>
|
||||
private static void EmitResult(
|
||||
IReadOnlyList<ResourceSample> all,
|
||||
long readinessMs,
|
||||
int toolCount,
|
||||
int concurrency,
|
||||
int sampleMs,
|
||||
long steadyRequests,
|
||||
int steadySeconds,
|
||||
long soakRequests,
|
||||
int soakSeconds,
|
||||
string[] serverArgs)
|
||||
{
|
||||
var procCount = Environment.ProcessorCount;
|
||||
|
||||
var phases = new Dictionary<string, object?>();
|
||||
foreach (var name in new[] { "startup", "idle", "discovery" })
|
||||
{
|
||||
var summary = SummarizePhase(all, name, procCount);
|
||||
if (summary is not null)
|
||||
{
|
||||
phases[name] = summary;
|
||||
}
|
||||
}
|
||||
|
||||
var steadySummary = SummarizePhase(all, "steady_state", procCount);
|
||||
if (steadySummary is not null)
|
||||
{
|
||||
steadySummary["requests"] = steadyRequests;
|
||||
steadySummary["throughput_rps"] = steadySeconds > 0
|
||||
? Math.Round((double)steadyRequests / steadySeconds, 1)
|
||||
: 0d;
|
||||
phases["steady_state"] = steadySummary;
|
||||
}
|
||||
|
||||
var peak = new
|
||||
{
|
||||
working_set_mb = all.Count > 0 ? MaxMb(all, x => x.WorkingSetBytes) : 0d,
|
||||
private_mb = all.Count > 0 ? MaxMb(all, x => x.PrivateBytes) : 0d,
|
||||
handles = all.Count > 0 ? all.Max(x => x.HandleCount) : 0,
|
||||
};
|
||||
|
||||
var growth = ComputeGrowth(all);
|
||||
var soak = ComputeSoak(all, soakRequests, soakSeconds);
|
||||
|
||||
var result = new
|
||||
{
|
||||
server_args = string.Join(' ', serverArgs),
|
||||
tool_count = toolCount,
|
||||
processor_count = procCount,
|
||||
concurrency,
|
||||
sample_interval_ms = sampleMs,
|
||||
sample_count = all.Count,
|
||||
readiness_ms = readinessMs,
|
||||
phases,
|
||||
peak,
|
||||
growth,
|
||||
soak,
|
||||
};
|
||||
|
||||
Console.WriteLine(JsonSerializer.Serialize(result));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the mean/max working-set, private-bytes, handle, and CPU-percent summary for
|
||||
/// a single phase, or <c>null</c> when the phase produced no samples.
|
||||
/// </summary>
|
||||
private static Dictionary<string, object?>? SummarizePhase(
|
||||
IReadOnlyList<ResourceSample> all, string phase, int procCount)
|
||||
{
|
||||
var s = all.Where(x => x.Phase == phase).ToList();
|
||||
if (s.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Dictionary<string, object?>
|
||||
{
|
||||
["samples"] = s.Count,
|
||||
["working_set_mb"] = new { mean = MeanMb(s, x => x.WorkingSetBytes), max = MaxMb(s, x => x.WorkingSetBytes) },
|
||||
["private_mb"] = new { mean = MeanMb(s, x => x.PrivateBytes), max = MaxMb(s, x => x.PrivateBytes) },
|
||||
["handles"] = new { mean = Math.Round(s.Average(x => (double)x.HandleCount), 0), max = s.Max(x => x.HandleCount) },
|
||||
["cpu_percent"] = CpuPercent(s, procCount),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Computes the mean working-set and handle growth from idle to steady-state.</summary>
|
||||
private static object ComputeGrowth(IReadOnlyList<ResourceSample> all)
|
||||
{
|
||||
var idle = all.Where(x => x.Phase == "idle").ToList();
|
||||
var steady = all.Where(x => x.Phase == "steady_state").ToList();
|
||||
|
||||
double? workingSet = null;
|
||||
double? handles = null;
|
||||
if (idle.Count > 0 && steady.Count > 0)
|
||||
{
|
||||
workingSet = Math.Round((steady.Average(x => x.WorkingSetBytes) - idle.Average(x => x.WorkingSetBytes)) / BytesPerMb, 2);
|
||||
handles = Math.Round(steady.Average(x => (double)x.HandleCount) - idle.Average(x => (double)x.HandleCount), 0);
|
||||
}
|
||||
|
||||
return new { idle_to_steady_working_set_mb = workingSet, idle_to_steady_handles = handles };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the working-set and handle-count trend (least-squares slope + R²) across the
|
||||
/// soak phase and flags unbounded growth when a slope exceeds its threshold with a
|
||||
/// well-correlated (R² ≥ 0.5) upward trend. Returns <c>null</c> when no soak ran.
|
||||
/// </summary>
|
||||
private static object? ComputeSoak(IReadOnlyList<ResourceSample> all, long soakRequests, int soakSeconds)
|
||||
{
|
||||
var soak = all.Where(x => x.Phase == "soak").ToList();
|
||||
if (soak.Count < 2)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Exclude an initial warmup portion so the leak verdict reflects steady-state growth
|
||||
// rather than the JIT/cache/thread-pool ramp that occurs when sustained load begins.
|
||||
// Real soak runs should be long enough (minutes) that this window is a small fraction.
|
||||
var warmup = soak.Count / 4;
|
||||
var analyzed = soak.Skip(warmup).ToList();
|
||||
if (analyzed.Count < 2)
|
||||
{
|
||||
analyzed = soak;
|
||||
}
|
||||
|
||||
var leakMbPerMin = GetEnvDouble("PERF_LEAK_MB_PER_MIN", 5.0);
|
||||
var leakHandlesPerMin = GetEnvDouble("PERF_LEAK_HANDLES_PER_MIN", 20.0);
|
||||
|
||||
var xs = analyzed.Select(x => x.ElapsedSeconds).ToList();
|
||||
var (wsSlopePerSec, wsR2) = LinearTrend(xs, analyzed.Select(x => x.WorkingSetBytes / BytesPerMb).ToList());
|
||||
var (handleSlopePerSec, handleR2) = LinearTrend(xs, analyzed.Select(x => (double)x.HandleCount).ToList());
|
||||
|
||||
var wsSlopePerMin = Math.Round(wsSlopePerSec * 60, 3);
|
||||
var handleSlopePerMin = Math.Round(handleSlopePerSec * 60, 3);
|
||||
|
||||
var unbounded = (wsSlopePerMin > leakMbPerMin && wsR2 >= 0.5)
|
||||
|| (handleSlopePerMin > leakHandlesPerMin && handleR2 >= 0.5);
|
||||
|
||||
return new
|
||||
{
|
||||
duration_s = soakSeconds,
|
||||
requests = soakRequests,
|
||||
samples = soak.Count,
|
||||
analyzed_samples = analyzed.Count,
|
||||
working_set_slope_mb_per_min = wsSlopePerMin,
|
||||
working_set_r2 = Math.Round(wsR2, 3),
|
||||
handle_slope_per_min = handleSlopePerMin,
|
||||
handle_r2 = Math.Round(handleR2, 3),
|
||||
unbounded_growth = unbounded,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Average CPU utilization percentage over a phase, derived from the cumulative CPU
|
||||
/// time delta divided by wall-clock time and processor count. Returns <c>null</c> when
|
||||
/// fewer than two samples make the rate undefined.
|
||||
/// </summary>
|
||||
private static double? CpuPercent(IReadOnlyList<ResourceSample> s, int procCount)
|
||||
{
|
||||
if (s.Count < 2)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var wall = s[^1].ElapsedSeconds - s[0].ElapsedSeconds;
|
||||
if (wall <= 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var cpu = s[^1].CpuSeconds - s[0].CpuSeconds;
|
||||
return Math.Round(cpu / (wall * procCount) * 100.0, 1);
|
||||
}
|
||||
|
||||
/// <summary>Least-squares linear regression returning the slope and R² of ys over xs.</summary>
|
||||
private static (double Slope, double R2) LinearTrend(IReadOnlyList<double> xs, IReadOnlyList<double> ys)
|
||||
{
|
||||
var n = xs.Count;
|
||||
if (n < 2)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
double sx = 0, sy = 0, sxx = 0, sxy = 0;
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
sx += xs[i];
|
||||
sy += ys[i];
|
||||
sxx += xs[i] * xs[i];
|
||||
sxy += xs[i] * ys[i];
|
||||
}
|
||||
|
||||
var denom = n * sxx - sx * sx;
|
||||
if (denom == 0)
|
||||
{
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
var slope = (n * sxy - sx * sy) / denom;
|
||||
var intercept = (sy - slope * sx) / n;
|
||||
var meanY = sy / n;
|
||||
|
||||
double ssTot = 0, ssRes = 0;
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var dy = ys[i] - meanY;
|
||||
ssTot += dy * dy;
|
||||
var e = ys[i] - (slope * xs[i] + intercept);
|
||||
ssRes += e * e;
|
||||
}
|
||||
|
||||
var r2 = ssTot <= 0 ? 0 : 1 - ssRes / ssTot;
|
||||
return (slope, r2);
|
||||
}
|
||||
|
||||
private static double MeanMb(IEnumerable<ResourceSample> s, Func<ResourceSample, long> selector)
|
||||
=> Math.Round(s.Average(x => (double)selector(x)) / BytesPerMb, 2);
|
||||
|
||||
private static double MaxMb(IEnumerable<ResourceSample> s, Func<ResourceSample, long> selector)
|
||||
=> Math.Round(s.Max(selector) / BytesPerMb, 2);
|
||||
|
||||
private static int GetEnvInt(string name, int fallback)
|
||||
{
|
||||
var raw = Environment.GetEnvironmentVariable(name);
|
||||
return int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) ? value : fallback;
|
||||
}
|
||||
|
||||
private static double GetEnvDouble(string name, double fallback)
|
||||
{
|
||||
var raw = Environment.GetEnvironmentVariable(name);
|
||||
return double.TryParse(raw, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) ? value : fallback;
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Text.Json;
|
||||
using Azure.Mcp.Server;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Mcp.Core.Commands;
|
||||
|
||||
namespace Azure.Mcp.Server.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Instantiates the full server in-process and counts the number of registered commands
|
||||
/// plus the exact token budget using the GPT-4o tokenizer.
|
||||
///
|
||||
/// Mode:
|
||||
/// --count-tools Builds DI container, counts commands, reports name+description
|
||||
/// byte and token cost. For full-schema token cost (including
|
||||
/// inputSchema per tool) use --mcp-startup instead.
|
||||
/// </summary>
|
||||
internal static class ToolCountMeasurement
|
||||
{
|
||||
internal static async Task RunCountToolsAsync()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
Program.ConfigureServices(services);
|
||||
services.AddLogging(b => b.SetMinimumLevel(LogLevel.None));
|
||||
|
||||
await using var sp = services.BuildServiceProvider();
|
||||
await Program.InitializeServicesAsync(sp);
|
||||
|
||||
var commandFactory = sp.GetRequiredService<ICommandFactory>();
|
||||
var allCommands = commandFactory.AllCommands;
|
||||
|
||||
var commandSummaries = allCommands
|
||||
.Select(kvp => new { name = kvp.Key, description = kvp.Value.Description })
|
||||
.OrderBy(c => c.name)
|
||||
.ToList();
|
||||
|
||||
var summaryJson = JsonSerializer.Serialize(commandSummaries);
|
||||
var (bytes, exactTokens, approxTokens) = PerfTokenizer.Measure(summaryJson);
|
||||
|
||||
var result = new
|
||||
{
|
||||
timestamp = DateTimeOffset.UtcNow,
|
||||
commandCount = allCommands.Count,
|
||||
nameAndDescriptionJsonBytes = bytes,
|
||||
nameAndDescriptionTokens = new
|
||||
{
|
||||
exact_gpt4o_o200k_base = exactTokens,
|
||||
approx_bytes_div_4 = approxTokens,
|
||||
error_pct = exactTokens == 0
|
||||
? (double?)null
|
||||
: Math.Round(Math.Abs(approxTokens - exactTokens) / (double)exactTokens * 100, 1)
|
||||
},
|
||||
notes = "Counts name+description only. Use --mcp-startup for full inputSchema token cost."
|
||||
};
|
||||
|
||||
Console.WriteLine(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,10 @@
|
||||
<SelfContained>true</SelfContained>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Azure.Mcp.Server.PerformanceBenchmarks" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -38,7 +38,7 @@ internal class Program
|
||||
private static readonly string ServerAreaName =
|
||||
Array.Find(Areas, static a => a is Microsoft.Mcp.Core.Areas.Server.ServerSetup)?.Name ?? "server";
|
||||
|
||||
private static async Task<int> Main(string[] args)
|
||||
internal static async Task<int> Main(string[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -169,7 +169,7 @@ internal class Program
|
||||
}
|
||||
}
|
||||
|
||||
private static IAreaSetup[] RegisterAreas()
|
||||
internal static IAreaSetup[] RegisterAreas()
|
||||
{
|
||||
|
||||
return [
|
||||
|
||||
Reference in new issue
Block a user