diff --git a/eng/tools/CopilotCliTester/src/AgentRunner.cs b/eng/tools/CopilotCliTester/src/AgentRunner.cs index 9ba7d46b7..de0fa6536 100644 --- a/eng/tools/CopilotCliTester/src/AgentRunner.cs +++ b/eng/tools/CopilotCliTester/src/AgentRunner.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Diagnostics; using System.Text; using System.Text.RegularExpressions; using CopilotCliTester.Models; -using GitHub.Copilot.SDK; +using GitHub.Copilot; namespace CopilotCliTester; @@ -15,8 +14,8 @@ namespace CopilotCliTester; /// internal sealed partial class AgentRunner(CopilotClient client, string serverExecutablePath, string? outputDir = null, string? workspacePath = null) : IAsyncDisposable { - private static readonly string TimeStamp = DateTimeOffset.UtcNow.ToString("yyyyMMdd-HHmmss"); - private readonly Lock eventLock = new(); + private static readonly string s_timeStamp = DateTimeOffset.UtcNow.ToString("yyyyMMdd-HHmmss"); + private readonly Lock _eventLock = new(); private readonly string _outputDirectory = outputDir ?? Path.Combine(AppContext.BaseDirectory, "reports"); [GeneratedRegex(@"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")] @@ -60,10 +59,10 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe cliArgs.Add(Path.Combine(logDir, "copilot-logs")); } - var client = new CopilotClient(new CopilotClientOptions + var client = new CopilotClient(new() { - CliArgs = [.. cliArgs], - Cwd = workspace, + WorkingDirectory = workspace, + Connection = RuntimeConnection.ForStdio(args: cliArgs) }); return (client, workspace); @@ -94,6 +93,9 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe { Model = config.Model ?? CopilotTestConstants.ModelName, Streaming = true, + // Permit every tool (including MCP tools) without an allowlist. + GitHubMcpToolConfig = new() { EnableAllTools = true }, + // Auto-approve all tool executions without confirmation dialogs. OnPermissionRequest = PermissionHandler.ApproveAll, McpServers = BuildMcpServersConfig(serverExecutablePath), SystemMessage = systemMessage @@ -106,9 +108,9 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe var idleTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - session.On(ev => + session.On(ev => { - lock (eventLock) + lock (_eventLock) { if (isComplete) return; @@ -160,7 +162,7 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - lock (eventLock) + lock (_eventLock) { isComplete = true; } @@ -230,15 +232,15 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe private static string GenerateMarkdownReport(AgentRunConfig config, AgentMetadata metadata) { - var lines = new List(); - - lines.Add("# User Prompt"); - lines.Add(""); - lines.Add(config.Prompt); - lines.Add(""); - - lines.Add("# Assistant"); - lines.Add(""); + var lines = new List + { + "# User Prompt", + "", + config.Prompt, + "", + "# Assistant", + "" + }; // Store tool results keyed by toolCallId var toolResults = new Dictionary(); @@ -335,7 +337,7 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe private string BuildReportFilePath(AgentRunConfig config) { - var runDir = $"test-run-{TimeStamp}"; + var runDir = $"test-run-{s_timeStamp}"; var ns = config.Namespace ?? "unknown"; var tool = config.ToolName ?? $"test-{DateTime.UtcNow:yyyyMMdd-HHmmss}"; var file = $"{tool}-{DateTimeOffset.UtcNow:HHmmssfff}.md"; @@ -369,7 +371,7 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe }; } - private static bool TryMapEvent(object ev, out AgentSessionEvent mapped) + private static bool TryMapEvent(SessionEvent ev, out AgentSessionEvent mapped) { // Session idle if (ev is SessionIdleEvent) @@ -492,5 +494,4 @@ internal sealed partial class AgentRunner(CopilotClient client, string serverExe } private static string Truncate(string s, int max) => s.Length <= max ? s : s[..max] + "... (truncated)"; - } diff --git a/eng/tools/CopilotCliTester/src/AgentRunnerUtils.cs b/eng/tools/CopilotCliTester/src/AgentRunnerUtils.cs index 99f898831..161cc4c8d 100644 --- a/eng/tools/CopilotCliTester/src/AgentRunnerUtils.cs +++ b/eng/tools/CopilotCliTester/src/AgentRunnerUtils.cs @@ -3,7 +3,6 @@ using System.Text.Json; using CopilotCliTester.Models; -using GitHub.Copilot.SDK; namespace CopilotCliTester; @@ -13,12 +12,12 @@ namespace CopilotCliTester; internal static class AgentRunnerUtils { // Internal/meta tools we do NOT want to count as "the expected MCP tool" - private static readonly HashSet IgnoredTools = new(StringComparer.OrdinalIgnoreCase) + private static readonly HashSet s_ignoredTools = new(StringComparer.OrdinalIgnoreCase) { "report_intent" }; - private static readonly string[] prefixes = new[] { "azure-", "azure_" }; + private static readonly string[] s_prefixes = ["azure-", "azure_"]; /// /// Returns tool.execution_start events @@ -30,7 +29,7 @@ internal static class AgentRunnerUtils .Where(e => { var name = e.Data.TryGetValue("toolName", out var tn) ? tn?.ToString() : null; - return !string.IsNullOrWhiteSpace(name) && !IgnoredTools.Contains(name!); + return !string.IsNullOrWhiteSpace(name) && !s_ignoredTools.Contains(name!); }).ToList(); } @@ -47,7 +46,7 @@ internal static class AgentRunnerUtils return true; // Strip known single-segment namespace-proxy prefix instead of open-ended suffix match to avoid false positives (e.g., "subscription_list" matching "eventgrid_subscription_list") - foreach (var prefix in prefixes) + foreach (var prefix in s_prefixes) { if (resolved.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && string.Equals(resolved[prefix.Length..], expectedTool, StringComparison.OrdinalIgnoreCase)) diff --git a/eng/tools/CopilotCliTester/src/CopilotCliTester.csproj b/eng/tools/CopilotCliTester/src/CopilotCliTester.csproj index 27332e1cf..13d6d910e 100644 --- a/eng/tools/CopilotCliTester/src/CopilotCliTester.csproj +++ b/eng/tools/CopilotCliTester/src/CopilotCliTester.csproj @@ -6,6 +6,7 @@ CopilotCliTester enable enable + https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/ diff --git a/eng/tools/CopilotCliTester/src/Program.cs b/eng/tools/CopilotCliTester/src/Program.cs index dd3c13817..89d70e1d8 100644 --- a/eng/tools/CopilotCliTester/src/Program.cs +++ b/eng/tools/CopilotCliTester/src/Program.cs @@ -13,7 +13,7 @@ namespace CopilotCliTester; /// /// E2E Test Runner for Azure MCP tools using Copilot SDK. Runs prompts from e2eTestPrompts.md and verifies correct tools are invoked. /// -static class Program +internal static class Program { private static readonly TimeSpan PerAttemptTimeout = TimeSpan.FromMinutes(5); @@ -21,8 +21,8 @@ static class Program private static readonly Lock _reportLock = new(); private static readonly string ServerProjectRelativePath = Path.Combine("servers", "Azure.Mcp.Server", "src", "Azure.Mcp.Server.csproj"); - private static readonly string runId = DateTimeOffset.UtcNow.ToString("yyyyMMdd-HHmmss") + "-" + Guid.NewGuid().ToString("N").Substring(0, 6); - static async Task Main(string[] args) + private static readonly string s_runId = string.Concat(DateTimeOffset.UtcNow.ToString("yyyyMMdd-HHmmss"), "-", Guid.NewGuid().ToString("N").AsSpan(0, 6)); + private static async Task Main(string[] args) { var command = args.Length > 0 ? args[0].ToLowerInvariant() : "run"; @@ -34,7 +34,7 @@ static class Program }; } - static int ShowHelp() + private static int ShowHelp() { Console.WriteLine(""" Azure MCP E2E Test Runner (Copilot SDK) @@ -59,7 +59,7 @@ static class Program return 0; } - static async Task RunE2ETestsFromArgs(string[] args) + private static async Task RunE2ETestsFromArgs(string[] args) { string? namespaceFilter = null, tool = null, outputDir = CopilotTestConstants.OutputDirectory, model = CopilotTestConstants.ModelName, promptsFile = null; int max = CopilotTestConstants.MaxPrompts, retries = CopilotTestConstants.MaxRetryAttempts, parallel = CopilotTestConstants.Parallel; @@ -180,13 +180,13 @@ static class Program return await RunE2ETests(namespaceFilter, tool, max, retries, onePerTool, outputDir, model, parallel, threshold, failOnNoMatch, promptsFile); } - static void CleanStaleWorkspaces() + private static void CleanStaleWorkspaces() { try { var staleDirectoryDeleteThreshold = DateTimeOffset.UtcNow.AddMinutes(-20); var allDirectories = Directory.GetDirectories(Path.GetTempPath(), $"mcp-test-*") - .Where(dir => !dir.Contains(runId) && Directory.GetCreationTimeUtc(dir) < staleDirectoryDeleteThreshold); + .Where(dir => !dir.Contains(s_runId) && Directory.GetCreationTimeUtc(dir) < staleDirectoryDeleteThreshold); var count = 0; foreach (var directory in allDirectories) { @@ -211,7 +211,7 @@ static class Program } } - static async Task RunE2ETests(string? namespaceFilter, string? tool, int max, int retries, bool onePerTool, string outputDir, string model, int parallel, double threshold, bool failOnNoMatch, string? promptsFile = null) + private static async Task RunE2ETests(string? namespaceFilter, string? tool, int max, int retries, bool onePerTool, string outputDir, string model, int parallel, double threshold, bool failOnNoMatch, string? promptsFile = null) { CleanStaleWorkspaces(); @@ -318,7 +318,7 @@ static class Program try { // Create a dedicated client per task. When the client is disposed, it kills its CLI process tree, ensuring child azmcp.exe processes are cleaned up. - var (client, workspacePath) = AgentRunner.CreateSharedClient(runId, debug, outputDir); + var (client, workspacePath) = AgentRunner.CreateSharedClient(s_runId, debug, outputDir); await using var runner = new AgentRunner(client, serverExecutablePath, outputDir, workspacePath); var result = await ProcessPromptAsync(runner, prompt, prompt.Namespace, testContext, model, retries); AppendResultToMarkdown(reportFile, result); @@ -379,7 +379,7 @@ static class Program /// /// Processes a single prompt with retry logic and returns the test result. /// - static async Task ProcessPromptAsync( + private static async Task ProcessPromptAsync( AgentRunner runner, TestPrompt prompt, string namespaceName, @@ -477,7 +477,7 @@ static class Program /// /// Initializes the live markdown report with header and table columns. /// - static void InitializeMarkdownReport(string filePath) + private static void InitializeMarkdownReport(string filePath) { using var writer = new StreamWriter(filePath, append: false); writer.WriteLine("# Azure MCP E2E Test Report"); @@ -493,7 +493,7 @@ static class Program /// /// Appends a single test result row to the live markdown report. /// - static void AppendResultToMarkdown(string filePath, TestResult result) + private static void AppendResultToMarkdown(string filePath, TestResult result) { lock (_reportLock) { @@ -521,7 +521,7 @@ static class Program /// /// Appends summary and failed tests sections to the live markdown report. /// - static void AppendMarkdownSummary(string filePath, List results, TimeSpan duration) + private static void AppendMarkdownSummary(string filePath, List results, TimeSpan duration) { var passed = results.Count(r => r.Status == TestStatus.Pass); var failed = results.Count(r => r.Status == TestStatus.Fail); @@ -571,7 +571,7 @@ static class Program } } - static void WriteLineLock(string message) + private static void WriteLineLock(string message) { lock (_consoleLock) { @@ -579,9 +579,9 @@ static class Program } } - static (string? TestContextPath, string? PromptsPath) LoadFiles() + private static (string? TestContextPath, string? PromptsPath) LoadFiles() { - string? root = null; + string? root; try { root = AgentRunnerUtils.FindRepoRoot(Directory.GetCurrentDirectory()); @@ -666,13 +666,8 @@ static class Program } // After build, the Debug output should exist - var builtPath = FindBuildExecutable(repoRoot); - - if (builtPath is null) - { - throw new InvalidOperationException( + var builtPath = FindBuildExecutable(repoRoot) ?? throw new InvalidOperationException( "Build succeeded but server executable could not be located in any output directory."); - } if (!File.Exists(builtPath)) { @@ -684,7 +679,7 @@ static class Program return builtPath; } - private static readonly string[] configurations = ["Debug", "Release"]; + private static readonly string[] s_configurations = ["Debug", "Release"]; private static string? FindBuildExecutable(string repoRoot) { @@ -692,7 +687,7 @@ static class Program var binDirectory = Path.Combine(repoRoot, "servers", "Azure.Mcp.Server", "src", "bin"); // Check well-known output paths (bin/{Debug,Release}/*/) and pick the most recent - var knownPath = configurations + var knownPath = s_configurations .Select(config => Path.Combine(binDirectory, config)) .Where(Directory.Exists) .SelectMany(configDir => Directory.GetFiles(configDir, exeName, SearchOption.AllDirectories)) @@ -711,7 +706,7 @@ static class Program return null; } - var knownConfigDirs = configurations + var knownConfigDirs = s_configurations .Select(config => Path.Combine(binDirectory, config)) .ToHashSet(StringComparer.OrdinalIgnoreCase);