diff --git a/tools/Fabric.Mcp.Tools.Docs/src/Services/FabricPublicApiService.cs b/tools/Fabric.Mcp.Tools.Docs/src/Services/FabricPublicApiService.cs index 72ec32ac0..a4a294b61 100644 --- a/tools/Fabric.Mcp.Tools.Docs/src/Services/FabricPublicApiService.cs +++ b/tools/Fabric.Mcp.Tools.Docs/src/Services/FabricPublicApiService.cs @@ -82,16 +82,40 @@ public class FabricPublicApiService( /// Builds a regex pattern for matching item definition resources. /// Workload names from directory listings are camelCase (e.g., "kqlDatabase"), /// while item definition files use kebab-case (e.g., "kql-database-definition.md"). - /// This inserts an optional hyphen at each camelCase boundary to match either form. + /// This method handles three kinds of naming mismatches: + /// 1. camelCase boundaries → optional hyphens plus extra kebab-case segments + /// (e.g., "mirroredAzureDatabricksCatalog" matches "mirrored-azuredatabricks-unitycatalog") + /// 2. All-lowercase workload names → optional hyphens between every character + /// (e.g., "sparkjob" matches "spark-job") + /// 3. Trailing "definition" in workload names is stripped since the file suffix + /// already includes "-definition.md" (e.g., "sparkjobdefinition" → "sparkjob") /// internal static string BuildItemDefinitionPattern(string workloadType) { + // Strip trailing "definition" since the file naming convention already includes "-definition.md" + if (workloadType.EndsWith("definition", StringComparison.OrdinalIgnoreCase)) + { + workloadType = workloadType[..^10]; + } + var sb = new StringBuilder("item-definitions/"); for (int i = 0; i < workloadType.Length; i++) { - if (i > 0 && char.IsUpper(workloadType[i])) + if (i > 0) { - sb.Append("-?"); + if (char.IsUpper(workloadType[i])) + { + // At camelCase boundaries, allow extra lowercase chars and hyphens + // to handle naming inconsistencies (e.g., "AzureDatabricksCatalog" + // matching "azuredatabricks-unitycatalog") + sb.Append("[-a-z]*?"); + } + else + { + // Between consecutive lowercase chars, allow optional hyphen + // to handle kebab-case variants (e.g., "sparkjob" matching "spark-job") + sb.Append("-?"); + } } sb.Append(char.ToLowerInvariant(workloadType[i])); diff --git a/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Commands/CombinedWorkflowTests.cs b/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Commands/CombinedWorkflowTests.cs new file mode 100644 index 000000000..2770c6793 --- /dev/null +++ b/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Commands/CombinedWorkflowTests.cs @@ -0,0 +1,393 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Buffers; +using System.CommandLine; +using System.Net; +using System.Text.Json; +using Fabric.Mcp.Tools.Docs.Commands.BestPractices; +using Fabric.Mcp.Tools.Docs.Commands.PublicApis; +using Fabric.Mcp.Tools.Docs.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Mcp.Core.Models.Command; + +namespace Fabric.Mcp.Tools.Docs.Tests.Commands; + +/// +/// Shared fixture for combined workflow tests. Creates the service provider +/// and command instances once, shared across all tests in the class. +/// +public sealed class CombinedWorkflowFixture : IDisposable +{ + public ServiceProvider ServiceProvider { get; } + public ListWorkloadsCommand ListWorkloadsCommand { get; } + public GetWorkloadApisCommand GetWorkloadApisCommand { get; } + public GetWorkloadDefinitionCommand GetWorkloadDefinitionCommand { get; } + public GetExamplesCommand GetExamplesCommand { get; } + + private readonly LoggerFactory _loggerFactory; + + public CombinedWorkflowFixture() + { + _loggerFactory = new LoggerFactory(); + ListWorkloadsCommand = new ListWorkloadsCommand(_loggerFactory.CreateLogger()); + GetWorkloadApisCommand = new GetWorkloadApisCommand(_loggerFactory.CreateLogger()); + GetWorkloadDefinitionCommand = new GetWorkloadDefinitionCommand(_loggerFactory.CreateLogger()); + GetExamplesCommand = new GetExamplesCommand(_loggerFactory.CreateLogger()); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(); + services.AddSingleton(); + ServiceProvider = services.BuildServiceProvider(); + } + + public void Dispose() + { + ServiceProvider.Dispose(); + _loggerFactory.Dispose(); + } +} + +/// +/// Combined workflow tests that exercise real service implementations (no mocks). +/// Commands use the real EmbeddedResourceProviderService and FabricPublicApiService +/// backed by assembly-embedded resources. +/// +public class CombinedWorkflowTests(CombinedWorkflowFixture fixture) : IClassFixture +{ + private readonly ServiceProvider _serviceProvider = fixture.ServiceProvider; + private readonly ListWorkloadsCommand _listWorkloadsCommand = fixture.ListWorkloadsCommand; + private readonly GetWorkloadApisCommand _getWorkloadApisCommand = fixture.GetWorkloadApisCommand; + private readonly GetWorkloadDefinitionCommand _getWorkloadDefinitionCommand = fixture.GetWorkloadDefinitionCommand; + private readonly GetExamplesCommand _getExamplesCommand = fixture.GetExamplesCommand; + + [Fact] + public async Task ListWorkloads_ThenGetApisForEach_ReturnsApiSpecsForAllWorkloads() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Step 1: List all workloads using the real service + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + Assert.NotNull(listResult.Results); + + var workloads = DeserializeWorkloads(listResult); + Assert.NotEmpty(workloads); + + // Step 2: For each workload, get its API spec + foreach (var workload in workloads) + { + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + + Assert.Equal(HttpStatusCode.OK, apiResult.Status); + Assert.NotNull(apiResult.Results); + } + } + + [Fact] + public async Task ListWorkloads_ThenGetDefinitionForEach_ReturnsOrReportsNotFound() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Step 1: List all workloads + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + + var workloads = DeserializeWorkloads(listResult); + Assert.NotEmpty(workloads); + + // Step 2: For each workload, get its item definition. + // Not all workloads have embedded item definitions, so we accept OK or NotFound. + var successCount = 0; + foreach (var workload in workloads) + { + var defContext = new CommandContext(_serviceProvider); + var defResult = await _getWorkloadDefinitionCommand.ExecuteAsync( + defContext, + _getWorkloadDefinitionCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + + Assert.True( + defResult.Status == HttpStatusCode.OK || defResult.Status == HttpStatusCode.NotFound, + $"Unexpected status {defResult.Status} for workload '{workload}': {defResult.Message}"); + + if (defResult.Status == HttpStatusCode.OK) + { + Assert.NotNull(defResult.Results); + successCount++; + } + } + + // The number of workloads that successfully returned a definition should + // match the number of *-definition.md files in Resources/item-definitions + // that are reachable from the listed workloads. + // When a new definition file is added, the corresponding workload must + // also be present in the API specs for the definition to be discoverable. + var assembly = typeof(EmbeddedResourceProviderService).Assembly; + var allDefinitionResources = assembly.GetManifestResourceNames() + .Where(name => name.Contains("item-definitions/") && name.EndsWith("-definition.md", StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + Assert.True(allDefinitionResources.Length > 0, "Expected embedded item-definition resources to exist"); + + // Count how many definition files are reachable from listed workloads + // via the same pattern used by GetWorkloadItemDefinition + var matchedDefinitions = new HashSet(); + foreach (var workload in workloads) + { + var pattern = FabricPublicApiService.BuildItemDefinitionPattern(workload); + var regex = new System.Text.RegularExpressions.Regex(pattern); + foreach (var resource in allDefinitionResources) + { + if (regex.IsMatch(resource)) + { + matchedDefinitions.Add(resource); + } + } + } + + // Every reachable definition should have been successfully retrieved + Assert.Equal(matchedDefinitions.Count, successCount); + + // Flag any orphaned definition files that no workload can reach. + // dbtjob and hlscohort have no corresponding workload API spec directories. + var knownOrphans = new HashSet(StringComparer.OrdinalIgnoreCase) { "dbtjob-definition.md", "hlscohort-definition.md" }; + var orphaned = allDefinitionResources + .Except(matchedDefinitions) + .Where(r => !knownOrphans.Contains(Path.GetFileName(r)!)) + .ToArray(); + Assert.True(orphaned.Length == 0, + $"The following {orphaned.Length} item-definition file(s) have no matching workload in the API specs: " + + string.Join(", ", orphaned.Select(Path.GetFileName))); + } + + [Fact] + public async Task ListWorkloads_ThenGetApisAndDefinitionsForEach_ReturnsAllData() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Step 1: List all workloads + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + + var workloads = DeserializeWorkloads(listResult); + Assert.NotEmpty(workloads); + + // Step 2: For each workload, get both API spec and item definition + foreach (var workload in workloads) + { + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + + Assert.Equal(HttpStatusCode.OK, apiResult.Status); + Assert.NotNull(apiResult.Results); + + var defContext = new CommandContext(_serviceProvider); + var defResult = await _getWorkloadDefinitionCommand.ExecuteAsync( + defContext, + _getWorkloadDefinitionCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + + // Item definitions may not exist for every workload + Assert.True( + defResult.Status == HttpStatusCode.OK || defResult.Status == HttpStatusCode.NotFound, + $"Unexpected status {defResult.Status} for workload '{workload}'"); + } + } + + [Fact] + public async Task ListWorkloads_ThenGetApisDefinitionsAndExamples_FullWorkflow() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Step 1: List workloads + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + + var workloads = DeserializeWorkloads(listResult); + Assert.NotEmpty(workloads); + + // Step 2: For each workload, get APIs, definitions, and examples + foreach (var workload in workloads) + { + // API spec - should always succeed for listed workloads + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + Assert.Equal(HttpStatusCode.OK, apiResult.Status); + Assert.NotNull(apiResult.Results); + + // Item definition - may or may not exist + var defContext = new CommandContext(_serviceProvider); + var defResult = await _getWorkloadDefinitionCommand.ExecuteAsync( + defContext, + _getWorkloadDefinitionCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + Assert.True( + defResult.Status == HttpStatusCode.OK || defResult.Status == HttpStatusCode.NotFound, + $"Unexpected definition status {defResult.Status} for workload '{workload}'"); + + // Examples - should always succeed (returns empty dict if none exist) + var exContext = new CommandContext(_serviceProvider); + var exResult = await _getExamplesCommand.ExecuteAsync( + exContext, + _getExamplesCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + Assert.Equal(HttpStatusCode.OK, exResult.Status); + Assert.NotNull(exResult.Results); + } + } + + [Fact] + public async Task ListWorkloads_DoesNotReturnCommon() + { + // The service filters out the "common" pseudo-workload + var cancellationToken = TestContext.Current.CancellationToken; + + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + + var workloads = DeserializeWorkloads(listResult); + Assert.DoesNotContain("common", workloads, StringComparer.OrdinalIgnoreCase); + } + + [Fact] + public async Task GetApis_WithCommonWorkloadType_ReturnsNotFound() + { + // "common" is explicitly rejected with a helpful message + var cancellationToken = TestContext.Current.CancellationToken; + + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", "common"]), + cancellationToken); + + Assert.Equal(HttpStatusCode.NotFound, apiResult.Status); + Assert.Contains("common", apiResult.Message); + Assert.Contains("platform", apiResult.Message); + } + + [Fact] + public async Task GetApis_WithNonexistentWorkload_ReturnsError() + { + var cancellationToken = TestContext.Current.CancellationToken; + + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", "this-workload-does-not-exist"]), + cancellationToken); + + Assert.NotEqual(HttpStatusCode.OK, apiResult.Status); + } + + [Fact] + public async Task ListWorkloads_ThenGetApisForEach_ApiSpecContainsValidJson() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Step 1: List workloads + var listContext = new CommandContext(_serviceProvider); + var listResult = await _listWorkloadsCommand.ExecuteAsync( + listContext, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, listResult.Status); + + var workloads = DeserializeWorkloads(listResult); + Assert.NotEmpty(workloads); + + // Step 2: For each workload, verify the API spec result contains parseable JSON content + foreach (var workload in workloads) + { + var apiContext = new CommandContext(_serviceProvider); + var apiResult = await _getWorkloadApisCommand.ExecuteAsync( + apiContext, + _getWorkloadApisCommand.GetCommand().Parse(["--workload-type", workload]), + cancellationToken); + + Assert.Equal(HttpStatusCode.OK, apiResult.Status); + Assert.NotNull(apiResult.Results); + + // Serialize the result to JSON and verify it contains an apiSpecification field + var json = SerializeResults(apiResult); + using var doc = JsonDocument.Parse(json); + Assert.True(doc.RootElement.TryGetProperty("apiSpecification", out var apiSpecElement), + $"API result for workload '{workload}' should contain 'apiSpecification'"); + var apiSpecJson = apiSpecElement.GetString(); + Assert.False(string.IsNullOrEmpty(apiSpecJson), + $"API specification for workload '{workload}' should not be empty"); + + // Verify the swagger spec inside apiSpecification is valid JSON + using var swaggerDoc = JsonDocument.Parse(apiSpecJson!); + Assert.NotEqual(JsonValueKind.Undefined, swaggerDoc.RootElement.ValueKind); + } + } + + [Fact] + public async Task ListWorkloads_ResultsAreConsistentAcrossMultipleInvocations() + { + // Verifies that calling the same tool twice returns consistent results + var cancellationToken = TestContext.Current.CancellationToken; + + var context1 = new CommandContext(_serviceProvider); + var result1 = await _listWorkloadsCommand.ExecuteAsync( + context1, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + var context2 = new CommandContext(_serviceProvider); + var result2 = await _listWorkloadsCommand.ExecuteAsync( + context2, _listWorkloadsCommand.GetCommand().Parse([]), cancellationToken); + + Assert.Equal(HttpStatusCode.OK, result1.Status); + Assert.Equal(HttpStatusCode.OK, result2.Status); + + var workloads1 = DeserializeWorkloads(result1); + var workloads2 = DeserializeWorkloads(result2); + + Assert.Equal(workloads1.OrderBy(w => w), workloads2.OrderBy(w => w)); + } + + private static string SerializeResults(CommandResponse response) + { + Assert.NotNull(response.Results); + var buffer = new ArrayBufferWriter(); + using var writer = new Utf8JsonWriter(buffer); + response.Results.Write(writer); + writer.Flush(); + return System.Text.Encoding.UTF8.GetString(buffer.WrittenSpan); + } + + private static IEnumerable DeserializeWorkloads(CommandResponse response) + { + var json = SerializeResults(response); + using var doc = JsonDocument.Parse(json); + var workloadsArray = doc.RootElement.GetProperty("Workloads"); + return workloadsArray.EnumerateArray().Select(e => e.GetString()!).ToArray(); + } +} diff --git a/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Services/FabricPublicApiServiceTests.cs b/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Services/FabricPublicApiServiceTests.cs index 158e526e8..175351ac2 100644 --- a/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Services/FabricPublicApiServiceTests.cs +++ b/tools/Fabric.Mcp.Tools.Docs/tests/Fabric.Mcp.Tools.Docs.UnitTests/Services/FabricPublicApiServiceTests.cs @@ -260,6 +260,8 @@ public class FabricPublicApiServiceTests [InlineData("digitalTwinBuilder")] [InlineData("graphQLApi")] [InlineData("semanticModel")] + [InlineData("sparkjobdefinition")] + [InlineData("mirroredAzureDatabricksCatalog")] public void GetFabricWorkloadItemDefinition_WithCamelCaseWorkload_ReturnsDefinition(string workloadType) { // Act @@ -270,10 +272,12 @@ public class FabricPublicApiServiceTests } [Theory] - [InlineData("notebook", "item-definitions/notebook-definition\\.md")] - [InlineData("kqlDatabase", "item-definitions/kql-?database-definition\\.md")] - [InlineData("cosmosDbDatabase", "item-definitions/cosmos-?db-?database-definition\\.md")] - [InlineData("graphQLApi", "item-definitions/graph-?q-?l-?api-definition\\.md")] + [InlineData("notebook", "item-definitions/n-?o-?t-?e-?b-?o-?o-?k-definition\\.md")] + [InlineData("kqlDatabase", "item-definitions/k-?q-?l[-a-z]*?d-?a-?t-?a-?b-?a-?s-?e-definition\\.md")] + [InlineData("cosmosDbDatabase", "item-definitions/c-?o-?s-?m-?o-?s[-a-z]*?d-?b[-a-z]*?d-?a-?t-?a-?b-?a-?s-?e-definition\\.md")] + [InlineData("graphQLApi", "item-definitions/g-?r-?a-?p-?h[-a-z]*?q[-a-z]*?l[-a-z]*?a-?p-?i-definition\\.md")] + [InlineData("sparkjobdefinition", "item-definitions/s-?p-?a-?r-?k-?j-?o-?b-definition\\.md")] + [InlineData("mirroredAzureDatabricksCatalog", "item-definitions/m-?i-?r-?r-?o-?r-?e-?d[-a-z]*?a-?z-?u-?r-?e[-a-z]*?d-?a-?t-?a-?b-?r-?i-?c-?k-?s[-a-z]*?c-?a-?t-?a-?l-?o-?g-definition\\.md")] public void BuildItemDefinitionPattern_ConvertsCorrectly(string workloadType, string expectedPattern) { // Act