[ResilienceManagement] RO validate failover command (namespace: resilience) (#3354)

This commit is contained in:
adishiritwick authored and GitHub committed 2026-08-26 18:08:42 -07:00
1 parent d2654901bd
commit 9b4e834e77
20 files changed
+1032 -89

No files matched your search

+1
View File
@@ -530,6 +530,7 @@
"usgov",
"usgovcloudapi",
"usgovernment",
"validateforfailover",
"vally",
"varchar",
"vecc",
@@ -0,0 +1,3 @@
changes:
- section: "Features Added"
description: "Added the resilience recovery plan validate-for-failover tool to report per-resource failover qualification and blocking reasons."
@@ -3840,6 +3840,15 @@ azmcp resilience recoveryplan create --service-group <service-group> \
azmcp resilience recoveryplan delete --service-group <service-group> \
--recovery-plan <recovery-plan>
# Validate which recovery-plan resources are qualified for failover from the specified source locations.
# Optionally limit validation to selected full recovery-resource IDs and provide execution consent.
# ❌ Destructive | ❌ Idempotent | ❌ OpenWorld | ❌ ReadOnly | ❌ Secret | ❌ LocalRequired
azmcp resilience recoveryplan validateforfailover --service-group <service-group> \
--recovery-plan <recovery-plan> \
--source-locations <source-location> [<source-location> ...] \
[--selected-resource-ids <recovery-resource-id> [<recovery-resource-id> ...]] \
[--user-consent <Unspecified|Allowed>]
# Configure recovery-plan resource inclusions, exclusions, removals, recovery groups, identities, and protection settings. At least one JSON array is required.
# First inclusion requires matching protection type and settings. CustomRunbook requires failover and reprotect runbook resource IDs.
# AzureSiteRecovery is supported for virtual machines and requires disk reprotect details. Existing configuration is preserved on sparse updates.
@@ -969,6 +969,10 @@ The `Interaction` column describes whether a prompt can invoke its tool immediat
| resilience_recoveryplan_delete | Recovery plan <recovery_plan_name> is no longer needed. Delete it from resilience service group <service_group> | none |
| resilience_recoveryplan_get | List all resilience recovery plans in service group <service_group> | none |
| resilience_recoveryplan_get | Get the details of recovery plan <recovery_plan_name> in service group <service_group> | none |
| resilience_recoveryplan_validateforfailover | Validate recovery plan <recovery_plan_name> for failover in service group <service_group>, but I have not specified a source location or selected recovery-resource ID | clarification-required |
| resilience_recoveryplan_validateforfailover | Validate which resources in recovery plan <recovery_plan_name> in service group <service_group> can fail over from <source_location> and report blocking reasons | none |
| resilience_recoveryplan_validateforfailover | Check whether recovery resource <recovery_resource_id> in recovery plan <recovery_plan_name> is qualified for failover without requiring a source location or executing failover | none |
| resilience_recoveryplan_validateforfailover | Validate recovery plan <recovery_plan_name> for failover from <source_location>, supply required user consent, and return per-resource qualification results without updating resources | none |
| resilience_recoveryplan_resource_update | Include and configure recovery resource <recovery_resource_id> in recovery plan <recovery_plan_name> in service group <service_group> with selected protection solution type <protection_solution_type> and settings <protection_settings_json> | none |
| resilience_recoveryplan_resource_update | Add recovery resource <recovery_resource_id> to recovery plan <recovery_plan_name> in service group <service_group>. Protect it with CustomRunbook using failover runbook <failover_runbook_resource_id> and reprotect runbook <reprotect_runbook_resource_id> | none |
| resilience_recoveryplan_resource_update | Include virtual machine recovery resource <recovery_resource_id> in recovery plan <recovery_plan_name> in service group <service_group> using AzureSiteRecovery protection settings <protection_settings_json> with disk reprotection, staging storage, and a test failover virtual network | none |
@@ -83,6 +83,39 @@
"resilience_recoveryplan_resource_update"
]
},
{
"name": "validate_azure_resilience_recovery_plan_for_failover",
"description": "Validate which resources in an Azure Resilience Management recovery plan are qualified for failover from specified source locations.",
"toolMetadata": {
"destructive": {
"value": false,
"description": "This tool does not delete or modify recovery plan resources."
},
"idempotent": {
"value": false,
"description": "Each invocation starts a new failover validation operation."
},
"openWorld": {
"value": false,
"description": "This tool's domain of interaction is closed and well-defined, limited to a specific set of entities."
},
"readOnly": {
"value": false,
"description": "This tool starts a validation operation and may update operation state."
},
"secret": {
"value": false,
"description": "This tool does not handle sensitive or secret information."
},
"localRequired": {
"value": false,
"description": "This tool is available in both local and remote server modes."
}
},
"mappedToolList": [
"resilience_recoveryplan_validateforfailover"
]
},
{
"name": "check_azure_resilience_management_readiness",
"description": "Check whether an Azure Resilience Management recovery plan and its protected resources are ready for recovery operations in an Azure service group.",
@@ -959,37 +992,37 @@
]
},
{
"name": "apply_azure_advisor_recommendations",
"description": "Get rules that can help apply Advisor recommendation to create or modify IaaC files (like ARM, Bicep) for Azure resources.",
"toolMetadata": {
"destructive": {
"value": false,
"description": "This tool performs only additive updates without deleting or modifying existing resources."
},
"idempotent": {
"value": true,
"description": "Running this operation multiple times with the same arguments produces the same result without additional effects."
},
"openWorld": {
"value": false,
"description": "This tool's domain of interaction is closed and well-defined, limited to a specific set of entities."
},
"readOnly": {
"value": true,
"description": "This tool only performs read operations without modifying any state or data."
},
"secret": {
"value": false,
"description": "This tool does not handle sensitive or secret information."
},
"localRequired": {
"value": false,
"description": "This tool is available in both local and remote server modes."
}
"name": "apply_azure_advisor_recommendations",
"description": "Get rules that can help apply Advisor recommendation to create or modify IaaC files (like ARM, Bicep) for Azure resources.",
"toolMetadata": {
"destructive": {
"value": false,
"description": "This tool performs only additive updates without deleting or modifying existing resources."
},
"mappedToolList": [
"advisor_recommendation_apply"
]
"idempotent": {
"value": true,
"description": "Running this operation multiple times with the same arguments produces the same result without additional effects."
},
"openWorld": {
"value": false,
"description": "This tool's domain of interaction is closed and well-defined, limited to a specific set of entities."
},
"readOnly": {
"value": true,
"description": "This tool only performs read operations without modifying any state or data."
},
"secret": {
"value": false,
"description": "This tool does not handle sensitive or secret information."
},
"localRequired": {
"value": false,
"description": "This tool is available in both local and remote server modes."
}
},
"mappedToolList": [
"advisor_recommendation_apply"
]
},
{
"name": "get_azure_retail_pricing",
@@ -0,0 +1,132 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Net;
using Azure.Core;
using Azure.Mcp.Tools.ResilienceManagement.Models;
using Azure.Mcp.Tools.ResilienceManagement.Options.Recovery.Plans;
using Azure.Mcp.Tools.ResilienceManagement.Services;
using Azure.ResourceManager.ResilienceManagement;
using Microsoft.Extensions.Logging;
using Microsoft.Mcp.Core.Commands;
using Microsoft.Mcp.Core.Models.Command;
namespace Azure.Mcp.Tools.ResilienceManagement.Commands.Recovery.Plans;
[CommandMetadata(
Id = "96622339-b89f-4764-b15f-793bd52d11bf",
Name = "validateforfailover",
Title = "Validate Resilience Recovery Plan for Failover",
Description = "Validates a resilience recovery plan for failover using customer-provided source locations, selected recovery-resource IDs, or both. If the customer provides neither selector, ask which source locations or recovery-resource IDs to validate; never infer them from prior context or resource metadata. Use this tool to check failover qualification or readiness, identify blocking reasons per recovery resource, or supply user consent. This validation-only operation does not execute failover or update recovery resources.",
Destructive = false,
Idempotent = false,
OpenWorld = false,
ReadOnly = false,
Secret = false,
LocalRequired = false)]
public sealed class RecoveryPlanValidateForFailoverCommand(
ILogger<RecoveryPlanValidateForFailoverCommand> logger,
IResilienceManagementService resilienceManagementService)
: AuthenticatedCommand<RecoveryPlanValidateForFailoverOptions, RecoveryPlanValidateForFailoverResult>
{
private readonly ILogger<RecoveryPlanValidateForFailoverCommand> _logger = logger;
private readonly IResilienceManagementService _resilienceManagementService = resilienceManagementService;
public override void ValidateOptions(RecoveryPlanValidateForFailoverOptions options, ValidationResult validationResult)
{
base.ValidateOptions(options, validationResult);
RecoveryPlanValidation.ValidateServiceGroup(options.ServiceGroup, validationResult);
RecoveryPlanValidation.ValidateName(options.RecoveryPlan, validationResult);
bool hasSourceLocations = options.SourceLocations is { Length: > 0 };
bool hasSelectedResourceIds = options.SelectedResourceIds is { Length: > 0 };
if (!hasSourceLocations && !hasSelectedResourceIds)
{
validationResult.Errors.Add("Provide at least one --source-locations or --selected-resource-ids value.");
}
if (hasSourceLocations && options.SourceLocations!.Any(string.IsNullOrWhiteSpace))
{
validationResult.Errors.Add("Each --source-locations value must be a non-empty Azure location.");
}
if (options.UserConsent is not null &&
!string.Equals(options.UserConsent, "Unspecified", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(options.UserConsent, "Allowed", StringComparison.OrdinalIgnoreCase))
{
validationResult.Errors.Add("--user-consent must be Unspecified or Allowed when specified.");
}
foreach (string resourceId in options.SelectedResourceIds ?? [])
{
if (string.IsNullOrWhiteSpace(resourceId) || !IsRecoveryResourceIdForPlan(resourceId, options.ServiceGroup, options.RecoveryPlan))
{
validationResult.Errors.Add("Each --selected-resource-ids value must be a full recovery-resource ID under the requested service group and recovery plan.");
break;
}
}
}
public override async Task<CommandResponse> ExecuteAsync(CommandContext context, RecoveryPlanValidateForFailoverOptions options, CancellationToken cancellationToken)
{
try
{
RecoveryPlanValidateForFailoverResult result = await _resilienceManagementService.ValidateRecoveryPlanForFailoverAsync(
options.ServiceGroup,
options.RecoveryPlan,
options.SourceLocations ?? [],
options.SelectedResourceIds,
options.UserConsent,
options.Tenant,
options.RetryPolicy,
cancellationToken);
context.Response.Results = ResponseResult.Create(
result,
ResilienceManagementJsonContext.Default.RecoveryPlanValidateForFailoverResult);
}
catch (Exception ex)
{
_logger.LogError(ex,
"Error validating recovery plan for failover. ServiceGroup: {ServiceGroup}, RecoveryPlan: {RecoveryPlan}.",
options.ServiceGroup, options.RecoveryPlan);
HandleException(context, ex);
}
return context.Response;
}
private static bool IsRecoveryResourceIdForPlan(string resourceId, string serviceGroup, string recoveryPlan)
{
try
{
var parsed = new ResourceIdentifier(resourceId);
return parsed.ResourceType == RecoveryMembersResource.ResourceType &&
string.Equals(parsed.Parent?.Name, recoveryPlan, StringComparison.OrdinalIgnoreCase) &&
string.Equals(parsed.Parent?.Parent?.Name, serviceGroup, StringComparison.OrdinalIgnoreCase);
}
catch (Exception ex) when (ex is ArgumentException or FormatException)
{
return false;
}
}
protected override string GetErrorMessage(Exception ex) => ex switch
{
TimeoutException =>
"The recovery plan failover validation timed out before it completed. Retry the operation.",
RequestFailedException reqEx when reqEx.Status == (int)HttpStatusCode.Forbidden =>
"Authorization failed validating the recovery plan for failover. Verify you have access to the recovery plan and service group.",
RequestFailedException reqEx when reqEx.Status == (int)HttpStatusCode.NotFound =>
"Recovery plan not found. Verify the recovery plan and service group exist and you have access.",
RequestFailedException =>
"The failover validation request failed. Verify the recovery plan, source locations, selected resources, and request parameters, then try again.",
_ => base.GetErrorMessage(ex)
};
protected override HttpStatusCode GetStatusCode(Exception ex) => ex switch
{
TimeoutException => HttpStatusCode.GatewayTimeout,
_ => base.GetStatusCode(ex)
};
}
@@ -58,8 +58,13 @@ namespace Azure.Mcp.Tools.ResilienceManagement.Commands;
[JsonSerializable(typeof(RecoveryPlanIdentityInfo))]
[JsonSerializable(typeof(RecoveryPlanGroupInfo))]
[JsonSerializable(typeof(RecoveryPlanDeleteCommand.RecoveryPlanDeleteCommandResult))]
[JsonSerializable(typeof(RecoveryPlanValidateForFailoverResult))]
[JsonSerializable(typeof(RecoveryPlanFailoverQualification))]
[JsonSerializable(typeof(RecoveryPlanUpdateResourcesCommand.RecoveryPlanUpdateResourcesCommandResult))]
[JsonSerializable(typeof(RecoveryPlanReadinessResult))]
[JsonSerializable(typeof(RecoveryPlanReadinessError))]
[JsonSerializable(typeof(RecoveryPlanReadinessFailedTask))]
[JsonSerializable(typeof(RecoveryPlanReadinessFailedResource))]
[JsonSerializable(typeof(RecoveryPlanUpdateResourcesResult))]
[JsonSerializable(typeof(RecoveryPlanUpdateResourcesFailedResource))]
[JsonSerializable(typeof(RecoveryPlanUpdateResourcesError))]
@@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace Azure.Mcp.Tools.ResilienceManagement.Models;
public sealed record RecoveryPlanFailoverQualification(
string RecoveryResourceId,
string RecoveryResourceUniqueId,
string? AzureResourceId,
string? AzureResourceLocation,
string QualificationState,
IReadOnlyList<string> NotQualifiedReasons,
IReadOnlyList<string> ResourcePhysicalZones,
string? InclusionState,
string? ProtectionStatus,
bool? IsAttentionRequired,
IReadOnlyList<string> AttentionReasons);
@@ -0,0 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace Azure.Mcp.Tools.ResilienceManagement.Models;
public sealed record RecoveryPlanValidateForFailoverResult(
string OperationId,
IReadOnlyList<RecoveryPlanFailoverQualification> RecoveryResourceQualifications);
@@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Azure.Mcp.Core.Options;
using Microsoft.Mcp.Core.Options;
namespace Azure.Mcp.Tools.ResilienceManagement.Options.Recovery.Plans;
public sealed class RecoveryPlanValidateForFailoverOptions
{
[Option(Description = ResilienceManagementOptionDescriptions.ServiceGroup)]
public required string ServiceGroup { get; set; }
[Option(Description = "The name of the recovery plan to validate for failover.")]
public required string RecoveryPlan { get; set; }
[Option(Description = "Customer-provided Azure locations from which resources would fail over, such as eastus or westus2-az3. Provide source locations, selected resource IDs, or both. Do not infer omitted locations; ask the customer.")]
public string[]? SourceLocations { get; set; }
[Option(Description = "Customer-provided full recovery-resource IDs to validate. Provide selected resource IDs, source locations, or both. Do not infer omitted IDs; ask the customer.")]
public string[]? SelectedResourceIds { get; set; }
[Option(Description = "Optional execution consent. Allowed values are Unspecified and Allowed.")]
public string? UserConsent { get; set; }
[Option(Description = OptionDescriptions.Tenant)]
public string? Tenant { get; set; }
[OptionContainer(Prefix = "retry")]
public RetryPolicyOptions? RetryPolicy { get; set; }
}
@@ -41,6 +41,7 @@ public class ResilienceManagementSetup : IAreaSetup
services.AddSingleton<RecoveryPlanGetCommand>();
services.AddSingleton<RecoveryPlanCreateCommand>();
services.AddSingleton<RecoveryPlanDeleteCommand>();
services.AddSingleton<RecoveryPlanValidateForFailoverCommand>();
services.AddSingleton<RecoveryPlanUpdateResourcesCommand>();
services.AddSingleton<RecoveryPlanCheckReadinessCommand>();
services.AddSingleton<RecoveryResourceGetCommand>();
@@ -107,9 +108,10 @@ public class ResilienceManagementSetup : IAreaSetup
recoveryPlans.AddCommand<RecoveryPlanGetCommand>(serviceProvider);
recoveryPlans.AddCommand<RecoveryPlanCreateCommand>(serviceProvider);
recoveryPlans.AddCommand<RecoveryPlanDeleteCommand>(serviceProvider);
recoveryPlans.AddCommand<RecoveryPlanValidateForFailoverCommand>(serviceProvider);
recoveryPlans.AddCommand<RecoveryPlanCheckReadinessCommand>(serviceProvider);
// Create resource subgroup under recovery plan
// Create resource subgroup under recoveryplan
var recoveryResources = new CommandGroup("resource", "Resilience recovery resource operations - Commands for listing, getting, and updating the resources (members) of a resilience recovery plan.");
recoveryPlans.AddSubGroup(recoveryResources);
@@ -42,6 +42,8 @@ public interface IResilienceManagementService
Task<RecoveryPlanUpdateResourcesResult> UpdateRecoveryPlanResourcesAsync(string serviceGroup, string recoveryPlan, UpdateRecoveryResourcesContent content, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default);
Task<RecoveryPlanValidateForFailoverResult> ValidateRecoveryPlanForFailoverAsync(string serviceGroup, string recoveryPlan, IReadOnlyList<string> sourceLocations, IReadOnlyList<string>? selectedResourceIds = null, string? userConsent = null, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default);
Task<RecoveryPlanReadinessResult> CheckRecoveryPlanReadinessAsync(string serviceGroup, string recoveryPlan, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default);
Task<IEnumerable<ResourceSummary>> ListRecoveryResourcesAsync(string serviceGroup, string recoveryPlan, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default);
@@ -10,6 +10,7 @@ using Azure.ResourceManager;
using Azure.ResourceManager.Models;
using Azure.ResourceManager.ResilienceManagement;
using Azure.ResourceManager.ResilienceManagement.Models;
using Microsoft.Mcp.Core.Helpers;
using Microsoft.Mcp.Core.Options;
namespace Azure.Mcp.Tools.ResilienceManagement.Services;
@@ -17,8 +18,8 @@ namespace Azure.Mcp.Tools.ResilienceManagement.Services;
public sealed class ResilienceManagementService(IAzureService azureService)
: BaseAzureResourceService(azureService), IResilienceManagementService
{
private static readonly TimeSpan ReadinessJobPollingInterval = TimeSpan.FromSeconds(30);
private static readonly TimeSpan ReadinessTimeout = TimeSpan.FromMinutes(10);
private static readonly TimeSpan RecoveryPlanPollingInterval = TimeSpan.FromSeconds(30);
private static readonly TimeSpan RecoveryPlanOperationTimeout = TimeSpan.FromMinutes(10);
public async Task<IEnumerable<ResourceSummary>> ListGoalTemplatesAsync(string serviceGroup, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default)
{
@@ -428,7 +429,7 @@ public sealed class ResilienceManagementService(IAzureService azureService)
recoveryPlan,
data,
cancellationToken);
await WaitForLroCompletionAsync(operation, cancellationToken);
await WaitForRecoveryPlanLroCompletionAsync(operation, cancellationToken);
return CreateRecoveryPlanInfo(operation.Value.Data);
}
@@ -486,7 +487,7 @@ public sealed class ResilienceManagementService(IAzureService azureService)
try
{
var operation = await existingPlan.Value.DeleteAsync(WaitUntil.Started, cancellationToken);
await WaitForLroCompletionAsync(operation, cancellationToken);
await WaitForRecoveryPlanLroCompletionAsync(operation, cancellationToken);
return true;
}
catch (RequestFailedException ex) when (ex.Status == 404)
@@ -515,17 +516,169 @@ public sealed class ResilienceManagementService(IAzureService azureService)
Guid.NewGuid().ToString(),
content,
cancellationToken);
await WaitForLroCompletionAsync(operation, cancellationToken);
await WaitForRecoveryPlanLroCompletionAsync(operation, cancellationToken);
return CreateRecoveryPlanUpdateResourcesResult(operation.Value.FailedResources);
}
public Task<RecoveryPlanValidateForFailoverResult> ValidateRecoveryPlanForFailoverAsync(string serviceGroup, string recoveryPlan, IReadOnlyList<string> sourceLocations, IReadOnlyList<string>? selectedResourceIds = null, string? userConsent = null, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default)
{
return ExecuteWithTimeoutAsync(
token => ValidateRecoveryPlanForFailoverCoreAsync(serviceGroup, recoveryPlan, sourceLocations, selectedResourceIds, userConsent, tenant, retryPolicy, token),
"recovery plan failover validation",
RecoveryPlanOperationTimeout,
cancellationToken);
}
private async Task<RecoveryPlanValidateForFailoverResult> ValidateRecoveryPlanForFailoverCoreAsync(string serviceGroup, string recoveryPlan, IReadOnlyList<string> sourceLocations, IReadOnlyList<string>? selectedResourceIds, string? userConsent, string? tenant, RetryPolicyOptions? retryPolicy, CancellationToken cancellationToken)
{
ArmClient armClient = await CreateArmClientAsync(tenantIdOrName: tenant, retryPolicy: retryPolicy, cancellationToken: cancellationToken);
var recoveryPlanId = RecoveryPlanResource.CreateResourceIdentifier(serviceGroup, recoveryPlan);
RecoveryPlanResource recoveryPlanResource = await armClient.GetRecoveryPlanResource(recoveryPlanId).GetAsync(cancellationToken);
var properties = new FailoverRequestProperties(sourceLocations);
foreach (string selectedResourceId in selectedResourceIds ?? [])
{
properties.SelectedResourceIds.Add(new ResourceIdentifier(selectedResourceId));
}
if (userConsent is not null)
{
properties.ExecutionConfigurationsUserConsent = new UserConsent(userConsent);
}
var content = new ResilienceManagementFailoverContent(FailoverDirectionTypes.FromSpecificLocations)
{
FailoverRequestProperties = properties
};
string operationId = Guid.NewGuid().ToString();
ArmOperation<ValidateForRecoveryOperationBaseResult> operation = await recoveryPlanResource.ValidateForFailoverAsync(
WaitUntil.Started,
operationId,
content,
cancellationToken);
await WaitForRecoveryPlanLroCompletionAsync(operation, cancellationToken);
return CreateRecoveryPlanValidateForFailoverResult(
operationId,
operation.GetRawResponse().Content,
operation.Value.RecoveryResourceQualifications);
}
internal static RecoveryPlanValidateForFailoverResult CreateRecoveryPlanValidateForFailoverResult(
string operationId,
BinaryData operationResponse,
IEnumerable<RecoveryResourceQualification>? fallbackQualifications)
{
using JsonDocument document = JsonDocument.Parse(operationResponse);
if (!document.RootElement.TryGetProperty("properties", out JsonElement properties))
{
return CreateRecoveryPlanValidateForFailoverResult(operationId, fallbackQualifications);
}
string resultJson = properties.ValueKind switch
{
JsonValueKind.String => properties.GetString() ?? string.Empty,
JsonValueKind.Object => properties.GetRawText(),
_ => throw new JsonException("The ValidateForFailover operation result has an invalid properties value.")
};
using JsonDocument resultDocument = JsonDocument.Parse(resultJson);
if (!resultDocument.RootElement.TryGetProperty("recoveryResourceQualifications", out JsonElement qualifications))
{
return CreateRecoveryPlanValidateForFailoverResult(operationId, fallbackQualifications);
}
List<RecoveryPlanFailoverQualification> mappedQualifications = qualifications
.EnumerateArray()
.Select(qualification =>
{
JsonElement resource = GetOptionalObject(qualification, "recoveryResource");
JsonElement resourceProperties = GetOptionalObject(resource, "properties");
JsonElement details = GetOptionalObject(qualification, "operationQualificationDetails");
return new RecoveryPlanFailoverQualification(
GetOptionalString(resource, "id") ?? string.Empty,
GetOptionalString(resourceProperties, "recoveryResourceUniqueId") ?? GetOptionalString(resource, "name") ?? string.Empty,
GetOptionalString(resourceProperties, "resourceId"),
GetOptionalString(resourceProperties, "resourceLocation"),
GetOptionalString(details, "qualificationState") ?? "Unknown",
GetStringList(details, "notQualifiedReasons"),
GetStringList(resourceProperties, "resourcePhysicalZones"),
GetOptionalString(resourceProperties, "inclusionState"),
GetOptionalString(resourceProperties, "protectionStatus"),
GetOptionalBoolean(resourceProperties, "needsAttention"),
GetStringList(resourceProperties, "attentionReasons"));
})
.ToList();
return new RecoveryPlanValidateForFailoverResult(operationId, mappedQualifications);
}
private static JsonElement GetOptionalObject(JsonElement element, string propertyName) =>
element.ValueKind == JsonValueKind.Object &&
element.TryGetProperty(propertyName, out JsonElement property) &&
property.ValueKind == JsonValueKind.Object
? property
: default;
private static string? GetOptionalString(JsonElement element, string propertyName) =>
element.ValueKind == JsonValueKind.Object &&
element.TryGetProperty(propertyName, out JsonElement property) &&
property.ValueKind == JsonValueKind.String
? property.GetString()
: null;
private static bool? GetOptionalBoolean(JsonElement element, string propertyName) =>
element.ValueKind == JsonValueKind.Object &&
element.TryGetProperty(propertyName, out JsonElement property) &&
property.ValueKind is JsonValueKind.True or JsonValueKind.False
? property.GetBoolean()
: null;
private static IReadOnlyList<string> GetStringList(JsonElement element, string propertyName) =>
element.ValueKind == JsonValueKind.Object &&
element.TryGetProperty(propertyName, out JsonElement property) &&
property.ValueKind == JsonValueKind.Array
? property.EnumerateArray()
.Where(item => item.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(item.GetString()))
.Select(item => item.GetString()!)
.ToList()
: [];
internal static RecoveryPlanValidateForFailoverResult CreateRecoveryPlanValidateForFailoverResult(
string operationId,
IEnumerable<RecoveryResourceQualification>? recoveryResourceQualifications)
{
List<RecoveryPlanFailoverQualification> qualifications = (recoveryResourceQualifications ?? [])
.Select(qualification =>
{
RecoveryMembersData? resource = qualification?.RecoveryResource;
RecoveryResourceProperties? resourceProperties = resource?.Properties;
OperationQualificationDetails? details = qualification?.OperationQualificationDetails;
return new RecoveryPlanFailoverQualification(
resource?.Id?.ToString() ?? string.Empty,
resourceProperties?.RecoveryResourceUniqueId ?? resource?.Name ?? string.Empty,
resourceProperties?.ResourceId?.ToString(),
resourceProperties?.ResourceLocation?.ToString(),
details?.QualificationState.ToString() ?? "Unknown",
details?.NotQualifiedReasons?.Where(reason => !string.IsNullOrWhiteSpace(reason)).ToList() ?? [],
resourceProperties?.ResourcePhysicalZones?.Where(zone => !string.IsNullOrWhiteSpace(zone)).ToList() ?? [],
resourceProperties?.InclusionState?.ToString(),
resourceProperties?.ProtectionStatus?.ToString(),
resourceProperties?.IsAttentionRequired,
resourceProperties?.AttentionReasons?.Where(reason => !string.IsNullOrWhiteSpace(reason)).ToList() ?? []);
})
.ToList();
return new RecoveryPlanValidateForFailoverResult(operationId, qualifications);
}
public Task<RecoveryPlanReadinessResult> CheckRecoveryPlanReadinessAsync(string serviceGroup, string recoveryPlan, string? tenant = null, RetryPolicyOptions? retryPolicy = null, CancellationToken cancellationToken = default)
{
return ExecuteWithTimeoutAsync(
token => CheckRecoveryPlanReadinessCoreAsync(serviceGroup, recoveryPlan, tenant, retryPolicy, token),
"recovery plan readiness check",
ReadinessTimeout,
RecoveryPlanOperationTimeout,
cancellationToken);
}
@@ -539,7 +692,7 @@ public sealed class ResilienceManagementService(IAzureService azureService)
HashSet<string> existingRecoveryJobIds = await GetRecoveryJobIdsAsync(recoveryJobs, cancellationToken);
string operationId = Guid.NewGuid().ToString();
ArmOperation operation = await recoveryPlanResource.CheckReadinessAsync(WaitUntil.Started, operationId, cancellationToken);
await WaitForLroCompletionAsync(operation, cancellationToken);
await WaitForRecoveryPlanLroCompletionAsync(operation, cancellationToken);
ResourceIdentifier recoveryJobResourceId = TryGetRecoveryJobResourceId(
operation.GetRawResponse().Content,
serviceGroup,
@@ -607,6 +760,28 @@ public sealed class ResilienceManagementService(IAzureService azureService)
}
}
private static async Task WaitForRecoveryPlanLroCompletionAsync(Operation operation, CancellationToken cancellationToken)
{
#if DEBUG
if (EnvironmentHelpers.IsPlaybackTesting())
{
await WaitForLroCompletionAsync(operation, cancellationToken);
return;
}
#endif
while (true)
{
_ = await operation.UpdateStatusAsync(cancellationToken);
if (operation.HasCompleted)
{
return;
}
await Task.Delay(RecoveryPlanPollingInterval, cancellationToken);
}
}
internal static ResourceIdentifier GetRecoveryJobResourceId(BinaryData responseContent, string serviceGroup, string recoveryPlan)
{
return TryGetRecoveryJobResourceId(responseContent, serviceGroup, recoveryPlan)
@@ -683,7 +858,7 @@ public sealed class ResilienceManagementService(IAzureService azureService)
throw new InvalidOperationException("Multiple recovery jobs were created by concurrent readiness operations.");
}
await Task.Delay(ReadinessJobPollingInterval, cancellationToken);
await Task.Delay(RecoveryPlanPollingInterval, cancellationToken);
}
}
@@ -704,8 +879,8 @@ public sealed class ResilienceManagementService(IAzureService azureService)
token => GetRecoveryJobIfAvailableAsync(recoveryJob, token),
job => IsTerminalJobStatus(GetRequiredProperties(job.Data.Properties, "recovery job").Status?.ToString() ?? string.Empty),
$"readiness recovery job '{recoveryJobId}'",
ReadinessJobPollingInterval,
ReadinessTimeout,
RecoveryPlanPollingInterval,
RecoveryPlanOperationTimeout,
cancellationToken);
}
@@ -761,11 +936,11 @@ public sealed class ResilienceManagementService(IAzureService azureService)
}
}
private static RecoveryPlanReadinessError? CreateReadinessError(JobErrorInfo? error)
internal static RecoveryPlanReadinessError? CreateReadinessError(JobErrorInfo? error)
{
return error is null
? null
: new RecoveryPlanReadinessError(error.ErrorCode, error.ErrorMessage, error.Recommendations);
: new RecoveryPlanReadinessError(error.ErrorCode, error.ErrorMessage, error.Recommendations ?? []);
}
private static List<RecoveryPlanReadinessFailedTask> GetFailedTasks(IReadOnlyList<JobTaskDetail>? tasks)
@@ -0,0 +1,183 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Net;
using Azure.Mcp.Tools.ResilienceManagement.Commands;
using Azure.Mcp.Tools.ResilienceManagement.Commands.Recovery.Plans;
using Azure.Mcp.Tools.ResilienceManagement.Models;
using Azure.Mcp.Tools.ResilienceManagement.Services;
using Microsoft.Mcp.Core.Options;
using Microsoft.Mcp.Tests.Client;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using Xunit;
namespace Azure.Mcp.Tools.ResilienceManagement.Tests.Recovery.Plans;
public sealed class RecoveryPlanValidateForFailoverCommandTests : CommandUnitTestsBase<RecoveryPlanValidateForFailoverCommand, IResilienceManagementService>
{
private const string RecoveryResourceId = "/providers/Microsoft.Management/serviceGroups/sg1/providers/Microsoft.AzureResilienceManagement/recoveryPlans/plan1/recoveryResources/12345678-9012-3456-7890-123456789012";
[Fact]
public void Constructor_InitializesCommandCorrectly()
{
var command = Command.GetCommand();
Assert.Equal("validateforfailover", command.Name);
Assert.Contains("failover qualification", command.Description, StringComparison.OrdinalIgnoreCase);
Assert.Contains("source locations", command.Description, StringComparison.OrdinalIgnoreCase);
}
[Theory]
[InlineData("--service-group sg1 --recovery-plan plan1", "source-locations or --selected-resource-ids")]
[InlineData("--service-group sg1 --recovery-plan plan1 --source-locations eastus --user-consent Denied", "Unspecified or Allowed")]
public async Task ExecuteAsync_RejectsInvalidInput(string args, string expectedMessage)
{
var response = await ExecuteCommandAsync(args);
Assert.Equal(HttpStatusCode.BadRequest, response.Status);
Assert.Contains(expectedMessage, response.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task ExecuteAsync_RejectsRecoveryResourceFromAnotherPlan()
{
const string otherPlanResourceId = "/providers/Microsoft.Management/serviceGroups/sg1/providers/Microsoft.AzureResilienceManagement/recoveryPlans/other-plan/recoveryResources/12345678-9012-3456-7890-123456789012";
var response = await ExecuteCommandAsync(
"--service-group", "sg1",
"--recovery-plan", "plan1",
"--source-locations", "eastus",
"--selected-resource-ids", otherPlanResourceId);
Assert.Equal(HttpStatusCode.BadRequest, response.Status);
Assert.Contains("under the requested service group and recovery plan", response.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task ExecuteAsync_AcceptsSelectedResourcesWithoutSourceLocations()
{
var expected = new RecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
[]);
Service.ValidateRecoveryPlanForFailoverAsync(
"sg1",
"plan1",
Arg.Is<IReadOnlyList<string>>(locations => locations.Count == 0),
Arg.Is<IReadOnlyList<string>>(resourceIds => resourceIds.SequenceEqual(new[] { RecoveryResourceId })),
null,
null,
null,
Arg.Any<CancellationToken>())
.Returns(expected);
var response = await ExecuteCommandAsync(
"--service-group", "sg1",
"--recovery-plan", "plan1",
"--selected-resource-ids", RecoveryResourceId);
RecoveryPlanValidateForFailoverResult result = ValidateAndDeserializeResponse(
response,
ResilienceManagementJsonContext.Default.RecoveryPlanValidateForFailoverResult);
Assert.Equal(expected.OperationId, result.OperationId);
Assert.Empty(result.RecoveryResourceQualifications);
}
[Fact]
public async Task ExecuteAsync_ForwardsRequestAndReturnsQualifications()
{
var expected = new RecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
[new RecoveryPlanFailoverQualification(
RecoveryResourceId,
"12345678-9012-3456-7890-123456789012",
"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1",
"eastus",
"NotQualified",
["ProtectionNotComplete"],
["eastus-az1"],
"Included",
"Protected",
false,
[])]);
Service.ValidateRecoveryPlanForFailoverAsync(
"sg1",
"plan1",
Arg.Is<IReadOnlyList<string>>(locations => locations.SequenceEqual(new[] { "eastus" })),
Arg.Is<IReadOnlyList<string>>(resourceIds => resourceIds.SequenceEqual(new[] { RecoveryResourceId })),
"Allowed",
null,
null,
Arg.Any<CancellationToken>())
.Returns(expected);
var response = await ExecuteCommandAsync(
"--service-group", "sg1",
"--recovery-plan", "plan1",
"--source-locations", "eastus",
"--selected-resource-ids", RecoveryResourceId,
"--user-consent", "Allowed");
RecoveryPlanValidateForFailoverResult result = ValidateAndDeserializeResponse(
response,
ResilienceManagementJsonContext.Default.RecoveryPlanValidateForFailoverResult);
Assert.Equal(expected.OperationId, result.OperationId);
RecoveryPlanFailoverQualification qualification = Assert.Single(result.RecoveryResourceQualifications);
Assert.Equal("NotQualified", qualification.QualificationState);
Assert.Equal(["ProtectionNotComplete"], qualification.NotQualifiedReasons);
}
[Theory]
[InlineData(HttpStatusCode.Forbidden, "Authorization failed")]
[InlineData(HttpStatusCode.NotFound, "not found")]
[InlineData(HttpStatusCode.BadRequest, "request failed")]
public async Task ExecuteAsync_SanitizesRequestFailedException(HttpStatusCode status, string expectedMessage)
{
const string providerDetails = "Sensitive provider details: request-id=123; endpoint=https://example.invalid";
Service.ValidateRecoveryPlanForFailoverAsync(
Arg.Any<string>(),
Arg.Any<string>(),
Arg.Any<IReadOnlyList<string>>(),
Arg.Any<IReadOnlyList<string>?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<RetryPolicyOptions?>(),
Arg.Any<CancellationToken>())
.ThrowsAsync(new RequestFailedException((int)status, providerDetails));
var response = await ExecuteCommandAsync(
"--service-group", "sg1",
"--recovery-plan", "plan1",
"--source-locations", "eastus");
Assert.Equal(status, response.Status);
Assert.Contains(expectedMessage, response.Message, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain(providerDetails, response.Message);
}
[Fact]
public async Task ExecuteAsync_MapsTimeoutExceptionToGatewayTimeout()
{
const string internalDetails = "Internal polling timeout details";
Service.ValidateRecoveryPlanForFailoverAsync(
Arg.Any<string>(),
Arg.Any<string>(),
Arg.Any<IReadOnlyList<string>>(),
Arg.Any<IReadOnlyList<string>?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<RetryPolicyOptions?>(),
Arg.Any<CancellationToken>())
.ThrowsAsync(new TimeoutException(internalDetails));
var response = await ExecuteCommandAsync(
"--service-group", "sg1",
"--recovery-plan", "plan1",
"--source-locations", "eastus");
Assert.Equal(HttpStatusCode.GatewayTimeout, response.Status);
Assert.Contains("timed out", response.Message, StringComparison.OrdinalIgnoreCase);
Assert.Contains("Retry", response.Message, StringComparison.OrdinalIgnoreCase);
Assert.DoesNotContain(internalDetails, response.Message);
}
}
@@ -1,6 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
// cspell:ignore LIFECYCLESERVICEGROUPNAME
// cspell:ignore LIFECYCLESERVICEGROUPNAME PLANLIFECYCLESERVICEGROUPNAME
using System.Text.Json;
using System.Text.Json.Nodes;
@@ -178,7 +178,7 @@ public class ResilienceManagementCommandTests(
[Fact]
public async Task Should_delete_drill()
{
var serviceGroup = RegisterOrRetrieveDeploymentOutputVariable("serviceGroupName", "SERVICEGROUPNAME");
var serviceGroup = RegisterOrRetrieveDeploymentOutputVariable("serviceGroupName", "LIFECYCLESERVICEGROUPNAME");
var drillName = RegisterOrRetrieveDeploymentOutputVariable("drillName", "DELETEDRILLNAME");
var result = await CallToolAsync(
@@ -449,7 +449,7 @@ public class ResilienceManagementCommandTests(
[CustomMatcher(compareBody: false)]
public async Task Should_create_update_and_delete_recovery_plan()
{
var serviceGroup = RegisterOrRetrieveDeploymentOutputVariable("lifecycleServiceGroupName", "LIFECYCLESERVICEGROUPNAME");
var serviceGroup = RegisterOrRetrieveDeploymentOutputVariable("lifecycleServiceGroupName", "PLANLIFECYCLESERVICEGROUPNAME");
var recoveryPlan = RegisterOrRetrieveVariable("lifecycleRecoveryPlanName", $"mcp-lifecycle-{Guid.NewGuid().ToString("N")[..8]}");
bool recoveryPlanExists = false;
@@ -637,6 +637,67 @@ public class ResilienceManagementCommandTests(
Assert.Equal("Excluded", inclusionState);
}
[Fact]
public async Task Should_validate_recovery_plan_for_failover()
{
var serviceGroup = RegisterOrRetrieveDeploymentOutputVariable("serviceGroupName", "SERVICEGROUPNAME");
var recoveryPlan = RegisterOrRetrieveDeploymentOutputVariable("recoveryPlanName", "RECOVERYPLANNAME");
var listedResources = await CallToolAsync(
"resilience_recoveryplan_resource_get",
new()
{
{ "tenant", Settings.TenantId },
{ "service-group", serviceGroup },
{ "recovery-plan", recoveryPlan }
});
string? resourceId = null;
string? sourceLocation = null;
foreach (JsonElement resource in listedResources.AssertProperty("recoveryResources").EnumerateArray())
{
string candidateResourceId = resource.AssertProperty("id").GetString()!;
var resourceResult = await CallToolAsync(
"resilience_recoveryplan_resource_get",
new()
{
{ "tenant", Settings.TenantId },
{ "service-group", serviceGroup },
{ "recovery-plan", recoveryPlan },
{ "name", candidateResourceId.Split('/').Last() }
});
JsonElement physicalZones = resourceResult
.AssertProperty("recoveryResource")
.AssertProperty("properties")
.AssertProperty("resourcePhysicalZones");
sourceLocation = physicalZones.ValueKind == JsonValueKind.Array
? physicalZones.EnumerateArray().Select(zone => zone.GetString()).FirstOrDefault(zone => !string.IsNullOrWhiteSpace(zone))
: null;
if (sourceLocation is not null)
{
resourceId = candidateResourceId;
break;
}
}
Assert.False(string.IsNullOrEmpty(resourceId), "The Zonal recovery plan must contain a resource with a physical zone.");
Assert.False(string.IsNullOrEmpty(sourceLocation), "A physical source zone is required for Zonal failover validation.");
var result = await CallToolAsync(
"resilience_recoveryplan_validateforfailover",
new()
{
{ "tenant", Settings.TenantId },
{ "service-group", serviceGroup },
{ "recovery-plan", recoveryPlan },
{ "source-locations", new[] { sourceLocation } },
{ "selected-resource-ids", new[] { resourceId } }
});
Assert.True(Guid.TryParse(result.AssertProperty("operationId").GetString(), out _));
JsonElement qualifications = result.AssertProperty("recoveryResourceQualifications");
Assert.Equal(JsonValueKind.Array, qualifications.ValueKind);
Assert.NotEmpty(qualifications.EnumerateArray());
}
[Fact]
public async Task Should_get_recovery_job()
{
@@ -339,6 +339,138 @@ public sealed class ResilienceManagementServiceTests
Assert.Equal("The recovery resource configuration is invalid.", failedResult.Error?.Message);
}
[Fact]
public void CreateReadinessError_DefaultsMissingRecommendationsToEmptyList()
{
JobErrorInfo providerError = ModelReaderWriter.Read<JobErrorInfo>(BinaryData.FromObjectAsJson(new
{
errorCode = "NotReady",
errorMessage = "The recovery plan is not ready."
}))!;
RecoveryPlanReadinessError? result = ResilienceManagementService.CreateReadinessError(providerError);
Assert.NotNull(result);
Assert.Empty(result.Recommendations);
}
[Fact]
public void CreateRecoveryPlanValidateForFailoverResult_MapsQualificationDetails()
{
ValidateForRecoveryOperationBaseResult sdkResult = ModelReaderWriter.Read<ValidateForRecoveryOperationBaseResult>(BinaryData.FromObjectAsJson(new
{
recoveryResourceQualifications = new[]
{
new
{
recoveryResource = new
{
id = "/providers/Microsoft.Management/serviceGroups/sg1/providers/Microsoft.AzureResilienceManagement/recoveryPlans/plan1/recoveryResources/12345678-9012-3456-7890-123456789012",
name = "12345678-9012-3456-7890-123456789012",
type = "Microsoft.AzureResilienceManagement/recoveryPlans/recoveryResources",
properties = new
{
recoveryResourceUniqueId = "12345678-9012-3456-7890-123456789012",
resourceId = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm",
resourceLocation = "eastus",
resourcePhysicalZones = new[] { "eastus-az1" },
inclusionState = "Included",
protectionStatus = "Protected",
needsAttention = true,
attentionReasons = new[] { "Replication lag" }
}
},
operationQualificationDetails = new
{
qualificationState = "NotQualified",
notQualifiedReasons = new[] { "ProtectionNotComplete" }
}
}
}
}))!;
RecoveryPlanValidateForFailoverResult result = ResilienceManagementService.CreateRecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
sdkResult.RecoveryResourceQualifications);
Assert.Equal("11111111-1111-1111-1111-111111111111", result.OperationId);
RecoveryPlanFailoverQualification qualification = Assert.Single(result.RecoveryResourceQualifications);
Assert.Equal("12345678-9012-3456-7890-123456789012", qualification.RecoveryResourceUniqueId);
Assert.Equal("eastus", qualification.AzureResourceLocation);
Assert.Equal("NotQualified", qualification.QualificationState);
Assert.Equal(["ProtectionNotComplete"], qualification.NotQualifiedReasons);
Assert.Equal(["eastus-az1"], qualification.ResourcePhysicalZones);
Assert.Equal("Included", qualification.InclusionState);
Assert.Equal("Protected", qualification.ProtectionStatus);
Assert.True(qualification.IsAttentionRequired);
Assert.Equal(["Replication lag"], qualification.AttentionReasons);
}
[Fact]
public void CreateRecoveryPlanValidateForFailoverResult_ParsesOperationStatusProperties()
{
BinaryData operationResponse = BinaryData.FromObjectAsJson(new
{
status = "Succeeded",
properties = """
{"recoveryResourceQualifications":[{"recoveryResource":{"id":"/providers/Microsoft.Management/serviceGroups/sg1/providers/Microsoft.AzureResilienceManagement/recoveryPlans/plan1/recoveryResources/resource1","name":"resource1","properties":{"recoveryResourceUniqueId":"resource1"}},"operationQualificationDetails":{"qualificationState":"Qualified","notQualifiedReasons":[]}}]}
"""
});
RecoveryPlanValidateForFailoverResult result = ResilienceManagementService.CreateRecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
operationResponse,
[]);
RecoveryPlanFailoverQualification qualification = Assert.Single(result.RecoveryResourceQualifications);
Assert.Equal("resource1", qualification.RecoveryResourceUniqueId);
Assert.Equal("Qualified", qualification.QualificationState);
}
[Fact]
public void CreateRecoveryPlanValidateForFailoverResult_HandlesMissingQualifications()
{
RecoveryPlanValidateForFailoverResult result = ResilienceManagementService.CreateRecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
null);
Assert.Empty(result.RecoveryResourceQualifications);
}
[Fact]
public void CreateRecoveryPlanValidateForFailoverResult_HandlesMissingQualificationDetails()
{
ValidateForRecoveryOperationBaseResult sdkResult = ModelReaderWriter.Read<ValidateForRecoveryOperationBaseResult>(BinaryData.FromObjectAsJson(new
{
recoveryResourceQualifications = new[]
{
new
{
recoveryResource = new
{
id = "/providers/Microsoft.Management/serviceGroups/sg1/providers/Microsoft.AzureResilienceManagement/recoveryPlans/plan1/recoveryResources/12345678-9012-3456-7890-123456789012",
name = "12345678-9012-3456-7890-123456789012",
type = "Microsoft.AzureResilienceManagement/recoveryPlans/recoveryResources",
properties = new
{
recoveryResourceUniqueId = "12345678-9012-3456-7890-123456789012"
}
}
}
}
}))!;
RecoveryPlanValidateForFailoverResult result = ResilienceManagementService.CreateRecoveryPlanValidateForFailoverResult(
"11111111-1111-1111-1111-111111111111",
sdkResult.RecoveryResourceQualifications);
RecoveryPlanFailoverQualification qualification = Assert.Single(result.RecoveryResourceQualifications);
Assert.Equal("Unknown", qualification.QualificationState);
Assert.Empty(qualification.NotQualifiedReasons);
Assert.Empty(qualification.ResourcePhysicalZones);
Assert.Empty(qualification.AttentionReasons);
}
[Fact]
public void CreateRecoveryPlanInfo_MapsSupportedPlanFields()
{
@@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "",
"TagPrefix": "Azure.Mcp.Tools.ResilienceManagement.Tests",
"Tag": "Azure.Mcp.Tools.ResilienceManagement.Tests_5deb588ecf"
"Tag": "Azure.Mcp.Tools.ResilienceManagement.Tests_700e44ca6d"
}
@@ -107,9 +107,11 @@ function Remove-RecoveryPlans {
$serviceGroupName = Get-OutputValue -Name 'serviceGroupName'
$lifecycleServiceGroupName = Get-OutputValue -Name 'lifecycleServiceGroupName'
$planLifecycleServiceGroupName = Get-OutputValue -Name 'planLifecycleServiceGroupName'
$usagePlanName = Get-OutputValue -Name 'usagePlanName'
$enrollmentName = Get-OutputValue -Name 'enrollmentName'
$lifecycleEnrollmentName = Get-OutputValue -Name 'lifecycleEnrollmentName'
$planLifecycleEnrollmentName = Get-OutputValue -Name 'planLifecycleEnrollmentName'
$goalAssignmentName = Get-OutputValue -Name 'goalAssignmentName'
$goalTemplateName = Get-OutputValue -Name 'goalTemplateName'
$drillName = Get-OutputValue -Name 'drillName'
@@ -133,19 +135,28 @@ foreach ($requiredOutput in $requiredOutputs.GetEnumerator()) {
$serviceGroupId = "/providers/Microsoft.Management/serviceGroups/$serviceGroupName"
$lifecycleServiceGroupId = "/providers/Microsoft.Management/serviceGroups/$lifecycleServiceGroupName"
$planLifecycleServiceGroupId = "/providers/Microsoft.Management/serviceGroups/$planLifecycleServiceGroupName"
$resilienceBase = "$serviceGroupId/providers/Microsoft.AzureResilienceManagement"
$usagePlanId = "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.AzureResilienceManagement/usagePlans/$usagePlanName"
Remove-Resource -ResourceId "$resilienceBase/drills/$drillName" -ApiVersion $resilienceApiVersion
Remove-RecoveryPlans -ServiceGroupId $serviceGroupId
Remove-RecoveryPlans -ServiceGroupId $lifecycleServiceGroupId
if (![string]::IsNullOrWhiteSpace($planLifecycleServiceGroupName)) {
Remove-RecoveryPlans -ServiceGroupId $planLifecycleServiceGroupId
}
Remove-Resource -ResourceId "$resilienceBase/goalAssignments/$goalAssignmentName" -ApiVersion $resilienceApiVersion
Remove-Resource -ResourceId "$resilienceBase/goalTemplates/$goalTemplateName" -ApiVersion $resilienceApiVersion
Remove-Resource -ResourceId "$usagePlanId/enrollments/$enrollmentName" -ApiVersion $resilienceApiVersion
Remove-Resource -ResourceId "$usagePlanId/enrollments/$lifecycleEnrollmentName" -ApiVersion $resilienceApiVersion
if (![string]::IsNullOrWhiteSpace($planLifecycleEnrollmentName)) {
Remove-Resource -ResourceId "$usagePlanId/enrollments/$planLifecycleEnrollmentName" -ApiVersion $resilienceApiVersion
}
Remove-Resource -ResourceId "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Relationships/serviceGroupMember/rhub-rg-member" -ApiVersion $membershipApiVersion
Remove-Resource -ResourceId "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Relationships/serviceGroupMember/rhub-lifecycle-rg-member" -ApiVersion $membershipApiVersion
Remove-Resource -ResourceId "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Relationships/serviceGroupMember/rhub-plan-lifecycle-rg-member" -ApiVersion $membershipApiVersion
foreach ($serviceGroupIdToDelete in @($lifecycleServiceGroupId, $serviceGroupId)) {
foreach ($serviceGroupIdToDelete in @($planLifecycleServiceGroupName ? $planLifecycleServiceGroupId : $null, $lifecycleServiceGroupId, $serviceGroupId)) {
if ($serviceGroupIdToDelete -notmatch '/$') {
Remove-Resource -ResourceId $serviceGroupIdToDelete -ApiVersion $serviceGroupApiVersion
}
@@ -40,6 +40,8 @@ $usagePlanName = $DeploymentOutputs['USAGEPLANNAME']
$enrollmentName = $DeploymentOutputs['ENROLLMENTNAME']
$lifecycleEnrollmentName = $DeploymentOutputs['LIFECYCLEENROLLMENTNAME']
$lifecycleServiceGroupName = $DeploymentOutputs['LIFECYCLESERVICEGROUPNAME']
$planLifecycleEnrollmentName = $DeploymentOutputs['PLANLIFECYCLEENROLLMENTNAME']
$planLifecycleServiceGroupName = $DeploymentOutputs['PLANLIFECYCLESERVICEGROUPNAME']
$goalTemplateName = $DeploymentOutputs['GOALTEMPLATENAME']
$goalAssignmentName = $DeploymentOutputs['GOALASSIGNMENTNAME']
$recoveryPlanName = $DeploymentOutputs['RECOVERYPLANNAME']
@@ -55,6 +57,8 @@ $resilienceApiVersion = '2026-04-01-preview'
$serviceGroupId = "/providers/Microsoft.Management/serviceGroups/$serviceGroupName"
$serviceGroupResilienceBase = "$serviceGroupId/providers/Microsoft.AzureResilienceManagement"
$lifecycleServiceGroupId = "/providers/Microsoft.Management/serviceGroups/$lifecycleServiceGroupName"
$lifecycleServiceGroupResilienceBase = "$lifecycleServiceGroupId/providers/Microsoft.AzureResilienceManagement"
$planLifecycleServiceGroupId = "/providers/Microsoft.Management/serviceGroups/$planLifecycleServiceGroupName"
function Invoke-ResilienceRestPut {
param(
@@ -74,12 +78,28 @@ function Invoke-ResilienceRestPut {
function Invoke-ResilienceRestPost {
param(
[string] $Path,
[hashtable] $Body
[hashtable] $Body,
[string] $OperationId
)
Write-Host "POST $Path"
if ($Body) {
$payload = $Body | ConvertTo-Json -Depth 20 -Compress
if ($OperationId) {
$payloadPath = [System.IO.Path]::GetTempFileName()
try {
[System.IO.File]::WriteAllText($payloadPath, $payload)
$responseContent = az rest --method POST --url "https://management.azure.com$Path" --headers "operation-id=$OperationId" "Content-Type=application/json" --body "@$payloadPath" --output json 2>&1
if ($LASTEXITCODE -ne 0) {
throw "POST $Path failed: $responseContent"
}
return $responseContent
}
finally {
Remove-Item $payloadPath -Force
}
}
$response = Invoke-AzRestMethod -Method POST -Path $Path -Payload $payload
}
else {
@@ -121,7 +141,13 @@ function Wait-ResilienceProvisioning {
return
}
if ($state -in @('Failed', 'Canceled')) {
throw "Provisioning of $Path ended in state '$state'."
$errorDetails = ($response.Content | ConvertFrom-Json).properties.errorDetails
$errorMessage = if ($errorDetails) {
" ErrorCode: $($errorDetails.code). Message: $($errorDetails.message)"
} else {
''
}
throw "Provisioning of $Path ended in state '$state'.$errorMessage"
}
Start-Sleep -Seconds 15
@@ -135,6 +161,13 @@ function Add-RecoveryContributorRole {
[string] $Scope
)
$currentAccount = (Get-AzContext).Account
$currentUserObjectId = $currentAccount.ExtendedProperties.HomeAccountId.Split('.')[0]
if ($TestApplicationOid -eq $currentUserObjectId) {
Write-Host "Using the automatic service-group roles for the signed-in test user at $Scope"
return
}
$roleName = 'Azure Resilience Management Recovery Contributor'
$assignment = Get-AzRoleAssignment -ObjectId $TestApplicationOid -Scope $Scope -RoleDefinitionName $roleName -ErrorAction SilentlyContinue
if (!$assignment) {
@@ -168,8 +201,22 @@ Invoke-ResilienceRestPut -Path $lifecycleServiceGroupPath -Body @{
} | Out-Null
Wait-ResilienceProvisioning -Path $lifecycleServiceGroupPath -WaitForAuthorization
# Recovery-plan lifecycle tests use a separate service group because only one plan of each
# type can exist in a service group and the drill delete fixture reserves the lifecycle group.
$planLifecycleServiceGroupPath = "$planLifecycleServiceGroupId`?api-version=$serviceGroupApiVersion"
Invoke-ResilienceRestPut -Path $planLifecycleServiceGroupPath -Body @{
properties = @{
displayName = $planLifecycleServiceGroupName
parent = @{
resourceId = "/providers/Microsoft.Management/serviceGroups/$tenantId"
}
}
} | Out-Null
Wait-ResilienceProvisioning -Path $planLifecycleServiceGroupPath -WaitForAuthorization
Add-RecoveryContributorRole -Scope $serviceGroupId
Add-RecoveryContributorRole -Scope $lifecycleServiceGroupId
Add-RecoveryContributorRole -Scope $planLifecycleServiceGroupId
# 2) Add the resource group as a member of the service group so its resources
# (e.g. the storage account) surface as goal/recovery resource targets.
@@ -180,6 +227,20 @@ Invoke-ResilienceRestPut -Path $membershipPath -Body @{
}
} | Out-Null
$lifecycleMembershipPath = "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Relationships/serviceGroupMember/rhub-lifecycle-rg-member`?api-version=$membershipApiVersion"
Invoke-ResilienceRestPut -Path $lifecycleMembershipPath -Body @{
properties = @{
targetId = $lifecycleServiceGroupId
}
} | Out-Null
$planLifecycleMembershipPath = "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Relationships/serviceGroupMember/rhub-plan-lifecycle-rg-member`?api-version=$membershipApiVersion"
Invoke-ResilienceRestPut -Path $planLifecycleMembershipPath -Body @{
properties = @{
targetId = $planLifecycleServiceGroupId
}
} | Out-Null
# 3) Enroll the service group into the usage plan (the usage plan is created by the bicep template).
$enrollmentPath = "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.AzureResilienceManagement/usagePlans/$usagePlanName/enrollments/$enrollmentName`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPut -Path $enrollmentPath -Body @{
@@ -197,6 +258,14 @@ Invoke-ResilienceRestPut -Path $lifecycleEnrollmentPath -Body @{
} | Out-Null
Wait-ResilienceProvisioning -Path $lifecycleEnrollmentPath
$planLifecycleEnrollmentPath = "/subscriptions/$subscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.AzureResilienceManagement/usagePlans/$usagePlanName/enrollments/$planLifecycleEnrollmentName`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPut -Path $planLifecycleEnrollmentPath -Body @{
properties = @{
serviceGroupId = $planLifecycleServiceGroupId
}
} | Out-Null
Wait-ResilienceProvisioning -Path $planLifecycleEnrollmentPath
# 4) Create a goal template on the service group.
$goalTemplatePath = "$serviceGroupResilienceBase/goalTemplates/$goalTemplateName`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPut -Path $goalTemplatePath -Body @{
@@ -234,30 +303,49 @@ if ($existingGoalAssignment.StatusCode -eq 404) {
throw "GET $goalAssignmentPath failed with status $($existingGoalAssignment.StatusCode): $($existingGoalAssignment.Content)"
}
# 6) Create a recovery plan on the service group.
# 6) Create or validate the recovery plan on the service group. Do not PUT an
# existing plan because recovery group IDs are referenced by its recovery resources.
$recoveryPlanPath = "$serviceGroupResilienceBase/recoveryPlans/$recoveryPlanName`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPut -Path $recoveryPlanPath -Body @{
identity = @{
type = 'SystemAssigned'
}
properties = @{
planDescription = 'Recovery plan for live testing.'
planType = 'Zonal'
recoveryGroupsSetting = @{
defaultGroup = @{
properties = @{
description = 'Default recovery group'
groupUniqueId = (New-Guid).Guid
orderId = 0
preActions = @()
postActions = @()
}
}
additionalGroups = @()
$existingRecoveryPlan = Invoke-AzRestMethod -Method GET -Path $recoveryPlanPath
if ($existingRecoveryPlan.StatusCode -eq 404) {
Invoke-ResilienceRestPut -Path $recoveryPlanPath -Body @{
identity = @{
type = 'SystemAssigned'
}
properties = @{
planDescription = 'Recovery plan for live testing.'
planType = 'Zonal'
recoveryGroupsSetting = @{
defaultGroup = @{
properties = @{
description = 'Default recovery group'
groupUniqueId = (New-Guid).Guid
orderId = 0
preActions = @()
postActions = @()
}
}
additionalGroups = @()
}
}
} | Out-Null
Wait-ResilienceProvisioning -Path $recoveryPlanPath
} elseif ($existingRecoveryPlan.StatusCode -eq 200) {
$recoveryPlan = $existingRecoveryPlan.Content | ConvertFrom-Json
if ($recoveryPlan.properties.provisioningState -ne 'Succeeded') {
$errorDetails = $recoveryPlan.properties.errorDetails
throw "Existing recovery plan '$recoveryPlanName' is in provisioning state '$($recoveryPlan.properties.provisioningState)'. ErrorCode: $($errorDetails.code). Message: $($errorDetails.message)"
}
} | Out-Null
Wait-ResilienceProvisioning -Path $recoveryPlanPath
if ($recoveryPlan.properties.planType -ne 'Zonal') {
throw "Existing recovery plan '$recoveryPlanName' does not match the requested Zonal test configuration."
}
if ([string]::IsNullOrWhiteSpace($recoveryPlan.properties.recoveryGroupsSetting.defaultGroup.properties.groupUniqueId)) {
throw "Existing recovery plan '$recoveryPlanName' does not have a valid default recovery group."
}
Write-Host "Recovery plan '$recoveryPlanName' already exists with the requested configuration."
} else {
throw "GET $recoveryPlanPath failed with status $($existingRecoveryPlan.StatusCode): $($existingRecoveryPlan.Content)"
}
# 7) Run a readiness check on the recovery plan so it has a recorded validation status.
$checkReadinessPath = "$serviceGroupResilienceBase/recoveryPlans/$recoveryPlanName/checkReadiness`?api-version=$resilienceApiVersion"
@@ -308,10 +396,6 @@ if ($existingDrill.StatusCode -eq 404) {
properties = @{
drillType = 'Zonal'
rbacSetupMode = 'AutomatedBuiltinRoles'
metricsProperties = @{
identity = @{ type = 'SystemAssigned' }
metricsToTrack = @()
}
recoveryPlanProperties = @{
recoveryPlanId = $recoveryPlanId
identity = @{ type = 'SystemAssigned' }
@@ -325,6 +409,9 @@ if ($existingDrill.StatusCode -eq 404) {
identity = @{ type = 'SystemAssigned' }
chaosResourceIdentityForFaults = @{ type = 'SystemAssigned' }
}
monitoringProperties = @{
identity = @{ type = 'SystemAssigned' }
}
}
} | Out-Null
Wait-ResilienceProvisioning -Path $drillPath
@@ -337,8 +424,7 @@ elseif ($existingDrill.StatusCode -eq 200) {
$drill.properties.drillAssetProperties.region -ne $location -or
$drill.properties.drillAssetProperties.resourceGroup -ne $ResourceGroupName -or
$drill.properties.recoveryPlanProperties.recoveryPlanId -ne $recoveryPlanId -or
-not $drill.properties.metricsProperties.identity -or
$drill.properties.metricsProperties.metricsToTrack.Count -ne 0 -or
$drill.properties.monitoringProperties.identity.type -ne 'SystemAssigned' -or
$drill.properties.provisioningState -ne 'Succeeded') {
throw "Existing drill '$drillName' does not match the requested test configuration."
}
@@ -348,8 +434,35 @@ else {
throw "GET $drillPath failed with status $($existingDrill.StatusCode): $($existingDrill.Content)"
}
# 8b) Create an isolated drill used exclusively by the delete live test.
$deleteDrillPath = "$serviceGroupResilienceBase/drills/$deleteDrillName`?api-version=$resilienceApiVersion"
# 8b) Create an isolated recovery plan and drill used exclusively by the delete live test.
$deleteRecoveryPlanPath = "$lifecycleServiceGroupResilienceBase/recoveryPlans/$recoveryPlanName`?api-version=$resilienceApiVersion"
if ((Invoke-AzRestMethod -Method GET -Path $deleteRecoveryPlanPath).StatusCode -eq 404) {
Invoke-ResilienceRestPut -Path $deleteRecoveryPlanPath -Body @{
identity = @{
type = 'SystemAssigned'
}
properties = @{
planDescription = 'Recovery plan for the drill delete live test.'
planType = 'Zonal'
recoveryGroupsSetting = @{
defaultGroup = @{
properties = @{
description = 'Default recovery group'
groupUniqueId = (New-Guid).Guid
orderId = 0
preActions = @()
postActions = @()
}
}
additionalGroups = @()
}
}
} | Out-Null
Wait-ResilienceProvisioning -Path $deleteRecoveryPlanPath
}
$deleteDrillPath = "$lifecycleServiceGroupResilienceBase/drills/$deleteDrillName`?api-version=$resilienceApiVersion"
$deleteRecoveryPlanId = "$lifecycleServiceGroupResilienceBase/recoveryPlans/$recoveryPlanName"
if ((Invoke-AzRestMethod -Method GET -Path $deleteDrillPath).StatusCode -eq 404) {
Invoke-ResilienceRestPut -Path $deleteDrillPath -Body @{
identity = @{
@@ -358,12 +471,8 @@ if ((Invoke-AzRestMethod -Method GET -Path $deleteDrillPath).StatusCode -eq 404)
properties = @{
drillType = 'Zonal'
rbacSetupMode = 'AutomatedBuiltinRoles'
metricsProperties = @{
identity = @{ type = 'SystemAssigned' }
metricsToTrack = @()
}
recoveryPlanProperties = @{
recoveryPlanId = $recoveryPlanId
recoveryPlanId = $deleteRecoveryPlanId
identity = @{ type = 'SystemAssigned' }
}
drillAssetProperties = @{
@@ -375,6 +484,9 @@ if ((Invoke-AzRestMethod -Method GET -Path $deleteDrillPath).StatusCode -eq 404)
identity = @{ type = 'SystemAssigned' }
chaosResourceIdentityForFaults = @{ type = 'SystemAssigned' }
}
monitoringProperties = @{
identity = @{ type = 'SystemAssigned' }
}
}
} | Out-Null
Wait-ResilienceProvisioning -Path $deleteDrillPath
@@ -392,7 +504,7 @@ while (-not $drillResource -and (Get-Date) -lt $deadline) {
$drillResources = ($response.Content | ConvertFrom-Json).value
$drillResource = $drillResources | Where-Object {
$_.properties.targetResourceId -eq $storageAccountId
$_.properties.resourceId -eq $storageAccountId
} | Select-Object -First 1
if (-not $drillResource) {
@@ -415,8 +527,8 @@ if ($drillResource.properties.faultProperties) {
$includeResource['faultProperties'] = $drillResource.properties.faultProperties
}
$addResourcesPath = "$serviceGroupResilienceBase/drills/$drillName/addOrUpdateResources`?api-version=$resilienceApiVersion&operationId=$((New-Guid).Guid)"
Invoke-ResilienceRestPost -Path $addResourcesPath -Body @{
$addResourcesPath = "$serviceGroupResilienceBase/drills/$drillName/addOrUpdateResources`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPost -Path $addResourcesPath -OperationId (New-Guid).Guid -Body @{
faultDurationInMin = 1
forceInclusionAndUpdate = 'Enable'
resourceLists = @{
@@ -455,8 +567,8 @@ if ($existingRunsResponse.StatusCode -lt 400) {
$existingRunIds = @((($existingRunsResponse.Content | ConvertFrom-Json).value).id)
}
$startPath = "$serviceGroupResilienceBase/drills/$drillName/start`?api-version=$resilienceApiVersion&operationId=$((New-Guid).Guid)"
Invoke-ResilienceRestPost -Path $startPath -Body @{
$startPath = "$serviceGroupResilienceBase/drills/$drillName/start`?api-version=$resilienceApiVersion"
Invoke-ResilienceRestPost -Path $startPath -OperationId (New-Guid).Guid -Body @{
mode = 'Failover'
} | Out-Null
@@ -10,12 +10,15 @@ var enrollmentName = take('en${uniqueSuffix}', 24)
var serviceGroupName = 'sgr${uniqueSuffix}'
var lifecycleEnrollmentName = take('el${uniqueSuffix}', 24)
var lifecycleServiceGroupName = take('sgl${uniqueSuffix}', 24)
var planLifecycleEnrollmentName = take('ep${uniqueSuffix}', 24)
var planLifecycleServiceGroupName = take('sgp${uniqueSuffix}', 24)
var goalTemplateName = take('gt${uniqueSuffix}', 24)
var goalAssignmentName = take('ga${uniqueSuffix}', 24)
var recoveryPlanName = take('rp${uniqueSuffix}', 24)
var drillName = take('dr${uniqueSuffix}', 24)
var deleteDrillName = take('dd${uniqueSuffix}', 24)
var storageAccountName = toLower(take('st${uniqueSuffix}', 24))
var managedDiskName = take('md${uniqueSuffix}', 80)
// The test identity is automatically granted access to this resource group by the
// test harness (New-TestResources.ps1), so no explicit role assignment is created here.
@@ -46,6 +49,23 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' = {
}
}
resource managedDisk 'Microsoft.Compute/disks@2024-03-02' = {
name: managedDiskName
location: resourceGroup().location
zones: [
'1'
]
sku: {
name: 'Standard_LRS'
}
properties: {
creationData: {
createOption: 'Empty'
}
diskSizeGB: 4
}
}
// Usage plan (resource-group scoped). This resource type is only available in the 'global' location.
resource usagePlan 'Microsoft.AzureResilienceManagement/usagePlans@2026-04-01-preview' = {
name: usagePlanName
@@ -60,6 +80,8 @@ output enrollmentName string = enrollmentName
output serviceGroupName string = serviceGroupName
output lifecycleEnrollmentName string = lifecycleEnrollmentName
output lifecycleServiceGroupName string = lifecycleServiceGroupName
output planLifecycleEnrollmentName string = planLifecycleEnrollmentName
output planLifecycleServiceGroupName string = planLifecycleServiceGroupName
output goalTemplateName string = goalTemplateName
output goalAssignmentName string = goalAssignmentName
output recoveryPlanName string = recoveryPlanName