From 8960c04adffda58da239703867573941fafa8d7c Mon Sep 17 00:00:00 2001 From: Krishna Nithin Date: Thu, 1 Jun 2017 09:41:09 -0700 Subject: [PATCH] Flag to enable no cleanup on failures --- CanaryValidator/Canary.Tests.ps1 | 86 +++++++++++++++------------ CanaryValidator/Canary.Utilities.psm1 | 10 ++++ 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/CanaryValidator/Canary.Tests.ps1 b/CanaryValidator/Canary.Tests.ps1 index 30e3344f..c9facb4f 100644 --- a/CanaryValidator/Canary.Tests.ps1 +++ b/CanaryValidator/Canary.Tests.ps1 @@ -86,6 +86,11 @@ param ( [Parameter(ParameterSetName="tenant", Mandatory=$false)] [ValidateNotNullOrEmpty()] [switch]$NoCleanup, + [parameter(HelpMessage="Specifies whether Canary needs to clean up resources when a failure is encountered")] + [Parameter(ParameterSetName="default", Mandatory=$false)] + [Parameter(ParameterSetName="tenant", Mandatory=$false)] + [ValidateNotNullOrEmpty()] + [switch]$NoCleanupOnFailure, [parameter(HelpMessage="Specifies the path for log files")] [Parameter(ParameterSetName="default", Mandatory=$false)] [Parameter(ParameterSetName="tenant", Mandatory=$false)] @@ -851,63 +856,66 @@ while ($runCount -le $NumberOfIterations) if (-not $NoCleanup) { - Invoke-Usecase -Name 'DeleteVMWithPrivateIP' -Description "Delete the VM with private IP" -UsecaseBlock ` + if (-not ($NoCleanupOnFailure -and (Get-CanaryFailureStatus))) { - if ($vmObject = Get-AzureRmVM -ResourceGroupName $CanaryVMRG -Name $privateVMName -ErrorAction Stop) + Invoke-Usecase -Name 'DeleteVMWithPrivateIP' -Description "Delete the VM with private IP" -UsecaseBlock ` { - $deleteVM = $vmObject | Remove-AzureRmVM -Force -ErrorAction Stop - if (-not (($deleteVM.StatusCode -eq "OK") -and ($deleteVM.IsSuccessStatusCode))) + if ($vmObject = Get-AzureRmVM -ResourceGroupName $CanaryVMRG -Name $privateVMName -ErrorAction Stop) { - throw [System.Exception]"Failed to delete the VM $privateVMName" + $deleteVM = $vmObject | Remove-AzureRmVM -Force -ErrorAction Stop + if (-not (($deleteVM.StatusCode -eq "OK") -and ($deleteVM.IsSuccessStatusCode))) + { + throw [System.Exception]"Failed to delete the VM $privateVMName" + } } } - } - Invoke-Usecase -Name 'DeleteVMResourceGroup' -Description "Delete the resource group that contains all the VMs and corresponding resources" -UsecaseBlock ` - { - if ($removeRG = Get-AzureRmResourceGroup -Name $CanaryVMRG -ErrorAction Stop) + Invoke-Usecase -Name 'DeleteVMResourceGroup' -Description "Delete the resource group that contains all the VMs and corresponding resources" -UsecaseBlock ` { - $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop + if ($removeRG = Get-AzureRmResourceGroup -Name $CanaryVMRG -ErrorAction Stop) + { + $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop + } } - } - Invoke-Usecase -Name 'DeleteUtilitiesResourceGroup' -Description "Delete the resource group that contains all the utilities and corresponding resources" -UsecaseBlock ` - { - if ($removeRG = Get-AzureRmResourceGroup -Name $CanaryUtilitiesRG -ErrorAction Stop) + Invoke-Usecase -Name 'DeleteUtilitiesResourceGroup' -Description "Delete the resource group that contains all the utilities and corresponding resources" -UsecaseBlock ` { - $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop + if ($removeRG = Get-AzureRmResourceGroup -Name $CanaryUtilitiesRG -ErrorAction Stop) + { + $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop + } } - } - if (($TenantAdminCredentials) -or ($listAvl)) - { - Invoke-Usecase -Name 'TenantRelatedcleanup' -Description "Remove all the tenant related stuff" -UsecaseBlock ` + if (($TenantAdminCredentials) -or ($listAvl)) { - Invoke-Usecase -Name 'DeleteTenantSubscriptions' -Description "Remove all the tenant related subscriptions" -UsecaseBlock ` + Invoke-Usecase -Name 'TenantRelatedcleanup' -Description "Remove all the tenant related stuff" -UsecaseBlock ` { - if ($subs = Get-AzureRmTenantSubscription -ErrorAction Stop | Where-Object DisplayName -eq $tenantSubscriptionName) - { - Remove-AzureRmTenantSubscription -TargetSubscriptionId $subs.SubscriptionId -ErrorAction Stop - } - if ($subs = Get-AzureRmTenantSubscription -ErrorAction Stop | Where-Object DisplayName -eq $canaryDefaultTenantSubscription) + Invoke-Usecase -Name 'DeleteTenantSubscriptions' -Description "Remove all the tenant related subscriptions" -UsecaseBlock ` { - Remove-AzureRmTenantSubscription -TargetSubscriptionId $subs.SubscriptionId -ErrorAction Stop - } - } + if ($subs = Get-AzureRmTenantSubscription -ErrorAction Stop | Where-Object DisplayName -eq $tenantSubscriptionName) + { + Remove-AzureRmTenantSubscription -TargetSubscriptionId $subs.SubscriptionId -ErrorAction Stop + } + if ($subs = Get-AzureRmTenantSubscription -ErrorAction Stop | Where-Object DisplayName -eq $canaryDefaultTenantSubscription) + { + Remove-AzureRmTenantSubscription -TargetSubscriptionId $subs.SubscriptionId -ErrorAction Stop + } + } - Invoke-Usecase -Name 'LoginToAzureStackEnvAsSvcAdminForCleanup' -Description "Login to $SvcAdminEnvironmentName as service administrator to remove the subscription resource group" -UsecaseBlock ` - { - Add-AzureRmAccount -EnvironmentName $SvcAdminEnvironmentName -Credential $ServiceAdminCredentials -TenantId $TenantID -ErrorAction Stop - } + Invoke-Usecase -Name 'LoginToAzureStackEnvAsSvcAdminForCleanup' -Description "Login to $SvcAdminEnvironmentName as service administrator to remove the subscription resource group" -UsecaseBlock ` + { + Add-AzureRmAccount -EnvironmentName $SvcAdminEnvironmentName -Credential $ServiceAdminCredentials -TenantId $TenantID -ErrorAction Stop + } - Invoke-Usecase -Name 'DeleteSubscriptionResourceGroup' -Description "Delete the resource group that contains subscription resources" -UsecaseBlock ` - { - if ($removeRG = Get-AzureRmResourceGroup -Name $subscriptionRGName -ErrorAction Stop) + Invoke-Usecase -Name 'DeleteSubscriptionResourceGroup' -Description "Delete the resource group that contains subscription resources" -UsecaseBlock ` { - $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop - } - } - } + if ($removeRG = Get-AzureRmResourceGroup -Name $subscriptionRGName -ErrorAction Stop) + { + $removeRG | Remove-AzureRmResourceGroup -Force -ErrorAction Stop + } + } + } + } } } diff --git a/CanaryValidator/Canary.Utilities.psm1 b/CanaryValidator/Canary.Utilities.psm1 index d8e4f652..9875788f 100644 --- a/CanaryValidator/Canary.Utilities.psm1 +++ b/CanaryValidator/Canary.Utilities.psm1 @@ -151,6 +151,16 @@ function Get-CanaryLonghaulResult @{Expression={$pCount = ($_.Group | Where-Object Result -eq "PASS").Count; $times = ($_.Group | Where-Object Result -eq "PASS" | ForEach-Object {((Get-Date $_.EndTime) - (Get-Date $_.StartTime)).TotalMilliseconds}); $avgTime = ($times | Measure-Object -Average).Average; $sd = 0; foreach ($time in $times){$sd += [math]::Pow(($time - $avgTime), 2)}; [math]::Round(([math]::Round([math]::Sqrt($sd/$pCount), 0)/$avgTime), 0) * 100};Label="RelativeStdDev`n[Goal: <50%]"; Align = "Left"} } +function Get-CanaryFailureStatus +{ + $logContent = Get-Content -Raw -Path $Global:JSONLogFile | ConvertFrom-Json + if ($logContent.Usecases.Result -contains "FAIL") + { + return $true + } + return $false +} + function Start-Scenario { [CmdletBinding()]