Support.AksArc.psm1
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # Support.AksArc Module # ######################################################################################### <# .SYNOPSIS Imports the latest version of the Azs.Support module from the nuget directory. .PARAMETER AcceptEula If EULA is accepted, the disclaimer will not be shown. .PARAMETER SkipUpdate Skips the automatic update method from being executed. #> param( [CmdletBinding(DefaultParameterSetName = "AcceptEula")] [Parameter(Mandatory = $false , ParameterSetName = "AcceptEula", Position = 1)] [Bool]$AcceptEula = $false, [Parameter(Mandatory = $false, ParameterSetName = "SkipUpdate", Position = 2)] [Bool]$SkipUpdate = $false ) #requires -runasadministrator #requires -version 5.1 #requires -module Moc # Constants for reusability and maintainability $script:KnownIssuesPanicPattern = ".*\(*Client\)\.cleanupVirtualHardDiskNode" $script:CloudAgentServiceName = "wssdcloudagent" function Get-SupportAksArcKnownIssues { <# .SYNOPSIS Returns known issues in the MOC environment. .DESCRIPTION Returns a collection of known issues in the MOC environment, including their test and remediation functions. .OUTPUTS Array of PSObjects containing known issue definitions .EXAMPLE Get-SupportAksArcKnownIssues #> [CmdletBinding()] [OutputType([PSObject[]])] param() $knownIssues = @( @{ 'TestId' = "MOC-0001" 'TestName' = "Null Pointer Dereference in VirtualHardDisk" 'Description' = "Validates MOC Cloud Agent Panics" 'TestFunction' = "Test-SupportAksArcKnownIssues_Panics" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_Panics" }, @{ 'TestId' = "MOC-0002" 'TestName' = "Multiple MOC Cloud Agent Instances" 'Description' = "Validates that there is only one instance of the MOC Cloud Agent running." 'TestFunction' = "Test-SupportAksArcKnownIssues_MultipleCloudAgentInstances" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_MultipleCloudAgentInstances" }, @{ 'TestId' = "MOC-0004" 'TestName' = "MOC Powershell Stuck in Updating State" 'Description' = "Validates that the MOC Powershell is not stuck in an updating state." 'TestFunction' = "Test-SupportAksArcKnownIssues_StuckInUpdating" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_StuckInUpdating" }, @{ 'TestId' = "MOC-0005" 'TestName' = "MOC Nodes Out of Sync with Cluster Nodes" 'Description' = "Validates that all MOC nodes are in sync with the cluster nodes." 'TestFunction' = "Test-SupportAksArcKnownIssues_MocNodesOutOfSyncWithClusterNodes" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_MocNodesOutOfSyncWithClusterNodes" }, @{ 'TestId' = "MOC-0006" 'TestName' = "MOC Cloud Agent Not Running" 'Description' = "Validates that the MOC Cloud Agent service is running on all nodes." 'TestFunction' = "Test-SupportAksArcKnownIssues_MocCloudAgentNotRunning" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_MocCloudAgentNotRunning" }, @{ 'TestId' = "MOC-0007" 'TestName' = "Windows Event Log Not Running" 'Description' = "Validates that the Windows Event Log service is running on all nodes." 'TestFunction' = "Test-SupportAksArcKnownIssues_EventLogNotRunning" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_EventLogNotRunning" }, @{ 'TestId' = "MOC-0008" 'TestName' = "Gallery Image Stuck in Deleting State" 'Description' = "Validates that no gallery images are stuck in deleting state." 'TestFunction' = "Test-SupportAksArcKnownIssues_GalleryImageStuckInDeleting" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_GalleryImageStuckInDeleting" }, @{ 'TestId' = "MOC-0009" 'TestName' = "MOC Agent Not Responding" 'Description' = "Validates if the Virtual Machine is stuck in a pending state." 'TestFunction' = "Test-SupportAksArcKnownIssues_VirtualMachineStuckInPending" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_VirtualMachineStuckInPending" }, @{ 'TestId' = "MOC-0010" 'TestName' = "MOC Not on Latest Patch Version" 'Description' = "Checks if the current MOC version is the latest patch version." 'TestFunction' = "Test-SupportAksArcKnownIssues_MocNotOnLatestPatch" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_MocNotOnLatestPatch" }, @{ 'TestId' = "MOC-0011" 'TestName' = "MOC Nodes Not Active" 'Description' = "Checks if any MOC nodes are not in the 'Active' state." 'TestFunction' = "Test-SupportAksArcKnownIssues_MocNodesNotActive" 'RemediationFunction' = "Invoke-SupportAksArcRemediation_MocNodesNotActive" } ) | ForEach-Object { New-Object -TypeName PSObject -Property $_ } return $knownIssues } function Test-SupportAksArcKnownIssues { <# .SYNOPSIS Tests for known issues in the MOC environment. .DESCRIPTION Executes tests for known issues in the MOC environment. If a TestId is provided, only that specific test will be run. Otherwise, all known issue tests are executed. .PARAMETER TestId Optional. The specific test ID to run. If not provided, all tests will be executed. .OUTPUTS Array of test results .EXAMPLE Test-SupportAksArcKnownIssues .EXAMPLE Test-SupportAksArcKnownIssues -TestId "MOC-0001" #> [CmdletBinding()] [OutputType([PSObject[]])] param( [ValidatePattern("^MOC-\d{4}$")] [string]$TestId ) try { $knownIssues = Get-SupportAksArcKnownIssues $results = @() foreach ($issue in $knownIssues) { if ([string]::IsNullOrWhiteSpace($TestId) -or ($issue.TestId -eq $TestId)) { Write-Host "Running test: $($issue.TestName) with ID: $($issue.TestId)" Write-Verbose "Invoking test function for $($issue.TestName)..." try { $result = & $issue.TestFunction Write-Verbose "Test result for $($issue.TestName): $($result.Status) - $($result.Message)" $results += $result } catch { Write-Warning "Test $($issue.TestId) failed with error: $_" $results += @{ TestName = $issue.TestName Status = "Error" Message = "Test execution failed: $_" } } } } return $results } catch { Write-Error "Failed to execute known issue tests: $_" throw } } function Invoke-SupportAksArcRemediation { <# .SYNOPSIS Remediates known issues in the MOC environment. .DESCRIPTION Executes remediation for known issues in the MOC environment. If a TestId is provided, only that specific issue will be remediated. Otherwise, all failed tests will be remediated. .PARAMETER TestId Optional. The specific issue ID to remediate. If not provided, all failed tests will be remediated. .OUTPUTS None .EXAMPLE Invoke-SupportAksArcRemediation .EXAMPLE Invoke-SupportAksArcRemediation -TestId "MOC-0001" #> [CmdletBinding()] param( [ValidatePattern("^MOC-\d{4}$")] [string]$TestId ) try { $knownIssues = Get-SupportAksArcKnownIssues foreach ($issue in $knownIssues) { if ([string]::IsNullOrWhiteSpace($TestId) -or ($issue.TestId -eq $TestId)) { Write-Verbose "Checking issue: $($issue.TestName) with ID: $($issue.TestId)" try { $testResult = & $issue.TestFunction if ($testResult.Status -ne "Passed") { Write-Host "Remediation for $($issue.TestName) is required." Write-Verbose "Invoking remediation for $($issue.TestName)..." if (& $issue.RemediationFunction) { Write-Host "Remediation for $($issue.TestName) succeeded." } else { Write-Warning "Remediation for $($issue.TestName) failed." } } else { Write-Host "No remediation required for $($issue.TestName)." } } catch { Write-Error "Failed to remediate $($issue.TestName): $_" } } } } catch { Write-Error "Failed to execute remediation: $_" throw } } function Test-SupportAksArcKnownIssues_Panics { <# .SYNOPSIS Tests for known MOC Cloud Agent panics. .DESCRIPTION Validates if any known panics exist in the MOC Cloud Agent logs. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_Panics #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC Cloud Agent Panics" Status = "Passed" Message = "No panics found in the MOC Cloud Agent." } try { $panics = Get-MocCloudAgentPanic -ErrorAction Stop if ($panics.Count -gt 0) { $result.Status = "Failed" $result.Message = "Panics found in the MOC Cloud Agent: $($panics | ConvertTo-Json -Compress)" } } catch { Write-Error "Failed to check for panics: $_" throw } return $result } function Get-MocCloudAgentPanic { <# .SYNOPSIS Retrieves panic information from MOC Cloud Agent logs. .DESCRIPTION Searches through MOC Cloud Agent log files for known panic patterns and returns details about any panics found. .OUTPUTS Array of panic information objects .EXAMPLE Get-MocCloudAgentPanic #> [CmdletBinding()] [OutputType([PSObject[]])] param() try { $panicLocation = [IO.Path]::Combine((Get-MocConfig).cloudConfigLocation, "log") $matchedPanics = @() Get-ChildItem -Path $panicLocation -Filter "panic*" -ErrorAction Stop | Sort-Object LastWriteTime -Descending | Select-Object -First 10 | ForEach-Object { Write-Verbose "Analyzing panic file: $($_.FullName)" $panicDetails = Get-PanicDetails -FilePath $_.FullName if ($panicDetails.StackTrace -match $script:KnownIssuesPanicPattern) { Write-Verbose "Found known panic pattern in $($_.Name)" $matchedPanics += $panicDetails } } return $matchedPanics } catch { Write-Error "Failed to retrieve panic information: $_" throw } } function Get-PanicDetails { <# .SYNOPSIS Extracts detailed information from a panic log file. .DESCRIPTION Parses a panic log file to extract structured information about the panic, including message, timestamp, process ID, goroutine ID, and stack trace. .PARAMETER FilePath The path to the panic log file to analyze. .OUTPUTS PSObject containing structured panic information .EXAMPLE Get-PanicDetails -FilePath "path/to/panic.log" #> [CmdletBinding()] [OutputType([PSObject])] param( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [System.IO.FileInfo]$FilePath ) try { $panicInfo = @{ PanicMessage = "" Timestamp = "" PID = "" GoroutineID = "" StackTrace = @() } $lines = Get-Content -Path $FilePath -ErrorAction Stop foreach ($line in $lines) { switch -Regex ($line) { '^Panic:\s+(.*)' { $panicInfo.PanicMessage = $matches[1].Trim() } '^Time:\s+(.*)' { $panicInfo.Timestamp = $matches[1].Trim() } '^PID:\s+(\d+)' { $panicInfo.PID = $matches[1].Trim() } '^goroutine\s+(\d+)\s+\[.*\]:' { $panicInfo.GoroutineID = $matches[1].Trim() } default { $panicInfo.StackTrace += $line.Trim() } } } return [PSCustomObject]$panicInfo } catch { Write-Error "Failed to extract panic details from $FilePath : $_" throw } } function Invoke-SupportAksArcRemediation_Panics { <# .SYNOPSIS Remediates MOC Cloud Agent panics. .DESCRIPTION Updates the MOC version to address panic issues. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_Panics #> [CmdletBinding()] [OutputType([bool])] param() if (-not (Test-AzureLocal)) { Write-Verbose "Remediation for panics is only applicable to Azure Local Solution." return $true } try { Write-Verbose "Remediating MOC Cloud Agent Panics..." return Invoke-SupportAksArcRemediation_MocNotOnLatestPatch } catch { Write-Error "Failed to remediate panics: $_" return $false } } function Invoke-UpdateMocHotFix { <# .SYNOPSIS Updates MOC to the latest hotfix version. .DESCRIPTION Checks for the latest patch version of MOC and updates it. .OUTPUTS Boolean indicating update success .EXAMPLE Invoke-UpdateMocHotFix #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Updating MOC to the latest hotfix version..." $nextPatchVersion = Get-NextPatchVersionOfMoc Update-Moc -Version $nextPatchVersion -skipValidationCheck -ErrorAction Stop return $true } catch { Write-Error "Failed to update MOC hotfix: $_" return $false } } function Invoke-UpdateMocPatchRelease { <# .SYNOPSIS Updates MOC to the latest patch release. .DESCRIPTION Checks for the next minor version of MOC and updates it. .OUTPUTS Boolean indicating update success .EXAMPLE Invoke-UpdateMocPatchRelease #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Updating MOC to the next minor version..." $nextMinorVersion = Get-NextMinorVersionofMoc Update-Moc -Version $nextMinorVersion -skipValidationCheck -ErrorAction Stop return $true } catch { Write-Error "Failed to update MOC patch release: $_" return $false } } function Get-NextPatchVersionOfMoc { <# .SYNOPSIS Gets the next available patch version for the current MOC minor version. .DESCRIPTION Retrieves the latest available patch version for the current MOC minor version. .OUTPUTS String containing the version number .EXAMPLE Get-NextPatchVersionOfMoc #> [CmdletBinding()] [OutputType([string])] param() try { $version = Get-MocVersion return Get-LatestVersionOfMoc -version $version -ErrorAction Stop } catch { Write-Error "Failed to get next patch version: $_" throw } } function Get-NextMinorVersionofMoc { <# .SYNOPSIS Gets the next available minor version for MOC. .DESCRIPTION Increments the current minor version by 1 and returns the latest patch version for that minor version. .OUTPUTS String containing the version number .EXAMPLE Get-NextMinorVersionofMoc #> [CmdletBinding()] [OutputType([string])] param() $version = Get-MocVersion try { $versions = $version.Split('.') | Select-Object -First 3 $versions[1] = [int]$versions[1] + 1 $newVersion = $versions -join '.' return Get-LatestVersionOfMoc -version $newVersion -ErrorAction Stop } catch { # If unable to get the next minor version, fallback to the current version return $version } } function Get-LatestVersionOfMoc { <# .SYNOPSIS Gets the latest available version for a given MOC version prefix. .DESCRIPTION Searches the manifest cache for the latest version matching the specified version prefix. .PARAMETER version The version prefix to match against (e.g., "1.2" to find the latest 1.2.x version) .OUTPUTS String containing the complete version number .EXAMPLE Get-LatestVersionOfMoc -version "1.2" #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string]$version ) try { $versionSelectSubString = ($version.Split('.') | Select-Object -First 2) -join '.' $catalog = Get-Content -Path (Get-MocConfig).manifestCache -ErrorAction Stop | ConvertFrom-Json # Sorting is not required, as we expect the release to be ordered by version from latest to oldest $matchingVersion = $catalog.ProductStreamRefs[0].ProductReleases | Where-Object { $_.Version -match "^$([regex]::Escape($versionSelectSubString))\." } | Select-Object -First 1 -ExpandProperty Version if (-not $matchingVersion) { throw "No matching version found in manifest" } return $matchingVersion } catch { Write-Error "Failed to get latest version: $_" throw } } function Test-SupportAksArcKnownIssues_MultipleCloudAgentInstances { <# .SYNOPSIS Tests for multiple running instances of the MOC Cloud Agent. .DESCRIPTION Checks cluster nodes for multiple running instances of the MOC Cloud Agent service. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_MultipleCloudAgentInstances #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate Multiple MOC Cloud Agent Instances" Status = "Passed" Message = "No multiple instances of MOC Cloud Agent found." } try { if (-not (Test-FailoverCluster)) { Write-Verbose "Not a failover cluster environment, skipping test" return $result } $instances = Get-ClusterNode | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { Get-Service $using:script:CloudAgentServiceName } } | Where-Object { $_.Status -eq 'Running' } | Select-Object -Unique -Property PSComputerName if ($instances.Count -gt 1) { $result.Status = "Failed" $result.Message = "Multiple instances of MOC Cloud Agent found: $($instances.Count)" } } catch { Write-Error "Failed to check for multiple instances: $_" throw } return $result } function Invoke-SupportAksArcRemediation_MultipleCloudAgentInstances { <# .SYNOPSIS Remediates multiple MOC Cloud Agent instances. .DESCRIPTION Stops all running instances and ensures only one instance is running on the correct node. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_MultipleCloudAgentInstances #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Remediating multiple MOC Cloud Agent instances..." if (-not (Test-FailoverCluster)) { Write-Verbose "Not a failover cluster environment, no remediation needed" return $true } $instances = Get-ClusterNode | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { Get-Service $using:script:CloudAgentServiceName } } | Where-Object { $_.Status -eq 'Running' } | Select-Object -Unique if ($instances.Count -eq 1) { Write-Verbose "Only one instance running, no remediation needed" return $true } Write-Verbose "Stopping all MOC Cloud Agent instances..." Get-ClusterNode | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { Stop-Service $using:script:CloudAgentServiceName -Force } } Write-Verbose "Starting cluster group to ensure MOC Cloud Agent runs on correct node..." Start-ClusterGroup (Get-MocConfig).clusterRoleName return $true } catch { Write-Error "Failed to remediate multiple instances: $_" return $false } } function Test-SupportAksArcKnownIssues_StuckInUpdating { <# .SYNOPSIS Tests if MOC is stuck in updating state. .DESCRIPTION Checks if the MOC install state is stuck in "Updating". .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_StuckInUpdating #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC Powershell Not Stuck in Updating" Status = "Passed" Message = "MOC Powershell is not stuck in updating state." } try { $agentStatus = (Get-MocConfig).InstallState if ($agentStatus -eq "Updating") { $result.Status = "Failed" $result.Message = "MOC Powershell is stuck in updating state." } } catch { Write-Error "Failed to check updating state: $_" throw } return $result } function Invoke-SupportAksArcRemediation_CleanupStaleDirectories { <# .SYNOPSIS Cleans up stale MOC entity directories. .DESCRIPTION Removes any unexpected or outdated directories from the MOC configuration location. This cleanup is needed when there are many files and directories that need to be purged as part of remediation. .OUTPUTS None .EXAMPLE Invoke-SupportAksArcRemediation_CleanupStaleDirectories #> [CmdletBinding()] param() Write-Verbose "Cleaning up MOC entity directories..." try { Set-Location (Get-MocConfig).cloudConfigLocation -ErrorAction Stop [string[]]$expectedNames = @( "certificate", "cluster", "container", "galleryimage", "group", "identity", "key", "keyvault", "loadbalancer", "location", "macpool", "networkinterface", "node", "role", "roleassignment", "secret", "virtualharddisk", "virtualmachine", "virtualmachineimage", "virtualnetwork" ) # Create a HashSet for efficient lookups $set = [System.Collections.Generic.HashSet[string]]::new($expectedNames) # Find all log and versioned directories $subDirs = @() $logDir = Get-ChildItem -Directory -Filter "log" -ErrorAction Stop if ($null -eq $logDir) { Write-Error "Log directory not found in expected location" return } $subDirs += $logDir.FullName # Get version-specific directories $versionDirs = Get-ChildItem -Directory -Filter "v0*" -ErrorAction Stop $subDirs += $versionDirs.FullName # Add log directories from version directories foreach($versionDir in $versionDirs) { $filterPath = Join-Path -Path $versionDir -ChildPath "log" $childDir = Get-ChildItem -Directory -Filter $filterPath -ErrorAction SilentlyContinue if ($null -ne $childDir) { $subDirs += $childDir.FullName } } # Clean up entity folders in each directory foreach($subDir in $subDirs) { Write-Verbose "Cleaning up entity directories in $subDir" Set-Location $subDir -ErrorAction Stop $entityDirs = Get-ChildItem -Directory -ErrorAction Stop foreach($dir in $entityDirs) { if (-not $set.Contains($dir.Name)) { Write-Verbose "Skipping non-entity directory: $($dir.FullName)" continue } try { Write-Verbose "Removing entity directory: $($dir.FullName)" Remove-Item -Path $dir.FullName -Recurse -Force -ErrorAction Stop } catch { Write-Warning "Failed to remove $($dir.FullName): $_" } } } } catch { Write-Error "Failed to cleanup stale directories: $_" } } function Invoke-SupportAksArcRemediation_StuckInUpdating { <# .SYNOPSIS Remediates MOC stuck in updating state. .DESCRIPTION Sets the MOC install state to UpdateFailed to allow future updates. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_StuckInUpdating #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Remediating MOC Powershell stuck in updating state..." Invoke-SupportAksArcRemediation_CleanupStaleDirectories Write-Verbose "Cleaning up HostAgent on upgrade hang" # Stop all mochostagent processes to ensure we remove handles to the service Get-ClusterNode | % { Invoke-Command -ComputerName $_.Name -ScriptBlock { hostname Stop-Process -Name mochostagent -Force } } # Initiate deletion of MocHostAgent service from each node. Get-ClusterNode | % { Invoke-Command -ComputerName $_.Name -ScriptBlock { hostname $service = Get-WmiObject -Class Win32_Service -Filter "Name='mochostagent'" if ($null -ne $service) { $service.delete() | Out-Null } } } Write-Verbose "Updating MOC install state to UpdateFailed..." Import-Module Moc Set-MocConfigValue -name "installState" -value ([InstallState]::UpdateFailed) -ErrorAction Stop return $true } catch { Write-Error "Failed to remediate updating state: $_" return $false } } function Test-SupportAksArcKnownIssues_MocNodesOutOfSyncWithClusterNodes { <# .SYNOPSIS Tests if MOC nodes are out of sync with cluster nodes. .DESCRIPTION Validates that all MOC nodes are in sync with the cluster nodes. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_MocNodesOutOfSyncWithClusterNodes #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC Nodes Sync with Cluster Nodes" Status = "Passed" Message = "All MOC nodes are in sync with cluster nodes." } if (-not (Test-FailoverCluster)) { Write-Verbose "Not a failover cluster environment, skipping test" return $result } try { Import-Module Moc $testResults = Test-MocNodesMatchFailoverClusterNodes -Cluster (Get-Cluster).Name if ($testResults.Status -eq "FAILURE") { $result.Status = "Failed" $result.Message = $testResults.Description } } catch { Write-Error "Failed to check node sync: $_" throw } return $result } function Invoke-SupportAksArcRemediation_MocNodesOutOfSyncWithClusterNodes { <# .SYNOPSIS Remediates MOC nodes that are out of sync with cluster nodes. .DESCRIPTION Synchronizes MOC nodes with the cluster nodes to ensure consistency. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_MocNodesOutOfSyncWithClusterNodes #> [CmdletBinding()] [OutputType([bool])] param() throw "Not Implemented" } function Test-SupportAksArcKnownIssues_MocCloudAgentNotRunning { <# .SYNOPSIS Tests if MOC Cloud Agent is running. .DESCRIPTION Validates that the MOC Cloud Agent service is running on all nodes. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_MocCloudAgentNotRunning #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC Cloud Agent Running" Status = "Passed" Message = "MOC Cloud Agent is running " } if (-not (Test-FailoverCluster)) { Write-Verbose "Not a failover cluster environment, skipping test" return $result } if ((Get-ClusterResource -Name "MOC Cloud Agent Service").State -eq "Failed" -or (Get-ClusterResource -Name "MOC Cloud Agent Service").State -eq "Offline") { $result.Status = "Failed" $result.Message = "MOC Cloud Agent is not running." } return $result } function Invoke-SupportAksArcRemediation_MocCloudAgentNotRunning { <# .SYNOPSIS Remediates MOC Cloud Agent not running. .DESCRIPTION Starts the MOC Cloud Agent service on all nodes to ensure it is running. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_MocCloudAgentNotRunning #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Starting MOC Cloud Agent service on all nodes..." Start-ClusterResource -Name "MOC Cloud Agent Service" -ErrorAction Stop return $true } catch { # If Start-ClusterResource has failed, Its mostly the service is crashing return Invoke-SupportAksArcRemediation_Panics } } function Test-SupportAksArcKnownIssues_EventLogNotRunning { <# .SYNOPSIS Tests if MOC Event Log is running. .DESCRIPTION Validates that the Windows Event Log service is running on all nodes. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_EventLogNotRunning #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate Windows Event Log Running" Status = "Passed" Message = "Windows Event Log is running " } if (-not (Test-FailoverCluster)) { Write-Verbose "Not a failover cluster environment, skipping test" return $result } Get-ClusterNode | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { # Check if EventLog service is running $service = Get-Service -Name "EventLog" -ErrorAction SilentlyContinue if ($null -eq $service) { return $false # Service does not exist } return $service.Status -eq "Running" } | ForEach-Object { if (-not $_) { $result.Status = "Failed" $result.Message = "Windows Event Log Service is not running on node: $($_.Name)" } } } return $result } function Invoke-SupportAksArcRemediation_EventLogNotRunning { <# .SYNOPSIS Remediates Windows Event Log not running. .DESCRIPTION Starts the Windows Event Log service on all nodes to ensure it is running. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_EventLogNotRunning #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Starting Windows Event Log service on all nodes..." Get-ClusterNode | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { Start-Service -Name "EventLog" -ErrorAction Stop } } return $true } catch { Write-Error "Failed to remediate Event Log not running: $_" return $false } } function Test-SupportAksArcKnownIssues_GalleryImageStuckInDeleting { <# .SYNOPSIS Tests if any gallery images are stuck in deleting state. .DESCRIPTION Validates that no gallery images are stuck in deleting state. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_GalleryImageStuckInDeleting #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate Gallery Image Stuck In Deleting" Status = "Passed" Message = "No gallery images are stuck in deleting state." } # Check for any gallery images in deleting state $stuckImages = (Get-MocGalleryImage -location MocLocation) | Where-Object { $_.properties.statuses.ProvisionState -match "DELETE_FAILED" } if ($stuckImages) { $result.Status = "Failed" $result.Message = "The gallery images are stuck in deleting state and need remediation: $($stuckImages | Select-Object -ExpandProperty name -Unique -Join ', ')" } return $result } function Invoke-SupportAksArcRemediation_GalleryImageStuckInDeleting { <# .SYNOPSIS Remediates gallery images stuck in deleting state. .DESCRIPTION Deletes gallery images that are stuck in deleting state. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_GalleryImageStuckInDeleting #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Remediating gallery images stuck in deleting state..." $stuckImages = Get-MocGalleryImage -location MocLocation | Where-Object { $_.properties.statuses.ProvisionState -match "DELETE_FAILED" } foreach ($image in $stuckImages) { Write-Verbose "Deleting stuck gallery image: $($image.name)" try { # This shoud pass. Remove-MocGalleryImage -Name $image.name -Location MocLocation -Force -ErrorAction Stop } catch { # If they are on 2411 or less, this would fail if ($_.Exception.Message -match "is missing from nodeEntityMap: Failed") { Write-Warning "Gallery image $($image.name) is still in deleting state, trying next remediation" Invoke-SupportAksArcRemediation_MocNotOnLatestPatch # Re-attempt the mitigation Remove-MocGalleryImage -Name $image.name -Location MocLocation -Force -ErrorAction Stop } else { Write-Error "Failed to delete gallery image $($image.name): $_" return $false } <#Do this if a terminating exception happens#> } } return $true } catch { Write-Error "Failed to remediate gallery images stuck in deleting state: $_" return $false } } function Test-SupportAksArcKnownIssues_VirtualMachineStuckInPending { <# .SYNOPSIS Tests if any virtual machines are stuck in pending state. .DESCRIPTION Validates that no virtual machines are stuck in pending state. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_VirtualMachineStuckInPending #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate Virtual Machine Stuck In Pending" Status = "Passed" Message = "No virtual machines are stuck in pending state." } # Check for any virtual machines in pending state Get-MocGroup -Location MocLocation | ForEach-Object { $group = $_ # Check if any VMs are in Delete Failed State # Check if the VMs are stuck in Pending State $stuckVMs = (Get-MocVirtualMachine -location MocLocation -group $group.Name) | Where-Object { $_.properties.statuses.ProvisionState -match "DELETE_FAILED" -and $_.properties.statuses.Error -match "Pending State" } if ($stuckVMs) { $result.Status = "Failed" $result.Message = "The virtual machines are stuck in pending state and need remediation: $($stuckVMs | Select-Object -ExpandProperty name -Unique -Join ', ')" } } return $result } function Invoke-SupportAksArcRemediation_VirtualMachineStuckInPending { <# .SYNOPSIS Remediates virtual machines stuck in pending state. .DESCRIPTION Deletes virtual machines that are stuck in pending state. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_VirtualMachineStuckInPending #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Remediating virtual machines stuck in pending state..." $currentVersion = Get-MocVersion if ($currentVersion -like "1.10.*" -or $currentVersion -like "1.9.*") { # Jump to 1.11.x $jumpToVersion = Get-LatestVersionOfMoc -version "1.11.2.10205" -ErrorAction Stop Update-Moc -Version $jumpToVersion -ErrorAction Stop } else { Write-Verbose "Simply apply hotfix version" Invoke-SupportAksArcRemediation_MocNotOnLatestPatch } return $true } catch { Write-Error "Failed to remediate virtual machines stuck in pending state: $_" return $false } } function Test-SupportAksArcKnownIssues_MocNotOnLatestPatch { <# .SYNOPSIS Checks if the current MOC version is the latest patch version. .DESCRIPTION Compares the current installed MOC version with the latest available patch version, ensuring the minor versions are the same and patch versions are different. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_MocNotOnLatestPatch #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC is on Latest Patch Version" Status = "Passed" Message = "MOC is on the latest patch version." } try { $currentVersion = Get-MocVersion $latestVersion = Get-NextPatchVersionOfMoc if ([Version]$currentVersion -lt [Version]$latestVersion) { $result.Status = "Failed" $result.Message = "MOC is not on the latest patch version. Current: $currentVersion, Latest: $latestVersion" } } catch { Write-Error "Failed to check latest patch version: $_" throw } return $result } function Invoke-SupportAksArcRemediation_MocNotOnLatestPatch { <# .SYNOPSIS Remediates when MOC is not on the latest patch version. .DESCRIPTION Updates MOC to the latest available patch version. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_MocNotOnLatestPatch #> [CmdletBinding()] [OutputType([bool])] param() try { Write-Verbose "Remediating: Updating MOC to the latest patch version..." Invoke-SupportAksArcRemediation_CleanupStaleDirectories return Invoke-UpdateMocHotFix } catch { Write-Error "Failed to remediate MOC not on latest patch: $_" return $false } } function Test-SupportAksArcKnownIssues_MocNodesNotActive { <# .SYNOPSIS Checks if any MOC nodes are not in the 'Active' state. .DESCRIPTION Validates that all MOC nodes are in the 'Active' state. .OUTPUTS Test result object containing status and message .EXAMPLE Test-SupportAksArcKnownIssues_MocNodesNotActive #> [CmdletBinding()] [OutputType([PSObject])] param() $result = [PSCustomObject]@{ TestName = "Validate MOC Nodes Not Active" Status = "Passed" Message = "All MOC nodes are in the 'Active' state." } try { $nodes = Get-MocNode -Location MocLocation $inactiveNodes = $nodes | Where-Object { $_.properties.statuses.RunningState -ne "Active" } if ($inactiveNodes) { $result.Status = "Failed" $result.Message = "The following MOC nodes are not in 'Active' state: $($($inactiveNodes | Select-Object -ExpandProperty name) -Join ', ')" } } catch { Write-Error "Failed to check MOC node states: $_" throw } return $result } function Invoke-SupportAksArcRemediation_MocNodesNotActive { <# .SYNOPSIS Remediates MOC nodes that are not in the 'Active' state. .DESCRIPTION Attempts to bring MOC nodes that are not in the 'Active' state back to 'Active' by restarting the node service. .OUTPUTS Boolean indicating remediation success .EXAMPLE Invoke-SupportAksArcRemediation_MocNodesNotActive #> [CmdletBinding()] [OutputType([bool])] param() try { $nodes = Get-MocNode -Location MocLocation $inactiveNodes = $nodes | Where-Object { $_.properties.statuses.RunningState -ne "Active" } if (-not $inactiveNodes) { Write-Verbose "All MOC nodes are already in 'Active' state." return $true } Repair-Moc return $true } catch { Write-Error "Failed to remediate MOC nodes not active: $_" return $false } } ############################################ function Test-AzureLocal { # Detecting Azure Local based on the moc catalog used return ((Get-MocConfig).catalog -eq "aks-hci-asz-stable-catalogs-int") } function Test-FailoverCluster { # Detecting Failover Cluster based on the presence of the Get-Cluster cmdlet return (Get-Command "Get-Cluster" -ErrorAction SilentlyContinue) -and (Get-Cluster -ErrorAction SilentlyContinue) } function Show-Disclaimer { @' THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. '@ | Write-Warning } function Test-LatestVersion() { Write-Host "Checking if you are on the latest version" Update-Module -Name "Support.AksArc" } if ($PSBoundParameters['AcceptEula'] -ne $true){ Show-Disclaimer } # if $SkipUpdate or specific version is defined, then skip the auto-update method if ($PSBoundParameters['SkipUpdate'] -eq $true){ "SkipUpdate: {0}. Skipping auto-update" -f $SkipUpdate | Write-Host -ForegroundColor Yellow } else { Test-LatestVersion } # SIG # Begin signature block # MIIoOwYJKoZIhvcNAQcCoIIoLDCCKCgCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDUaW699uFacS6O # iq5yq6vTrUReo7/7Tk5DYyz/1GJKSqCCDYUwggYDMIID66ADAgECAhMzAAAEA73V # lV0POxitAAAAAAQDMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTEzWhcNMjUwOTExMjAxMTEzWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQCfdGddwIOnbRYUyg03O3iz19XXZPmuhEmW/5uyEN+8mgxl+HJGeLGBR8YButGV # LVK38RxcVcPYyFGQXcKcxgih4w4y4zJi3GvawLYHlsNExQwz+v0jgY/aejBS2EJY # oUhLVE+UzRihV8ooxoftsmKLb2xb7BoFS6UAo3Zz4afnOdqI7FGoi7g4vx/0MIdi # kwTn5N56TdIv3mwfkZCFmrsKpN0zR8HD8WYsvH3xKkG7u/xdqmhPPqMmnI2jOFw/ # /n2aL8W7i1Pasja8PnRXH/QaVH0M1nanL+LI9TsMb/enWfXOW65Gne5cqMN9Uofv # ENtdwwEmJ3bZrcI9u4LZAkujAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU6m4qAkpz4641iK2irF8eWsSBcBkw # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMjkyNjAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AFFo/6E4LX51IqFuoKvUsi80QytGI5ASQ9zsPpBa0z78hutiJd6w154JkcIx/f7r # EBK4NhD4DIFNfRiVdI7EacEs7OAS6QHF7Nt+eFRNOTtgHb9PExRy4EI/jnMwzQJV # NokTxu2WgHr/fBsWs6G9AcIgvHjWNN3qRSrhsgEdqHc0bRDUf8UILAdEZOMBvKLC # rmf+kJPEvPldgK7hFO/L9kmcVe67BnKejDKO73Sa56AJOhM7CkeATrJFxO9GLXos # oKvrwBvynxAg18W+pagTAkJefzneuWSmniTurPCUE2JnvW7DalvONDOtG01sIVAB # +ahO2wcUPa2Zm9AiDVBWTMz9XUoKMcvngi2oqbsDLhbK+pYrRUgRpNt0y1sxZsXO # raGRF8lM2cWvtEkV5UL+TQM1ppv5unDHkW8JS+QnfPbB8dZVRyRmMQ4aY/tx5x5+ # sX6semJ//FbiclSMxSI+zINu1jYerdUwuCi+P6p7SmQmClhDM+6Q+btE2FtpsU0W # +r6RdYFf/P+nK6j2otl9Nvr3tWLu+WXmz8MGM+18ynJ+lYbSmFWcAj7SYziAfT0s # IwlQRFkyC71tsIZUhBHtxPliGUu362lIO0Lpe0DOrg8lspnEWOkHnCT5JEnWCbzu # iVt8RX1IV07uIveNZuOBWLVCzWJjEGa+HhaEtavjy6i7MIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGgwwghoIAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAQDvdWVXQ87GK0AAAAA # BAMwDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIIiJ # xe3eGOm9bNODqfa1vc2PZWtf4MizNQnDI1LE/430MEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAasjDixfCuSjidVrZFb/omPhaNkN91Zsc9q4a # Ob16bAhR1tYnb69d2lOKydMUjBeFCVoJ8Tu1fWkz0WVvF329U7vak//Q279HcqNo # h7JtozxsxjzHCksBldnAkALJKurhOlyGtgx24X163L/0BYFbh73Xa6nnXTAIvAk8 # znfTB0A4P0MWTRZf5VRzlb0iS7sz4gu6cwImGP7K5a0B0zFJ9WiuTFB7gFm1RtHv # L0pCojM/OsZWbpSAdNE+PmczU0KoIFM56cSXgpdrxu2MdCK73On33w7VycxaR8CO # h3fZxEcgsIH6XnQp/BbZzsf+NPov7m/OioFTLf+WknxA/3ne2qGCF5YwgheSBgor # BgEEAYI3AwMBMYIXgjCCF34GCSqGSIb3DQEHAqCCF28wghdrAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFSBgsqhkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCB7BTmeg2l0LLNJuCm+6teIzEidCVHSvzac # nB+bJ7pfdwIGaEszemhXGBMyMDI1MDYxMzAxMDQxNC45ODNaMASAAgH0oIHRpIHO # MIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQL # ExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxk # IFRTUyBFU046OTYwMC0wNUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1l # LVN0YW1wIFNlcnZpY2WgghHsMIIHIDCCBQigAwIBAgITMwAAAgTY4A4HlzJYmAAB # AAACBDANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAx # MDAeFw0yNTAxMzAxOTQyNDdaFw0yNjA0MjIxOTQyNDdaMIHLMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l # cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTYwMC0w # NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Uw # ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDw3Sbcee2d66vkWGTIXhfG # qqgQGxQXTnq44XlUvNzFSt7ELtO4B939jwZFX7DrRt/4fpzGNkFdGpc7EL5S86qK # Yv360eXjW+fIv1lAqDD31d/p8Ai9/AZz8M95zo0rDpK2csz9WAyR9FtUDx52VOs9 # qP3/pgpHvgUvD8s6/3KNITzms8QC1tJ3TMw1cRn9CZgVIYzw2iD/ZvOW0sbF/DRd # gM8UdtxjFIKTXTaI/bJhsQge3TwayKQ2j85RafFFVCR5/ChapkrBQWGwNFaPzpmY # N46mPiOvUxriISC9nQ/GrDXUJWzLDmchrmr2baABJevvw31UYlTlLZY6zUmjkgaR # fpozd+Glq9TY2E3Dglr6PtTEKgPu2hM6v8NiU5nTvxhDnxdmcf8UN7goeVlELXbO # m7j8yw1xM9IyyQuUMWkorBaN/5r9g4lvYkMohRXEYB0tMaOPt0FmZmQMLBFpNRVn # XBTa4haXvn1adKrvTz8VlfnHxkH6riA/h2AlqYWhv0YULsEcHnaDWgqA29ry+jH0 # 97MpJ/FHGHxk+d9kH2L5aJPpAYuNmMNPB7FDTPWAx7Apjr/J5MhUx0i07gV2brAZ # 9J9RHi+fMPbS+Qm4AonC5iOTj+dKCttVRs+jKKuO63CLwqlljvnUCmuSavOX54IX # OtKcFZkfDdOZ7cE4DioP1QIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFBp1dktAcGpW # /Km6qm+vu4M1GaJfMB8GA1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8G # A1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y3JsL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBs # BggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0 # LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy # MDIwMTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUH # AwgwDgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4ICAQBecv6sRw2HTLMy # UC1WJJ+FR+DgA9Jkv0lGsIt4y69CmOj8R63oFbhSmcdpakxqNbr8v9dyTb4RDyNq # tohiiXbtrXmQK5X7y/Q++F0zMotTtTpTPvG3eltyV/LvO15mrLoNQ7W4VH58aLt0 # 30tORxs8VnAQQF5BmQQMOua+EQgH4f1F4uF6rl3EC17JBSJ0wjHSea/n0WYiHPR0 # qkz/NRAf8lSUUV0gbIMawGIjn7+RKyCr+8l1xdNkK/F0UYuX3hG0nE+9Wc0L4A/e # nluUN7Pa9vOV6Vi3BOJST0RY/ax7iZ45leM8kqCw7BFPcTIkWzxpjr2nCtirnkw7 # OBQ6FNgwIuAvYNTU7r60W421YFOL5pTsMZcNDOOsA01xv7ymCF6zknMGpRHuw0Rb # 2BAJC9quU7CXWbMbAJLdZ6XINKariSmCX3/MLdzcW5XOycK0QhoRNRf4WqXRshEB # aY2ymJvHO48oSSY/kpuYvBS3ljAAuLN7Rp8jWS7t916paGeE7prmrP9FJsoy1LFK # mFnW+vg43ANhByuAEXq9Cay5o7K2H5NFnR5wj/SLRKwK1iyUX926i1TEviEiAh/P # VyJbAD4koipig28p/6HDuiYOZ0wUkm/a5W8orIjoOdU3XsJ4i08CfNp5I73CsvB5 # QPYMcLpF9NO/1LvoQAw3UPdL55M5HTCCB3EwggVZoAMCAQICEzMAAAAVxedrngKb # SZkAAAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQI # EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv # ZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmlj # YXRlIEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIy # NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXI # yjVX9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjo # YH1qUoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1y # aa8dq6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v # 3byNpOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pG # ve2krnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viS # kR4dPf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYr # bqgSUei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlM # jgK8QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSL # W6CmgyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AF # emzFER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIu # rQIDAQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIE # FgQUKqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWn # G1M1GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEW # M2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5 # Lmh0bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBi # AEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV # 9lbLj+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3Js # Lm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAx # MC0wNi0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2 # LTIzLmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv # 6lwUtj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZn # OlNN3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1 # bSNU5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4 # rPf5KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU # 6ZGyqVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDF # NLB62FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/ # HltEAY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdU # CbFpAUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKi # excdFYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTm # dHRbatGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZq # ELQdVTNYs6FwZvKhggNPMIICNwIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJp # Y2EgT3BlcmF0aW9uczEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjk2MDAtMDVF # MC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMK # AQEwBwYFKw4DAhoDFQC6PYHRw9+9SH+1pwy6qzVG3k9lbqCBgzCBgKR+MHwxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jv # c29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6/Wx8zAi # GA8yMDI1MDYxMjIwMDcxNVoYDzIwMjUwNjEzMjAwNzE1WjB2MDwGCisGAQQBhFkK # BAExLjAsMAoCBQDr9bHzAgEAMAkCAQACAQMCAf8wBwIBAAICEw0wCgIFAOv3A3MC # AQAwNgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEK # MAgCAQACAwGGoDANBgkqhkiG9w0BAQsFAAOCAQEARq9wOAXXARbqBsDscQ6ter3u # dp/AL/Fpur8fREk4uIzXGEGY2dwGdVfJ+87ZwtgFP9kA3FeVAEoMa1pjST9iSDdR # GvIaRytk7u1S0aucVVnKAdTOABIB7OkOAXgSVm0cXq+Kii4fSVhonIIz+wJmQfwU # p9igLtZYAuVcG7WiON6ANvg6HfLK43pnQl74iTJir4204SidnnQjpQLBGcK+Cdp4 # uie+iR3E9+Gd0nDUQWqJGOYJEz7M0VyqEm0qZnWSsr2VjtNz+vqPV8UmutFYN4tG # 2IfqnIWp2bRP5ZiyTp3IvF4A5A0izReuFb75nFYEhwIkkMKoETp/ziQOAP6XWTGC # BA0wggQJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAC # BNjgDgeXMliYAAEAAAIEMA0GCWCGSAFlAwQCAQUAoIIBSjAaBgkqhkiG9w0BCQMx # DQYLKoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEIMcHxrGkJU4adLai0n1AWZQ3 # /zPqDglbk4VGAAfVzOMoMIH6BgsqhkiG9w0BCRACLzGB6jCB5zCB5DCBvQQg+e14 # Zf1bCrxV0kzqaN/HUYQmy7v/qRTqXRJLmtx5uf4wgZgwgYCkfjB8MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQg # VGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAgTY4A4HlzJYmAABAAACBDAiBCDxHHr/ # eRiKGCECvHC9g/ACtu4yLayoKHye3aqPzgLmSTANBgkqhkiG9w0BAQsFAASCAgCM # NHwG+NrY3dk5f9jqDmtpY67GISDc/SSy/XV3SnpWVZzldH+s7NJRNeEJ8Qv8iePx # uRejvSSNd0Nz4B+lmPR3g6WtdXvRbOBs1a/Kl4ztscEfvqYKlIFIVVgBqepceo8F # 4D3a8pLLj7q39RHKYz9zyAsx5m805HS+gMT+GrEjnJ9w7WgxlAHyveM06nR59sdX # dQdf7aoiPxUycVW9nCn44cUXDaNeipEvFRiNkQbnvu39w9KG2EvXONHWaHkZbTFH # A6bgWUVwHf/9dpkmUOcivCuAI+ze7Rr6U+iulLIKnojtUY6BZbSqDBtG4oUqJOgX # fAurDsjVEt30+Av9gRhJHvZEU5esw3lUl36ouiuOaCtNO5fu1VVLQge6Zb77bjUU # tRlJItBdYvFmjM6kgQFHDLVYwu9bgjTsjsz1GU8m0F7nJN4WEawGCstKpGdIMt8P # jB11fd8dv5gH7B0S1zOsDJ4IaMQgbykR0ZV4BjzCZWSmuUdy2iiKFr5SaxhxV3Be # Sw+seIQEeAY5L8emhgLabmDGcoh23RGHPmBAPzTbnW/0+BPytPP64go8Nd3nY0lx # 4VPVwqbwJ6PaLZs4CKRr9a+N9w5WPre5KM7jG56V/4nsK8gWBie3eQVdiaiqX98M # tydD37W0HGBWuT9hx++G3gyETiY3xJydpomAMrlffQ== # SIG # End signature block |