SdnDiagnostics

4.2501.30.156

modules/SdnDiag.Health.psm1

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

Import-Module $PSScriptRoot\SdnDiag.Common.psm1
Import-Module $PSScriptRoot\SdnDiag.Server.psm1
Import-Module $PSScriptRoot\SdnDiag.NetworkController.psm1
Import-Module $PSScriptRoot\SdnDiag.NetworkController.FC.psm1
Import-Module $PSScriptRoot\SdnDiag.NetworkController.SF.psm1
Import-Module $PSScriptRoot\SdnDiag.Utilities.psm1

$configurationData = Import-PowerShellDataFile -Path "$PSScriptRoot\SdnDiag.Health.Config.psd1"
New-Variable -Name 'SdnDiagnostics_Health' -Scope 'Script' -Force -Value @{
    Cache  = @{}
    Config = $configurationData
}

# confirm that the current system is supported to generate health faults
$displayVersion = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion" -Name 'DisplayVersion'
$productName = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion" -Name 'ProductName'
if ($productName.ProductName -iin $script:SdnDiagnostics_Health.Config.HealthFaultSupportedProducts){
    $productSupported = $true
}
if ($displayVersion.DisplayVersion -iin $script:SdnDiagnostics_Health.Config.HealthFaultSupportedBuilds){
    $versionSupported = $true
}
if ($versionSupported -and $productSupported){
    $script:SdnDiagnostics_Health.Config.HealthFaultEnabled = $true
}

##########################
#### CLASSES & ENUMS #####
##########################

class SdnFaultInfo {
    [datetime] $OccurrenceTime = [System.DateTime]::UtcNow
    [string] $KeyFaultingObjectDescription
    [string] $KeyFaultingObjectID
    [string] $KeyFaultingObjectType
    [string] $FaultingObjectLocation
    [string] $FaultDescription
    [string] $FaultActionRemediation
}

##########################
#### FAULT HELPERS #####
##########################

# pInvoke definition for fault APIs
$signature = @'
[DllImport("hcihealthutils.dll", CharSet = CharSet.Unicode, SetLastError = false)]
public static extern int HciModifyFault(
    string entityType,
    string entityKey,
    string entityDescription,
    string entityLocation,
    string entityUniqueKey,
    uint action,
    string faultType,
    uint urgency,
    string title,
    string description,
    string actions,
    uint flag);
 
[DllImport("hcihealthutils.dll", CharSet = CharSet.Unicode, SetLastError = false)]
public static extern int HciModifyRelationship(
    string entityType,
    string entityKey,
    string entityDescription,
    string entityLocation,
    string entityUniqueKey,
    uint action,
    string parentEntityType,
    string parenetEntityKey,
    string parentEntityDescription,
    string parentEntityLocation,
    string parentEntityUniqueKey,
    string groupKey,
    uint urgency,
    uint relationshipType,
    uint flag);
'@

function ValidateFault {
    param(
        [SdnFaultInfo] $Fault
    )

    if ([string]::IsNullOrEmpty($Fault.KeyFaultingObjectDescription)) {
        throw "KeyFaultingObjectDescription is required"
    }

    if ([string]::IsNullOrEmpty($Fault.KeyFaultingObjectID)) {
        throw "KeyFaultingObjectID is required"
    }

    if ([string]::IsNullOrEmpty($Fault.KeyFaultingObjectType)) {
        throw "KeyFaultingObjectType is required"
    }
}
function LogWmiHealthFault {

    <#
        .SYNOPSIS
        Logs the WMI version of the health fault
 
        .PARAMETER fault
        The fault to log
    #>

    param(
        [object] $fault
    )
    Write-Verbose " WmiFault:"
    Write-Verbose " (FaultId) $($fault.FaultId)"
    Write-Verbose " (FaultingObjectDescription) $($fault.FaultingObjectDescription)"
    Write-Verbose " (FaultingObjectLocation) $($fault.FaultingObjectLocation)"
    Write-Verbose " (FaultingObjectType) $($fault.FaultingObjectType)"
    Write-Verbose " (FaultingObjectUniqueId) $($fault.FaultingObjectUniqueId)"
    Write-Verbose " (FaultTime) $($fault.FaultTime)"
    Write-Verbose " (FaultType) $($fault.FaultType)"
    Write-Verbose " (Reason) $($fault.Reason)"
}

function ConvertFaultListToPsObjectList {

    <#
        .SYNOPSIS
        Converts a list of faults to a list of PSObjects
        (used by ASZ modules to emit telemetry events )
 
        .PARAMETER faults
        The list of faults to convert
    #>

    param(
        [SdnFaultInfo[]] $faults,

        [ValidateSet("Create", "Delete")]
        [string] $faultType
    )

    $faultList = @()
    foreach ($fault in $faults) {
        # convert properties of class SdnFaultInfo
        $faultList += [PSCustomObject]@{
            OccurrenceTime               = $fault.OccurrenceTime
            KeyFaultingObjectDescription = $fault.KeyFaultingObjectDescription
            KeyFaultingObjectID          = $fault.KeyFaultingObjectID
            KeyFaultingObjectType        = $fault.KeyFaultingObjectType
            FaultingObjectLocation       = $fault.FaultingObjectLocation
            FaultDescription             = $fault.FaultDescription
            FaultActionRemediation       = $fault.FaultActionRemediation
            OperationType                = $faultType
        }
    }

    return $faultList
}

function ConvertFaultToPsObject {

    <#
        .SYNOPSIS
        Converts a fault to a PSObject
        (used by ASZ modules to emit telemetry events )
 
        .PARAMETER healthFault
        The fault to convert
 
        .PARAMETER faultOpType
        The operation type of the fault
    #>

    param(
        [SdnFaultInfo] $healthFault,

        [ValidateSet("Create", "Delete")]
        [string] $faultOpType
    )

    # convert properties of class SdnFaultInfo
    $faultObject = [PSCustomObject]@{
        OccurrenceTime               = $healthFault.OccurrenceTime
        KeyFaultingObjectDescription = $healthFault.KeyFaultingObjectDescription
        KeyFaultingObjectID          = $healthFault.KeyFaultingObjectID
        KeyFaultingObjectType        = $healthFault.KeyFaultingObjectType
        FaultingObjectLocation       = $healthFault.FaultingObjectLocation
        FaultDescription             = $healthFault.FaultDescription
        FaultActionRemediation       = $healthFault.FaultActionRemediation
        OperationType                = $faultOpType
    }

    return $faultObject
}

function LogHealthFault {

    <#
        .SYNOPSIS
        Logs the health fault
 
        .PARAMETER fault
        The fault to log
    #>

    param(
        [object] $healthFault
    )
    Write-Verbose " HealthFault:"
    Write-Verbose " (KeyFaultingObjectDescription) $($healthFault.KeyFaultingObjectDescription)"
    Write-Verbose " (KeyFaultingObjectID) $($healthFault.KeyFaultingObjectID)"
    Write-Verbose " (KeyFaultingObjectType) $($healthFault.KeyFaultingObjectType)"
    Write-Verbose " (FaultingObjectLocation) $($healthFault.FaultingObjectLocation)"
    Write-Verbose " (FaultDescription) $($healthFault.FaultDescription)"
    Write-Verbose " (FaultActionRemediation) $($healthFault.FaultActionRemediation)"
}

function LogHealthFaultToEventLog {

    <#
        .SYNOPSIS
        Logs the health fault to the event log
 
        .PARAMETER fault
        The fault to log
    #>

    [CmdletBinding()]
    param(
        [object] $fault,

        [ValidateSet("Create", "Delete")]
        [string] $operation
    )

    if ([string]::IsNullOrEmpty($operation) ) {
        $operation = ""
    }

    $eventLogMessage = "SDN HealthServiceHealth Fault: $($fault.FaultDescription)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Faulting Object Description: $($fault.KeyFaultingObjectDescription)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Faulting Object ID: $($fault.KeyFaultingObjectID)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Faulting Object Type: $($fault.KeyFaultingObjectType)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Faulting Object Location: $($fault.FaultingObjectLocation)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Fault Action Remediation: $($fault.FaultActionRemediation)"
    $eventLogMessage += "`r`n"
    $eventLogMessage += "Fault Operation: $($operation)"
    $eventLogJson = (ConvertTo-Json -InputObject $fault -Depth 5)

    $eventInstance = [System.Diagnostics.EventInstance]::new(1, 1)
    $evtObject = New-Object System.Diagnostics.EventLog;
    $evtObject.Log = $LOG_NAME
    $evtObject.Source = $LOG_SOURCE

    Write-Verbose "Source : $($LOG_SOURCE) Log : $($LOG_NAME) Message : $($eventLogMessage)"
    $evtObject.WriteEvent($eventInstance, @($eventLogMessage, $eventLogJson, $operation))
}

function CreateorUpdateFault {
    param(
        [SdnFaultInfo] $Fault
    )

    if (-NOT $script:SdnDiagnostics_Health.Config.HealthFaultEnabled) {
        return
    }

    ValidateFault -Fault $Fault
    InitFaults

    Write-Verbose "CreateorUpdateFault:"

    LogHealthFault -healthFault $Fault
    LogHealthFaultToEventLog -fault $Fault -operation Create

    if ([string]::IsNullOrEmpty($script:subsystemId)) {
        $script:subsystemId = (get-storagesubsystem Cluster*).UniqueId
        $script:entityTypeSubSystem = "Microsoft.Health.EntityType.Subsystem"
    }
    $retValue = [Microsoft.NetworkHud.FunctionalTests.Module.HciHealthUtils]::HciModifyFault( `
        $Fault.KeyFaultingObjectDescription, # $entityType, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $Fault.KeyFaultingObjectDescription, # "E Desc", `
        $Fault.FaultingObjectLocation, # $entityLocation, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $HCI_MODIFY_FAULT_ACTION_MODIFY, #action `
        $Fault.KeyFaultingObjectType, # $faultType, `
        $HEALTH_URGENCY_UNHEALTHY, # `
        "Fault Title", `
        $Fault.FaultDescription, # fault description
        $Fault.FaultActionRemediation, # fault remediation action
        $HCI_MODIFY_FAULT_FLAG_NONE) | Out-Null

    $retValue = [Microsoft.NetworkHud.FunctionalTests.Module.HciHealthUtils]::HciModifyRelationship(
        $Fault.KeyFaultingObjectDescription, # $entityType, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $Fault.KeyFaultingObjectDescription, # $entityDescription
        $Fault.FaultingObjectLocation, # $entityLocation, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $HCI_MODIFY_RELATIONSHIP_ACTION_MODIFY, `
        $script:entityTypeSubSystem, `
        $script:subsystemId, `
        $null, `
        $null, `
        $script:subsystemId, `
        "TestGroupKey", `
        $HEALTH_URGENCY_UNHEALTHY, `
        $HEALTH_RELATIONSHIP_COLLECTION, `
        $HCI_MODIFY_RELATIONSHIP_FLAG_NONE) | Out-Null
}

function DeleteFaultBy {
    <#
        .SYNOPSIS
        Deletes a fault by its key properties, those with empty or a * will be ignored while comaprison for a broader clear operation
 
        .PARAMETER KeyFaultingObjectDescription
        The description of the faulting object
 
        .PARAMETER KeyFaultingObjectID
        The unique ID of the faulting object
 
        .PARAMETER KeyFaultingObjectType
        The type of the faulting object
 
        .PARAMETER FaultingObjectLocation
        The location of the faulting object
    #>
    param(
        [string] $KeyFaultingObjectDescription,
        [string] $KeyFaultingObjectID,
        [string] $KeyFaultingObjectType,
        [string] $FaultingObjectLocation,
        [switch] $Verbose
    )

    if (-NOT $script:SdnDiagnostics_Health.Config.HealthFaultEnabled) {
        return
    }

    Write-Verbose "DeleteFault: "
    Write-Verbose "(KeyFaultingObjectDescription) $($KeyFaultingObjectDescription)"
    Write-Verbose "(KeyFaultingObjectID) $($KeyFaultingObjectID)"
    Write-Verbose "(KeyFaultingObjectType) $($KeyFaultingObjectType)"
    Write-Verbose "(FaultingObjectLocation) $($FaultingObjectLocation)"

    InitFaults

    # get all the system faults
    $faults = Get-HealthFault
    [bool] $match = $true
    [string[]] $matchFaultsId = @()
    foreach ($fault in $faults) {
        # delete the one(s) that match the filter
        # KeyFaultingObjectDescription, KeyFaultingObjectID, KeyFaultingObjectType may be empty , in which case
        # we will not consider them for comparison
        $match = [string]::IsNullOrEmpty($KeyFaultingObjectDescription) -or $KeyFaultingObjectDescription -eq "*" -or `
            $KeyFaultingObjectDescription -eq $fault.FaultingObjectDescription;

        Write-Verbose "KeyFaultingObjectDescription $match"

        $match = $match -and ([string]::IsNullOrEmpty($KeyFaultingObjectID) -or $KeyFaultingObjectID -eq "*" -or `
                $KeyFaultingObjectID -eq $fault.FaultingObjectUniqueId)
        Write-Verbose "KeyFaultingObjectID $match"

        $match = $match -and ([string]::IsNullOrEmpty($KeyFaultingObjectType) -or $KeyFaultingObjectType -eq "*" -or `
                $KeyFaultingObjectType -eq $fault.FaultingObjectType)
        Write-Verbose "KeyFaultingObjectType $match"

        if ($match) {
            Write-Verbose "Deleting fault (ID) $($fault.FaultId)"
            $matchFaultsId += $fault.FaultId
        }
    }
    if ($matchFaultsId.Count -eq 0) {
        Write-Verbose "No faults found to delete"
        return
    }
    else {
        Write-Verbose "Found $($matchFaultsId.Count) faults to delete"
    }

    foreach ($faultId in $matchFaultsId) {
        DeleteFaultById -faultUniqueID $faultId
    }
}

function DeleteFaultById {
    <#
        .SYNOPSIS
        Deletes a fault by its unique ID
 
        .PARAMETER faultUniqueID
        The unique ID of the fault to delete
    #>
    param(
        [string] $faultUniqueID
    )

    if (-NOT $script:SdnDiagnostics_Health.Config.HealthFaultEnabled) {
        return
    }

    if ([string]::IsNullOrEmpty($faultUniqueID)) {
        throw "Empty faultID"
    }

    InitFaults
    Write-Verbose "DeleteFaultById $faultId"
    $fault = Get-HealthFault | ? { $_.FaultId -eq $faultUniqueID }

    if ($null -eq $fault) {
        throw "Fault with ID $faultUniqueID not found"
    }
    else {
        LogWmiHealthFault -fault $fault
    }

    [Microsoft.NetworkHud.FunctionalTests.Module.HciHealthUtils]::HciModifyFault( `
            $fault.FaultingObjectType, `
            $fault.FaultingObjectUniqueId, `
            "", `
            $fault.FaultingObjectUniqueId, `
            $fault.FaultingObjectUniqueId, `
            $HCI_MODIFY_FAULT_ACTION_REMOVE, `
            $fault.FaultType, `
            $HEALTH_URGENCY_UNHEALTHY, `
            "", `
            "", `
            "", `
            $HCI_MODIFY_FAULT_FLAG_NONE) | Out-Null
}

function ShowFaultSet {
    <#
        .SYNOPSIS
        Shows the fault set
 
        .PARAMETER faultset
        The fault set to show
    #>

    param([object[]]$faultset)

    Write-Verbose "Success Faults (for rest res):"
    if ($PSCmdlet.MyInvocation.BoundParameters["Verbose"].IsPresent) {
        if ($null -eq $faultset[0] -or $faultset[0].Count -eq 0) {
            Write-Verbose "(none)"
            return
        }
        foreach ($faultInst in $faultset[0]) {
            LogHealthFault -healthFault $faultInst
        }
    }

    Write-Verbose "Failure Faults (for rest res):"
    if ($PSCmdlet.MyInvocation.BoundParameters["Verbose"].IsPresent) {
        if ($null -eq $faultset[1] -or $faultset[1].Count -eq 0) {
            Write-Verbose "(none)"
            return
        }
        foreach ($faultInst in $faultset[1]) {
            LogHealthFault -healthFault $faultInst
        }
    }
}

function UpdateFaultSet {

    <#
        .SYNOPSIS
        Updates the fault set and returns the health test object
 
        .PARAMETER successFaults
        The set of faults that were successful
 
        .PARAMETER failureFaults
        The set of faults that failed
    #>

    param(
        [object[]]$successFaults,
        [object[]]$failureFaults
    )

    $healthTest = New-SdnHealthTest

    if ($null -ne $failureFaults -and $failureFaults.Count -gt 0) {
        $healthTest.Result = "FAIL"
    }

    foreach ($fault in $successFaults) {
        DeleteFaultBy -KeyFaultingObjectDescription $fault.KeyFaultingObjectDescription
        $convFault = ConvertFaultToPsObject -healthFault $fault -faultOpType "Delete"
        $healthTest.HealthFault += $convFault
    }

    foreach ($fault in $failureFaults) {
        CreateOrUpdateFault -Fault $fault
        $convFault = ConvertFaultToPsObject -healthFault $fault -faultOpType "Create"
        $healthTest.HealthFault += $convFault
    }

    $healthTest
}

function DeleteFault {
    <#
        .SYNOPSIS
        Deletes a fault
 
        .PARAMETER Fault
        The fault to delete
    #>
    [CmdletBinding()]
    param(
        [SdnFaultInfo] $Fault
    )

    if (-NOT$script:SdnDiagnostics_Health.Config.HealthFaultEnabled) {
        return
    }

    ValidateFault -Fault $Fault
    InitFaults

    Write-Verbose "DeleteFault $($Fault.KeyFaultingObjectDescription) $($Fault.KeyFaultingObjectID) $($Fault.KeyFaultingObjectType)"

    if ([string]::IsNullOrEmpty($script:subsystemId)) {
        $script:subsystemId = (get-storagesubsystem Cluster*).UniqueId
        $script:entityTypeSubSystem = "Microsoft.Health.EntityType.Subsystem"
    }
    [Microsoft.NetworkHud.FunctionalTests.Module.HciHealthUtils]::HciModifyFault( `
        $Fault.KeyFaultingObjectDescription, # $entityType, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $Fault.KeyFaultingObjectDescription, # "E Desc", `
        $Fault.FaultingObjectLocation, # $entityLocation, `
        $Fault.KeyFaultingObjectID, # $entityId, `
        $HCI_MODIFY_FAULT_ACTION_REMOVE, #action `
        $Fault.KeyFaultingObjectType, # $faultType, `
        $HEALTH_URGENCY_UNHEALTHY, # `
        "Fault Title", `
        $Fault.FaultDescription, # fault description
        $Fault.FaultActionRemediation, # fault remediation action
        $HCI_MODIFY_FAULT_FLAG_NONE) | Out-Null
}

function Start-HealthFaultsTranscript {
    <#
        .SYNOPSIS
        Initializes the health runner transcript
    #>

    $logLocation = GetLogLocation

    if ($null -eq $logLocation) {
        return $false
    }
    else {
        $fullLogPath = Join-Path -Path $logLocation -ChildPath "SdnHealthTranscript.log"
        Start-Transcript -Path $fullLogPath -Append -ErrorAction SilentlyContinue
        $script:TranscriptStarted = $true
        return $true
    }
}

function StopHealthRunnerTranscript {
    <#
        .SYNOPSIS
        Stops the health runner transcript
    #>

    if ($script:TranscriptStarted) {
        Write-Host "Stopping transcript"
        Stop-Transcript -ErrorAction SilentlyContinue
        $script:TranscriptStarted = $false
    }
}

function InitFaults {
    <#
        .SYNOPSIS
        Initializes defaults and constants for fault handling
    #>

    [CmdletBinding()]
    param()

    Write-Verbose "InitFaults"
    if (-not ("Microsoft.NetworkHud.FunctionalTests.Module.HciHealthUtils" -as [type])) {
        Add-Type -MemberDefinition $signature -Name "HciHealthUtils" -Namespace "Microsoft.NetworkHud.FunctionalTests.Module" | Out-Null
        Write-Verbose "Registered HCI fault utilities"
    }

    New-Variable -Name 'HCI_MODIFY_FAULT_ACTION_MODIFY' -Scope 'Script' -Force -Value 0
    New-Variable -Name 'HCI_MODIFY_FAULT_ACTION_REMOVE' -Scope 'Script' -Force -Value 1

    New-Variable -Name 'HCI_MODIFY_RELATIONSHIP_ACTION_MODIFY' -Scope 'Script' -Force -Value 0
    New-Variable -Name 'HCI_MODIFY_RELATIONSHIP_ACTION_REMOVE' -Scope 'Script' -Force -Value 1

    New-Variable -Name 'HEALTH_RELATIONSHIP_UNKNOWN' -Scope 'Script' -Force -Value 0
    New-Variable -Name 'HEALTH_RELATIONSHIP_COMPOSITION' -Scope 'Script' -Force -Value 1
    New-Variable -Name 'HEALTH_RELATIONSHIP_CONTAINMENT' -Scope 'Script' -Force -Value 2
    New-Variable -Name 'HEALTH_RELATIONSHIP_COLLECTION' -Scope 'Script' -Force -Value 3

    New-Variable -Name 'HEALTH_URGENCY_UNKNOWN' -Scope 'Script' -Force -Value 255
    New-Variable -Name 'HEALTH_URGENCY_HEALTHY' -Scope 'Script' -Force -Value 0
    New-Variable -Name 'HEALTH_URGENCY_WARNING' -Scope 'Script' -Force -Value 1
    New-Variable -Name 'HEALTH_URGENCY_UNHEALTHY' -Scope 'Script' -Force -Value 2

    New-Variable -Name 'HCI_MODIFY_FAULT_FLAG_NONE' -Scope 'Script' -Force -Value 0
    New-Variable -Name 'HCI_MODIFY_RELATIONSHIP_FLAG_NONE' -Scope 'Script' -Force -Value 0

    New-Variable -Name 'LOG_NAME' -Scope 'Script' -Force -Value 'SdnHealthService'
    New-Variable -Name 'LOG_CHANNEL' -Scope 'Script' -Force -Value 'Admin'
    New-Variable -Name 'LOG_SOURCE' -Scope 'Script' -Force -Value 'HealthService'

    [bool] $eventLogFound = $false
    try {
        $evtLog = Get-EventLog -LogName $script:LOG_NAME -Source $script:LOG_SOURCE -ErrorAction SilentlyContinue
        if ($null -ne $evtLog) {
            $eventLogFound = $true
        }
    }
    catch {
        #get-eventlog throws even on erroraction silentlycontinue
    }

    try {
        if ($eventLogFound -eq $false) {
            New-EventLog -LogName $script:LOG_NAME -Source $script:LOG_SOURCE -ErrorAction SilentlyContinue
        }
    }
    catch {
        #failure to create event log is non-fatal
    }
}

function IsSdnFcClusterServiceRole {

    <#
        .SYNOPSIS
        Checks if the provided service role is an SDN cluster service role
 
        .PARAMETER ServiceName
        The name of the service role to check
    #>

    param([string] $ServiceName)

    # Define the list of valid service roles
    $validServiceRoles = @(
        "ApiService",
        "ControllerService",
        "FirewallService",
        "FnmService",
        "GatewayManager",
        "ServiceInsertion",
        "VSwitchService"
    )

    # Check if the provided service role name is in the list
    return $validServiceRoles -contains $ServiceName
}
function IsSdnService {

    <#
        .SYNOPSIS
        Checks if the provided service name is an SDN agent service
 
        .PARAMETER serviceName
        The name of the service to check
    #>

    param([string] $serviceName)

    return $serviceName -in @( "NCHostAgent", "SlbHostAgent")
}

function IsCurrentNodeClusterOwner {
    <#
        .SYNOPSIS
        Checks if the current node is the owner of the cluster
 
        .NOTES
        This function is used to determine if the current node is the owner of the cluster. This is used to determine if the current node is the primary node in a cluster.
    #>

    $activeNode = Get-ClusterResource -ErrorAction Ignore | Where-Object { $_.OwnerGroup -eq "Cluster Group" -and $_.ResourceType -eq "IP Address" -and $_.Name -eq "Cluster IP Address" }

    if ( $null -eq $activeNode ) {
        Write-Verbose "Active $($activeNode.OwnerNode)"

        # todo : generate a fault on failing to generate a fault (or switch to different algorithm for picking the primary node)
        return $false
    }

    return ($activeNode.OwnerNode -eq $env:COMPUTERNAME)
}

function GetFaultFromConfigurationState {
    <#
        .SYNOPSIS
        Generates a fault from the configuration state
 
        .PARAMETER resources
        The resources to generate the fault from
    #>

    param(
        [object[]] $resources
    )

    $healthFaults = @()
    # successful faults are just a stub holder for the resource
    # these are not created, but used for clearing out any older unhealthy states
    # these have KeyFaultingObjectType set to string.empty
    $successFaults = @()

    foreach ($resource in $resources) {

        ##########################################################################################
        ## ServiceState Fault Template (ServerResource)
        ##########################################################################################
        # $KeyFaultingObjectDescription (SDN ID) : [ResourceRef]
        # $KeyFaultingObjectID (ARC ID) : [ResourceMetadataID (if available) else ResourceRef]
        # $KeyFaultingObjectType (CODE) : "ConfgiStateCode" (if 2 more errors are found with same other properties will be concat)
        # $FaultingObjectLocation (SOURCE) : "Source (if keys of 2 errors collide they will be concatanated)"
        # $FaultDescription (MESSAGE) : "ConfigStateMessage (2 or more if errors collide)."
        # $FaultActionRemediation (ACTION) : "See <href> for more information on how to resolve this issue."
        # * Config state faults issued only from the primary Node
        ##########################################################################################


        if ($null -ne $resource.Properties.ConfigurationState -and $null -ne $resource.Properties.ConfigurationState.DetailedInfo -and `
                $resource.Properties.ConfigurationState.DetailedInfo.Count -gt 0) {

            foreach ($detailedInfo in $resource.Properties.ConfigurationState.DetailedInfo) {

                # supression check for some of the known configuration states
                if (IsConfigurationStateSkipped -Source $detailedInfo.Source -Message $detailedInfo.Message -Code $detailedInfo.Code) {
                    continue
                }

                # handle success cases
                if ($detailedInfo.Code -eq "Success") {

                    $successFault = [SdnFaultInfo]::new()
                    $successFault.KeyFaultingObjectDescription = $resource.ResourceRef
                    $successFault.KeyFaultingObjectID = $resource.ResourceRef
                    $successFault.KeyFaultingObjectType = [string]::Empty
                    $successFault.FaultingObjectLocation = [string]::Empty
                    $successFault.FaultDescription = [string]::Empty
                    $successFaults += $successFault

                }
                else {

                    # find any existing overlapping fault
                    $existingFault = $healthFaults | Where-Object { $_.KeyFaultingObjectDescription -eq $resource.ResourceRef -and `
                            $_.KeyFaultingObjectType -eq $detailedInfo.Code }

                    if ($null -ne $existingFault) {

                        $existingFault.FaultDescription += ("; " + $detailedInfo.Message)
                        $existingFault.FaultingObjectLocation += ("; " + $detailedInfo.Source)

                    }
                    else {

                        $healthFault = [SdnFaultInfo]::new()
                        $healthFault.KeyFaultingObjectDescription = $resource.ResourceRef
                        $healthFault.KeyFaultingObjectType = $detailedInfo.Code
                        $healthFault.FaultingObjectLocation = $detailedInfo.Source
                        $healthFault.FaultDescription += $detailedInfo.Message

                        # add resource metadata if available
                        if ($null -ne $resource.Properties.ResourceMetadata) {
                            $healthFault.KeyFaultingObjectID = $resource.Properties.ResourceMetadata
                        }
                        else {
                            $healthFault.KeyFaultingObjectID = $resource.ResourceRef
                        }
                    }
                    $healthFaults += $healthFault
                }
            }
        }
        else {
            # if configuration state is not available, we will clear out any existing faults
            if ($healthFaults.Count -eq 0) {
                $successFault = [SdnFaultInfo]::new()
                $successFault.KeyFaultingObjectDescription = $resource.ResourceRef
                $successFault.KeyFaultingObjectType = [string]::Empty
                $successFault.FaultingObjectLocation = [string]::Empty
                $successFault.FaultDescription = [string]::Empty
                $successFault.KeyFaultingObjectID = $resource.ResourceRef
                $successFaults += $successFault
            }
        }
    }

    foreach ($fault in $healthFaults) {
        LogWmiHealthFault -fault $fault
    }

    @($successFaults, $healthFaults)
}

function IsConfigurationStateSkipped {

    <#
        .SYNOPSIS
        Checks if the configuration state should be skipped
 
        .PARAMETER Source
        The source of the configuration state
 
        .PARAMETER Message
        The message of the configuration state
 
        .PARAMETER Code
        The code of the configuration state
    #>

    param(
        [string] $Source,
        [string] $Message,
        [string] $Code
    )

    if ($Source -eq "SoftwareLoadbalancerManager") {
        if ($Code -eq "HostNotConnectedToController") {
            return $true
        }
    }

    $false
}

##########################
#### ARG COMPLETERS ######
##########################

$argScriptBlock = @{
    Role = {
        param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameters)
        $result = (Get-SdnFabricInfrastructureResult)
        if ([string]::IsNullOrEmpty($wordToComplete)) {
            return ($result.Role | Sort-Object -Unique)
        }

        return $result.Role | Where-Object { $_ -like "*$wordToComplete*" } | Sort-Object
    }
    Name = {
        param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameters)
        $result = (Get-SdnFabricInfrastructureResult).RoleTest.HealthTest
        if ([string]::IsNullOrEmpty($wordToComplete)) {
            return ($result.Name | Sort-Object -Unique)
        }

        return $result | Where-Object { $_.Name -like "*$wordToComplete*" } | Sort-Object
    }
}

Register-ArgumentCompleter -CommandName 'Get-SdnFabricInfrastructureResult' -ParameterName 'Role' -ScriptBlock $argScriptBlock.Role
Register-ArgumentCompleter -CommandName 'Get-SdnFabricInfrastructureResult' -ParameterName 'Name' -ScriptBlock $argScriptBlock.Name

##########################
####### FUNCTIONS ########
##########################

function New-SdnHealthTest {
    param (
        [Parameter(Mandatory = $false)]
        [System.String]$Name = (Get-PSCallStack)[0].Command
    )

    $object = [PSCustomObject]@{
        Name           = $Name
        Result         = 'PASS' # default to PASS. Allowed values are PASS, WARN, FAIL
        OccurrenceTime = [System.DateTime]::UtcNow
        Properties     = @()
        Remediation    = @()
        HealthFault    = [PSCustomObject]@()
    }

    return $object
}

function New-SdnRoleHealthReport {
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$Role
    )

    $object = [PSCustomObject]@{
        Role           = $Role
        ComputerName   = $env:COMPUTERNAME
        Result         = 'PASS' # default to PASS. Allowed values are PASS, WARN, FAIL
        OccurrenceTime = [System.DateTime]::UtcNow
        HealthTest     = @() # array of New-SdnHealthTest objects
    }

    return $object
}

function New-SdnFabricHealthReport {
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$Role
    )

    $object = [PSCustomObject]@{
        OccurrenceTime = [System.DateTime]::UtcNow
        Role           = $Role
        Result         = 'PASS' # default to PASS. Allowed values are PASS, WARN, FAIL
        RoleTest       = @() # array of New-SdnRoleHealthReport objects
    }

    return $object
}


function Get-HealthData {
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$Property,

        [Parameter(Mandatory = $true)]
        [System.String]$Id
    )

    $results = $script:SdnDiagnostics_Health.Config[$Property]
    return ($results[$Id])
}

function Write-HealthValidationInfo {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [String]$ComputerName,

        [Parameter(Mandatory = $true)]
        [String]$Name,

        [Parameter(Mandatory = $false)]
        [String[]]$Remediation
    )

    $details = Get-HealthData -Property 'HealthValidations' -Id $Name

    $outputString += "`r`n`r`n"
    $outputString += "--------------------------`r`n"
    $outputString += "[$ComputerName] $Name"
    $outputString += "`r`n`r`n"
    $outputString += "Description:`t$($details.Description)`r`n"
    $outputString += "Impact:`t`t`t$($details.Impact)`r`n"

    if (-NOT [string]::IsNullOrEmpty($Remediation)) {
        $outputString += "Remediation:`r`n`t - $($Remediation -join "`r`n`t - ")`r`n"
    }

    if (-NOT [string]::IsNullOrEmpty($details.PublicDocUrl)) {
        $outputString += "`r`n"
        $outputString += "Additional information can be found at $($details.PublicDocUrl).`r`n"
    }

    $outputString += "`r`n--------------------------`r`n"

    $outputString | Write-Host -ForegroundColor Yellow
}

function Debug-SdnFabricInfrastructure {
    <#
    .SYNOPSIS
        Executes a series of fabric validation tests to validate the state and health of the underlying components within the SDN fabric.
    .PARAMETER NetworkController
        Specifies the name or IP address of the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER ComputerName
        Type the NetBIOS name, an IP address, or a fully qualified domain name of one or more remote computers.
    .PARAMETER Role
        The specific SDN role(s) to perform tests and validations for. If ommitted, defaults to all roles.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER NcRestCertificate
        Specifies the client certificate that is used for a secure web request to Network Controller REST API.
        Enter a variable that contains a certificate or a command or expression that gets the certificate.
    .PARAMETER NcRestCredential
        Specifies a user account that has permission to perform this action against the Network Controller REST API. The default is the current user.
    .EXAMPLE
        PS> Debug-SdnFabricInfrastructure
    .EXAMPLE
        PS> Debug-SdnFabricInfrastructure -NetworkController 'NC01' -Credential (Get-Credential) -NcRestCredential (Get-Credential)
    #>

    [CmdletBinding(DefaultParameterSetName = 'Role')]
    param (
        [Parameter(Mandatory = $false, ParameterSetName = 'Role')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ComputerName')]
        [System.String]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false, ParameterSetName = 'Role')]
        [ValidateSet('Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')]
        [String[]]$Role = ('Gateway', 'LoadBalancerMux', 'NetworkController', 'Server'),

        [Parameter(Mandatory = $true, ParameterSetName = 'ComputerName')]
        [System.String[]]$ComputerName,

        [Parameter(Mandatory = $false, ParameterSetName = 'Role')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ComputerName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Role')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ComputerName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Role')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ComputerName')]
        [X509Certificate]$NcRestCertificate
    )

    $script:SdnDiagnostics_Health.Cache = $null
    $aggregateHealthReport = @()
    if (Test-ComputerNameIsLocal -ComputerName $NetworkController) {
        Confirm-IsNetworkController
    }

    if ($PSBoundParameters.ContainsKey('NcRestCertificate')) {
        $restCredParam = @{ NcRestCertificate = $NcRestCertificate }
    }
    else {
        $restCredParam = @{ NcRestCredential = $NcRestCredential }
    }

    $environmentInfo = Get-SdnInfrastructureInfo -NetworkController $NetworkController -Credential $Credential @restCredParam
    if ($null -eq $environmentInfo) {
        throw New-Object System.NullReferenceException("Unable to retrieve environment details")
    }

    try {
        # if we opted to specify the ComputerName rather than Role, we need to determine which role
        # the computer names are associated with
        if ($PSCmdlet.ParameterSetName -ieq 'ComputerName') {
            $Role = @()
            $ComputerName | ForEach-Object {
                $computerRole = $_ | Get-SdnRole -EnvironmentInfo $environmentInfo
                if ($computerRole) {
                    $Role += $computerRole
                }
            }
        }

        $Role = $Role | Sort-Object -Unique
        foreach ($object in $Role) {
            "Processing tests for {0} role" -f $object.ToString() | Trace-Output -Level:Verbose
            $config = Get-SdnModuleConfiguration -Role $object.ToString()

            $roleHealthReport = New-SdnFabricHealthReport -Role $object.ToString()
            $sdnFabricDetails = [PSCustomObject]@{
                ComputerName    = $null
                NcUrl           = $environmentInfo.NcUrl
                Role            = $config
                EnvironmentInfo = $environmentInfo
            }

            # check to see if we were provided a specific computer(s) to test against
            # otherwise we will want to pick up the node name(s) from the environment info
            if ($ComputerName) {
                $sdnFabricDetails.ComputerName = $ComputerName
            }
            else {
                # in scenarios where there are not mux(es) or gateway(s) then we need to gracefully handle this
                # and move to the next role for processing
                if ($null -ieq $environmentInfo[$object.ToString()]) {
                    "Unable to locate fabric nodes for {0}. Skipping health tests." -f $object.ToString() | Trace-Output -Level:Warning
                    continue
                }

                $sdnFabricDetails.ComputerName = $environmentInfo[$object.ToString()]
            }

            $restApiParams = @{
                NcUri = $sdnFabricDetails.NcUrl
            }
            $restApiParams += $restCredParam

            # before proceeding with tests, ensure that the computer objects we are testing against are running the latest version of SdnDiagnostics
            Install-SdnDiagnostics -ComputerName $sdnFabricDetails.ComputerName -Credential $Credential

            $params = @{
                ComputerName = $sdnFabricDetails.ComputerName
                Credential   = $Credential
                ScriptBlock  = $null
                ArgumentList = @($restApiParams)
            }

            switch ($object) {
                'Gateway' { $params.ScriptBlock = { param($boundParams) Debug-SdnGateway @boundParams } }
                'LoadBalancerMux' { $params.ScriptBlock = { param($boundParams) Debug-SdnLoadBalancerMux @boundParams } }
                'NetworkController' { $params.ScriptBlock = { param($boundParams) Debug-SdnNetworkController @boundParams } }
                'Server' { $params.ScriptBlock = { param($boundParams) Debug-SdnServer @boundParams } }
            }

            $healthReport = Invoke-SdnCommand @params

            # evaluate the results of the tests and determine if any completed with Warning or FAIL
            # if so, we will want to set the Result of the report to reflect this
            foreach ($test in $healthReport) {
                if ($test.Result -ieq 'WARN') {
                    $roleHealthReport.Result = 'WARN'
                }
                if ($test.Result -ieq 'FAIL') {
                    $roleHealthReport.Result = 'FAIL'
                    break
                }
            }

            $roleHealthReport.RoleTest += $healthReport
            $aggregateHealthReport += $roleHealthReport
        }
    }
    catch {
        $_ | Trace-Exception
        $_ | Write-Error
    }
    finally {
        if ($aggregateHealthReport) {

            # enumerate all the roles that were tested so we can determine if any completed with Warning or FAIL
            $aggregateHealthReport | ForEach-Object {
                if ($_.Result -ine 'PASS') {

                    # enumerate all the individual role tests performed so we can determine if any completed that are not PASS
                    $_.RoleTest | ForEach-Object {
                        $c = $_.ComputerName
                        $_.HealthTest | ForEach-Object {

                            # enum only the health tests that failed
                            if ($_.Result -ine 'PASS') {
                                # add the remediation steps to an array list so we can pass it to the Write-HealthValidationInfo function
                                # otherwise if we pass it directly, it will be treated as a single string
                                $remediationList = [System.Collections.ArrayList]::new()
                                $_.Remediation | ForEach-Object { [void]$remediationList.Add($_) }

                                Write-HealthValidationInfo -ComputerName $c -Name $_.Name -Remediation $remediationList
                            }
                        }
                    }
                }
            }

            # save the aggregate health report to cache so we can use it for further analysis
            $script:SdnDiagnostics_Health.Cache = $aggregateHealthReport
        }
    }

    if ($script:SdnDiagnostics_Health.Cache) {
        "Results for fabric health have been saved to cache for further analysis. Use 'Get-SdnFabricInfrastructureResult' to examine the results." | Trace-Output
        return $script:SdnDiagnostics_Health.Cache
    }
}

function GetLogLocation {

    <#
        .SYNOPSIS
        Gets the log location file path for SDN Health, returns null if none is set
    #>

    $RegistryPath = "HKLM:\SOFTWARE\Microsoft\SdnHealth"
    $logPath = Get-ItemProperty -Path $RegistryPath -Name LogPath -ErrorAction SilentlyContinue
    if ($null -ne $logPath) {
        return $logPath.LogPath
    }
    else {
        return $null
    }
}
function SetLogLocation {
    <#
        .SYNOPSIS
        Sets the location of the log path for the SDN diagnostics module
 
        .PARAMETER logPath
        The path to the log file
    #>
    param(
        [string] $logPath
    )

    $RegistryPath = "HKLM:\SOFTWARE\Microsoft\SdnHealth"

    if (-not (Test-Path $RegistryPath)) {
        New-Item -Path $RegistryPath -Force | Out-Null
    }

    if ([string]::IsNullOrEmpty($logPath)) {
        Remove-ItemProperty -Path $RegistryPath -Name logPath -ErrorAction SilentlyContinue
    }
    else {
        New-ItemProperty -Path $RegistryPath -Name logPath -Value $logPath -Force | Out-Null
    }
}

function Start-SdnHealthFault {
    <#
        .SYNOPSIS
        Executes a series of fabric validation tests to validate the state and health of the underlying components within the SDN fabric.
 
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [bool] $Poll = $false,

        [Parameter(Mandatory = $false)]
        [int] $PollIntervalSeconds = 30
    )

    Write-Verbose "Starting SDN Health Faults"
    [bool] $transcriptStarted = $false
    try {

        # todo : change logpath
        $transcriptFile = Join-Path -Path  $Env:TEMP  -ChildPath "SdnDiag.log"
        Start-Transcript -Path $transcriptFile -Append
        $transcriptStarted = $true

        do {

            # Test encapoverhead settings
            Test-SdnEncapOverhead

            # Test all SDN Services
            $validServiceRoles = @(
                "ApiService",
                "ControllerService",
                "FirewallService",
                "FnmService",
                "GatewayManager",
                "ServiceInsertion",
                "VSwitchService"
            )
            Test-SdnClusterServiceState -ServiceName $validServiceRoles

            # Test all agent services
            $agentServices = @(
                'NcHostAgent',
                'SlbHostAgent'
            )
            Test-SdnServiceState -ServiceName $agentServices

            # Test certificate related faults
            Test-SdnNonSelfSignedCertificateInTrustedRootStore

            # Test tenant configuration states
            Test-SdnConfigurationState

            if ($Poll) {
                Start-Sleep -Seconds $PollIntervalSeconds
            }

        } until($Poll -eq $false);
    }
    catch {
        $_ | Write-Error
    }
    finally {
        if ($transcriptStarted) {
            Stop-Transcript
        }
    }
}

function GetSdnResourceFromNc {
    <#
        .SYNOPSIS
        Wrapper around Get-SdnResource which attempts using different available certificates
        NOTE: this is specifically for ASZ env because the nc cmdlets do not work there
 
        .PARAMETER NcUri
        The base URI of the Network Controller. (https://<nc rest name>)
 
        .PARAMETER Resource
        The resource to retrieve from the Network Controller.
 
        .PARAMETER ApiVersion
        (optional) The version of the resource to retrieve from the Network Controller.
        note: if nothing is specified, v1 is queried
 
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [string] $NcUri,

        [Parameter(Mandatory = $true)]
        [ValidateSet('Servers', 'NetworkInterfaces', 'VirtualNetworks', 'LogicalNetworks')]
        [String]$ResourceType,

        [Parameter(Mandatory = $false)]
        [String]$ApiVersion = 'v1'
    )

    $certs = @()
    $certs += $null
    $resources = $null
    $NcUri = $NcUri.TrimEnd('/')

    $sdnRequestParams = @{
        NcUri       = $NcUri
        ResourceRef = $ResourceType
        ApiVersion  = $ApiVersion
        NcRestCertificate = $null
    }

    try {
        $certs += Get-SdnServerCertificate
        [System.Array]::Reverse($certs)
        foreach ($cert in $certs) {
            if ($null -ieq $cert) {
                $sdnRequestParams = @{
                    NcUri       = $NcUri
                    ResourceRef = $ResourceType
                    ApiVersion  = $ApiVersion
                }
            }
            else {
                $sdnRequestParams = @{
                    NcUri       = $NcUri
                    ResourceRef = $ResourceType
                    ApiVersion  = $ApiVersion
                    NcRestCertificate = $cert
                }

                Write-Verbose "Retrieving $NcUri with certificate $($cert.Subject) thumbprint $($cert.Thumbprint)"
            }

            try {
                $resources = Get-SdnResource @sdnRequestParams
                if ($resources) {
                    Write-Verbose "Retrieved $($resources.Count) resources for $ResourceType"
                    return $resources
                }
            }
            catch [System.Net.WebException] {
                if ( $_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::Unauthorized ) {
                    continue
                }
                else {
                    Write-Error $_
                    break
                }
            }
            catch {
                Write-Error $_
                # dont try other certificates
                break
            }
        }

        return $null
    }
    catch {
        Write-Error $_
    }
}

function Get-SdnFabricInfrastructureResult {
    <#
        .SYNOPSIS
            Returns the results that have been saved to cache as part of running Debug-SdnFabricInfrastructure.
        .PARAMETER Role
            The name of the SDN role that you want to return test results from within the cache.
        .PARAMETER Name
            The name of the test results you want to examine.
        .EXAMPLE
            PS> Get-SdnFabricInfrastructureResult
        .EXAMPLE
            PS> Get-SdnFabricInfrastructureResult -Role Server
        .EXAMPLE
            PS> Get-SdnFabricInfrastructureResult -Role Server -Name 'Test-SdnServiceState'
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$Role,

        [Parameter(Mandatory = $false)]
        [System.String]$Name
    )

    $cacheResults = $script:SdnDiagnostics_Health.Cache

    if ($PSBoundParameters.ContainsKey('Role')) {
        if ($cacheResults) {
            $cacheResults = $cacheResults | Where-Object { $_.Role -eq $Role }
        }
    }

    if ($PSBoundParameters.ContainsKey('Name')) {
        if ($cacheResults) {
            $cacheResults = $cacheResults.HealthValidation | Where-Object { $_.Name -eq $Name }
        }
    }

    return $cacheResults
}

function Debug-SdnNetworkController {
    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    Confirm-IsNetworkController
    $healthReport = New-SdnRoleHealthReport -Role 'NetworkController'

    try {
        # execute tests for network controller, regardless of the cluster type
        $healthReport.HealthTest += @(
            Test-SdnNonSelfSignedCertificateInTrustedRootStore
        )

        # execute tests based on the cluster type
        switch ($Global:SdnDiagnostics.EnvironmentInfo.ClusterConfigType) {
            'FailoverCluster' {
                $healthReport.HealthTest += @(
                    Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'FcDiagnostics'
                )
            }
            'ServiceFabric' {
                $config_sf = Get-SdnModuleConfiguration -Role 'NetworkController_SF'
                [string[]]$services_sf = $config_sf.properties.services.Keys
                $healthReport.HealthTest += @(
                    Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'SDN Diagnostics Task'
                    Test-SdnServiceState -ServiceName $services_sf
                    Test-SdnServiceFabricApplicationHealth
                    Test-SdnServiceFabricClusterHealth
                    Test-SdnServiceFabricNodeStatus
                )
            }
        }

        # enumerate all the tests performed so we can determine if any completed with WARN or FAIL
        # if any of the tests completed with WARN, we will set the aggregate result to WARN
        # if any of the tests completed with FAIL, we will set the aggregate result to FAIL and then break out of the foreach loop
        # we will skip tests with PASS, as that is the default value
        foreach ($test in $healthReport.HealthTest) {
            if ($test.Result -eq 'WARN') {
                $healthReport.Result = $test.Result
            }
            elseif ($test.Result -eq 'FAIL') {
                $healthReport.Result = $test.Result
                break
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $healthReport.Result = 'FAIL'
    }

    return $healthReport
}

function Debug-SdnServer {
    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    Confirm-IsServer
    $config = Get-SdnModuleConfiguration -Role 'Server'
    [string[]]$services = $config.properties.services.Keys
    $healthReport = New-SdnRoleHealthReport -Role 'Server'

    $ncRestParams = $PSBoundParameters
    $serverResource = Get-SdnResource @ncRestParams -Resource:Servers

    try {
        # execute tests based on the cluster type
        switch ($Global:SdnDiagnostics.EnvironmentInfo.ClusterConfigType) {
            'ServiceFabric' {
                $healthReport.HealthTest += @(
                    Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'SDN Diagnostics Task'
                )
            }
            'FailoverCluster' {
                $healthReport.HealthTest += @(
                    Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'FcDiagnostics'
                )
            }
        }

        # these tests are executed locally and have no dependencies on network controller rest API being available
        $healthReport.HealthTest += @(
            Test-SdnNonSelfSignedCertificateInTrustedRootStore
            Test-SdnEncapOverhead
            Test-VfpDuplicateMacAddress
            Test-VMNetAdapterDuplicateMacAddress
            Test-SdnServiceState -ServiceName $services
            Test-SdnProviderNetwork
            Test-SdnHostAgentConnectionStateToApiService
            Test-SdnNetworkControllerApiNameResolution -NcUri $NcUri
        )

        # these tests have dependencies on network controller rest API being available
        # and will only be executed if we have been able to get the data from the network controller
        if ($serverResource) {
            $healthReport.HealthTest += @(
                Test-ServerHostId -InstanceId $serverResource.InstanceId
            )
        }

        # enumerate all the tests performed so we can determine if any completed with WARN or FAIL
        # if any of the tests completed with WARN, we will set the aggregate result to WARN
        # if any of the tests completed with FAIL, we will set the aggregate result to FAIL and then break out of the foreach loop
        # we will skip tests with PASS, as that is the default value
        foreach ($test in $healthReport.HealthTest) {
            if ($test.Result -eq 'WARN') {
                $healthReport.Result = $test.Result
            }
            elseif ($test.Result -eq 'FAIL') {
                $healthReport.Result = $test.Result
                break
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $healthReport.Result = 'FAIL'
    }

    return $healthReport
}

function Debug-SdnLoadBalancerMux {
    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    Confirm-IsLoadBalancerMux
    $config = Get-SdnModuleConfiguration -Role 'LoadBalancerMux'
    [string[]]$services = $config.properties.services.Keys
    $healthReport = New-SdnRoleHealthReport -Role 'LoadBalancerMux'

    $ncRestParams = $PSBoundParameters

    try {
        $muxCertRegKey = Get-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Services\SlbMux" -Name MuxCert
        $virtualServers = Get-SdnResource -Resource VirtualServers @ncRestParams
        $muxVirtualServer = $virtualServers | Where-Object { $_.properties.connections.managementaddresses -contains $muxCertRegKey.MuxCert }
        $loadBalancerMux = Get-SdnLoadBalancerMux @ncRestParams | Where-Object { $_.properties.virtualserver.resourceRef -ieq $muxVirtualServer.resourceRef }
        $peerRouters = $loadBalancerMux.properties.routerConfiguration.peerRouterConfigurations.routerIPAddress

        $healthReport.HealthTest += @(
            Test-SdnNonSelfSignedCertificateInTrustedRootStore
            Test-SdnServiceState -ServiceName $services
            Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'SDN Diagnostics Task'
            Test-SdnMuxConnectionStateToSlbManager
            Test-SdnNetworkControllerApiNameResolution -NcUri $NcUri
        )

        # these tests have dependencies on network controller rest API being available
        # and will only be executed if we have been able to get the data from the network controller
        if ($muxVirtualServer) {
            $healthReport.HealthTest += @(
                Test-SdnMuxConnectionStateToRouter -RouterIPAddress $peerRouters
            )
        }

        # enumerate all the tests performed so we can determine if any completed with WARN or FAIL
        # if any of the tests completed with WARN, we will set the aggregate result to WARN
        # if any of the tests completed with FAIL, we will set the aggregate result to FAIL and then break out of the foreach loop
        # we will skip tests with PASS, as that is the default value
        foreach ($test in $healthReport.HealthTest) {
            if ($test.Result -eq 'WARN') {
                $healthReport.Result = $test.Result
            }
            elseif ($test.Result -eq 'FAIL') {
                $healthReport.Result = $test.Result
                break
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $healthReport.Result = 'FAIL'
    }

    return $healthReport
}

function Debug-SdnGateway {
    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'RestCredential')]
        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    Confirm-IsRasGateway
    $config = Get-SdnModuleConfiguration -Role 'Gateway'
    [string[]]$services = $config.properties.services.Keys
    $healthReport = New-SdnRoleHealthReport -Role 'Gateway'

    $ncRestParams = @{
        NcUri = $NcUri
    }
    switch ($PSCmdlet.ParameterSetName) {
        'RestCredential' { $ncRestParams += @{ NcRestCredential = $NcRestCredential } }
        'RestCertificate' { $ncRestParams += @{ NcRestCertificate = $NcRestCertificate } }
    }

    try {
        $healthReport.HealthTest += @(
            Test-SdnNonSelfSignedCertificateInTrustedRootStore
            Test-SdnDiagnosticsCleanupTaskEnabled -TaskName 'SDN Diagnostics Task'
            Test-SdnServiceState -ServiceName $services
        )

        # enumerate all the tests performed so we can determine if any completed with Warning or FAIL
        # if any of the tests completed with Warning, we will set the aggregate result to Warning
        # if any of the tests completed with FAIL, we will set the aggregate result to FAIL and then break out of the foreach loop
        # we will skip tests with PASS, as that is the default value
        foreach ($test in $healthReport.HealthTest) {
            if ($test.Result -eq 'Warning') {
                $healthReport.Result = $test.Result
            }
            elseif ($test.Result -eq 'FAIL') {
                $healthReport.Result = $test.Result
                break
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $healthReport.Result = 'FAIL'
    }

    return ( $healthReport )
}

###################################
#### COMMON HEALTH VALIDATIONS ####
###################################

function Test-SdnNonSelfSignedCertificateInTrustedRootStore {
    <#
    .SYNOPSIS
        Validate the Cert in Host's Root CA Store to detect if any Non Root Cert exist
    #>

    [CmdletBinding()]
    param ()

    Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) invoked"
    $sdnHealthTest = New-SdnHealthTest
    $array = @()

    try {
        $rootCerts = Get-ChildItem -Path 'Cert:LocalMachine\Root' | Where-Object { $_.Issuer -ne $_.Subject }
        if ($rootCerts -or $rootCerts.Count -gt 0) {
            $sdnHealthTest.Result = 'FAIL'

            $rootCerts | ForEach-Object {
                $sdnHealthTest.Remediation += "Remove Certificate Thumbprint: $($_.Thumbprint) Subject: $($_.Subject)"
                $array += [PSCustomObject]@{
                    Thumbprint = $_.Thumbprint
                    Subject    = $_.Subject
                    Issuer     = $_.Issuer
                }
            }
        }
        $sdnHealthTest.Properties = $array


        ##########################################################################################
        ## ServiceState Fault Template
        ##########################################################################################
        # $KeyFaultingObjectDescription (SDN ID) : [HostName]
        # $KeyFaultingObjectID (ARC ID) : [HostName]
        # $KeyFaultingObjectType (CODE) : "NonSelfSignedCertificateInTrustedRootStore"
        # $FaultingObjectLocation (SOURCE) : "CertificateConfiguration"
        # $FaultDescription (MESSAGE) : "A non self signed ceritificate was found in trusted root store. This may lead to authentication problems."
        # $FaultActionRemediation (ACTION) : "Investigate and remove certificate with subject [SubjectNamesCsv]"
        # * Fault may be issued from each node
        ##########################################################################################
        if ($null -ne $array.Subject -and $array.Subject.Count -gt 0) {
            $subjectNames = [string]::Join(",", $array.Subject)
        }
        else {
            $subjectNames = ""
        }
        $healthFault = [SdnFaultInfo]::new()
        $healthFault.KeyFaultingObjectDescription = $Env:COMPUTERNAME
        $healthFault.KeyFaultingObjectID = $Env:COMPUTERNAME
        $healthFault.KeyFaultingObjectType = "NonSelfSignedCertificateInTrustedRootStore"
        $healthFault.FaultingObjectLocation = "CertificateConfiguration"
        $healthFault.FaultDescription = "A non self signed ceritificate was found in trusted root store. This may lead to authentication problems."
        $healthFault.FaultActionRemediation = "Investigate and remove certificate with subject(s) $($subjectNames)."

        if ( $rootCerts -or $rootCerts.Count -gt 0) {
            CreateorUpdateFault -Fault $healthFault
            $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Create"
            $sdnHealthTest.HealthFault += $convFault
        }
        else {
            DeleteFault -Fault $healthFault
            $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Delete"
            $sdnHealthTest.HealthFault += $convFault
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }
    finally {
        Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) exiting"
    }

    return $sdnHealthTest
}

function Test-SdnServiceState {
    [CmdletBinding()]
    param (

        [Parameter(Mandatory = $true)]
        [String[]]$ServiceName
    )

    Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) invoked for $($ServiceName)"
    $sdnHealthTest = New-SdnHealthTest
    $failureDetected = $false
    $array = @()

    try {
        foreach ($service in $ServiceName) {
            $result = Get-Service -Name $service -ErrorAction Ignore
            if ($result) {
                $array += [PSCustomObject]@{
                    ServiceName = $result.Name
                    Status      = $result.Status
                }

                if ($result.Status -ine 'Running') {
                    $failureDetected = $true
                    $sdnHealthTest.Remediation += "[$service] Start the service"
                }
            }
            else {
                $failureDetected = $true
            }

            ##########################################################################################
            ## ServiceState Fault Template
            ##########################################################################################
            # $KeyFaultingObjectDescription (SDN ID) : [HostName]
            # $KeyFaultingObjectID (ARC ID) : [ServiceName]
            # $KeyFaultingObjectType (CODE) : [ServiceDown]
            # $FaultingObjectLocation (SOURCE) : [ServiceName]
            # $FaultDescription (MESSAGE) : Service [ServiceName] is not up.
            # $FaultActionRemediation (ACTION) : [ServiceName] Start the service
            # *ServiceState faults will be reported from each node
            ##########################################################################################

            $healthFault = [SdnFaultInfo]::new()
            $healthFault.KeyFaultingObjectDescription = $Env:COMPUTERNAME
            $healthFault.KeyFaultingObjectID = $service
            $healthFault.KeyFaultingObjectType = "ServiceDown"
            $healthFault.FaultingObjectLocation = $service
            $healthFault.FaultDescription = "Service $($service) is not up."
            $healthFault.FaultActionRemediation = "Start the cluster service role $($service) from failover cluster manager"

            if ($result.Status -ine 'Running') {
                Write-Verbose "Creating fault for $($service) status $($result.Status)"
                CreateorUpdateFault -Fault $healthFault
                $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Create"
                $sdnHealthTest.HealthFault += $convFault
            }
            else {
                Write-Verbose "No fault(s) on $($service) clearing any existing ones"
                DeleteFault -Fault $healthFault
                $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Delete"
                $sdnHealthTest.HealthFault += $convFault
            }
        }

        if ($failureDetected) {
            $sdnHealthTest.Result = 'FAIL'
        }

        if ($array) {
            $sdnHealthTest.Properties = $array
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }
    finally {
        Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) exiting"
    }

    return $sdnHealthTest
}


function Test-SdnClusterServiceState {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [String[]]$ServiceName
    )

    $isCurrentNodeClusterOwner = IsCurrentNodeClusterOwner
    if ($isCurrentNodeClusterOwner -eq $false) {
        Write-Verbose "This node is not the cluster owner. Skipping health tests."
        return
    }

    Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) invoked"
    $sdnHealthTest = New-SdnHealthTest
    $failureDetected = $false
    $array = @()

    try {
        foreach ($service in $ServiceName) {
            $result = Get-ClusterGroup -Name $service -ErrorAction Ignore
            if ($result) {
                $array += [PSCustomObject]@{
                    ServiceName = $result.Name
                    Status      = $result.State
                }
                Write-Verbose "$service state $($result.State)"
                if ($result.State -ine 'Online') {
                    $failureDetected = $true
                    $sdnHealthTest.Remediation += "[$service] Start the service"
                }

                ##########################################################################################
                ## FailoverClusterServiceState Fault Template
                ##########################################################################################
                # $KeyFaultingObjectDescription (SDN ID) : [ServiceName]
                # $KeyFaultingObjectID (ARC ID) : [ServiceName]
                # $KeyFaultingObjectType (CODE) : ServiceUnavailable
                # $FaultingObjectLocation (SOURCE) : [ServiceName]
                # $FaultDescription (MESSAGE) : Service [ServiceName] is not up.
                # $FaultActionRemediation (ACTION) : [ServiceName] Start the service
                # *ServiceState faults will be reported only on one (primary) cluster node
                ##########################################################################################

                $healthFault = [SdnFaultInfo]::new()
                $healthFault.KeyFaultingObjectDescription = $service
                $healthFault.KeyFaultingObjectID = $service
                $healthFault.KeyFaultingObjectType = "ServiceUnavailable"
                $healthFault.FaultingObjectLocation = $service
                $healthFault.FaultDescription = "Service $($service) is $($result.State) on Failover Cluster"
                $healthFault.FaultActionRemediation = "Start the cluster service role $($service)"

                if ($result.State -ine 'Online') {
                    Write-Verbose "Creating fault for $($service)"
                    CreateorUpdateFault -Fault $healthFault
                    $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Create"
                    $sdnHealthTest.HealthFault += $convFault
                }
                else {
                    Write-Verbose "No fault(s) on $($service)"
                    DeleteFault -Fault $healthFault
                    $convFault = ConvertFaultToPsObject -healthFault $healthFault -faultOpType "Delete"
                    $sdnHealthTest.HealthFault += $convFault
                }
            }
            else {
                $sdnHealthTest.Result = 'FAIL'
            }
        }

        if ($failureDetected) {
            $sdnHealthTest.Result = 'FAIL'
        }
        $sdnHealthTest.Properties = $array
    }
    catch {
        $_ | Trace-Exception
        $_ | Write-Error
    }
    finally {
        Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) exiting"
    }

    return $sdnHealthTest
}

function Test-SdnDiagnosticsCleanupTaskEnabled {
    <#
    .SYNOPSIS
        Ensures the scheduled task responsible for etl compression is enabled and running
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateSet('FcDiagnostics', 'SDN Diagnostics Task')]
        [String]$TaskName
    )

    $sdnHealthTest = New-SdnHealthTest

    try {
        # check to see if logging is enabled on the registry key
        $isLoggingEnabled = Get-ItemPropertyValue -Path "HKLM:\Software\Microsoft\NetworkController\Sdn\Diagnostics\Parameters" -Name 'IsLoggingEnabled' -ErrorAction Ignore

        # in this scenario, logging is currently disabled so scheduled task will not be available
        if ($isLoggingEnabled ) {
            try {
                $result = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
                if ($result.State -ieq 'Disabled') {
                    $sdnHealthTest.Result = 'FAIL'
                    $sdnHealthTest.Remediation += "Use 'Repair-SdnDiagnosticsScheduledTask -TaskName $TaskName'."
                }
            }
            catch {
                $_ | Trace-Exception
                $sdnHealthTest.Result = 'FAIL'
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnNetworkControllerApiNameResolution {
    <#
    .SYNOPSIS
        Validates that the Network Controller API is resolvable via DNS
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri
    )

    $sdnHealthTest = New-SdnHealthTest

    try {
        # check to see if the Uri is an IP address or a DNS name
        # if it is a DNS name, we need to ensure that it is resolvable
        # if it is an IP address, we can skip the DNS resolution check
        $isIpAddress = [System.Net.IPAddress]::TryParse($NcUri.Host, [ref]$null)
        if (-NOT $isIpAddress) {
            $dnsResult = Resolve-DnsName -Name $NcUri.Host -ErrorAction Ignore
            if ($null -eq $dnsResult) {
                $sdnHealthTest.Result = 'FAIL'
                $sdnHealthTest.Remediation += "Ensure that the DNS server(s) are reachable and DNS record exists."
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}


###################################
#### SERVER HEALTH VALIDATIONS ####
###################################

function Test-SdnEncapOverhead {
    <#
    .SYNOPSIS
        Validate EncapOverhead configuration on the network adapter
    #>

    [CmdletBinding()]
    param ()

    Confirm-IsServer
    Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) invoked"

    [int]$encapOverheadExpectedValue = 160
    [int]$jumboPacketExpectedValue = 1674 # this is default 1514 MTU + 160 encap overhead
    $sdnHealthTest = New-SdnHealthTest
    [bool] $misconfigurationFound = $false
    [string[]] $misconfiguredNics = @()

    try {
        # check to see if provider addresses are configured
        # if not, we know that workloads have not been deployed and we can skip this test
        # as none of the settings will be configured
        $providerAddreses = Get-SdnProviderAddress
        if ($null -ieq $providerAddreses -or $providerAddreses.Count -eq 0) {
            return $sdnHealthTest
        }

        $encapOverheadResults = Get-SdnNetAdapterEncapOverheadConfig
        if ($null -eq $encapOverheadResults) {
            # skip generation of fault if we cannot determine status confidently
            $sdnHealthTest.Result = 'FAIL'
        }
        else {
            $encapOverheadResults | ForEach-Object {
                # if encapoverhead is not enabled, this is most commonly due to network adapter firmware or driver
                # recommendations are to update the firmware and driver to the latest version and make sure not using default inbox drivers
                if ($_.EncapOverheadEnabled -eq $false) {

                    # in this scenario, encapoverhead is disabled and we have the expected jumbo packet value
                    # packets will be allowed to traverse the network without being dropped after adding VXLAN/GRE headers
                    if ($_.JumboPacketValue -ge $jumboPacketExpectedValue) {
                        # will not do anything as configuring the jumbo packet is viable workaround if encapoverhead is not supported on the network adapter
                        # this is a PASS scenario
                    }

                    # in this scenario, encapoverhead is disabled and we do not have the expected jumbo packet value
                    # this will result in a failure on the test as it will result in packets being dropped if we exceed default MTU
                    if ($_.JumboPacketValue -lt $jumboPacketExpectedValue) {
                        $sdnHealthTest.Result = 'FAIL'
                        $sdnHealthTest.Remediation += "[$($_.NetAdapterInterfaceDescription)] Ensure the latest firmware and drivers are installed to support EncapOverhead. Configure JumboPacket to $jumboPacketExpectedValue if EncapOverhead is not supported."
                        $misconfigurationFound = $true
                        $misconfiguredNics += $_.NetAdapterInterfaceDescription
                    }
                }

                # in this case, the encapoverhead is enabled but the value is less than the expected value
                if ($_.EncapOverheadEnabled -and $_.EncapOverheadValue -lt $encapOverheadExpectedValue) {
                    $sdnHealthTest.Result = 'FAIL'
                    $sdnHealthTest.Remediation += "[$($_.NetAdapterInterfaceDescription)] Ensure the latest firmware and drivers are installed to support EncapOverhead. Configure JumboPacket to $jumboPacketExpectedValue if EncapOverhead is not supported."
                    $misconfigurationFound = $true
                    $misconfiguredNics += $_.NetAdapterInterfaceDescription
                }

                $FAULTNAME = "InvalidEncapOverheadConfiguration"
                ##########################################################################################
                ## EncapOverhead Fault Template
                ##########################################################################################
                # $KeyFaultingObjectDescription (SDN ID) : [HostName]
                # $KeyFaultingObjectID (ARC ID) : [NetworkAdapterIfDesc]
                # $KeyFaultingObjectType (CODE) : InvalidEncapOverheadConfiguration
                # $FaultingObjectLocation (SOURCE) : [HostName]
                # $FaultDescription (MESSAGE) : EncapOverhead is not enabled or configured correctly for <AdapterNames> on host <HostName>.
                # $FaultActionRemediation (ACTION) : JumboPacket should be enabled & EncapOverhead must be configured to support SDN. Please check NetworkATC configuration for configuring optimal networking configuration.
                # *EncapOverhead Faults will be reported from each node
                ##########################################################################################

                $sdnHealthFault = [SdnFaultInfo]::new()
                $sdnHealthFault.KeyFaultingObjectDescription = $env:COMPUTERNAME
                $sdnHealthFault.KeyFaultingObjectID = $_.NetAdapterInterfaceDescription
                $sdnHealthFault.KeyFaultingObjectType = $FAULTNAME
                $sdnHealthFault.FaultingObjectLocation = $env:COMPUTERNAME
                $sdnHealthFault.FaultDescription = "EncapOverhead is not enabled or configured correctly for $($_.NetAdapterInterfaceDescription) on host $env:COMPUTERNAME."
                $sdnHealthFault.FaultActionRemediation = "JumboPacket should be enabled & EncapOverhead must be configured to support SDN. Please check NetworkATC configuration for configuring optimal networking configuration."

                if ($misconfigurationFound -eq $true) {
                    CreateorUpdateFault -Fault $sdnHealthFault
                    $sdnHealthTest.HealthFault += ConvertFaultToPsObject -healthFault $sdnHealthFault -faultOpType "Create"
                }
                else {
                    Write-Verbose "No fault(s) on EncapOverhead, clearing any existing ones"
                    # clear all existing faults for host($FAULTNAME)
                    # todo: validate multiple hosts reporting the same fault
                    DeleteFaultBy -KeyFaultingObjectDescription $env:COMPUTERNAME -KeyFaultingObjectType $FAULTNAME
                    $sdnHealthTest.HealthFault += ConvertFaultToPsObject -healthFault $sdnHealthFault -faultOpType "Delete"
                }
            }
        }

        if ($misconfiguredNics) {
            $sdnHealthTest.Properties = $misconfiguredNics
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }
    finally {
        Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) exiting"
    }

    return $sdnHealthTest
}

function Test-ServerHostId {
    <#
    .SYNOPSIS
        Queries the NCHostAgent HostID registry key value ensure the HostID matches known InstanceID
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [string[]]$InstanceId
    )

    Confirm-IsServer

    $sdnHealthTest = New-SdnHealthTest
    $regkeyPath = 'HKLM:\SYSTEM\CurrentControlSet\Services\NcHostAgent\Parameters'

    try {
        $regHostId = Get-ItemProperty -Path $regkeyPath -Name 'HostId' -ErrorAction Ignore
        if ($null -ieq $regHostId) {
            $sdnHealthTest.Result = 'FAIL'
        }
        else {
            if ($regHostId.HostId -inotin $InstanceId) {
                $sdnHealthTest.Result = 'FAIL'
                $sdnHealthTest.Remediation += "Update the HostId registry under $regkeyPath to match the correct InstanceId from the NC Servers API."
                $sdnHealthTest.Properties = [PSCustomObject]@{
                    HostID = $regHostId
                }
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-VfpDuplicateMacAddress {
    [CmdletBinding()]
    param ()

    Confirm-IsServer
    $sdnHealthTest = New-SdnHealthTest

    try {
        $vfpPorts = Get-SdnVfpVmSwitchPort
        $duplicateObjects = $vfpPorts | Where-Object { $_.MACaddress -ne '00-00-00-00-00-00' -and $null -ne $_.MacAddress } | Group-Object -Property MacAddress | Where-Object { $_.Count -ge 2 }
        if ($duplicateObjects) {
            $sdnHealthTest.Result = 'FAIL'

            $duplicateObjects | ForEach-Object {
                $sdnHealthTest.Remediation += "[$($_.Name)] Resolve the duplicate MAC address issue with VFP."
            }
        }

        $sdnHealthTest.Properties = [PSCustomObject]@{
            DuplicateVfpPorts = $duplicateObjects.Group
            VfpPorts          = $vfpPorts
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-VMNetAdapterDuplicateMacAddress {
    [CmdletBinding()]
    param ()

    Confirm-IsServer
    $sdnHealthTest = New-SdnHealthTest

    try {
        $vmNetAdapters = Get-SdnVMNetworkAdapter
        $duplicateObjects = $vmNetAdapters | Group-Object -Property MacAddress | Where-Object { $_.Count -ge 2 }
        if ($duplicateObjects) {
            $sdnHealthTest.Result = 'FAIL'

            $duplicateObjects | ForEach-Object {
                $sdnHealthTest.Remediation += "[$($_.Name)] Resolve the duplicate MAC address issue with VMNetworkAdapters."
            }
        }

        $sdnHealthTest.Properties = [PSCustomObject]@{
            DuplicateVMNetworkAdapters = $duplicateObjects.Group
            VMNetworkAdapters          = $vmNetAdapters
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnProviderNetwork {

    <#
        .SYNOPSIS
            Validate the health of the provider network by pinging the provider addresses.
    #>

    [CmdletBinding()]
    param ()

    Confirm-IsServer
    $sdnHealthTest = New-SdnHealthTest

    try {
        $addressMapping = Get-SdnOvsdbAddressMapping
        if (-NOT ($null -eq $addressMapping -or $addressMapping.Count -eq 0)) {
            $providerAddreses = $addressMapping.ProviderAddress | Sort-Object -Unique
            $connectivityResults = Test-SdnProviderAddressConnectivity -ProviderAddress $providerAddreses

            foreach ($destination in $connectivityResults) {
                $failureDetected = $false
                $sourceIPAddress = $destination.SourceAddress[0]
                $destinationIPAddress = $destination.DestinationAddress[0]
                $jumboPacketResult = $destination | Where-Object { $_.BufferSize -gt 1472 }
                $standardPacketResult = $destination | Where-Object { $_.BufferSize -le 1472 }

                if ($destination.Status -ine 'Success') {
                    $remediationMsg = $null
                    $failureDetected = $true

                    # if both jumbo and standard icmp tests fails, indicates a failure in the physical network
                    if ($jumboPacketResult.Status -ieq 'Failure' -and $standardPacketResult.Status -ieq 'Failure') {
                        $remediationMsg = "Unable to ping Provider Addresses. Ensure ICMP enabled on $sourceIPAddress and $destinationIPAddress. If issue persists, investigate physical network."
                        $sdnHealthTest.Remediation += $remediationMsg
                    }

                    # if standard MTU was success but jumbo MTU was failure, indication that jumbo packets or encap overhead has not been setup and configured
                    # either on the physical nic or within the physical switches between the provider addresses
                    if ($jumboPacketResult.Status -ieq 'Failure' -and $standardPacketResult.Status -ieq 'Success') {
                        $remediationMsg = "Ensure the physical network between $sourceIPAddress and $destinationIPAddress are configured to support VXLAN or NVGRE encapsulated packets with minimum MTU of 1660."
                        $sdnHealthTest.Remediation += $remediationMsg
                    }
                }
            }
        }

        if ($failureDetected) {
            $sdnHealthTest.Result = 'FAIL'
        }
        if ($connectivityResults) {
            $sdnHealthTest.Properties = [PSCustomObject]@{
                PingResult = $connectivityResults
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnHostAgentConnectionStateToApiService {
    <#
        SYNOPSIS
            Validate the health of the Network Controller Host Agent connection to the Network Controller API Service.
    #>

    [CmdletBinding()]
    param()

    Confirm-IsServer
    $sdnHealthTest = New-SdnHealthTest

    try {
        $tcpConnection = Get-NetTCPConnection -RemotePort 6640 -ErrorAction Ignore
        if ($null -eq $tcpConnection -or $tcpConnection.State -ine 'Established') {
            $sdnHealthTest.Result = 'FAIL'
        }

        if ($tcpConnection) {
            if ($tcpConnection.ConnectionState -ine 'Connected') {
                $serviceState = Get-Service -Name NCHostAgent -ErrorAction Stop
                if ($serviceState.Status -ine 'Running') {
                    $sdnHealthTest.Result = 'WARN'
                    $sdnHealthTest.Remediation += "Ensure the NCHostAgent service is running."
                }
                else {
                    $sdnHealthTest.Result = 'FAIL'
                    $sdnHealthTest.Remediation += "Ensure that Network Controller ApiService is healthy and operational. Investigate and fix TCP / TLS connectivity issues."
                }
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

###################################
###### NC HEALTH VALIDATIONS ######
###################################

function Test-SdnServiceFabricApplicationHealth {
    <#
    .SYNOPSIS
        Validate the health of the Network Controller application within Service Fabric.
    #>

    [CmdletBinding()]
    param ()

    $sdnHealthTest = New-SdnHealthTest

    try {
        $applicationHealth = Get-SdnServiceFabricApplicationHealth -ErrorAction Stop
        if ($applicationHealth.AggregatedHealthState -ine 'Ok') {
            $sdnHealthTest.Result = 'FAIL'
            $sdnHealthTest.Remediation += "Examine the Service Fabric Application Health for Network Controller to determine why the health is not OK."
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnServiceFabricClusterHealth {
    <#
    .SYNOPSIS
        Validate the health of the Network Controller cluster within Service Fabric.
    #>

    [CmdletBinding()]
    param ()

    $sdnHealthTest = New-SdnHealthTest

    try {
        $clusterHealth = Get-SdnServiceFabricClusterHealth -ErrorAction Stop
        if ($clusterHealth.AggregatedHealthState -ine 'Ok') {
            $sdnHealthTest.Result = 'FAIL'
            $sdnHealthTest.Remediation += "Examine the Service Fabric Cluster Health for Network Controller to determine why the health is not OK."
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnServiceFabricNodeStatus {
    <#
    .SYNOPSIS
        Validate the health of the Network Controller nodes within Service Fabric.
    #>

    [CmdletBinding()]
    param ()

    $sdnHealthTest = New-SdnHealthTest

    try {
        $ncNodes = Get-SdnServiceFabricNode -NodeName $env:COMPUTERNAME -ErrorAction Stop
        if ($null -eq $ncNodes) {
            $sdnHealthTest.Result = 'FAIL'
        }
        else {
            if ($ncNodes.NodeStatus -ine 'Up') {
                $sdnHealthTest.Result = 'FAIL'
                $sdnHealthTest.Remediation = 'Examine the Service Fabric Nodes for Network Controller to determine why the node is not Up.'
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnResourceConfigurationState {
    <#
    .SYNOPSIS
        Validate that the configurationState of the resources.
    #>

    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true)]
        [string]$Resource,

        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    $sdnHealthTest = New-SdnHealthTest
    $array = @()

    try {
        "Validating configuration state of {0}" -f $SdnEnvironmentObject.Role.ResourceName | Trace-Output

        $sdnResources = Get-SdnResource @PSBoundParameters
        foreach ($object in $sdnResources) {

            # if we have a resource that is not in a success state, we will skip validation
            # as we do not expect configurationState to be accurate if provisioningState is not Success
            if ($object.properties.provisioningState -ine 'Succeeded') {
                continue
            }

            # examine the configuration state of the resources and display errors to the screen
            $errorMessages = @()
            switch ($object.properties.configurationState.Status) {
                'Warning' {
                    # if we already have a failure, we will not change the result to warning
                    if ($sdnHealthTest.Result -ne 'FAIL') {
                        $sdnHealthTest.Result = 'WARNING'
                    }

                    $traceLevel = 'Warning'
                }

                'Failure' {
                    $sdnHealthTest.Result = 'FAIL'
                    $traceLevel = 'Error'
                }

                'InProgress' {
                    # if we already have a failure, we will not change the result to warning
                    if ($sdnHealthTest.Result -ne 'FAIL') {
                        $sdnHealthTest.Result = 'WARNING'
                    }

                    $traceLevel = 'Warning'
                }

                'Uninitialized' {
                    # in scenarios where state is redundant, we will not fail the test
                    if ($object.properties.state -ieq 'Redundant') {
                        # do nothing
                    }
                    else {
                        # if we already have a failure, we will not change the result to warning
                        if ($sdnHealthTest.Result -ne 'FAIL') {
                            $sdnHealthTest.Result = 'WARNING'
                        }

                        $traceLevel = 'Warning'
                    }
                }

                default {
                    $traceLevel = 'Verbose'
                }
            }

            if ($object.properties.configurationState.detailedInfo) {
                foreach ($detail in $object.properties.configurationState.detailedInfo) {
                    switch ($detail.code) {
                        'Success' {
                            # do nothing
                        }

                        default {
                            $errorMessages += $detail.message
                            try {
                                $errorDetails = Get-HealthData -Property 'ConfigurationStateErrorCodes' -Id $detail.code
                                $sdnHealthTest.Remediation += "[{0}] {1}" -f $object.resourceRef, $errorDetails.Action
                            }
                            catch {
                                "Unable to locate remediation actions for {0}" -f $detail.code | Trace-Output -Level:Warning
                                $remediationString = "[{0}] Examine the configurationState property to determine why configuration failed." -f $object.resourceRef
                                $sdnHealthTest.Remediation += $remediationString
                            }
                        }
                    }
                }

                # print the overall configuration state to screen, with each of the messages that were captured
                # as part of the detailedinfo property
                if ($errorMessages) {
                    $msg = "{0} is reporting configurationState status {1}:`n`t- {2}" -f $object.resourceRef, $object.properties.configurationState.Status, ($errorMessages -join "`n`t- ")
                }
                else {
                    $msg = "{0} is reporting configurationState status {1}" -f $object.resourceRef, $object.properties.configurationState.Status
                }

                $msg | Trace-Output -Level $traceLevel.ToString()
            }

            $details = [PSCustomObject]@{
                resourceRef        = $object.resourceRef
                configurationState = $object.properties.configurationState
            }

            $array += $details
        }

        $sdnHealthTest.Properties = $array
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnResourceProvisioningState {
    <#
    .SYNOPSIS
        Validate that the provisioningState of the resources.
    #>

    [CmdletBinding(DefaultParameterSetName = 'RestCredential')]
    param (
        [Parameter(Mandatory = $true)]
        [string]$Resource,

        [Parameter(Mandatory = $true)]
        [ValidateScript({
                if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                    throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
                }
                return $true
            })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    $sdnHealthTest = New-SdnHealthTest
    $array = @()

    try {
        "Validating provisioning state of {0}" -f $Resource | Trace-Output

        $sdnResources = Get-SdnResource @PSBoundParameters
        foreach ($object in $sdnResources) {
            # examine the provisioning state of the resources and display errors to the screen
            $msg = "{0} is reporting provisioning state: {1}" -f $object.resourceRef, $object.properties.provisioningState

            switch ($object.properties.provisioningState) {
                'Failed' {
                    $sdnHealthTest.Result = 'FAIL'
                    $msg | Trace-Output -Level:Error

                    $sdnHealthTest.Remediation += "[$($object.resourceRef)] Examine the Network Controller logs to determine why provisioning is $($object.properties.provisioningState)."
                }

                'Updating' {
                    # if we already have a failure, we will not change the result to warning
                    if ($sdnHealthTest.Result -ne 'FAIL') {
                        $sdnHealthTest.Result = 'WARNING'
                    }

                    # since we do not know what operations happened prior to this, we will log a warning
                    # and ask the user to monitor the provisioningState
                    $msg | Trace-Output -Level:Warning
                    $sdnHealthTest.Remediation += "[$($object.resourceRef)] Is reporting $($object.properties.provisioningState). Monitor to ensure that provisioningState moves to Succeeded."
                }

                default {
                    # this should cover scenario where provisioningState is 'Deleting' or Succeeded
                    $msg | Trace-Output -Level:Verbose
                }
            }

            $details = [PSCustomObject]@{
                resourceRef       = $object.resourceRef
                provisioningState = $object.properties.provisioningState
            }

            $array += $details
        }

        $sdnHealthTest.Properties = $array
        return $sdnHealthTest
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnConfigurationState {

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false, ParameterSetName = 'RestCredential')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestCertificate')]
        [X509Certificate]$NcRestCertificate
    )

    Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) invoked"
    try {
        $isCurrentNodeClusterOwner = IsCurrentNodeClusterOwner
        if ($false -eq $isCurrentNodeClusterOwner) {
            Write-Verbose "This node is not the cluster owner. Skipping health tests."
            return
        }

        # servers
        $items = Get-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Services\NcHostAgent\Parameters\
        $NcUri = "https://$($items.PeerCertificateCName)"

        $configStateHealths = @()

        # generate faults for servers
        $servers = GetSdnResourceFromNc -ResourceType 'Servers' -NcUri $NcUri
        $faultSet = GetFaultFromConfigurationState -resources $servers
        ShowFaultSet -faultset $faultSet
        $serverHealthTest = UpdateFaultSet -successFaults $faultSet[0] -FailureFaults $faultSet[1]
        $serverHealthTest.Name = "servers"
        $configStateHealths += $serverHealthTest

        # generate faults for vnics
        $vnics = GetSdnResourceFromNc -Resource 'NetworkInterfaces' -NcUri $NcUri
        $faultSet = GetFaultFromConfigurationState -resources $vnics
        ShowFaultSet -faultset $faultSet
        $vnicHealthTest = UpdateFaultSet -successFaults $faultSet[0] -FailureFaults $faultSet[1]
        $vnicHealthTest.Name = "networkinterfaces"
        $configStateHealths += $vnicHealthTest

        # generate faults for lnets
        $vnics = GetSdnResourceFromNc -Resource 'LogicalNetworks' -NcUri $NcUri
        $faultSet = GetFaultFromConfigurationState -resources $vnics
        ShowFaultSet -faultset $faultSet
        $vnicHealthTest = UpdateFaultSet -successFaults $faultSet[0] -FailureFaults $faultSet[1]
        $vnicHealthTest.Name = "logicalnetworks"
        $configStateHealths += $vnicHealthTest
    }
    catch {
        $_ | Write-Error
    }
    finally {
        Write-Verbose "$($PSCmdlet.MyInvocation.MyCommand.Name) exiting"
    }
}

###################################
##### MUX HEALTH VALIDATIONS ######
###################################

function Test-SdnMuxConnectionStateToRouter {
    <#
    SYNOPSIS
        Validates the TCP connectivity for BGP endpoint to the routers.
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory = $true)]
        [string[]]$RouterIPAddress
    )

    Confirm-IsLoadBalancerMux
    $sdnHealthTest = New-SdnHealthTest

    try {
        foreach ($router in $RouterIPAddress) {
            $tcpConnection = Get-NetTCPConnection -RemotePort 179 -RemoteAddress $router -ErrorAction Ignore
            if ($null -eq $tcpConnection -or $tcpConnection.State -ine 'Established') {
                $sdnHealthTest.Result = 'FAIL'
                $sdnHealthTest.Remediation += "Examine the TCP connectivity for router $router to determine why TCP connection is not established."
            }
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

function Test-SdnMuxConnectionStateToSlbManager {
    <#
        SYNOPSIS
        Validates the TCP / TLS connectivity to the SlbManager service.
    #>

    [CmdletBinding()]
    param()

    Confirm-IsLoadBalancerMux
    $sdnHealthTest = New-SdnHealthTest

    try {
        $tcpConnection = Get-NetTCPConnection -LocalPort 8560 -ErrorAction Ignore
        if ($null -eq $tcpConnection -or $tcpConnection.State -ine 'Established') {
            $sdnHealthTest.Result = 'FAIL'
            $sdnHealthTest.Remediation += "Move SlbManager service primary role to another node. Examine the TCP / TLS connectivity for the SlbManager service."
        }
    }
    catch {
        $_ | Trace-Exception
        $sdnHealthTest.Result = 'FAIL'
    }

    return $sdnHealthTest
}

# SIG # Begin signature block
# MIIoQgYJKoZIhvcNAQcCoIIoMzCCKC8CAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBefnpVJPym9ZFD
# q7+1RfXOkf1sL2ayI/vWKCnjfTiZJKCCDXYwggX0MIID3KADAgECAhMzAAAEBGx0
# Bv9XKydyAAAAAAQEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTE0WhcNMjUwOTExMjAxMTE0WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQC0KDfaY50MDqsEGdlIzDHBd6CqIMRQWW9Af1LHDDTuFjfDsvna0nEuDSYJmNyz
# NB10jpbg0lhvkT1AzfX2TLITSXwS8D+mBzGCWMM/wTpciWBV/pbjSazbzoKvRrNo
# DV/u9omOM2Eawyo5JJJdNkM2d8qzkQ0bRuRd4HarmGunSouyb9NY7egWN5E5lUc3
# a2AROzAdHdYpObpCOdeAY2P5XqtJkk79aROpzw16wCjdSn8qMzCBzR7rvH2WVkvF
# HLIxZQET1yhPb6lRmpgBQNnzidHV2Ocxjc8wNiIDzgbDkmlx54QPfw7RwQi8p1fy
# 4byhBrTjv568x8NGv3gwb0RbAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU8huhNbETDU+ZWllL4DNMPCijEU4w
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMjkyMzAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAIjmD9IpQVvfB1QehvpC
# Ge7QeTQkKQ7j3bmDMjwSqFL4ri6ae9IFTdpywn5smmtSIyKYDn3/nHtaEn0X1NBj
# L5oP0BjAy1sqxD+uy35B+V8wv5GrxhMDJP8l2QjLtH/UglSTIhLqyt8bUAqVfyfp
# h4COMRvwwjTvChtCnUXXACuCXYHWalOoc0OU2oGN+mPJIJJxaNQc1sjBsMbGIWv3
# cmgSHkCEmrMv7yaidpePt6V+yPMik+eXw3IfZ5eNOiNgL1rZzgSJfTnvUqiaEQ0X
# dG1HbkDv9fv6CTq6m4Ty3IzLiwGSXYxRIXTxT4TYs5VxHy2uFjFXWVSL0J2ARTYL
# E4Oyl1wXDF1PX4bxg1yDMfKPHcE1Ijic5lx1KdK1SkaEJdto4hd++05J9Bf9TAmi
# u6EK6C9Oe5vRadroJCK26uCUI4zIjL/qG7mswW+qT0CW0gnR9JHkXCWNbo8ccMk1
# sJatmRoSAifbgzaYbUz8+lv+IXy5GFuAmLnNbGjacB3IMGpa+lbFgih57/fIhamq
# 5VhxgaEmn/UjWyr+cPiAFWuTVIpfsOjbEAww75wURNM1Imp9NJKye1O24EspEHmb
# DmqCUcq7NqkOKIG4PVm3hDDED/WQpzJDkvu4FrIbvyTGVU01vKsg4UfcdiZ0fQ+/
# V0hf8yrtq9CkB8iIuk5bBxuPMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGiIwghoeAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAAQEbHQG/1crJ3IAAAAABAQwDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIIPDzdas7vxERLfesplV4ZIX
# 5vs9M2d7rKj1dPmfIPTPMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEAWVMwJshfvKPVQBmKKZxTyaQatsZtDKmv6G1g7vtxDLeF37ErUpC1nOnH
# /C3/KUFNE3NCin2YgJiDTRpWCbGJhWkjDHw67cukBinSNM4ZJMmJRoMGYxEuiFBA
# +IWUmTXv6qc7uu42Ru/X4DcW5eAcmJ4ErmXO2v/+Sz19rnAiZgllisyCttbc4EA4
# p5LB2Z4CxVrTKiiErLZpyxPnCC8UOZaewfIO9AYOSvA60t/dVG9Pk4eETI6JA64r
# BPvPGufHzCxH51S7dtoERRf9zfVHFYjiLwDf/ElGkPNAs4bcaMVoRiIFXQ78fzzv
# juXbh/pCykB2UR3yWrIBft23XBNAU6GCF6wwgheoBgorBgEEAYI3AwMBMYIXmDCC
# F5QGCSqGSIb3DQEHAqCCF4UwgheBAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsq
# hkiG9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCCapt1FBldY0eDYUsd0jfaGQ83pgkxq5xzdCXSRzwZRIgIGZ5qn08z2
# GBMyMDI1MDEzMDAxNTczNC43NjdaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT
# Tjo0MDFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg
# U2VydmljZaCCEfowggcoMIIFEKADAgECAhMzAAAB/tCowns0IQsBAAEAAAH+MA0G
# CSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u
# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp
# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI0
# MDcyNTE4MzExOFoXDTI1MTAyMjE4MzExOFowgdMxCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9w
# ZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjQwMUEt
# MDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl
# MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvLwhFxWlqA43olsE4PCe
# gZ4mSfsH2YTSKEYv8Gn3362Bmaycdf5T3tQxpP3NWm62YHUieIQXw+0u4qlay4AN
# 3IonI+47Npi9fo52xdAXMX0pGrc0eqW8RWN3bfzXPKv07O18i2HjDyLuywYyKA9F
# mWbePjahf9Mwd8QgygkPtwDrVQGLyOkyM3VTiHKqhGu9BCGVRdHW9lmPMrrUlPWi
# YV9LVCB5VYd+AEUtdfqAdqlzVxA53EgxSqhp6JbfEKnTdcfP6T8Mir0HrwTTtV2h
# 2yDBtjXbQIaqycKOb633GfRkn216LODBg37P/xwhodXT81ZC2aHN7exEDmmbiWss
# jGvFJkli2g6dt01eShOiGmhbonr0qXXcBeqNb6QoF8jX/uDVtY9pvL4j8aEWS49h
# KUH0mzsCucIrwUS+x8MuT0uf7VXCFNFbiCUNRTofxJ3B454eGJhL0fwUTRbgyCbp
# LgKMKDiCRub65DhaeDvUAAJT93KSCoeFCoklPavbgQyahGZDL/vWAVjX5b8Jzhly
# 9gGCdK/qi6i+cxZ0S8x6B2yjPbZfdBVfH/NBp/1Ln7xbeOETAOn7OT9D3UGt0q+K
# iWgY42HnLjyhl1bAu5HfgryAO3DCaIdV2tjvkJay2qOnF7Dgj8a60KQT9QgfJfwX
# nr3ZKibYMjaUbCNIDnxz2ykCAwEAAaOCAUkwggFFMB0GA1UdDgQWBBRvznuJ9SU2
# g5l/5/b+5CBibbHF3TAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBf
# BgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3Bz
# L2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmww
# bAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29m
# dC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El
# MjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUF
# BwMIMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAgEAiT4NUvO2lw+0
# dDMtsBuxmX2o3lVQqnQkuITAGIGCgI+sl7ZqZOTDd8LqxsH4GWCPTztc3tr8AgBv
# sYIzWjFwioCjCQODq1oBMWNzEsKzckHxAzYo5Sze7OPkMA3DAxVq4SSR8y+TRC2G
# cOd0JReZ1lPlhlPl9XI+z8OgtOPmQnLLiP9qzpTHwFze+sbqSn8cekduMZdLyHJk
# 3Niw3AnglU/WTzGsQAdch9SVV4LHifUnmwTf0i07iKtTlNkq3bx1iyWg7N7jGZAB
# RWT2mX+YAVHlK27t9n+WtYbn6cOJNX6LsH8xPVBRYAIRVkWsMyEAdoP9dqfaZzwX
# GmjuVQ931NhzHjjG+Efw118DXjk3Vq3qUI1re34zMMTRzZZEw82FupF3viXNR3DV
# OlS9JH4x5emfINa1uuSac6F4CeJCD1GakfS7D5ayNsaZ2e+sBUh62KVTlhEsQRHZ
# RwCTxbix1Y4iJw+PDNLc0Hf19qX2XiX0u2SM9CWTTjsz9SvCjIKSxCZFCNv/zpKI
# lsHx7hQNQHSMbKh0/wwn86uiIALEjazUszE0+X6rcObDfU4h/O/0vmbF3BMR+45r
# AZMAETJsRDPxHJCo/5XGhWdg/LoJ5XWBrODL44YNrN7FRnHEAAr06sflqZ8eeV3F
# uDKdP5h19WUnGWwO1H/ZjUzOoVGiV3gwggdxMIIFWaADAgECAhMzAAAAFcXna54C
# m0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UE
# CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z
# b2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp
# Y2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMy
# MjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQH
# EwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNV
# BAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0B
# AQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51
# yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY
# 6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9
# cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN
# 7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDua
# Rr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74
# kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2
# K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5
# TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZk
# i1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9Q
# BXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3Pmri
# Lq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUC
# BBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJl
# pxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIB
# FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9y
# eS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUA
# YgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU
# 1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2Ny
# bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIw
# MTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDov
# L3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0w
# Ni0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/yp
# b+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulm
# ZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM
# 9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECW
# OKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4
# FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3Uw
# xTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPX
# fx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVX
# VAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGC
# onsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU
# 5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEG
# ahC0HVUzWLOhcGbyoYIDVTCCAj0CAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT
# Tjo0MDFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg
# U2VydmljZaIjCgEBMAcGBSsOAwIaAxUAhGNHD/a7Q0bQLWVG9JuGxgLRXseggYMw
# gYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE
# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD
# VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsF
# AAIFAOtFJlMwIhgPMjAyNTAxMjkyMjEyMzVaGA8yMDI1MDEzMDIyMTIzNVowczA5
# BgorBgEEAYRZCgQBMSswKTAKAgUA60UmUwIBADAGAgEAAgEHMAcCAQACAhJaMAoC
# BQDrRnfTAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEA
# AgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQELBQADggEBAEVUfnwLBHflxbMB
# K2iYwJ9ALJO/wXQlqJW7fBajtBKvH5A7TjeT2yjkTTx3iUAJ7cBhvPrDjZb6jfvM
# XE8zS0BJs1sXhZd50riJEr28G0BpKF98+MCmpDkYsW4rs9UG3NQwCp2idDao/r9S
# qgx7qwdSffRVUuhpF7g6HZcEQET1ZsZIaUEc2mJt89ZAb4IJpnlvyMjaUyYbhvmY
# vFUp5Twm1QJoYq+0ba7D1yIvxocrdYBnSNxFvn8ErTJczV32TB4jVs03o6z+A2Hr
# cs2tnGLyYHgdHgS6GcaiXWLn6O1WU3KOdOznbndd7pnJ8LrbCmf52D32Q/3ti+Bk
# +kss/qUxggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz
# aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv
# cnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAx
# MAITMwAAAf7QqMJ7NCELAQABAAAB/jANBglghkgBZQMEAgEFAKCCAUowGgYJKoZI
# hvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCBlO6Kk4OQgutKq
# 2yYda02N30JfrMM+RKj08vdXfCH6LDCB+gYLKoZIhvcNAQkQAi8xgeowgecwgeQw
# gb0EIBGFzN38U8ifGNH3abaE9apz68Y4bX78jRa2QKy3KHR5MIGYMIGApH4wfDEL
# MAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1v
# bmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWlj
# cm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAH+0KjCezQhCwEAAQAAAf4w
# IgQgeWPZgga2qcYWbSJvB40nHo7XV9rP/XeGDP7Fx2wvFFEwDQYJKoZIhvcNAQEL
# BQAEggIAOAZu2wwDuPKoAA7CjN9NkcNhKemfs3v+WROUOJqBUk0kjCoqbywXXrgO
# P2BluHvkYwOzXEHqm1VGgRRzbQAamRUWbsDUd8eE/kg0Xebo8AVPs1OMjneeR+Wk
# +iFqS9ldgs+oKujsOZCdhk3mEUQntcPUDoCX2SLfN+KeJxbXNXtDgXmEEe96kDoM
# hVpiQ2xuBCnkwYGTU6lkO/E5dwYAVSlHh0E5QiqlxAD7sPvRsmKeGaEpL7HKrPKX
# f0pNjzKky5AUw3UbxQF65wLeA1LfWAymsKJVmv6OL2JH1dlgsWq/r8wIa8uyheSj
# EbDdNoBcAKx7DuGodTtKShXBMnQfcqC5yScLxfCpn4yA8YyHmKdouAj5mjwvNnfR
# LPCN5M9zANq7ZKr+SAl8NDSdbuYYCOnY1aukWY8/Rs8SOI7fqFynv6NHaNCtiANJ
# zF1vC1lkFlZbPG32dLZtfXsHThYVZJdJ8PVaU3HOpj1ncmP1eWQGE498y6pVLCuc
# 8mYVt+KUsdY9tbGly8cauy2L88Yl38IZ9xCXJ4rxOeZJ3GrZ8ecHkQr1/FR+Ndlx
# /NYgYgn8ML1Z4alDXHEkCgpEN/7TXVsCpLhuF94n04SgXhi83FdIo6oVj0fYk3YH
# WvH6Ba0rrzb5KnqcV5iHd6zZcvOhYY3L5kwWarJJE8uz8Aj78zc=
# SIG # End signature block