SdnDiagnostics

4.2402.952.23657

modules/SdnDiag.NetworkController/SdnDiag.NetworkController.psm1

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

Using module .\SdnDiag.NetworkController.Helper.psm1
Import-Module $PSScriptRoot\SdnDiag.NetworkController.Helper.psm1
Import-Module $PSScriptRoot\..\SdnDiag.Common\SdnDiag.Common.psm1
Import-Module $PSScriptRoot\..\SdnDiag.Utilities\SdnDiag.Utilities.psm1

# create local variable to store configuration data
$configurationData = Import-PowerShellDataFile -Path $PSScriptRoot\SdnDiag.NetworkController.Config.psd1
New-Variable -Name 'SdnDiagnostics_NC' -Scope 'Script' -Force -Value @{
    Config = $configurationData
}

##### FUNCTIONS AUTO-POPULATED BELOW THIS LINE DURING BUILD #####
function Connect-SlbManager {
    [CmdletBinding()]
    param(
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $slbClient = Get-SlbClient -ErrorAction Stop

    # we need identify the current primary replica for the slbmanager service
    # if the primary replica is on the local node, then we will use the loopback address
    $slbManagerPrimary = Get-SdnServiceFabricReplica -ServiceTypeName 'SlbManagerService' -Primary -Credential $Credential -ErrorAction Stop
    if ($null -ieq $slbManagerPrimary) {
        throw "Unable to return primary replica of SlbManagerService"
    }

    $slbManagerPrimaryNodeName = $slbManagerPrimary.ReplicaAddress.Split(':')[0]
    if (Test-ComputerNameIsLocal -ComputerName $slbManagerPrimaryNodeName) {
        $useLoopback = $true
    }

    # if we have already detected that we are using the loopback address, then we can just use that
    # otherwise we will test to check if the SlbManagerPrimary is an IP address or a hostname
    # if it is a hostname, then we will resolve it to an IP address
    if ($useLoopback) {
        $ipAddress = [System.Net.IPAddress]::Loopback
    }
    else {
        $isIpAddress = ($slbManagerPrimaryNodeName -as [IPAddress]) -as [Bool]
        if (!$isIpAddress) {
            [IPAddress]$ipAddress = [System.Net.Dns]::GetHostAddresses($slbManagerPrimaryNodeName)[0].IPAddressToString
            "Resolved {0} to {1}" -f $slbManagerPrimaryNodeName, $ipAddress.IPAddressToString | Trace-Output -Level:Verbose
        }
        else {
            [IPAddress]$ipAddress = $slbManagerPrimaryNodeName
        }
    }

    # create IPEndPoint object for the SlbManagerPrimary address and port 49001
    $endpoint = New-Object System.Net.IPEndPoint($ipAddress, 49001)
    $networkControllerNode = Get-SdnNetworkControllerNode -Name $env:COMPUTERNAME

    # check to see if we have a node certificate that will be used for establishing connectivity
    # otherwise if not using x509 between the NC nodes we can just use $null
    if ($networkControllerNode.NodeCertificate.Thumbprint) {
        $slbmConnection = $slbClient.ConnectToSlbManager($endpoint, $networkControllerNode.NodeCertificate.Thumbprint, $null)
    }
    else {
        $slbmConnection = $slbClient.ConnectToSlbManager($endpoint, $null, $null)
    }

    return $slbmConnection
}

function Copy-ServiceFabricManifestFromNetworkController {
    <#
    .SYNOPSIS
        Copy the Service Fabric Manifest Files from Network Controller.
    .PARAMETER NcNodeList
        The NcNodeList that retrieved via Get-SdnNetworkControllerInfoOffline.
    .PARAMETER ManifestFolder
        The Manifest Folder path for Manifest files copy to.
    .PARAMETER ManifestFolderNew
        The New Manifest Folder path for updated Manifest files.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolder,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolderNew,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if ($NcNodeList.Count -eq 0) {
            Trace-Output "No NC Node found" -Level:Error
            return
        }
        Trace-Output "Copying Manifest files from $($NcNodeList.IpAddressOrFQDN)" -Level:Verbose

        New-Item -Path $ManifestFolder -ItemType Directory -Force | Out-Null
        New-Item -Path $ManifestFolderNew -ItemType Directory -Force | Out-Null

        $fabricFolder = "$env:ProgramData\Microsoft\Service Fabric\$($NcNodeList[0].NodeName)\Fabric"
        Copy-FileFromRemoteComputer -Path "$fabricFolder\ClusterManifest.current.xml" -ComputerName $($NcNodeList[0].IpAddressOrFQDN) -Destination $ManifestFolder -Credential $Credential
        Copy-FileFromRemoteComputer -Path "$fabricFolder\Fabric.Data\InfrastructureManifest.xml" -ComputerName $($NcNodeList[0].IpAddressOrFQDN) -Destination $ManifestFolder -Credential $Credential

        $NcNodeList | ForEach-Object {
            $fabricFolder = "$env:ProgramData\Microsoft\Service Fabric\$($_.NodeName)\Fabric"

            $version = Invoke-PSRemoteCommand -ComputerName $_.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][String]$param1)
                $fabricPkgFile = Join-Path -Path $param1 -ChildPath "Fabric.Package.current.xml"
                $xml = [xml](Get-Content -Path $fabricPkgFile)
                $version = $xml.ServicePackage.DigestedConfigPackage.ConfigPackage.Version
                return $version
            } -ArgumentList $fabricFolder

            $fabricConfigDir = Join-Path -Path $fabricFolder -ChildPath $("Fabric.Config." + $version)
            $settingsFile = Join-Path -Path $fabricConfigDir -ChildPath "Settings.xml"
            New-Item -Path "$ManifestFolder\$($_.IpAddressOrFQDN)" -type Directory -Force | Out-Null
            New-Item -Path "$ManifestFolderNew\$($_.IpAddressOrFQDN)" -type Directory -Force | Out-Null

            Copy-FileFromRemoteComputer -Path $settingsFile -ComputerName $_.IpAddressOrFQDN -Destination "$ManifestFolder\$($_.IpAddressOrFQDN)" -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Copy-ServiceFabricManifestToNetworkController {
    <#
    .SYNOPSIS
        Copy the Service Fabric Manifest Files to Network Controller.
    .PARAMETER NcNodeList
        The NcNodeList that retrieved via Get-SdnNetworkControllerInfoOffline.
    .PARAMETER ManifestFolder
        The Manifest Folder path for Manifest files copy from.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolder,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if ($NcNodeList.Count -eq 0) {
            Trace-Output "No NC VMs found" -Level:Error
            return
        }
        Trace-Output "Copying Service Fabric Manifests to NC VMs: $($NcNodeList.IpAddressOrFQDN)"

        Trace-Output "Stopping Service Fabric Service"
        foreach ($nc in $NcNodeList.IpAddressOrFQDN) {
            Invoke-PSRemoteCommand -ComputerName $nc -Credential $Credential -ScriptBlock {
                Write-Host "[$(HostName)] Stopping Service Fabric Service"
                Stop-Service FabricHostSvc -Force
            }
        }


        $NcNodeList | ForEach-Object {
            $fabricFolder = "$env:ProgramData\Microsoft\Service Fabric\$($_.NodeName)\Fabric"

            $version = Invoke-PSRemoteCommand -ComputerName $_.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][String]$param1)
                $fabricPkgFile = Join-Path -Path $param1 -ChildPath "Fabric.Package.current.xml"
                $xml = [xml](Get-Content -Path $fabricPkgFile)
                $version = $xml.ServicePackage.DigestedConfigPackage.ConfigPackage.Version
                return $version
            } -ArgumentList $fabricFolder

            $fabricConfigDir = Join-Path -Path $fabricFolder -ChildPath $("Fabric.Config." + $version)
            $settingsFile = Join-Path -Path $fabricConfigDir -ChildPath "Settings.xml"

            Invoke-PSRemoteCommand -ComputerName $_.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                Set-ItemProperty -Path (Join-Path -Path $param1 -ChildPath "ClusterManifest.current.xml") -Name IsReadOnly -Value $false | Out-Null
                Set-ItemProperty -Path (Join-Path -Path $param1 -ChildPath "Fabric.Data\InfrastructureManifest.xml") -Name IsReadOnly -Value $false | Out-Null
                Set-ItemProperty -Path $param2 -Name IsReadOnly -Value $false | Out-Null
            } -ArgumentList @($fabricFolder, $settingsFile)

            Copy-FileToRemoteComputer -Path "$ManifestFolder\ClusterManifest.current.xml" -Destination "$fabricFolder\ClusterManifest.current.xml" -ComputerName $_.IpAddressOrFQDN -Credential $Credential
            Copy-FileToRemoteComputer -Path "$ManifestFolder\InfrastructureManifest.xml" -Destination "$fabricFolder\Fabric.Data\InfrastructureManifest.xml" -ComputerName $_.IpAddressOrFQDN -Credential $Credential
            Copy-FileToRemoteComputer -Path "$ManifestFolder\$($_.IpAddressOrFQDN)\settings.xml" -Destination $settingsFile -ComputerName $_.IpAddressOrFQDN -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-ManagementAddress {
    param (
        $ManagementAddress
    )

    $uniqueFQDN = @()
    $uniqueIPAddress = @()

    foreach ($ma in $ManagementAddress) {
        $isIpAddress = ($ma -as [IPAddress]) -as [Bool]
        if ($isIpAddress) {
            $uniqueIPAddress += $ma
        }
        else {
            $uniqueFQDN += $ma.ToLower()
        }
    }

    # if we have a mix of FQDN and IPAddress, defer to FQDN
    # use Sort-Object -Unique to remove duplicates from the list (case insensitive)
    if ($uniqueFQDN) {
        return ($uniqueFQDN | Sort-Object -Unique)
    }
    else {
        return ($uniqueIPAddress | Sort-Object -Unique)
    }
}

function Get-NetworkControllerConfigState {
    <#
    .SYNOPSIS
        Outputs a set of configuration state files for the network controller role.
    .PARAMETER OutputDirectory
        Specifies a specific path and folder in which to save the files.
    .EXAMPLE
        PS> Get-NetworkControllerConfigState -OutputDirectory "C:\Temp\CSS_SDN"
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [System.IO.FileInfo]$OutputDirectory
    )

    $currentErrorActionPreference = $ErrorActionPreference
    $ProgressPreference = 'SilentlyContinue'
    $ErrorActionPreference = 'SilentlyContinue'

    try {
        $config = Get-SdnModuleConfiguration -Role 'NetworkController'
        [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState"
        [System.IO.FileInfo]$ncAppDir = Join-Path $OutputDirectory.FullName -ChildPath "NCApp"
        [System.IO.FileInfo]$regDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "Registry"

        "Collect configuration state details for role {0}" -f $config.Name | Trace-Output

        if (-NOT (Initialize-DataCollection -Role 'NetworkController' -FilePath $OutputDirectory.FullName -MinimumMB 100)) {
            "Unable to initialize environment for data collection" | Trace-Output -Level:Error
            return
        }

        Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir.FullName
        Get-CommonConfigState -OutputDirectory $OutputDirectory.FullName

        # enumerate dll binary version for NC application
        $ncAppDirectories = Get-ChildItem -Path "$env:SystemRoot\NetworkController" -Directory
        foreach($directory in $ncAppDirectories){
            [System.String]$fileName = "FileInfo_{0}" -f $directory.BaseName
            Get-Item -Path "$($directory.FullName)\*" -Include *.dll,*.exe | Export-ObjectToFile -FilePath $ncAppDir.FullName -Name $fileName -FileType txt -Format List
        }
    }
    catch {
        $_ | Trace-Exception
    }

    $ProgressPreference = 'Continue'
    $ErrorActionPreference = $currentErrorActionPreference
}


function Get-NetworkControllerNodeInfoFromClusterManifest {
    <#
    .SYNOPSIS
        This function is used as fallback method in the event that normal Get-NetworkControllerNode cmdlet fails in scenarios where certs may be expired
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    "Attempting to retrieve NetworkControllerNode information via ClusterManifest" | Trace-Output
    $array = @()

    $clusterManifest = [xml](Get-SdnServiceFabricClusterManifest -NetworkController $NetworkController -Credential $Credential)
    $clusterManifest.ClusterManifest.Infrastructure.WindowsServer.NodeList.Node | ForEach-Object {
        $object = [PSCustomObject]@{
            Name = $_.NodeName
            Server = $_.IPAddressOrFQDN
            FaultDomain = $_.FaultDomain
            RestInterface = $null
            Status = $null
            NodeCertificate = $null
        }

        $certificate = ($clusterManifest.ClusterManifest.NodeTypes.NodeType | Where-Object Name -ieq $_.NodeName).Certificates.ServerCertificate.X509FindValue.ToString()
        $object | Add-Member -MemberType NoteProperty -Name NodeCertificateThumbprint -Value $certificate

        $array += $object
    }

    return $array
}

function Get-PublicIpReference {
    <##>
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [uri]$NcUri,

        [Parameter(Mandatory = $true)]
        [System.Object]$IpConfiguration,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {

        # check for an instance-level public IP address that is directly associated
        # with the ipconfiguration and return back to calling function
        if ($IpConfiguration.properties.publicIPAddress) {
            "Located {0} associated with {1}" -f $IpConfiguration.properties.publicIPAddress.resourceRef, $IpConfiguration.resourceRef | Trace-Output -Level:Verbose
            return ($IpConfiguration.properties.publicIPAddress.resourceRef)
        }
        else {
            "Unable to locate an instance-level public IP address associated with {0}" -f $IpConfiguration.resourceRef | Trace-Output -Level:Verbose
        }

        # NIC is connected to a load balancer with public IP association
        # or NIC is not associated to a public IP by any means and instead is connected via implicit load balancer attached to a virtual network
        "Checking for any backend address pool associated with {0}" -f $IpConfiguration.resourceRef | Trace-Output -Level:Verbose
        if ($IpConfiguration.properties.loadBalancerBackendAddressPools) {
            "Located backend address pool associations for {0}" -f $IpConfiguration.resourceRef | Trace-Output -Level:Verbose
            $loadBalancers = Get-SdnResource -NcUri $NcUri.AbsoluteUri -Resource:LoadBalancers -Credential $Credential
            $allBackendPoolRefs = @($IpConfiguration.properties.loadBalancerBackendAddressPools.resourceRef)

            $backendHash = [System.Collections.Hashtable]::new()
            foreach ($group in $loadBalancers.properties.backendAddressPools | Group-Object resourceRef) {
                [void]$backendHash.Add($group.Name, $group.Group)
            }

            foreach ($backendPoolRef in $allBackendPoolRefs) {
                "Checking for outboundNatRules for {0}" -f $backendPoolRef | Trace-Output -Level:Verbose
                $bePool = $backendHash[$backendPoolRef]

                if ($bePool.properties.outboundNatRules) {
                    "Located outboundNatRule associated with {0}" -f $bePool.resourceRef | Trace-Output -Level:Verbose

                    $obRuleRef = $bePool.properties.outboundNatRules[0].resourceRef
                    break
                }
            }

            if ($obRuleRef) {
                $natRule = $loadBalancers.properties.outboundNatRules | Where-Object { $_.resourceRef -eq $obRuleRef }
                $frontendConfig = $loadBalancers.properties.frontendIPConfigurations | Where-Object { $_.resourceRef -eq $natRule.properties.frontendIPConfigurations[0].resourceRef }

                "Located {0} associated with {0}" -f $frontendConfig.resourceRef, $natRule.resourceRef | Trace-Output -Level:Verbose
                return ($frontendConfig.properties.publicIPAddress.resourceRef)
            }
            else {
                "Unable to locate outboundNatRules associated with {0}" -f $IpConfiguration.properties.loadBalancerBackendAddressPools.resourceRef | Trace-Output -Level:Verbose
            }
        }
        else {
            "Unable to locate any backend pools associated with {0}" -f $IpConfiguration.resourceRef | Trace-Output -Level:Verbose
        }

        return $null
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnDipProbeInfoFromHost {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [IPAddress[]]$HostIPAddress,

        [Parameter(Mandatory = $false)]
        [System.String]$ProbeID = $null
    )

    $slbManager = Connect-SlbManager -ErrorAction Stop
    if ($slbManager) {
        $dipProbeInfo = $slbManager.GetDipProbeInfoFromHost($HostIPAddress, $ProbeID)
        return $dipProbeInfo
    }
}

function Get-SdnDiscovery {
    <#
    .SYNOPSIS
        Calls to the Discovery API endpoint to determine versioning and feature details
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceName 'Discovery'
        $result = Invoke-RestMethodWithRetry -Uri $uri -Method GET -UseBasicParsing -Credential $Credential -ErrorAction Stop
        return $result
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerInfoFromClusterManifest {
    <#
    .SYNOPSIS
        Get the Network Controller Configuration from network controller cluster manifest file. The function is used to retrieve information of the network controller when cluster down.
    .PARAMETER NetworkController
        Specifies the name the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    "Attempting to retrieve NetworkController information via ClusterManifest" | Trace-Output

    $clusterManifestXml = [xml](Get-SdnServiceFabricClusterManifest -NetworkController $NetworkController -Credential $Credential)
    $nodeList = $clusterManifestXml.ClusterManifest.Infrastructure.WindowsServer.NodeList.Node.NodeName
    $secretCertThumbprint = $clusterManifestXml.ClusterManifest.Certificates.SecretsCertificate.X509FindValue

    $splat = @{
        Path = 'Cert:\LocalMachine\My'
        Thumbprint = $secretCertThumbprint
    }

    if (Test-ComputerNameIsLocal -ComputerName $NetworkController) {
        $serverCertificate = Get-SdnCertificate @splat
    }
    else {
        $serverCertificate = Invoke-PSRemoteCommand -ComputerName $NetworkController -Credential $Credential -ScriptBlock {
            param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
            Get-SdnCertificate -Path $param1 -Thumbprint $param2
        } -ArgumentList @($splat.Path, $splat.Thumbprint)
    }

    $infraInfo = [PSCustomObject]@{
        Node = $nodeList
        ClientAuthentication = $null
        ClientCertificateThumbprint = $null
        ClientSecurityGroup = $null
        ServerCertificate = $serverCertificate
        RestIPAddress = $null
        RestName = $null
        Version = $null
    }

    return $infraInfo
}

function Get-SdnNetworkControllerInfoOffline {
    <#
    .SYNOPSIS
        Get the Network Controller Configuration from network controller cluster manifest file. The function is used to retrieve information of the network controller when cluster down.
    .PARAMETER NetworkController
        Specifies the name the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnNetworkControllerInfoOffline
    .EXAMPLE
        PS> Get-SdnNetworkControllerInfoOffline -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        $clusterManifestXml = [xml](Get-SdnServiceFabricClusterManifest -NetworkController $NetworkController -Credential $Credential)
        $NodeList = $clusterManifestXml.ClusterManifest.Infrastructure.WindowsServer.NodeList.Node
        $securitySection = $clusterManifestXml.ClusterManifest.FabricSettings.Section | Where-Object Name -eq "Security"
        $ClusterCredentialType = $securitySection.Parameter | Where-Object Name -eq "ClusterCredentialType"
        $secretCertThumbprint = $clusterManifestXml.ClusterManifest.Certificates.SecretsCertificate.X509FindValue

        $ncRestName = Invoke-PSRemoteCommand -ComputerName $NetworkController -Credential $Credential -ScriptBlock {
            param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
            $secretCert = Get-ChildItem -Path $param1 | Where-Object {$_.Thumbprint -ieq $param2}
            if($null -eq $secretCert) {
                return $null
            }
            else {
                return $secretCert.Subject.Replace("CN=","")
            }
        } -ArgumentList @('Cert:\LocalMachine\My', $secretCertThumbprint)

        $infraInfo = [PSCustomObject]@{
            ClusterCredentialType = $ClusterCredentialType.Value
            NodeList = $NodeList
            NcRestName = $ncRestName
            NcRestCertThumbprint = $secretCertThumbprint
        }

        return $infraInfo

    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerRestURL {
    <#
        .SYNOPSIS
        Queries Network Controller to identify the Rest URL endpoint that can be used to query the north bound API endpoint.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [String]$NetworkController,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        # if already populated into the cache, return the value
        if (-NOT ([System.String]::IsNullOrEmpty($Global:SdnDiagnostics.EnvironmentInfo.NcUrl))) {
            return $Global:SdnDiagnostics.EnvironmentInfo.NcUrl
        }

        $result = Get-SdnNetworkController -NetworkController $NetworkController -Credential $Credential
        if ($null -eq $result) {
            throw New-Object System.NullReferenceException("Unable to return information from Network Controller")
        }

        # use the Subject of the ServerCertificate object back for the NB API
        $endpoint = $result.ServerCertificate.Subject.Split('=')[1]
        $ncUrl = 'https://{0}' -f $endpoint

        return $ncUrl
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnVipState {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [IPAddress]$VirtualIPAddress
    )

    $slbManager = Connect-SlbManager -ErrorAction Stop
    if ($slbManager) {
        $vipState = $slbManager.GetVipState($VirtualIPAddress)
        return $vipState
    }
}

function Get-SdnVirtualServer {
    <#
    .SYNOPSIS
        Returns virtual server of a particular resource Id from network controller.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
 
    .PARAMETER ResourceRef
        Specifies Resource Ref of virtual server.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true)]
        [String]$ResourceRef,

        [Parameter(Mandatory = $false)]
        [switch]$ManagementAddressOnly,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        $result = Get-SdnResource -NcUri $NcUri.AbsoluteUri -ResourceRef $ResourceRef -Credential $Credential

        foreach ($obj in $result) {
            if ($obj.properties.provisioningState -ne 'Succeeded') {
                "{0} is reporting provisioningState: {1}" -f $obj.resourceId, $obj.properties.provisioningState | Trace-Output -Level:Warning
            }
        }

        if ($ManagementAddressOnly) {
            $connections = (Get-ManagementAddress -ManagementAddress $result.properties.connections.managementAddresses)
            return $connections
        }
        else {
            return $result
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SlbClient {
    [CmdletBinding()]
    param()

    # as we are dependent on the assemblies contained on Network Controller
    # we need to ensure we are running on Network Controller
    $config = Get-SdnModuleConfiguration -Role 'NetworkController'
    $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
    if (-NOT ($confirmFeatures)) {
        throw New-Object System.NotSupportedException("The current machine is not a NetworkController, run this on NetworkController.")
    }

    $rootDir = "$env:SystemRoot\NetworkController"
    $null = [Reflection.Assembly]::LoadFrom("$rootDir\SharedAssemblies\Microsoft.CloudNet.Slb.Utilities.SlbClient.dll");
    $null = [Reflection.Assembly]::LoadFrom("$rootDir\Framework\Microsoft.NetworkController.Utilities.dll");
    $null = [Reflection.Assembly]::LoadFrom("$rootDir\Framework\Microsoft.NetworkController.ServiceModule.dll");

    [Microsoft.Cloudnet.Slb.Utilities.SlbClient.SlbManagerConnectionFactory]::SlbClientInitializeWithDefaultSettings();
    [Microsoft.Cloudnet.Slb.Utilities.SlbClient.SlbManagerConnectionFactory]::UseInteractiveLogon = $false
    [Microsoft.Cloudnet.Slb.Utilities.SlbClient.SlbManagerConnectionFactory]::EnableBlockingNotifications = $true;

    $slbClient = [Microsoft.Cloudnet.Slb.Utilities.SlbClient.SlbClient]::new()
    return $slbClient
}


function Invoke-CertRotateCommand {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [ValidateSet('Set-NetworkController', 'Set-NetworkControllerCluster', 'Set-NetworkControllerNode')]
        [System.String]$Command,

        [Parameter(Mandatory = $false)]
        [System.String]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true)]
        [System.String]$Thumbprint,

        [Parameter(Mandatory = $false)]
        [Int]$TimeoutInMinutes = 30,

        [Parameter(Mandatory = $false)]
        [Int]$MaxRetry = 3
    )

    $stopWatch = [System.Diagnostics.Stopwatch]::StartNew()
    $retryAttempt = 0

    $params = @{
        'PassThru'  = $true
    }
    if ($Credential -ne [System.Management.Automation.PSCredential]::Empty -and $null -ne $Credential) {
        $params.Add('Credential', $Credential)
    }

    if (Test-ComputerNameIsLocal -ComputerName $NetworkController) {
        $cert = Get-SdnCertificate -Path 'Cert:\LocalMachine\My' -Thumbprint $Thumbprint
    }
    else {
        $params.Add('ComputerName', $NetworkController)
        $cert = Invoke-PSRemoteCommand -ComputerName $NetworkController -Credential $Credential -ScriptBlock {
            param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
            Get-SdnCertificate -Path $param1 -Thumbprint $param2
        } -ArgumentList @('Cert:\LocalMachine\My', $Thumbprint)
    }

    if ($null -eq $cert) {
        throw New-Object System.NullReferenceException("Unable to locate $($Thumbprint)")
    }
    if ($cert.Count -ge 2) {
        throw New-Object System.Exception("Duplicate certificates located that match $($Thumbprint)")
    }

    switch ($Command) {
        'Set-NetworkController' {
            $params.Add('ServerCertificate', $cert)
        }
        'Set-NetworkControllerCluster' {
            $params.Add('CredentialEncryptionCertificate', $cert)
        }
        'Set-NetworkControllerNode' {
            $ncNode = Get-SdnNetworkControllerNode -Name $NetworkController -Credential $Credential

            $params.Add('Name', $ncNode.Name)
            $params.Add('NodeCertificate', $cert)
        }
    }

    while ($true) {
        $retryAttempt++
        switch ($Command) {
            'Set-NetworkController' {
                $currentCertThumbprint = (Get-SdnNetworkControllerRestCertificate).Thumbprint
            }
            'Set-NetworkControllerCluster' {
                $currentCertThumbprint = (Get-NetworkControllerCluster).CredentialEncryptionCertificate.Thumbprint
            }
            'Set-NetworkControllerNode' {
                $currentCert = Invoke-PSRemoteCommand -ComputerName $NetworkController -Credential $Credential -ScriptBlock {
                    Get-SdnNetworkControllerNodeCertificate
                } -ErrorAction Stop
                $currentCertThumbprint = $currentCert.Thumbprint
            }
        }

        # if the certificate already matches what has been configured, then break out of the loop
        if ($currentCertThumbprint -ieq $Thumbprint) {
            "{0} has been updated to use certificate thumbprint {1}" -f $Command.Split('-')[1], $currentCertThumbprint | Trace-Output
            break
        }

        if ($stopWatch.Elapsed.TotalMinutes -ge $timeoutInMinutes) {
            throw New-Object System.TimeoutException("Rotate of certificate did not complete within the alloted time.")
        }

        if ($retryAttempt -ge $MaxRetry) {
            throw New-Object System.Exception("Rotate of certificate exceeded maximum number of retries.")
        }

        # if we have not started operation, or we hit a retryable error
        # then invoke the command to start the certificate rotate
        try {
            "Invoking {0} to configure thumbprint {1}" -f $Command, $cert.Thumbprint | Trace-Output
            "Command:{0} Params: {1}" -f $Command, ($params | ConvertTo-Json) | Trace-Output -Level:Verbose

            switch ($Command) {
                'Set-NetworkController' {
                    Set-NetworkController @params
                }
                'Set-NetworkControllerCluster' {
                    Set-NetworkControllerCluster @params
                }
                'Set-NetworkControllerNode' {
                    Set-NetworkControllerNode @params
                }
            }
        }
        catch [Microsoft.Management.Infrastructure.CimException] {
            switch -Wildcard ($_.Exception) {
                '*One or more errors occurred*' {
                    "Retryable exception caught`n`t$_" | Trace-Output -Level:Warning
                }

                default {
                    $stopWatch.Stop()
                    throw $_
                }
            }
        }
        catch [InvalidOperationException] {
            if ($_.FullyQualifiedErrorId -ilike "*UpdateInProgress*") {
                "Networkcontroller is being updated by another operation.`n`t{0}" -f $fullyQualifiedErrorId | Trace-Output -Level:Warning
                # Sleep 60s or longer before next retry if update in progress
                Start-Sleep -Seconds 60 * $retryAttempt
            }
            else {
                $stopWatch.Stop()
                throw $_
            }
        }
        catch {
            $stopWatch.Stop()
            throw $_
        }
    }

    $stopWatch.Stop()
    return $currentCertThumbprint
}

function Invoke-SdnNetworkControllerStateDump {
    <#
    .SYNOPSIS
        Executes a PUT operation against REST API endpoint for Network Controller to trigger a IMOS dump of Network Controller services.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ExecutionTimeout
        Specify the execution timeout (seconds) on how long you want to wait for operation to complete before cancelling operation. If omitted, defaults to 300 seconds.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [int]$ExecutionTimeOut = 300,

        [Parameter(Mandatory = $false)]
        [int]$PollingInterval = 1
    )

    try {
        $stopWatch = [system.diagnostics.stopwatch]::StartNew()
        [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceRef 'diagnostics/networkControllerState'

        # trigger IMOS dump
        "Generate In Memory Object State (IMOS) dump by executing PUT operation against {0}" -f $uri | Trace-Output
        $null = Invoke-WebRequestWithRetry -Method 'Put' -Uri $uri -Credential $Credential -Body "{}" -UseBasicParsing `
        -Headers @{"Accept"="application/json"} -Content "application/json; charset=UTF-8"

        # monitor until the provisionState for the object is not in 'Updating' state
        while ($true) {
            Start-Sleep -Seconds $PollingInterval
            if ($stopWatch.Elapsed.TotalSeconds -gt $ExecutionTimeOut) {
                throw New-Object System.TimeoutException("Operation did not complete within the specified time limit")
            }

            $result = Get-SdnResource -NcUri $NcUri.AbsoluteUri -ResourceRef 'diagnostics/networkControllerState' -Credential $Credential
            if ($result.properties.provisioningState -ine 'Updating') {
                break
            }
        }

        $stopWatch.Stop()

        if ($result.properties.provisioningState -ine 'Succeeded') {
            $msg = "Unable to generate IMOS dump. ProvisioningState: {0}" -f $result.properties.provisioningState
            throw New-Object System.Exception($msg)
        }

        return $true
    }
    catch {
        $_ | Trace-Exception
    }
}

function New-NetworkControllerClusterSecret {
    <#
    .SYNOPSIS
        Decrypt the current secret in ClusterManifest and Generate new one if decrypt success.
    .PARAMETER NcVMs
        The list of Network Controller VMs.
    .PARAMETER NcRestName
        The Network Controller REST Name in FQDN format.
    .PARAMETER ManifestFolder
        The Manifest Folder contains the orginal Manifest Files.
    .PARAMETER ManifestFolderNew
        The New Manifest Folder contains the new Manifest Files. Updated manifest file save here.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [String]
        $OldEncryptedSecret,
        [Parameter(Mandatory = $true)]
        [String]
        $NcRestCertThumbprint,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $decryptedText = Invoke-ServiceFabricDecryptText -CipherText $OldEncryptedSecret

    if($null -eq $decryptedText)
    {
        throw New-Object System.NotSupportedException("Failed to decrypt the secret.")
    }

    $newEncryptedSecret = Invoke-ServiceFabricEncryptText -CertThumbPrint $NcRestCertThumbprint -Text $decryptedText -StoreName MY -StoreLocation LocalMachine -CertStore

    $newDecryptedText = Invoke-ServiceFabricDecryptText -CipherText $newEncryptedSecret

    if ($newDecryptedText -eq $decryptedText) {
        "GOOD, new key and old key are same. Ready for use" | Trace-Output
    }
    else {
        throw New-Object System.NotSupportedException("Decrypted text by new certificate is not matching the old one. We cannot continue.")
    }
    if($null -eq $newEncryptedSecret)
    {
        throw New-Object System.NotSupportedException("Failed to encrypt the secret with new certificate")
    }

    return $newEncryptedSecret
}

function Start-SdnExpiredCertificateRotation {
    <#
    .SYNOPSIS
        Start the Network Controller Certificate Update.
    .DESCRIPTION
        Start the Network Controller Certificate Update.
        This will use the latest issued certificate on each of the NC VMs to replace existing certificates. Ensure below before execute this command:
        - NC Rest Certificate and NC Node certificate created on each NC and trusted.
        - "Network Service" account have read access to the private file of the new certificates.
        - NC Rest Certificate need to be trusted by all SLB MUX VMs and SDN Hosts.
 
        For Self-Signed Certificate. This can also be created by 'New-NetworkControllerCertificate'. To get more details, run 'Get-Help New-NetworkControllerCertificate'
 
        About SDN Certificate Requirement:
        https://docs.microsoft.com/en-us/windows-server/networking/sdn/security/sdn-manage-certs
 
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    .EXAMPLE
        Start-SdnExpiredCertificateRotation -NetworkController nc01
    #>

    param (
        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty
    )

    $NcUpdateFolder = "$(Get-WorkingDirectory)\NcCertUpdate_{0}" -f (Get-FormattedDateTimeUTC)
    $ManifestFolder = "$NcUpdateFolder\manifest"
    $ManifestFolderNew = "$NcUpdateFolder\manifest_new"

    $NcInfraInfo = Get-SdnNetworkControllerInfoOffline -Credential $Credential
    Trace-Output "Network Controller Infrastrucutre Info detected:"
    Trace-Output "ClusterCredentialType: $($NcInfraInfo.ClusterCredentialType)"
    Trace-Output "NcRestName: $($NcInfraInfo.NcRestName)"

    $NcNodeList = $NcInfraInfo.NodeList

    if ($null -eq $NcNodeList -or $NcNodeList.Count -eq 0) {
        Trace-Output "Failed to get NC Node List from NetworkController: $(HostName)" -Level:Error
    }

    Trace-Output "NcNodeList: $($NcNodeList.IpAddressOrFQDN)"

    Trace-Output "Validate CertRotateConfig"
    if(!(Test-SdnCertificateRotationConfig -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential)){
        Trace-Output "Invalid CertRotateConfig, please correct the configuration and try again" -Level:Error
        return
    }

    if ([String]::IsNullOrEmpty($NcInfraInfo.NcRestName)) {
        Trace-Output "Failed to get NcRestName using current secret certificate thumbprint. This might indicate the certificate not found on $(HOSTNAME). We won't be able to recover." -Level:Error
        throw New-Object System.NotSupportedException("Current NC Rest Cert not found, Certificate Rotation cannot be continue.")
    }

    $NcVms = $NcNodeList.IpAddressOrFQDN

    if (Test-Path $NcUpdateFolder) {
        $items = Get-ChildItem $NcUpdateFolder
        if ($items.Count -gt 0) {
            $confirmCleanup = Read-Host "The Folder $NcUpdateFolder not empty. Need to be cleared. Enter Y to confirm"
            if ($confirmCleanup -eq "Y") {
                $items | Remove-Item -Force -Recurse
            }
            else {
                return
            }
        }
    }

    foreach ($nc in $NcVms) {
        Invoke-Command -ComputerName $nc -ScriptBlock {
            Write-Host "[$(HostName)] Stopping Service Fabric Service"
            Stop-Service FabricHostSvc -Force
        }
    }

    Trace-Output "Step 1 Copy manifests and settings.xml from Network Controller"
    Copy-ServiceFabricManifestFromNetworkController -NcNodeList $NcNodeList -ManifestFolder $ManifestFolder -ManifestFolderNew $ManifestFolderNew -Credential $Credential

    # Step 2 Update certificate thumbprint
    Trace-Output "Step 2 Update certificate thumbprint and secret in manifest"
    Update-NetworkControllerCertificateInManifest -NcNodeList $NcNodeList -ManifestFolder $ManifestFolder -ManifestFolderNew $ManifestFolderNew -CertRotateConfig $CertRotateConfig -Credential $Credential

    # Step 3 Copy the new files back to the NC vms
    Trace-Output "Step 3 Copy the new files back to the NC vms"
    Copy-ServiceFabricManifestToNetworkController -NcNodeList $NcNodeList -ManifestFolder $ManifestFolderNew -Credential $Credential

    # Step 5 Start FabricHostSvc and wait for SF system service to become healty
    Trace-Output "Step 4 Start FabricHostSvc and wait for SF system service to become healty"
    Trace-Output "Step 4.1 Update Network Controller Certificate ACL to allow 'Network Service' Access"
    Update-NetworkControllerCertificateAcl -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential
    Trace-Output "Step 4.2 Start Service Fabric Host Service and wait"
    $clusterHealthy = Wait-ServiceFabricClusterHealthy -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential
    Trace-Output "ClusterHealthy: $clusterHealthy"
    if($clusterHealthy -ne $true){
        throw New-Object System.NotSupportedException("Cluster unheathy after manifest update, we cannot continue with current situation")
    }
    # Step 6 Invoke SF Cluster Upgrade
    Trace-Output "Step 5 Invoke SF Cluster Upgrade"
    Update-ServiceFabricCluster -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -ManifestFolderNew $ManifestFolderNew -Credential $Credential
    $clusterHealthy = Wait-ServiceFabricClusterHealthy -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential
    Trace-Output "ClusterHealthy: $clusterHealthy"
    if($clusterHealthy -ne $true){
        throw New-Object System.NotSupportedException("Cluster unheathy after cluster update, we cannot continue with current situation")
    }

    # Step 7 Fix NC App
    Trace-Output "Step 6 Fix NC App"
    Trace-Output "Step 6.1 Updating Network Controller Global and Cluster Config"
    if ($NcInfraInfo.ClusterCredentialType -eq "X509") {
        Update-NetworkControllerConfig -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential
    }

    # Step 7 Restart
    Trace-Output "Step 7 Restarting Service Fabric Cluster after configuration change"
    $clusterHealthy = Wait-ServiceFabricClusterHealthy -NcNodeList $NcNodeList -CertRotateConfig $CertRotateConfig -Credential $Credential -Restart

<# Trace-Output "Step 7.2 Rotate Network Controller Certificate"
    #$null = Invoke-CertRotateCommand -Command 'Set-NetworkController' -Credential $Credential -Thumbprint $NcRestCertThumbprint
 
    # Step 8 Update REST CERT credential
    Trace-Output "Step 8 Update REST CERT credential"
    # Step 8.1 Wait for NC App Healthy
    Trace-Output "Step 8.1 Wiating for Network Controller App Ready"
    #Wait-NetworkControllerAppHealthy -Interval 60
    Trace-Output "Step 8.2 Updating REST CERT Credential object calling REST API" #>
    #Update-NetworkControllerCredentialResource -NcUri "https://$($NcInfraInfo.NcRestName)" -NewRestCertThumbprint $NcRestCertThumbprint -Credential $NcRestCredential
}

function Update-NetworkControllerCertificateAcl {
    <#
    .SYNOPSIS
        Update the Network Controller Certificate to grant Network Service account read access to the private key.
    .PARAMETER NcNodeList
        The NcNodeList that retrieved via Get-SdnNetworkControllerInfoOffline.
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        $NcRestCertThumbprint = $CertRotateConfig["NcRestCert"]

        foreach ($ncNode in $NcNodeList) {
            $ncNodeCertThumbprint = $CertRotateConfig[$ncNode.NodeName.ToLower()]
            Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                Set-SdnCertificateAcl -Path $param1 -Thumbprint $param2
            } -ArgumentList @('Cert:\LocalMachine\My', $NcRestCertThumbprint)

            if ($CertRotateConfig["ClusterCredentialType"] -ieq "X509") {
                Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                    param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                    Set-SdnCertificateAcl -Path $param1 -Thumbprint $param2
                } -ArgumentList @('Cert:\LocalMachine\My', $ncNodeCertThumbprint)
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Update-NetworkControllerCertificateInManifest {
    <#
    .SYNOPSIS
        Update Network Controller Manifest File with new Network Controller Certificate.
    .PARAMETER NcVMs
        The list of Network Controller VMs.
    .PARAMETER ManifestFolder
        The Manifest Folder contains the orginal Manifest Files.
    .PARAMETER ManifestFolderNew
        The New Manifest Folder contains the new Manifest Files. Updated manifest file save here.
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolder,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolderNew,
        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    if ($NcNodeList.Count -eq 0) {
        throw New-Object System.NotSupportedException("NcNodeList is empty")
    }

    # Prepare the cert thumbprint to be used
    # Update certificates ClusterManifest.current.xml

    $clusterManifestXml = [xml](Get-Content "$ManifestFolder\ClusterManifest.current.xml")

    if ($null -eq $clusterManifestXml) {
        Trace-Output "ClusterManifest not found at $ManifestFolder\ClusterManifest.current.xml" -Level:Error
        throw
    }

    $NcRestCertThumbprint = $CertRotateConfig["NcRestCert"]

    # Update encrypted secret
    # Get encrypted secret from Cluster Manifest
    $fileStoreServiceSection = ($clusterManifestXml.ClusterManifest.FabricSettings.Section | Where-Object name -eq FileStoreService)
    $OldEncryptedSecret = ($fileStoreServiceSection.Parameter | Where-Object Name -eq "PrimaryAccountNTLMPasswordSecret").Value
    $newEncryptedSecret = New-NetworkControllerClusterSecret -OldEncryptedSecret $OldEncryptedSecret -NcRestCertThumbprint $NcRestCertThumbprint -Credential $Credential

    # Update new encrypted secret in Cluster Manifest
    ($fileStoreServiceSection.Parameter | Where-Object Name -eq "PrimaryAccountNTLMPasswordSecret").Value = "$newEncryptedSecret"
    ($fileStoreServiceSection.Parameter | Where-Object Name -eq "SecondaryAccountNTLMPasswordSecret").Value = "$newEncryptedSecret"

    # Update SecretsCertificate to new REST Cert

    Trace-Output "Updating SecretsCertificate with new rest cert thumbprint $NcRestCertThumbprint"
    $clusterManifestXml.ClusterManifest.Certificates.SecretsCertificate.X509FindValue = "$NcRestCertThumbprint"

    $securitySection = $clusterManifestXml.ClusterManifest.FabricSettings.Section | Where-Object Name -eq "Security"
    $ClusterCredentialType = $securitySection.Parameter | Where-Object Name -eq "ClusterCredentialType"

    $infrastructureManifestXml = [xml](Get-Content "$ManifestFolder\InfrastructureManifest.xml")

    # Update Node Certificate to new Node Cert if the ClusterCredentialType is X509 certificate
    if($ClusterCredentialType.Value -eq "X509")
    {
        foreach ($node in $clusterManifestXml.ClusterManifest.NodeTypes.NodeType) {
            $ncNode = $node.Name
            $ncNodeCertThumbprint = $CertRotateConfig[$ncNode.ToLower()]
            Write-Verbose "Updating node $ncNode with new thumbprint $ncNodeCertThumbprint"
            $node.Certificates.ClusterCertificate.X509FindValue = "$ncNodeCertThumbprint"
            $node.Certificates.ServerCertificate.X509FindValue = "$ncNodeCertThumbprint"
            $node.Certificates.ClientCertificate.X509FindValue = "$ncNodeCertThumbprint"
        }

        # Update certificates InfrastructureManifest.xml

        foreach ($node in $infrastructureManifestXml.InfrastructureInformation.NodeList.Node) {
            $ncNodeCertThumbprint = $CertRotateConfig[$node.NodeName.ToLower()]
            $node.Certificates.ClusterCertificate.X509FindValue = "$ncNodeCertThumbprint"
            $node.Certificates.ServerCertificate.X509FindValue = "$ncNodeCertThumbprint"
            $node.Certificates.ClientCertificate.X509FindValue = "$ncNodeCertThumbprint"
        }
    }

    # Update certificates for settings.xml
    foreach ($ncNode in $NcNodeList) {
        $ncVm = $ncNode.IpAddressOrFQDN
        $settingXml = [xml](Get-Content "$ManifestFolder\$ncVm\Settings.xml")
        if($ClusterCredentialType.Value -eq "X509")
        {
            $ncNodeCertThumbprint = $CertRotateConfig[$ncNode.NodeName.ToLower()]
            $fabricNodeSection = $settingXml.Settings.Section | Where-Object Name -eq "FabricNode"
            $parameterToUpdate = $fabricNodeSection.Parameter | Where-Object Name -eq "ClientAuthX509FindValue"
            $parameterToUpdate.Value = "$ncNodeCertThumbprint"
            $parameterToUpdate = $fabricNodeSection.Parameter | Where-Object Name -eq "ServerAuthX509FindValue"
            $parameterToUpdate.Value = "$ncNodeCertThumbprint"
            $parameterToUpdate = $fabricNodeSection.Parameter | Where-Object Name -eq "ClusterX509FindValue"
            $parameterToUpdate.Value = "$ncNodeCertThumbprint"
        }

        # Update encrypted secret in settings.xml
        $fileStoreServiceSection = $settingXml.Settings.Section | Where-Object Name -eq "FileStoreService"
        ($fileStoreServiceSection.Parameter | Where-Object Name -eq "PrimaryAccountNTLMPasswordSecret").Value = "$newEncryptedSecret"
        ($fileStoreServiceSection.Parameter | Where-Object Name -eq "SecondaryAccountNTLMPasswordSecret").Value = "$newEncryptedSecret"

        $settingXml.Save("$ManifestFolderNew\$ncVm\Settings.xml")
    }

    $infrastructureManifestXml.Save("$ManifestFolderNew\InfrastructureManifest.xml")
    $clusterManifestXml.Save("$ManifestFolderNew\ClusterManifest.current.xml")
}

function Update-NetworkControllerConfig {
    <#
    .SYNOPSIS
        Update the Network Controller Application Global Config with new certificate info. This to be run on Network Controller only.
    .PARAMETER NcNodeList
        The NcNodeList that retrieved via Get-SdnNetworkControllerInfoOffline.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    #>

    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $globalConfigUri = "GlobalConfiguration"
    $clusterConfigUri = "ClusterConfiguration"
    $globalConfigs = Get-SdnServiceFabricClusterConfig -Uri $globalConfigUri
    $clusterConfigs = Get-SdnServiceFabricClusterConfig -Uri $clusterConfigUri

    foreach ($ncNode in $NcNodeList) {
        $nodeCertThumbprint = $CertRotateConfig[$ncNode.NodeName.ToLower()]
        if($null -eq $nodeCertThumbprint){
            throw New-Object System.NotSupportedException("NodeCertificateThumbprint not found for $($ncNode.NodeName)")
        }
        $thumbprintPropertyName = "{0}.ClusterCertThumbprint" -f $ncNode.NodeName
        # Global Config property name like Global.Version.NodeName.ClusterCertThumbprint
        $thumbprintProperty = $globalConfigs | Where-Object Name -Match $thumbprintPropertyName

        if($null -ne $thumbprintProperty){
            "GlobalConfiguration: Property $($thumbprintProperty.Name) will be updated from $($thumbprintProperty.Value) to $nodeCertThumbprint" | Trace-Output
            Set-SdnServiceFabricClusterConfig -Uri $globalConfigUri -Name $thumbprintProperty.Name.ToString() -Value $nodeCertThumbprint
        }

        # Cluster Config property name like NodeName.ClusterCertThumbprint
        $thumbprintProperty = $clusterConfigs | Where-Object Name -ieq $thumbprintPropertyName

        # If NodeName.ClusterCertThumbprint exist (for Server 2022 +), Update
        if($null -ne $thumbprintProperty){
            "ClusterConfiguration: Property $($thumbprintProperty.Name) will be updated from $($thumbprintProperty.Value) to $nodeCertThumbprint" | Trace-Output
            Set-SdnServiceFabricClusterConfig -Uri $clusterConfigUri -Name $thumbprintProperty.Name.ToString() -Value $nodeCertThumbprint
        }

        $certProperty = $clusterConfigs | Where-Object Name -ieq $ncNode.NodeName
        if($null -ne $certProperty){
            $nodeCert = Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock{
                param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                return Get-SdnCertificate -Path $param1 -Thumbprint $param2
            } -ArgumentList @('Cert:\LocalMachine\My', $nodeCertThumbprint)

            "ClusterConfiguration: Property $($certProperty.Name) will be updated From :`n$($certProperty.Value) `nTo : `n$nodeCert" | Trace-Output
            Set-SdnServiceFabricClusterConfig -Uri $clusterConfigUri -Name $certProperty.Name.ToString() -Value $nodeCert.GetRawCertData()
        }
    }
}

function Update-NetworkControllerCredentialResource {
    <#
    .SYNOPSIS
        Update the Credential Resource in Network Controller with new certificate.
    .PARAMETER NcUri
        The Network Controller REST URI.
    .PARAMETER NewRestCertThumbprint
        The new Network Controller REST Certificate Thumbprint to be used by credential resource.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    param (
        [Parameter(Mandatory = $true)]
        [System.String]
        $NcUri,

        [Parameter(Mandatory = $true)]
        [System.String]
        $NewRestCertThumbprint,

        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $headers = @{"Accept"="application/json"}
    $content = "application/json; charset=UTF-8"
    $timeoutInMinutes = 10
    $array = @()

    $servers = Get-SdnServer -NcUri $NcUri -Credential $Credential
    foreach ($object in $servers) {
        "Processing X509 connections for {0}" -f $object.resourceRef | Trace-Output
        foreach ($connection in $servers.properties.connections | Where-Object {$_.credentialType -ieq 'X509Certificate'}) {
            $stopWatch = [System.Diagnostics.Stopwatch]::StartNew()

            $cred = Get-SdnResource -NcUri $NcUri -ResourceRef $connection.credential.resourceRef -Credential $Credential

            # if for any reason the certificate thumbprint has been updated, then skip the update operation for this credential resource
            if ($cred.properties.value -ieq $NewRestCertThumbprint) {
                "{0} has already updated to {1}" -f $cred.resourceRef, $NewRestCertThumbprint | Trace-Output
                continue
            }

            "{0} will be updated from {1} to {2}" -f $cred.resourceRef, $cred.properties.value, $NewRestCertThumbprint | Trace-Output
            $cred.properties.value = $NewRestCertThumbprint
            $credBody = $cred | ConvertTo-Json -Depth 100

            [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri -ResourceRef $cred.resourceRef
            $null = Invoke-WebRequestWithRetry -Method 'Put' -Uri $uri -Credential $Credential -UseBasicParsing `
            -Headers $headers -ContentType $content -Body $credBody

            while ($true) {
                if ($stopWatch.Elapsed.TotalMinutes -ge $timeoutInMinutes) {
                    $stopWatch.Stop()
                    throw New-Object System.TimeoutException("Update of $($cred.resourceRef) did not complete within the alloted time")
                }

                $result = Invoke-RestMethodWithRetry -Method 'Get' -Uri $uri -Credential $Credential -UseBasicParsing
                if ($result.properties.provisioningState -ieq 'Updating') {
                    "Status: {0}" -f $result.properties.provisioningState | Trace-Output
                    Start-Sleep -Seconds 15
                }
                elseif ($result.properties.provisioningState -ieq 'Failed') {
                    $stopWatch.Stop()
                    throw New-Object System.Exception("Failed to update $($cred.resourceRef)")
                }
                elseif ($result.properties.provisioningState -ieq 'Succeeded') {
                    "Successfully updated {0}" -f $cred.resourceRef | Trace-Output
                    break
                }
                else {
                    $stopWatch.Stop()
                    throw New-Object System.Exception("Failed to update $($cred.resourceRef) with $($result.properties.provisioningState)")
                }
            }

            $array += $result
        }
    }

    return $array
}

function Update-ServiceFabricCluster {
    <#
    .SYNOPSIS
        Upgrade the Service Fabric Cluster via Start-ServiceFabricClusterUpgrade and wait for the cluster to become healthy.
    .PARAMETER NcNodeList
        The list of Network Controller Nodes.
    .PARAMETER ClusterCredentialType
        X509, Windows or None.
    .PARAMETER ManifestFolderNew
        The New Manifest Folder contains the new Manifest Files.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,
        [Parameter(Mandatory = $true)]
        [String]
        $ManifestFolderNew,
        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    if ($NcNodeList.Count -eq 0) {
        throw New-Object System.NotSupportedException("NcNodeList is empty")
    }

    # Update the cluster manifest version to 1
    $clusterManifestXml = [xml](Get-Content "$ManifestFolderNew\ClusterManifest.current.xml")
    $currentVersionArray = $clusterManifestXml.ClusterManifest.Version.Split('.')
    $minorVersionIncrease = [int]$currentVersionArray[$currentVersionArray.Length - 1] + 1
    $currentVersionArray[$currentVersionArray.Length - 1] = $minorVersionIncrease
    $newVersionString = $currentVersionArray -Join '.'
    "Upgrade Service Fabric from $($clusterManifestXml.ClusterManifest.Version) to $newVersionString" | Trace-Output
    $clusterManifestXml.ClusterManifest.Version = $newVersionString
    $clusterManifestXml.Save("$ManifestFolderNew\ClusterManifest_new.xml")

    $currentNcNode = $null
    # Start Service Fabric Service for each NC
    foreach ($ncNode in $NcNodeList) {
        if(Test-ComputerNameIsLocal -ComputerName $ncNode.IpAddressOrFQDN){
            $currentNcNode = $ncNode
        }
    }
    $certThumb = $CertRotateConfig[$currentNcNode.NodeName.ToLower()]

    $clusterManifestPath = "$ManifestFolderNew\ClusterManifest_new.xml"

    if (!(Test-Path $clusterManifestPath)) {
        Throw "Path $clusterManifestPath not found"
    }

    "Upgrading Service Fabric Cluster with ClusterManifest at $clusterManifestPath" | Trace-Output

    # Sometimes access denied returned for the copy call, retry here to workaround this.
    $maxRetry = 3
    while($maxRetry -gt 0){
        try{
            if($CertRotateConfig["ClusterCredentialType"] -ieq "X509"){
                "Connecting to Service Fabric Cluster using cert with thumbprint: {0}" -f $certThumb | Trace-Output
                Connect-ServiceFabricCluster -X509Credential -FindType FindByThumbprint -FindValue $certThumb -ConnectionEndpoint "$($currentNcNode.IpAddressOrFQDN):49006" | Out-Null
            }
            else{
                Connect-ServiceFabricCluster | Out-Null
            }
            Copy-ServiceFabricClusterPackage -Config -ImageStoreConnectionString "fabric:ImageStore" -ClusterManifestPath  $clusterManifestPath -ClusterManifestPathInImageStore "ClusterManifest.xml"
            break
        }catch{
            "Copy-ServiceFabricClusterPackage failed with exception $_.Exception. Retry $(4 - $maxRetry)/3 after 60 seconds" | Trace-Output -Level:Warning
            Start-Sleep -Seconds 60
            $maxRetry --
        }
    }

    Register-ServiceFabricClusterPackage -Config -ClusterManifestPath "ClusterManifest.xml"
    Start-ServiceFabricClusterUpgrade -ClusterManifestVersion $NewVersionString -Config -UnmonitoredManual -UpgradeReplicaSetCheckTimeoutSec 30

    while ($true) {
        $upgradeStatus = Get-ServiceFabricClusterUpgrade
        "Current upgrade state: $($upgradeStatus.UpgradeState) UpgradeDomains: $($upgradeStatus.UpgradeDomains)" | Trace-Output
        if ($upgradeStatus.UpgradeState -eq "RollingForwardPending") {
            $nextNode = $upgradeStatus.NextUpgradeDomain
            "Next node to upgrade $nextNode" | Trace-Output
            try{
                Resume-ServiceFabricClusterUpgrade -UpgradeDomainName $nextNode
                # Catch exception for resume call, as sometimes, the upgrade status not updated intime caused duplicate resume call.
            }catch{
                "Exception in Resume-ServiceFabricClusterUpgrade $_.Exception" | Trace-Output -Level:Warning
            }
        }
        elseif ($upgradeStatus.UpgradeState -eq "Invalid" `
                -or $upgradeStatus.UpgradeState -eq "Failed") {
            Throw "Something wrong with the upgrade"
        }
        elseif ($upgradeStatus.UpgradeState -eq "RollingBackCompleted" `
                -or $upgradeStatus.UpgradeState -eq "RollingForwardCompleted") {
            "Upgrade has been completed" | Trace-Output
            break
        }
        else {
            "Waiting for current node upgrade to complete" | Trace-Output
        }

        Start-Sleep -Seconds 60
    }
}

function Wait-NetworkControllerAppHealthy {
    <#
    .SYNOPSIS
        Query the Network Controller App Health Status. Wait for the Network Controller App become healthy when $Interval specified.
    .PARAMETER NetworkController
        Specifies one of the Network Controller VM name.
    .PARAMETER Interval
        App healh status query interval until the App become healthy, default to 0 means no retry of the health status query.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]
        $NetworkController,
        [Parameter(Mandatory = $false)]
        [Int32]
        $Interval = 0,
        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        $scriptBlock = {
            param (
                [Int32]
                $Interval = 0
            )
            $isApplicationHealth = $false;
            Write-Host "[$(HostName)] Query Network Controller App Health"
            while($isApplicationHealth -ne $true){
                #Connect-ServiceFabricCluster -X509Credential -FindType FindByThumbprint -FindValue $certThumb -ConnectionEndpoint "$($NodeFQDN):49006" | Out-Null
                #Cluster should have been back to normal when reach here use default parameters to connect
                Connect-ServiceFabricCluster | Out-Null
                $clusterHealth = Get-ServiceFabricClusterHealth
                if ($clusterHealth.AggregatedHealthState -ne "Ok") {
                    if ($clusterHealth.NodeHealthStates -ne "Ok") {
                        Get-ServiceFabricNode -StatusFilter All | Format-Table Nodename, Nodestatus, HealthState, IpAddressOrFQDN, NodeUptime -autosize
                    }
                    $applicationStatus = Get-ServiceFabricApplication -ApplicationName fabric:/NetworkController
                    if ($applicationStatus.HealthState -ne "Ok") {
                        $applicationStatus | Format-Table ApplicationName, ApplicationStatus, HealthState -AutoSize
                        $services = Get-ServiceFabricService -ApplicationName fabric:/NetworkController
                        $allServiceHealth = $true;
                        foreach ($service in $services) {
                            if($service.HealthState -notlike "Ok"){
                                $allServiceHealth = $false;
                            }
                        }
                        if($allServiceHealth -and $services.Count -gt 0)
                        {
                            $isApplicationHealth = $true
                            break
                        }

                        $services | Format-Table ServiceName, ServiceStatus, HealthState -AutoSize
                    }
                    else {
                        $isApplicationHealth = $true
                    }

                    $systemStatus = Get-ServiceFabricService -ApplicationName fabric:/System
                    if ($systemStatus.HealthState -ne "Ok") {
                        $systemStatus | Format-Table ServiceName, ServiceStatus, HealthState -AutoSize
                    }
                }else{
                    $isApplicationHealth = $true;
                }

                Write-Host "[$(HostName)] Current Network Controller Health Status: $isApplicationHealth"
                if($Interval -gt 0)
                {
                    Start-Sleep -Seconds $Interval
                }else{
                    break
                }
            }
        }

        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController')))
        {
            Invoke-Command -ScriptBlock $scriptBlock -ArgumentList $Interval
        }
        else{
            Invoke-Command -ComputerName $NetworkController -ScriptBlock $scriptBlock -ArgumentList $Interval -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Wait-ServiceFabricClusterHealthy {
    <#
    .SYNOPSIS
        Start the FabricHostSvc on each of the Network Controller VM and wait for the service fabric service to become healthy.
    .PARAMETER NcVMs
        The list of Network Controller VMs.
    .PARAMETER ClusterCredentialType
        X509, Windows or None.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]
        $NcNodeList,

        [Parameter(Mandatory = $true)]
        [hashtable]
        $CertRotateConfig,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [switch]
        $Restart
    )

    try {
        $currentNcNode = $null

        # Start Service Fabric Service for each NC
        foreach ($ncNode in $NcNodeList) {
            if(Test-ComputerNameIsLocal -ComputerName $ncNode.IpAddressOrFQDN){
                $currentNcNode = $ncNode
            }

            Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][Switch]$param1)
                if ($param) {
                    Stop-Service -Name 'FabricHostSvc' -Force
                    Start-Sleep -Seconds 5
                }

                Start-Service -Name 'FabricHostSvc'
            } -ArgumentList $Restart
        }

        Trace-Output "Sleeping 60s to wait for Serice Fabric Service to be ready"
        Start-Sleep -Seconds 60
        "Waiting for service fabric service healthy" | Trace-Output
        $NodeFQDN = (get-ciminstance win32_computersystem).DNSHostName + "." + (get-ciminstance win32_computersystem).Domain
        $certThumb = $CertRotateConfig[$currentNcNode.NodeName.ToLower()]

        $maxRetry = 10
        $clusterConnected = $false
        while ($maxRetry -gt 0) {
            if(!$clusterConnected){
                try{
                    "Service fabric cluster connect attempt $(11 - $maxRetry)/10" | Trace-Output
                    if ($CertRotateConfig["ClusterCredentialType"] -ieq "X509") {
                        "Connecting to Service Fabric Cluster using cert with thumbprint: {0}" -f $certThumb | Trace-Output
                        Connect-ServiceFabricCluster -X509Credential -FindType FindByThumbprint -FindValue $certThumb  -ConnectionEndpoint "$($NodeFQDN):49006" | Out-Null
                    }
                    else {
                        Connect-ServiceFabricCluster | Out-Null
                    }
                    $clusterConnected = $true
                }catch{
                    $maxRetry --
                    continue
                }
            }

            if($clusterConnected){
                $services = @()
                $services = Get-ServiceFabricService -ApplicationName fabric:/System
                $allServiceHealth = $true
                if ($services.Count -eq 0) {
                    "No service fabric services retrieved yet" | Trace-Output -Level:Warning
                }

                foreach ($service in $services) {
                    if ($service.ServiceStatus -ne "Active" -or $service.HealthState -ne "Ok" ) {
                        "$($service.ServiceName) ServiceStatus: $($service.ServiceStatus) HealthState: $($service.HealthState)" | Trace-Output -Level:Warning
                        $allServiceHealth = $false
                    }
                }
                if ($allServiceHealth -and $services.Count -gt 0) {
                    "All service fabric service has been healthy" | Trace-Output -Level:Warning
                    return $allServiceHealth
                }

                Start-Sleep -Seconds 10
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnApiEndpoint {
    <#
    .SYNOPSIS
        Used to construct the URI endpoint for Network Controller NB API
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ApiVersion
        The API version to use when invoking against the NC REST API endpoint. By default, reads from $Global:SdnDiagnostics.EnvironmentInfo.RestApiVersion
        which defaults to 'v1' unless explicity overwritten, or 'Get-SdnInfrastructureInfo' is called.
    .PARAMETER ResourceName
        Network Controller resource exposed via NB API interface of Network Controller, as defined under https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-ncnbi/6dbabf43-0fcd-439c-81e2-7eb794f7c140.
    .PARAMETER OperationId
        Operation ID for diagnostics operation. This is optional and only used for certain resources.
    .PARAMETER ResourceRef
        The exact resource reference in format of /resourceName/{resourceId}/childObject/{resourceId}
    .EXAMPLE
        PS> Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceName 'VirtualNetworks'
    .EXAMPLE
        PS> Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceName '/virtualnetworks/contoso-vnet01'
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceName')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceName')]
        [System.String]$ApiVersion = $Global:SdnDiagnostics.EnvironmentInfo.RestApiVersion,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceName')]
        [System.String]$ResourceName,

        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceName')]
        [System.String]$OperationId,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [System.String]$ResourceRef
    )

    switch ($PSCmdlet.ParameterSetName) {
        'ResourceRef' {
            $ResourceRef = $ResourceRef.TrimStart('/')
            if ($resourceRef -ilike "Discovery*") {
                [System.String]$endpoint = "{0}/networking/{1}" -f $NcUri.AbsoluteUri.TrimEnd('/'), $ResourceRef
            }
            else {
                [System.String]$endpoint = "{0}/networking/{1}/{2}" -f $NcUri.AbsoluteUri.TrimEnd('/'), $ApiVersion, $ResourceRef
            }
        }
        'ResourceName' {
            $apiEndpointProperties = $Script:SdnDiagnostics_NC.Config.Properties.ApiResources[$ResourceName]
            if ([string]::IsNullOrEmpty($apiEndpointProperties.minVersion)) {
                [System.String]$endpoint = "{0}/networking/{1}" -f $NcUri.AbsoluteUri.TrimEnd('/'), $apiEndpointProperties.uri
            }
            else {
                [System.String]$endpoint = "{0}/networking/{1}/{2}" -f $NcUri.AbsoluteUri.TrimEnd('/'), $ApiVersion, $apiEndpointProperties.uri
            }

            if ($apiEndpointProperties.operationId -and (-NOT ([System.String]::IsNullOrEmpty($OperationId)))) {
                $endpoint = "{0}/{1}" -f $endpoint, $OperationId
            }
        }
    }

    $endpoint = $endpoint.TrimEnd('/')
    "Endpoint: {0}" -f $endpoint | Trace-Output -Level:Verbose

    return $endpoint
}


function Get-SdnAuditLog {
    <#
    .SYNOPSIS
        Collects the audit logs for Network Security Groups (NSG) from the hypervisor hosts
    .PARAMETER OutputDirectory
        Directory the results will be saved to. If ommitted, will default to the current working directory.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER NCRestCredential
        Specifies a user account that has permission to access the northbound NC API interface. The default is the current user.
    .PARAMETER ComputerName
        Type the NetBIOS name, an IP address, or a fully qualified domain name of one or more remote compute
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String]$OutputDirectory = "$(Get-WorkingDirectory)\AuditLogs",

        [Parameter(Mandatory = $true)]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ValueFromPipeline)]
        [System.String[]]$ComputerName,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    # verify that the environment we are on supports at least v3 API and later
    # as described in https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-ncnbi/dc23b547-9ec4-4cb3-ab20-a6bfe01ddafb
    $currentRestVersion = (Get-SdnResource -NcUri $NcUri.AbsoluteUri -Resource 'Discovery' -Credential $NcRestCredential).properties.currentRestVersion
    [int]$currentRestVersionInt = $currentRestVersion.Replace('V','').Replace('v','').Trim()
    if ($currentRestVersionInt -lt 3) {
        "Auditing requires API version 3 or later. Network Controller supports version {0}" -f $currentRestVersionInt | Trace-Output -Level:Warning
        return
    }

    # check to see that auditing has been enabled
    $auditSettingsConfig = Get-SdnResource -NcUri $NcUri.AbsoluteUri -Resource 'AuditingSettingsConfig' -ApiVersion $currentRestVersion -Credential $NcRestCredential
    if ([string]::IsNullOrEmpty($auditSettingsConfig.properties.outputDirectory)) {
        "Audit logging is not enabled" | Trace-Output
        return
    }
    else {
        "Audit logging location: {0}" -f $auditSettingsConfig.properties.outputDirectory | Trace-Output
    }

    # if $ComputerName was not specified, then attempt to locate the servers within the SDN fabric
    # only add the servers where auditingEnabled has been configured as 'Firewall'
    if ($null -eq $ComputerName) {
        $sdnServers = Get-SdnResource -Resource Servers -NcUri $NcUri.AbsoluteUri -Credential $NcRestCredential -ApiVersion $currentRestVersion `
        | Where-Object {$_.properties.auditingEnabled -ieq 'Firewall'}

        $ComputerName = ($sdnServers.properties.connections | Where-Object {$_.credentialType -ieq 'UsernamePassword'}).managementAddresses
    }

    $ComputerName | ForEach-Object {
        "Collecting audit logs from {0}" -f $_ | Trace-Output
        $outputDir = Join-Path -Path $OutputDirectory -ChildPath $_.ToLower()
        Copy-FileFromRemoteComputer -ComputerName $_ -Credential $Credential -Path $auditSettingsConfig.properties.outputDirectory -Destination $outputDir -Recurse -Force
    }
}

function Get-SdnGateway {
    <#
    .SYNOPSIS
        Returns a list of gateways from network controller.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ResourceId
        Specifies the unique identifier for the resource.
    .PARAMETER ResourceRef
        Specifies the resource reference for the resource.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER ManagementAddressOnly
        Optional parameter to only return back the Management Address value.
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential)
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef 'gateways/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnGateway -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef 'gateways/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    #>

    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [String]$ResourceId,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [String]$ResourceRef,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [switch]$ManagementAddressOnly
    )

    $params = @{
        NcUri = $NcUri
        Credential = $Credential
    }

    switch ($PSCmdlet.ParameterSetName) {
        'ResourceId' {
            $params.Add('Resource', 'Gateways')
            $params.Add('ResourceId', $ResourceId)
        }
        'ResourceRef' {
            $params.Add('ResourceRef', $ResourceRef)
        }
        default {
            $params.Add('Resource', 'Gateways')
        }
    }

    try {
        $result = Get-SdnResource @params
        if ($result) {
            foreach($obj in $result){
                if($obj.properties.provisioningState -ne 'Succeeded'){
                    "{0} is reporting provisioningState: {1}" -f $obj.resourceId, $obj.properties.provisioningState | Trace-Output -Level:Warning
                }
            }

            if($ManagementAddressOnly){
                $connections = @()
                foreach ($resource in $result) {
                    $virtualServerMgmtAddress = Get-SdnVirtualServer -NcUri $NcUri.AbsoluteUri -ResourceRef $resource.properties.virtualserver.ResourceRef -ManagementAddressOnly -Credential $Credential
                    $connections += $virtualServerMgmtAddress
                }

                return $connections
            }
            else {
                return $result
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnInfrastructureInfo {
    <#
    .SYNOPSIS
        Get the SDN infrastructure information from network controller. The function will update the $Global:SdnDiagnostics.EnvironmentInfo variable.
    .PARAMETER NetworkController
        Specifies the name or IP address of the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER NcRestCredential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER Force
        Switch parameter to force a refresh of the environment cache details
    .EXAMPLE
        PS> Get-SdnInfrastructureInfo
    .EXAMPLE
        PS> Get-SdnInfrastructureInfo -NetworkController 'NC01' -Credential (Get-Credential) -NcRestCredential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [ValidateScript({
            if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") {
                throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.")
            }
            return $true
        })]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [Switch]$Force
    )

    try {
        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        # if force is defined, purge the cache to force a refresh on the objects
        if ($PSBoundParameters.ContainsKey('Force')) {
            $Global:SdnDiagnostics.EnvironmentInfo.NcUrl = $null
            $global:SdnDiagnostics.EnvironmentInfo.NetworkController = $null
            $global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux = $null
            $Global:SdnDiagnostics.EnvironmentInfo.RasGateway = $null
            $Global:SdnDiagnostics.EnvironmentInfo.Server = $null
            $Global:SdnDiagnostics.EnvironmentInfo.FabricNodes = $null
        }

        # get the NC Northbound API endpoint
        if ($NcUri) {
            $Global:SdnDiagnostics.EnvironmentInfo.NcUrl = $NcUri.AbsoluteUri
        }
        elseif ([System.String]::IsNullOrEmpty($Global:SdnDiagnostics.EnvironmentInfo.NcUrl)) {
            $result = Get-SdnNetworkControllerRestURL -NetworkController $NetworkController -Credential $Credential

            if ($null -eq $result) {
                throw New-Object System.NullReferenceException("Unable to locate REST API endpoint for Network Controller. Please specify REST API with -RestUri parameter.")
            }

            $Global:SdnDiagnostics.EnvironmentInfo.NcUrl = $result
        }

        # get the supported rest API versions from network controller
        # as we default this to v1 on module import within $Global.SdnDiagnostics, will not check to see if null first
        $currentRestVersion = (Get-SdnDiscovery -NcUri $Global:SdnDiagnostics.EnvironmentInfo.NcUrl -Credential $NcRestCredential).properties.currentRestVersion
        if (-NOT [String]::IsNullOrEmpty($currentRestVersion)) {
            $Global:SdnDiagnostics.EnvironmentInfo.RestApiVersion = $currentRestVersion
        }

        # get the network controllers
        if ([System.String]::IsNullOrEmpty($global:SdnDiagnostics.EnvironmentInfo.NetworkController)) {
            [System.Array]$global:SdnDiagnostics.EnvironmentInfo.NetworkController = Get-SdnNetworkControllerNode -NetworkController $NetworkController -ServerNameOnly -Credential $Credential
        }

        # get the load balancer muxes
        if ([System.String]::IsNullOrEmpty($global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux)) {
            [System.Array]$global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux = Get-SdnLoadBalancerMux -NcUri $Global:SdnDiagnostics.EnvironmentInfo.NcUrl -ManagementAddressOnly -Credential $NcRestCredential
        }

        # get the gateways
        if ([System.String]::IsNullOrEmpty($Global:SdnDiagnostics.EnvironmentInfo.Gateway)) {
            [System.Array]$Global:SdnDiagnostics.EnvironmentInfo.Gateway = Get-SdnGateway -NcUri $Global:SdnDiagnostics.EnvironmentInfo.NcUrl -ManagementAddressOnly -Credential $NcRestCredential
        }

        # get the hypervisor hosts
        if ([System.String]::IsNullOrEmpty($Global:SdnDiagnostics.EnvironmentInfo.Server)) {
            [System.Array]$Global:SdnDiagnostics.EnvironmentInfo.Server = Get-SdnServer -NcUri $Global:SdnDiagnostics.EnvironmentInfo.NcUrl -ManagementAddressOnly -Credential $NcRestCredential
        }

        # populate the global cache that contains the names of the nodes for the roles defined above
        $fabricNodes = @()
        $fabricNodes += $global:SdnDiagnostics.EnvironmentInfo.NetworkController

        if($null -ne $Global:SdnDiagnostics.EnvironmentInfo.Server){
            $fabricNodes += $Global:SdnDiagnostics.EnvironmentInfo.Server
        }

        if($null -ne $Global:SdnDiagnostics.EnvironmentInfo.Gateway){
            $fabricNodes += $Global:SdnDiagnostics.EnvironmentInfo.Gateway
        }

        if($null -ne $Global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux){
            $fabricNodes += $Global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux
        }

        $Global:SdnDiagnostics.EnvironmentInfo.FabricNodes = $fabricNodes

        return $Global:SdnDiagnostics.EnvironmentInfo
    }
    catch {
        # Remove any cached info in case of exception as the cached info might be incorrect
        $Global:SdnDiagnostics.EnvironmentInfo.NcUrl = $null
        $global:SdnDiagnostics.EnvironmentInfo.NetworkController = $null
        $global:SdnDiagnostics.EnvironmentInfo.LoadBalancerMux = $null
        $Global:SdnDiagnostics.EnvironmentInfo.Gateway = $null
        $Global:SdnDiagnostics.EnvironmentInfo.Server = $null
        $Global:SdnDiagnostics.EnvironmentInfo.FabricNodes = $null
        $_ | Trace-Exception
    }
}

Set-Alias -Name "Get-SdnEnvironmentInfo" -Value "Get-SdnInfrastructureInfo" -Force

function Get-SdnInternalLoadBalancer {
    <#
        .SYNOPSIS
            Performs lookups and joins between OVSDB resources, load balancers and virtual networks to create internal load balancer object mappings
        .PARAMETER NcUri
            Specifies the Network Controller URI to connect to.
        .PARAMETER IPAddress
            Specify the private IP address of the Internal Load Balancer.
        .PARAMETER ProviderAddress
            Specify the provider address IP that is associated with the Internal Load Balancer.
        .PARAMETER Credential
            Specifies a user account that has permission to the Hyper-V Hosts within the SDN Fabric. The default is the current user.
        .PARAMETER NcRestCredential
            Specifies a user account that has permission to query the Network Controller NB API endpoint. The default is the current user.
        .EXAMPLE
            Get-SdnInternalLoadBalancer -NcUri https://nc.contoso.com -IPAddress 10.10.0.50
        .EXAMPLE
            Get-SdnInternalLoadBalancer -NcUri https://nc.contoso.com -Credential (Get-Credential)
    #>

    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $true, ParameterSetName = 'IPAddress')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ProviderAddress')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true, ParameterSetName = 'IPAddress')]
        [IPAddress]$IPAddress,

        [Parameter(Mandatory = $true, ParameterSetName = 'ProviderAddress')]
        [IPAddress]$ProviderAddress,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ProviderAddress')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ProviderAddress')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty
    )

    $array = @()
    $subnetHash = [System.Collections.Hashtable]::new()
    $frontendHash = [System.Collections.Hashtable]::new()

    try {
        $servers = Get-SdnServer -NcUri $NcUri -Credential $NcRestCredential -ManagementAddressOnly
        $ovsdbAddressMappings = Get-SdnOvsdbAddressMapping -ComputerName $servers -Credential $Credential | Where-Object {$_.mappingType -eq 'learning_disabled'}
        $loadBalancers = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -Resource LoadBalancers
        $virtualNetworks = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -Resource VirtualNetworks

        # if this returns null, this is due to no tenant internal load balancers have been provisioned on the system
        # in which case all the further processing is not needed
        if($null -eq $ovsdbAddressMappings){
            "No tenant Internal Load Balancer references found within OVSDB" | Trace-Output -Level:Verbose
            return $null
        }

        "Located {0} address mappings from OVSDB" -f $ovsdbAddressMappings.Count | Trace-Output -Level:Verbose
        # create a hash table based on the subnet instanceId contained within the virtual networks
        foreach($group in $virtualNetworks.properties.subnets | Group-Object InstanceID){
            [void]$subnetHash.Add($group.Name, $group.Group)
        }

        "Located {0} subnets" -f $subnetHash.Count | Trace-Output -Level:Verbose
        # create a hash table based on the resourceRef of the frontendIPConfigurations within the load balancers
        foreach($group in $loadBalancers.properties.frontendIPConfigurations | Group-Object resourceRef){
            [void]$frontendHash.Add($group.Name, $group.Group)
        }

        "Located {0} frontendIPConfigurations" -f $frontendHash.Count | Trace-Output -Level:Verbose
        foreach($ovsdbObject in $ovsdbAddressMappings){

            # leveraging the routing domain ID taken from the OVSDB objects we need to
            # do a lookup against the virtual network subnets to locate the associated ip configurations
            # once we have the ipconfiguration, we want to enumerate each load balancer to match on the customer ip address
            $tenantSubnet = $subnetHash[$ovsdbObject.RoutingDomainID.Guid]
            if($tenantSubnet){
                $loadBalancerResourceRef = $tenantSubnet.properties.ipConfigurations | Where-Object {$_.ResourceRef -like "/loadBalancers/*"}
                if($loadBalancerResourceRef){
                    foreach($resource in $loadBalancerResourceRef){
                        $internalLoadBalancer = $frontendHash[$resource.resourceRef]

                        # if the customer ip address does not match between load balancer and ovsdb then skip it as
                        # this is not the load balancer you are looking for
                        if($internalLoadBalancer){
                            if($internalLoadBalancer.properties.privateIPAddress -ne $ovsdbObject.CustomerAddress){
                                continue
                            }

                            # create a new object to add to the array list as we now have all the mappings we want
                            $array += [PSCustomObject]@{
                                ResourceRef = [String]$internalLoadBalancer.resourceRef
                                CustomerAddress = [IPAddress]$internalLoadBalancer.properties.privateIPAddress
                                ProviderAddress = [IPAddress]$ovsdbObject.ProviderAddress
                            }
                        }
                        else {
                            "Unable to locate Load Balancer Frontend IP Configuration for {0}" -f $resource.resourceRef | Trace-Output -Level:Warning
                        }
                    }
                }
                else {
                    "Unable to locate any Load Balancer objects within IP configurations for {0}" -f $tenantSubnet.resourceRef  | Trace-Output -Level:Warning
                }
            }
            else {
                "Unable to locate Virtual Network Subnet related to Routing Domain ID {0}" -f $ovsdbObject.RoutingDomainID | Trace-Output -Level:Warning
            }
        }

        if ($IPAddress) {
            return ($array | Where-Object {$_.CustomerAddress -eq $IPAddress})
        }

        if ($ProviderAddress) {
            return ($array | Where-Object {$_.ProviderAddress -eq $ProviderAddress})
        }

        return ($array | Sort-Object CustomerAddress -Unique)
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnLoadBalancerMux {
    <#
    .SYNOPSIS
        Returns a list of load balancer muxes from network controller
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ResourceId
        Specifies the unique identifier for the resource.
    .PARAMETER ResourceRef
        Specifies the resource reference for the resource.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER ManagementAddressOnly
        Optional parameter to only return back the Management Address value.
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential)
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef '/LoadBalancerMuxes/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnLoadBalancerMux -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef '/LoadBalancerMuxes/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    #>

    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [String]$ResourceId,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [String]$ResourceRef,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [switch]$ManagementAddressOnly
    )

    $params = @{
        NcUri = $NcUri
        Credential = $Credential
    }

    switch ($PSCmdlet.ParameterSetName) {
        'ResourceId' {
            $params.Add('Resource', 'LoadBalancerMuxes')
            $params.Add('ResourceId', $ResourceId)
        }
        'ResourceRef' {
            $params.Add('ResourceRef', $ResourceRef)
        }
        default {
            $params.Add('Resource', 'LoadBalancerMuxes')
        }
    }

    try {
        $result = Get-SdnResource @params
        if ($result) {
            foreach($obj in $result){
                if($obj.properties.provisioningState -ne 'Succeeded'){
                    "{0} is reporting provisioningState: {1}" -f $obj.resourceId, $obj.properties.provisioningState | Trace-Output -Level:Warning
                }
            }

            if($ManagementAddressOnly){
                $connections = @()
                foreach ($resource in $result) {
                    $virtualServerMgmtAddress = Get-SdnVirtualServer -NcUri $NcUri.AbsoluteUri -ResourceRef $resource.properties.virtualserver.ResourceRef -ManagementAddressOnly -Credential $Credential
                    $connections += $virtualServerMgmtAddress
                }

                return $connections
            }
            else {
                return $result
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkController {
    <#
    .SYNOPSIS
        Gets network controller application settings.
    .PARAMETER NetworkController
        Specifies the name or IP address of the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnNetworkController
    .EXAMPLE
        PS> Get-SdnNetworkController -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        try {
            if (Test-ComputerNameIsLocal -ComputerName $NetworkController) {
                $result = Get-NetworkController
            }
            else {
                $result = Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock { Get-NetworkController } -Credential $Credential
            }
        }
        catch {
            "Get-NetworkController failed with following exception: `n`t{0}`n" -f $_ | Trace-Output -Level:Error
            $result = Get-SdnNetworkControllerInfoFromClusterManifest -NetworkController $NetworkController -Credential $Credential
        }

        return $result
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerClusterInfo {
    <#
    .SYNOPSIS
        Gather the Network Controller cluster wide info from one of the Network Controller
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER OutputDirectory
        Directory location to save results. It will create a new sub-folder called NetworkControllerClusterInfo that the files will be saved to
    .EXAMPLE
        PS> Get-SdnNetworkControllerClusterInfo
    .EXAMPLE
        PS> Get-SdnNetworkControllerClusterInfo -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$NetworkController,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true)]
        [System.IO.FileInfo]$OutputDirectory
    )

    try {
        [System.IO.FileInfo]$outputDir = Join-Path -Path $OutputDirectory.FullName -ChildPath 'NetworkControllerClusterInfo'

        if (!(Test-Path -Path $outputDir.FullName -PathType Container)) {
            $null = New-Item -Path $outputDir.FullName -ItemType Directory -Force
        }

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock { Get-NetworkController } -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-NetworkController" -FileType txt

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock { Get-NetworkControllerNode } -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-NetworkControllerNode" -FileType txt

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock { Get-NetworkControllerCluster } -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-NetworkControllerCluster" -FileType txt

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock { Get-NetworkControllerReplica } -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-NetworkControllerReplica" -FileType txt

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock {  Get-SdnServiceFabricClusterConfig -Uri GlobalConfiguration} -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "NetworkControllerGlobalConfiguration" -FileType txt -Format Table

        Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock {  Get-SdnServiceFabricClusterConfig -Uri ClusterConfiguration} -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "NetworkControllerClusterConfiguration" -FileType txt -Format Table

        Get-SdnServiceFabricClusterHealth -NetworkController $NetworkController -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-SdnServiceFabricClusterHealth" -FileType txt

        Get-SdnServiceFabricApplicationHealth -NetworkController $NetworkController -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-SdnServiceFabricApplicationHealth" -FileType txt

        Get-SdnServiceFabricClusterManifest -NetworkController $NetworkController -Credential $Credential `
        | Out-File -FilePath "$($outputDir.FullName)\Get-SdnServiceFabricClusterManifest.xml"

        $ncServices = Get-SdnServiceFabricService -NetworkController $NetworkController -Credential $Credential
        $ncServices | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-SdnServiceFabricService" -FileType txt
        foreach ($service in $ncServices) {
            Get-SdnServiceFabricReplica -NetworkController $NetworkController -Credential $Credential -ServiceName $service.ServiceName `
            | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-SdnServiceFabricReplica_$($service.ServiceTypeName)" -FileType txt
        }

        Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -Credential $Credential -ScriptBlock {Get-ServiceFabricApplication} `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-ServiceFabricApplication" -FileType json

        Get-SdnServiceFabricNode -NetworkController $NetworkController -Credential $Credential `
        | Export-ObjectToFile -FilePath $outputDir.FullName -Name "Get-SdnServiceFabricNode" -FileType txt

    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerNode {
    <#
    .SYNOPSIS
        Returns a list of servers from network controller.
    .PARAMETER Name
        Specifies the friendly name of the node for the network controller. If not provided, settings are retrieved for all nodes in the deployment.
    .PARAMETER NetworkController
        Specifies the name or IP address of the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnNetworkControllerNode
    .EXAMPLE
        PS> Get-SdnNetworkControllerNode -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String]$Name,

        [Parameter(Mandatory = $false)]
        [System.String]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [switch]$ServerNameOnly
    )

    try {

        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        try {
            if (Test-ComputerNameIsLocal -ComputerName $NetworkController) {
                $result = Get-NetworkControllerNode -ErrorAction Stop
            }
            else {
                $result = Invoke-PSRemoteCommand -ComputerName $NetworkController -Credential $Credential -ScriptBlock {
                    Get-NetworkControllerNode -ErrorAction Stop
                } -ErrorAction Stop
            }

            # in this scenario if the results returned we will parse the objects returned and generate warning to user if node is not up
            # this property is only going to exist though if service fabric is healthy and underlying NC cmdlet can query node status
            foreach($obj in $result){
                if($obj.Status -ine 'Up'){
                    "{0} is reporting status {1}" -f $obj.Name, $obj.Status | Trace-Output -Level:Warning
                }

                # if we returned the object, we want to add a new property called NodeCertificateThumbprint as this will ensure consistent
                # output in scenarios where this operation fails due to NC unhealthy and we need to fallback to reading the cluster manifest
                $result | ForEach-Object {
                    if (!($_.PSOBject.Properties.name -contains "NodeCertificateThumbprint")) {
                        $_ | Add-Member -MemberType NoteProperty -Name 'NodeCertificateThumbprint' -Value $_.NodeCertificate.Thumbprint
                    }
                }
            }
        }
        catch {
            "Get-NetworkControllerNode failed with following exception: `n`t{0}`n" -f $_ | Trace-Output -Level:Error
            $result = Get-NetworkControllerNodeInfoFromClusterManifest  -NetworkController $NetworkController -Credential $Credential
        }

        if ($Name) {
            $result = $result | Where-Object { $_.Name.Split(".")[0] -ieq $Name.Split(".")[0] -or $_.Server -ieq $Name.Split(".")[0] }
        }

        if($ServerNameOnly){
            return [System.Array]$result.Server
        }
        else {
            return $result
        }

    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerNodeCertificate {
    <#
    .SYNOPSIS
        Returns the current Network Controller node certificate
    #>

    try {
        $networkControllerNode = Get-SdnNetworkControllerNode -Name $env:COMPUTERNAME

        # check to see if FindCertificateBy property exists as this was added in later builds
        # else if does not exist, default to Thumbprint for certificate
        if ($null -ne $networkControllerNode.FindCertificateBy) {
            "Network Controller is currently configured for FindCertificateBy: {0}" -f $networkControllerNode.FindCertificateBy | Trace-Output -Level:Verbose
            switch ($networkControllerNode.FindCertificateBy) {
                'FindBySubjectName' {
                    "`tFindBySubjectName: {0}" -f $networkControllerNode.NodeCertSubjectName | Trace-Output -Level:Verbose
                    $certificate = Get-SdnCertificate -Path 'Cert:\LocalMachine\My' -Subject $networkControllerNode.NodeCertSubjectName
                }

                'FindByThumbprint' {
                    "`FindByThumbprint: {0}" -f $networkControllerNode.NodeCertificateThumbprint | Trace-Output -Level:Verbose
                    $certificate = Get-SdnCertificate -Path 'Cert:\LocalMachine\My' -Thumbprint $networkControllerNode.NodeCertificateThumbprint
                }
            }
        }
        else {
            $certificate = Get-SdnCertificate -Path 'Cert:\LocalMachine\My' -Thumbprint $networkControllerNode.NodeCertificateThumbprint
        }

        if ($null -eq $certificate) {
            throw New-Object System.NullReferenceException("Unable to locate Network Controller Certificate")
        }

        return $certificate
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerRestCertificate {
    <#
    .SYNOPSIS
        Returns the current Network Controller REST Certificate
    #>

    try {

        $config = Get-SdnModuleConfiguration -Role 'NetworkController'
        $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
        if (-NOT ($confirmFeatures)) {
            "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
            return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
        }

        $networkController = Get-SdnNetworkController
        $ncRestCertThumprint = $($networkController.ServerCertificate.Thumbprint).ToString()
        $certificate = Get-SdnCertificate -Path 'Cert:\LocalMachine\My' -Thumbprint $ncRestCertThumprint

        if ($null -eq $certificate) {
            throw New-Object System.NullReferenceException("Unable to locate Network Controller Rest Certificate")
        }

        return $certificate
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkControllerState {
    <#
    .SYNOPSIS
        Gathers the Network Controller State dump files (IMOS) from each of the Network Controllers
    .PARAMETER NetworkController
        The computer name of the Network Controller used to retrieve Infrastructure Info and trigger IMOS generation.
    .PARAMETER OutputDirectory
        Directory location to save results. By default it will create a new sub-folder called NetworkControllerState that the files will be copied to
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER NcRestCredential
        Specifies a user account that has permission to access the northbound NC API interface. The default is the current user.
    .PARAMETER ExecutionTimeout
        Specify the execution timeout (seconds) on how long you want to wait for operation to complete before cancelling operation. If omitted, defaults to 300 seconds.
    .EXAMPLE
        PS> Get-SdnNcImosDumpFiles -NcUri "https://nc.contoso.com" -NetworkController $NetworkControllers -OutputDirectory "C:\Temp\CSS_SDN"
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$NetworkController,

        [Parameter(Mandatory = $true)]
        [System.IO.FileInfo]$OutputDirectory,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [int]$ExecutionTimeOut = 300
    )
    try {
        "Collecting In Memory Object State (IMOS) for Network Controller" | Trace-Output
        $config = Get-SdnModuleConfiguration -Role:NetworkController
        [System.IO.FileInfo]$netControllerStatePath = $config.properties.netControllerStatePath
        [System.IO.FileInfo]$outputDir = Join-Path -Path $OutputDirectory.FullName -ChildPath 'NetworkControllerState'

        if (!(Test-Path -Path $outputDir.FullName -PathType Container)) {
            $null = New-Item -Path $outputDir.FullName -ItemType Directory -Force
        }

        $scriptBlock = {
            param([Parameter(Position = 0)][String]$param1)
            try {
                if (Test-Path -Path $param1 -PathType Container) {
                    Get-Item -Path $param1 | Remove-Item -Recurse -Confirm:$false -Force -ErrorAction SilentlyContinue
                }

                $null = New-Item -Path $param1 -ItemType Container -Force
            }
            catch {
                $_ | Write-Error
            }
        }

        $infraInfo = Get-SdnInfrastructureInfo -NetworkController $NetworkController -Credential $Credential -NcRestCredential $NcRestCredential
        # invoke scriptblock to clean up any stale NetworkControllerState files
        Invoke-PSRemoteCommand -ComputerName $infraInfo.NetworkController -Credential $Credential -ScriptBlock $scriptBlock -ArgumentList $netControllerStatePath.FullName

        # invoke the call to generate the files
        # once the operation completes and returns true, then enumerate through the Network Controllers defined to collect the files
        $result = Invoke-SdnNetworkControllerStateDump -NcUri $infraInfo.NcUrl -Credential $NcRestCredential -ExecutionTimeOut $ExecutionTimeOut
        if ($result) {
            foreach ($ncVM in $infraInfo.NetworkController) {
                Copy-FileFromRemoteComputer -Path "$($config.properties.netControllerStatePath)\*" -ComputerName $ncVM -Destination $outputDir.FullName
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnNetworkInterfaceOutboundPublicIPAddress {
    <#
    .SYNOPSIS
        Gets the outbound public IP address that is used by a network interface.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ResourceId
        Specifies the unique identifier for the networkinterface resource.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnNetworkInterfaceOutboundPublicIPAddress -NcUri "https://nc.contoso.com" -ResourceId '8f9faf0a-837b-43cd-b4bf-dbe996993514'
    .EXAMPLE
        PS> Get-SdnNetworkInterfaceOutboundPublicIPAddress -NcUri "https://nc.contoso.com" -ResourceId '8f9faf0a-837b-43cd-b4bf-dbe996993514' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [uri]$NcUri,

        [Parameter(Mandatory = $true)]
        [System.String]$ResourceId,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )
    try {
        $arrayList = [System.Collections.ArrayList]::new()

        $networkInterface = Get-SdnResource -NcUri $NcUri.AbsoluteUri -Resource:NetworkInterfaces -Credential $Credential | Where-Object { $_.resourceId -ieq $ResourceId }
        if ($null -eq $networkInterface) {
            throw New-Object System.NullReferenceException("Unable to locate network interface within Network Controller")
        }

        foreach ($ipConfig in $networkInterface.properties.ipConfigurations) {
            $publicIpRef = Get-PublicIpReference -NcUri $NcUri.AbsoluteUri -IpConfiguration $ipConfig -Credential $Credential
            if ($publicIpRef) {
                $publicIpAddress = Get-SdnResource -NcUri $NcUri.AbsoluteUri -Credential $Credential -ResourceRef $publicIpRef
                if ($publicIpAddress) {
                    [void]$arrayList.Add(
                        [PSCustomObject]@{
                            IPConfigResourceRef      = $ipConfig.resourceRef
                            IPConfigPrivateIPAddress = $ipConfig.properties.privateIPAddress
                            PublicIPResourceRef      = $publicIpAddress.resourceRef
                            PublicIPAddress          = $publicIpAddress.properties.ipAddress
                        }
                    )
                }
            }
        }

        return $arrayList
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnPublicIPPoolUsageSummary {
    <#
    .SYNOPSIS
        Returns back the IP addresses associated with the public logical subnet IP pools within the Network Controller environment.
    .DESCRIPTION
        This function returns back a list of IP addresses that are consumed by the PublicIPAddresses and LoadBalancer resources that are derived from the public IP pools.
        This helps operators quickly locate which resources are associated with a public IP address, in addition to identify available vs non-available IP addresses.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER NcRestCredential
        Specifies a user account that has permission to access the northbound NC API interface. The default is the current user.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty
    )

    $array = @()

    try {
        $logicalNetworks = Get-SdnResource -NcUri $NcUri -Resource LogicalNetworks -Credential $NcRestCredential | Where-Object {$_.properties.subnets.properties.isPublic -ieq $true}
        $loadBalancers = Get-SdnResource -NcUri $NcUri -Resource LoadBalancers -Credential $NcRestCredential
        $publicIpAddresses = Get-SdnResource -NcUri $NcUri -Resource PublicIPAddresses -Credential $NcRestCredential

        foreach ($subnet in $logicalNetworks.properties.subnets) {
            foreach ($ipPool in $subnet.properties.ipPools) {
                # check to see if there was any loadbalancer frontend resources on the system and cross compare with the logical subnet ipPool
                # if they address falls within the ipPool range, then add to the array
                if ($loadBalancers) {
                    foreach ($loadBalancer in $loadBalancers) {
                        foreach ($frontEndConfig in $loadBalancer.properties.frontendIPConfigurations) {
                            if ($frontEndConfig.properties.privateIPAddress) {
                                if (Confirm-IpAddressInRange -IpAddress $frontEndConfig.properties.privateIPAddress -StartAddress $ipPool.properties.startIpAddress -EndAddress $ipPool.properties.EndIpAddress) {

                                    $object = [PSCustomObject]@{
                                        IPPool = $ipPool.ResourceId
                                        IPAddress = $frontEndConfig.properties.privateIPAddress
                                        ProvisioningState = $frontEndConfig.properties.provisioningState
                                        AllocationMethod = $frontEndConfig.properties.privateIPAllocationMethod
                                        ResourceType = 'FrontEndIpConfiguration'
                                        ResourceId = $frontEndConfig.resourceId
                                        InstanceId = $frontEndConfig.instanceId
                                        AssociatedResource = $loadBalancer.resourceRef
                                    }

                                    $array += $object
                                }
                            }
                        }
                    }
                }

                # check to see if there was any public IP address resources on the system and cross compare with the logical subnet ipPool
                # if they address falls within the ipPool range, then add to the array
                if ($publicIpAddresses) {
                    foreach ($publicIp in $publicIpAddresses) {
                        if (Confirm-IpAddressInRange -IpAddress $publicIp.properties.IpAddress -StartAddress $ipPool.properties.startIpAddress -EndAddress $ipPool.properties.EndIpAddress) {

                            $object = [PSCustomObject]@{
                                IPPool = $ipPool.ResourceId
                                IPAddress = $publicIp.properties.ipAddress
                                ProvisioningState = $publicIp.properties.provisioningState
                                AllocationMethod = $publicIp.properties.publicIPAllocationMethod
                                ResourceType = 'PublicIpAddress'
                                ResourceId = $publicIp.resourceId
                                InstanceId = $publicIp.instanceId
                                AssociatedResource = $publicIp.properties.ipConfiguration.resourceRef
                            }

                            $array += $object
                        }
                    }
                }
            }
        }

        return ($array | Sort-Object -Property 'IpAddress')
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnResource {
    <#
    .SYNOPSIS
        Invokes a web request to SDN API for the requested resource.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ResourceRef
        The resource ref of the object you want to perform the operation against.
    .PARAMETER Resource
        The resource type you want to perform the operation against.
    .PARAMETER ResourceName
    .PARAMETER InstanceID
    .PARAMETER ApiVersion
        The API version to use when invoking against the NC REST API endpoint.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnResource -Resource PublicIPAddresses
    .EXAMPLE
        PS> Get-SdnResource -NcUri "https://nc.$env:USERDNSDOMAIN" -ResourceRef "/publicIPAddresses/d9266251-a3ba-4ac5-859e-2c3a7c70352a"
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [Parameter(Mandatory = $true, ParameterSetName = 'Resource')]
        [Parameter(Mandatory = $true, ParameterSetName = 'InstanceID')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [System.String]$ResourceRef,

        [Parameter(Mandatory = $true, ParameterSetName = 'Resource')]
        [SdnApiResource]$Resource,

        [Parameter(Mandatory = $false, ParameterSetName = 'Resource')]
        [System.String]$ResourceId,

        [Parameter(Mandatory = $true, ParameterSetName = 'InstanceID')]
        [System.String]$InstanceId,

        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [Parameter(Mandatory = $false, ParameterSetName = 'Resource')]
        [System.String]$ApiVersion = $Global:SdnDiagnostics.EnvironmentInfo.RestApiVersion,

        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [Parameter(Mandatory = $false, ParameterSetName = 'Resource')]
        [Parameter(Mandatory = $false, ParameterSetName = 'InstanceID')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        switch ($PSCmdlet.ParameterSetName) {
            'InstanceId' {
                [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ApiVersion $ApiVersion -ResourceName 'internalResourceInstances'
                [System.String]$uri = "{0}/{1}" -f $uri, $InstanceId.Trim()
            }
            'ResourceRef' {
                [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ApiVersion $ApiVersion -ResourceRef $ResourceRef
            }
            'Resource' {
                [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ApiVersion $ApiVersion -ResourceName $Resource

                if ($ResourceID) {
                    [System.String]$uri = "{0}/{1}" -f $uri, $ResourceId.Trim()
                }
            }
        }

        "{0} {1}" -f $method, $uri | Trace-Output -Level:Verbose

        # gracefully handle System.Net.WebException responses such as 404 to throw warning
        # anything else we want to throw terminating exception and capture for debugging purposes
        try {
            $result = Invoke-RestMethodWithRetry -Uri $uri -Method 'GET' -UseBasicParsing -Credential $Credential -ErrorAction Stop
        }
        catch [System.Net.WebException] {
            "{0} ({1})" -f $_.Exception.Message, $_.Exception.Response.ResponseUri.AbsoluteUri | Write-Warning
            return $null
        }
        catch {
            throw $_
        }

        # if multiple objects are returned, they will be nested under a property called value
        # so we want to do some manual work here to ensure we have a consistent behavior on data returned back
        if ($result.value) {
            return $result.value
        }

        # in some instances if the API returns empty object, we will see it saved as 'nextLink' which is a empty string property
        # we need to return null instead otherwise the empty string will cause calling functions to treat the value as it contains data
        elseif ($result.PSObject.Properties.Name -ieq "nextLink" -and $result.PSObject.Properties.Name.Count -eq 1) {
            return $null
        }

        return $result
    }
    catch {
        "{0}`nAbsoluteUri:{1}`n{2}" -f $_.Exception, $_.TargetObject.Address.AbsoluteUri, $_.ScriptStackTrace | Trace-Output -Level:Error
    }
}

function Get-SdnServer {
    <#
    .SYNOPSIS
        Returns a list of servers from network controller.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER ResourceId
        Specifies the unique identifier for the resource.
    .PARAMETER ResourceRef
        Specifies the resource reference for the resource.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER ManagementAddressOnly
        Optional parameter to only return back the Management Address value.
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential)
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef 'Servers/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e'
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceId 'f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    .EXAMPLE
        PS> Get-SdnServer -NcUri 'https://NC.FQDN' -Credential (Get-Credential) -ResourceRef 'Servers/f5e3b3e0-1b7a-4b9e-8b9e-5b5e3b3e0f5e' -ManagementAddressOnly
    #>

    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceId')]
        [String]$ResourceId,

        [Parameter(Mandatory = $true, ParameterSetName = 'ResourceRef')]
        [String]$ResourceRef,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ParameterSetName = 'Default')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceId')]
        [Parameter(Mandatory = $false, ParameterSetName = 'ResourceRef')]
        [switch]$ManagementAddressOnly
    )

    $params = @{
        NcUri = $NcUri
        Credential = $Credential
    }

    switch ($PSCmdlet.ParameterSetName) {
        'ResourceId' {
            $params.Add('Resource', 'Servers')
            $params.Add('ResourceId', $ResourceId)
        }
        'ResourceRef' {
            $params.Add('ResourceRef', $ResourceRef)
        }
        default {
            $params.Add('Resource', 'Servers')
        }
    }

    try {
        $result = Get-SdnResource @params
        if ($result) {
            foreach($obj in $result){
                if($obj.properties.provisioningState -ne 'Succeeded'){
                    "{0} is reporting provisioningState: {1}" -f $obj.resourceId, $obj.properties.provisioningState | Trace-Output -Level:Warning
                }
            }

            if($ManagementAddressOnly){
                $connections = (Get-ManagementAddress -ManagementAddress $result.properties.connections.managementAddresses)
                return $connections
            }
            else {
                return $result
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricApplicationHealth {
    <#
    .SYNOPSIS
        Gets the health of a Service Fabric application from Network Controller.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER ApplicationName
        A service fabric application name that exists on the provided ring, such as fabric:/NetworkController.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricApplicationHealth -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false)]
        [String]$ApplicationName = 'fabric:/NetworkController',

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if ($NetworkController) {
            Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock { Get-ServiceFabricApplicationHealth -ApplicationName $using:ApplicationName } -Credential $Credential
        }
        else {
            Invoke-SdnServiceFabricCommand -ScriptBlock { Get-ServiceFabricApplicationHealth -ApplicationName $using:ApplicationName } -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricClusterConfig {
    <#
    .SYNOPSIS
        Gets Service Fabric Cluster Config Properties.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates. Default to local machine.
    .PARAMETER Uri
        The Uri to read properties from ClusterConfiguration, GlobalConfiguration
    .PARAMETER Name
        Property Name to filter the result. If not specified, it will return all properties.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricClusterConfig -NetworkController 'NC01' -Uri "ClusterConfiguration" -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $true)]
        [ValidateSet('GlobalConfiguration', 'ClusterConfiguration')]
        [String]$Uri,

        [Parameter(Mandatory = $false)]
        [String]$Name,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        Connect-ServiceFabricCluster | Out-Null
        $client = [System.Fabric.FabricClient]::new()
        $result = $null
        $absoluteUri = "fabric:/NetworkController/$Uri"
        $binaryMethod = [System.Fabric.NamedProperty].getmethod("GetValue").MakeGenericMethod([byte[]])
        $stringMethod = [System.Fabric.NamedProperty].getmethod("GetValue").MakeGenericMethod([string])

        $results = [System.Collections.ArrayList]::new()
        do {
            $result = $client.PropertyManager.EnumeratePropertiesAsync($absoluteUri, $true, $result).Result
            $result.GetEnumerator() | ForEach-Object {
                $propertyName = $_.Metadata.PropertyName

                $propertyObj = [PSCustomObject]@{
                    Name = $propertyName
                    Value = $null
                }
                if($_.Metadata.TypeId -ieq "string"){
                    $value = $stringMethod.Invoke($_, $null);
                    $propertyObj.Value = $value

                }elseif($_.Metadata.TypeId -ieq "binary"){
                    # only binary value exist is certificate
                    $value = $binaryMethod.Invoke($_, $null);
                    $certObj = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new($value)
                    $propertyObj.Value = $certObj
                }

                if($PSBoundParameters.ContainsKey('Name')){
                    if($propertyName -ieq $Name){
                        $results.Add($propertyObj) | Out-Null
                        # Property Name is uniqueue so when name found, return the list
                        return $results
                    }
                }else{
                    $results.Add($propertyObj) | Out-Null
                }
            }
        }
        while ($result.HasMoreData)
        return $results
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricClusterHealth {
    <#
    .SYNOPSIS
        Gets health information for a Service Fabric cluster from Network Controller.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricClusterHealth -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if ($NetworkController) {
            Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock { Get-ServiceFabricClusterHealth } -Credential $Credential
        }
        else {
            Invoke-SdnServiceFabricCommand -ScriptBlock { Get-ServiceFabricClusterHealth } -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricClusterManifest {
    <#
    .SYNOPSIS
        Gets the Service Fabric cluster manifest, including default configurations for reliable services from Network Controller.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricClusterManifest -NetworkController 'NC01'
    .EXAMPLE
        PS> Get-SdnServiceFabricClusterManifest -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String[]]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        # in instances where Service Fabric is down/offline we want to catch any exceptions returned by Invoke-SdnServiceFabricCommand
        # and then fallback to getting the cluster manifest information from the file system directly
        try {
            $clusterManifest = Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock { Get-ServiceFabricClusterManifest } -Credential $Credential
        }
        catch {
            "Unable to retrieve ClusterManifest directly from Service Fabric. Attempting to retrieve ClusterManifest from file system" | Trace-Output -Level:Warning

            # we want to loop through if multiple NetworkController objects were passed into the cmdlet
            foreach ($obj in $NetworkController) {
                $clusterManifestScript = {
                    $clusterManifestFile = Get-ChildItem -Path "$env:ProgramData\Microsoft\Service Fabric" -Recurse -Depth 2 -Filter "ClusterManifest.current.xml" -ErrorAction SilentlyContinue
                    if ($clusterManifestFile) {
                        $clusterManifest = Get-Content -Path $clusterManifestFile.FullName -ErrorAction SilentlyContinue
                        return $clusterManifest
                    }

                    return $null
                }

                if (Test-ComputerNameIsLocal -ComputerName $obj) {
                    $xmlClusterManifest = Invoke-Command -ScriptBlock $clusterManifestScript
                }
                else {
                    $xmlClusterManifest = Invoke-PSRemoteCommand -ComputerName $obj -Credential $Credential -ScriptBlock $clusterManifestScript
                }

                # once the cluster manifest has been retrieved from the file system break out of the loop
                if ($xmlClusterManifest) {
                    "Successfully retrieved ClusterManifest from {0}" -f $obj | Trace-Output
                    $clusterManifest = $xmlClusterManifest
                    break
                }
            }
        }

        if ($null -eq $clusterManifest) {
            throw New-Object System.NullReferenceException("Unable to retrieve ClusterManifest from Network Controller")
        }

        if ($clusterManifest) {
            # Convert to native Powershell XML
            $xmlClusterManifest = [xml]$clusterManifest

            # Although the strings are encrypted, they should be sanitized anyway
            # Change PrimaryAccountNTLMPasswordSecret and SecondaryAccountNTLMPasswordSecret to removed_for_security_reasons
            (($xmlClusterManifest.ClusterManifest.FabricSettings.Section | Where-Object {$_.Name -eq "FileStoreService"}).Parameter | Where-Object {$_.Name -eq "PrimaryAccountNTLMPasswordSecret"}).Value = "removed_for_security_reasons"
            (($xmlClusterManifest.ClusterManifest.FabricSettings.Section | Where-Object {$_.Name -eq "FileStoreService"}).Parameter | Where-Object {$_.Name -eq "SecondaryAccountNTLMPasswordSecret"}).Value = "removed_for_security_reasons"

            # If we want to keep newlines and indents, but return a string, we need to use the writer class
            # $xmlClusterManifest.OuterXml does not keep the formatting
            $stringWriter = New-Object System.IO.StringWriter
            $writer = New-Object System.Xml.XmlTextwriter($stringWriter)
            $writer.Formatting = [System.XML.Formatting]::Indented

            # Write the manifest to the StringWriter
            $xmlClusterManifest.WriteContentTo($writer)

            # Return the manifest as a string
            return $stringWriter.ToString()
        }

        return $clusterManifest
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricNode {
    <#
    .SYNOPSIS
        Gets information for all nodes in a Service Fabric cluster for Network Controller.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER NodeName
        Specifies the name of the Service Fabric node whose information is being returned. If not specified, the cmdlet will return information for all the nodes in the cluster.
    .EXAMPLE
        PS> Get-SdnServiceFabricNode -NetworkController 'Prefix-NC01' -Credential (Get-Credential)
    .EXAMPLE
        PS> Get-SdnServiceFabricNode -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -NodeName 'Prefix-NC02'
    .EXAMPLE
        PS> Get-SdnServiceFabricNode -NodeName 'Prefix-NC01'
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [System.String[]]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [System.String]$NodeName

    )

    try {

        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        if ($NodeName) {
            $sb = {
                Get-ServiceFabricNode -NodeName $using:NodeName
            }
        }
        else {
            $sb = {
                Get-ServiceFabricNode
            }
        }

        Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock $sb -Credential $Credential
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricPartition {
    <#
    .SYNOPSIS
        Gets information about the partitions of a specified Service Fabric partition or service from Network Controller.
    .PARAMETER ApplicationName
        A service fabric application name that exists on the provided ring, such as fabric:/NetworkController.
    .PARAMETER ServiceName
        A service fabric service name that is under the provided ApplicationName on the provided ring, such as fabric:/NetworkController/ApiService.
    .PARAMETER ServiceTypeName
        A service fabric service TypeName, such as VSwitchService.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricPartition -PartitionId 1a7a780e-dbfe-46d3-92fb-76908a95ce54
    .EXAMPLE
        PS> Get-SdnServiceFabricPartition -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceTypeName 'ApiService'
    .EXAMPLE
        PS> Get-SdnServiceFabricPartition -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceName 'fabric:/NetworkController/ApiService'
    #>

    [CmdletBinding(DefaultParameterSetName = 'NamedService')]
    param(
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ApplicationName = 'fabric:/NetworkController',

        [Parameter(Mandatory = $true, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [System.String]$ServiceName,

        [Parameter(Mandatory = $true, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ServiceTypeName,

        [Parameter(Mandatory = $true, ValueFromPipeline = $false, ParameterSetName = 'PartitionID')]
        [System.Guid]$PartitionId,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'PartitionID')]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'PartitionID')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        switch ($PSCmdlet.ParameterSetName) {
            'NamedService' {
                $sb = {
                    Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceName $using:ServiceName | Get-ServiceFabricPartition
                }
            }

            'NamedServiceTypeName' {
                $sb = {
                    Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceTypeName $using:ServiceTypeName | Get-ServiceFabricPartition
                }
            }

            'PartitionID' {
                $sb = {
                    Get-ServiceFabricPartition -PartitionId $using:PartitionId
                }
            }

            default {
                # no default
            }
        }

        if ($NetworkController) {
            return (Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock $sb -Credential $Credential)
        }
        else {
            return (Invoke-SdnServiceFabricCommand -ScriptBlock $sb -Credential $Credential)
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricReplica {
    <#
    .SYNOPSIS
        Gets Service Fabric replicas of a partition from Network Controller.
    .PARAMETER ApplicationName
        A service fabric application name that exists on the provided ring, such as fabric:/NetworkController.
    .PARAMETER ServiceName
        A service fabric service name that is under the provided ApplicationName on the provided ring, such as fabric:/NetworkController/ApiService.
    .PARAMETER ServiceTypeName
        A service fabric service TypeName, such as VSwitchService.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricReplica -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceTypeName 'ApiService'
    .EXAMPLE
        PS> Get-SdnServiceFabricReplica -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceName 'fabric:/NetworkController/ApiService'
    #>

    [CmdletBinding(DefaultParameterSetName = 'NamedService')]
    param(
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ApplicationName = 'fabric:/NetworkController',

        [Parameter(Mandatory = $true, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [System.String]$ServiceName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ServiceTypeName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [Switch]$Primary
    )

    try {
        switch ($PSCmdlet.ParameterSetName) {
            'NamedService' {
                $sb = {
                    Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceName $using:ServiceName | Get-ServiceFabricPartition | Get-ServiceFabricReplica
                }
            }

            'NamedServiceTypeName' {
                $sb = {
                    Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceTypeName $using:ServiceTypeName | Get-ServiceFabricPartition | Get-ServiceFabricReplica
                }
            }

            default {
                # no default
            }
        }

        if ($NetworkController) {
            $replica = Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock $sb -Credential $Credential
        }
        else {
            $replica = Invoke-SdnServiceFabricCommand -ScriptBlock $sb -Credential $Credential
        }

        # as network controller only leverages stateful service fabric services, we will have Primary and ActiveSecondary replicas
        # if the -Primary switch was declared, we only want to return the primary replica for that particular service
        if ($Primary) {
            return ($replica | Where-Object { $_.ReplicaRole -ieq 'Primary' })
        }
        else {
            return $replica
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnServiceFabricService {
    <#
    .SYNOPSIS
        Gets a list of Service Fabric services from Network Controller.
    .PARAMETER ApplicationName
        A service fabric application name that exists on the provided ring, such as fabric:/NetworkController.
    .PARAMETER ServiceName
        A service fabric service name that is under the provided ApplicationName on the provided ring, such as fabric:/NetworkController/ApiService.
    .PARAMETER ServiceTypeName
        A service fabric service TypeName, such as VSwitchService
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Get-SdnServiceFabricService -NetworkController 'Prefix-NC01' -Credential (Get-Credential)
    .EXAMPLE
        PS> Get-SdnServiceFabricService -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceTypeName 'ApiService'
    #>

    [CmdletBinding(DefaultParameterSetName = 'NamedService')]
    param(
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ApplicationName = 'fabric:/NetworkController',

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [System.String]$ServiceName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$ServiceTypeName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        switch ($PSCmdlet.ParameterSetName) {
            'NamedService' {
                if ($ServiceName) {
                    $sb = {
                        Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceName $using:ServiceName
                    }
                }
                else {
                    $sb = {
                        Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService
                    }
                }
            }

            'NamedServiceTypeName' {
                if ($ServiceTypeName) {
                    $sb = {
                        Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService -ServiceTypeName $using:ServiceTypeName
                    }
                }
                else {
                    $sb = {
                        Get-ServiceFabricApplication -ApplicationName $using:ApplicationName | Get-ServiceFabricService
                    }
                }
            }

            default {
                $sb = {
                    Get-ServiceFabricApplication | Get-ServiceFabricService
                }
            }
        }

        if ($NetworkController) {
            Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock $sb -Credential $Credential
        }
        else {
            Invoke-SdnServiceFabricCommand -ScriptBlock $sb -Credential $Credential
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnSlbStateInformation {
    <#
    .SYNOPSIS
        Generates an aggregated report of Virtual IPs (VIPs) in the environment and their current status as reported by Software Load Balancer and MUXes.
    .PARAMETER NcUri
         Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER VirtualIPAddress
        Specifies the VIP address to return information for. If omitted, returns all VIPs.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER ExecutionTimeout
        Specify the timeout duration to wait before automatically terminated. If omitted, defaults to 600 seconds.
    .PARAMETER PollingInterval
        Interval in which to query the state of the request to determine completion.
    .EXAMPLE
        Get-SdnSlbStateInformation -NcUri "https://nc.contoso.com"
    .EXAMPLE
        Get-SdnSlbStateInformation -NcUri "https://nc.contoso.com" -VirtualIPAddress 41.40.40.1
    .EXAMPLE
        Get-SdnSlbStateInformation -NcUri "https://nc.contoso.com" -Credential (Get-Credential)
    .EXAMPLE
        Get-SdnSlbStateInformation -NcUri "https://nc.contoso.com" -ExecutionTimeout 1200
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [uri]$NcUri,

        [Parameter(Mandatory = $false)]
        [IPAddress]$VirtualIPAddress,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [int]$ExecutionTimeOut = 600,

        [Parameter(Mandatory = $false)]
        [int]$PollingInterval = 5
    )

    try {
        [System.String]$uri = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceName 'SlbState'
        "Gathering SLB state information from {0}" -f $uri | Trace-Output -Level:Verbose

        $stopWatch = [system.diagnostics.stopwatch]::StartNew()

        $putResult = Invoke-WebRequestWithRetry -Method 'Put' -Uri $uri -Credential $Credential -Body "{}" -UseBasicParsing `
        -Content "application/json; charset=UTF-8" -Headers @{"Accept" = "application/json"}

        $resultObject = ConvertFrom-Json $putResult.Content
        "Response received $($putResult.Content)" | Trace-Output -Level:Verbose
        [System.String]$operationURI = Get-SdnApiEndpoint -NcUri $NcUri.AbsoluteUri -ResourceName 'SlbStateResults' -OperationId $resultObject.properties.operationId

        while ($true) {
            if ($stopWatch.Elapsed.TotalSeconds -gt $ExecutionTimeOut) {
                $msg = "Unable to get results for OperationId: {0}. Operation timed out" -f $operationId
                throw New-Object System.TimeoutException($msg)
            }

            Start-Sleep -Seconds $PollingInterval

            $stateResult = Invoke-WebRequestWithRetry -Uri $operationURI -UseBasicParsing -Credential $Credential
            $stateResult = $stateResult.Content | ConvertFrom-Json
            if ($stateResult.properties.provisioningState -ine 'Updating') {
                break
            }
        }

        $stopWatch.Stop()

        if ($stateResult.properties.provisioningState -ine 'Succeeded') {
            $msg = "Unable to get results for OperationId: {0}. {1}" -f $operationId, $stateResult.properties
            throw New-Object System.Exception($msg)
        }

        # if a VIP address is specified, return only the details for that VIP
        # must do some processing to get into the raw data
        if ($VirtualIPAddress) {
            $tenantDetails = $stateResult.properties.output.datagroups | Where-object { $_.name -eq 'Tenant' }
            $vipDetails = $tenantDetails.dataSections.dataunits | Where-object { $_.name -eq $VirtualIPAddress.IPAddressToString }
            return $vipDetails.value
        }

        return $stateResult.properties.output
    }
    catch {
        $_ | Trace-Exception
    }
}

function Get-SdnVipConfig {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [System.String]$VirtualIPAddress
    )

    $slbManager = Connect-SlbManager -ErrorAction Stop
    if ($slbManager) {
        $vipConfig = $slbManager.GetVipConfiguration($VirtualIPAddress)
        return $vipConfig
    }
}

function Invoke-SdnResourceDump {
    <#
    .SYNOPSIS
        Performs API request to all available northbound endpoints for NC and dumps out the resources to json file.
    .PARAMETER NcUri
        Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Invoke-SdnResourceDump
    .EXAMPLE
        PS> Invoke-SdnResourceDump -NcUri "https://nc.contoso.com"
    .EXAMPLE
        PS> Invoke-SdnResourceDump -NcUri "https://nc.contoso.com" -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [Uri]$NcUri,

        [Parameter(Mandatory = $true)]
        [System.IO.FileInfo]$OutputDirectory,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        "Generating resource dump for Network Controller NB API endpoints" | Trace-Output
        [System.IO.FileInfo]$outputDir = Join-Path -Path $OutputDirectory.FullName -ChildPath 'SdnApiResources'
        if (!(Test-Path -Path $outputDir.FullName -PathType Container)) {
            $null = New-Item -Path $outputDir.FullName -ItemType Directory -Force
        }

        $apiVersion = (Get-SdnDiscovery -NcUri $NcUri.AbsoluteUri -Credential $Credential).currentRestVersion
        if ($null -ieq $apiVersion) {
            $apiVersion = 'v1'
        }

        # objects returned from the apiResourse property are a hashtable, so need to work in key/value pairs
        $config = Get-SdnModuleConfiguration -Role:NetworkController
        [int]$apiVersionInt = $ApiVersion.Replace('v','').Replace('V','')
        foreach ($key in $config.properties.apiResources.Keys) {
            $value = $config.Properties.apiResources[$key]

            # skip any resources that are not designed to be exported
            if ($value.includeInResourceDump -ieq $false) {
                continue
            }

            [int]$minVersionInt = $value.minVersion.Replace('v','').Replace('V','')
            if ($minVersionInt -le $apiVersionInt) {
                $sdnResource = Get-SdnResource -NcUri $NcUri.AbsoluteUri -ResourceRef $value.uri -Credential $Credential
                if ($sdnResource) {
                    $sdnResource | Export-ObjectToFile -FilePath $outputDir.FullName -Name $key -FileType json -Depth 10
                }
            }
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Invoke-SdnServiceFabricCommand {
    <#
    .SYNOPSIS
        Connects to the service fabric ring that is used by Network Controller.
    .PARAMETER ScriptBlock
        Specifies the commands to run. Enclose the commands in braces ({ }) to create a script block. When using Invoke-Command to run a command remotely, any variables in the command are evaluated on the remote computer.
    .PARAMETER ArgumentList
        Supplies the values of parameters for the scriptblock. The parameters in the script block are passed by position from the array value supplied to ArgumentList. This is known as array splatting.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Invoke-SdnServiceFabricCommand -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ScriptBlock { Get-ServiceFabricClusterHealth }
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory = $false)]
        [System.String[]]$NetworkController = $env:COMPUTERNAME,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true)]
        [ScriptBlock]$ScriptBlock,

        [Parameter(Mandatory = $false)]
        [Object[]]$ArgumentList = $null
    )

    $params = @{
        ScriptBlock = $ScriptBlock
    }
    if ($ArgumentList) {
        $params.Add('ArgumentList', $ArgumentList)
    }

    if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
        $config = Get-SdnModuleConfiguration -Role 'NetworkController'
        $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
        if (-NOT ($confirmFeatures)) {
            "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
            return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
        }
    }

    foreach ($controller in $NetworkController) {

        $i = 0
        $maxRetry = 3

        # due to scenario as described in https://docs.microsoft.com/en-us/azure/service-fabric/service-fabric-troubleshoot-local-cluster-setup#cluster-connection-fails-with-object-is-closed
        # we want to catch any exception when connecting to service fabric cluster, and if necassary destroy and create a new remote pssession
        "Invoke Service Fabric cmdlets against {0}" -f $controller | Trace-Output -Level Verbose
        while ($i -lt $maxRetry) {
            $i++

            $session = New-PSRemotingSession -ComputerName $controller -Credential $Credential
            if (!$session) {
                "No session could be established to {0}" -f $controller | Trace-Output -Level:Error
                break
            }

            try {
                $connection = Invoke-Command -Session $session -ScriptBlock {
                    # The 3>$null 4>$null sends unwanted verbose and debug streams into the bit bucket
                    Connect-ServiceFabricCluster -TimeoutSec 15 3>$null 4>$null
                } -ErrorAction Stop
            }
            catch {
                "Unable to connect to Service Fabric Cluster. Attempt {0}/{1}`n`t{2}" -f $i, $maxRetry, $_ | Trace-Output -Level:Error
                "Terminating remote session {0} to {1}" -f $session.Name, $session.ComputerName | Trace-Output -Level:Warning
                Get-PSSession -Id $session.Id | Remove-PSSession
            }
        }

        # if we were not able to create a connection
        # we want to continue the foreach statement to connect to another network controller node (if provided)
        if (!$connection) {
            "Unable to connect to Service Fabric Cluster" | Trace-Output -Level:Error
            continue
        }

        # if we have the session created, we can then construct the remainder of the parameters for splatting purposes
        # and write some verbose details to the log for tracking purposes
        if ($session) {
            if (-NOT ($params.ContainsKey('Session'))) {
                $params.Add('Session', $session)
            }
            else {
                $params.Session = $session
            }

            "NetworkController: {0}, ScriptBlock: {1}" -f $controller, $ScriptBlock.ToString() | Trace-Output -Level:Verbose
            if ($params.ArgumentList) {
                "ArgumentList: {0}" -f ($params.ArgumentList | ConvertTo-Json).ToString() | Trace-Output -Level:Verbose
            }

            # if we get results from service fabric, then we want to break out of the loop
            # otherwise we will try again to see if state issue with service fabric or the particular node
            $sfResults = Invoke-Command @params
            if ($sfResults) {
                break
            }
        }
    }

    if (!$sfResults) {
        throw New-Object System.NullReferenceException("Unable to return results from service fabric")
    }

    if ($sfResults.GetType().IsPrimitive -or ($sfResults -is [String])) {
        return $sfResults
    }
    else {
        return ($sfResults | Select-Object -Property * -ExcludeProperty PSComputerName, RunspaceId)
    }
}

function Move-SdnServiceFabricReplica {
    <#
    .SYNOPSIS
        Moves the Service Fabric primary replica of a stateful service partition on Network Controller.
    .PARAMETER ApplicationName
        A service fabric application name that exists on the provided ring, such as fabric:/NetworkController.
    .PARAMETER ServiceName
        A service fabric service name that is under the provided ApplicationName on the provided ring, such as fabric:/NetworkController/ApiService.
    .PARAMETER ServiceTypeName
        A service fabric service TypeName, such as VSwitchService.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates.
    .PARAMETER NodeName
        Specifies the name of a Service Fabric node. The cmdlet moves the primary replica to the node that you specify.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS > Move-SdnServiceFabricReplica -NetworkController 'Prefix-NC01' -Credential (Get-Credential) -ServiceTypeName 'ApiService'
    #>

    [CmdletBinding(DefaultParameterSetName = 'NamedService')]
    param(
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [String]$ApplicationName = 'fabric:/NetworkController',

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [String]$ServiceName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [String]$ServiceTypeName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String[]]$NetworkController = $global:SdnDiagnostics.EnvironmentInfo.NetworkController,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.String]$NodeName,

        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedService')]
        [Parameter(Mandatory = $false, ValueFromPipeline = $false, ParameterSetName = 'NamedServiceTypeName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )
    try {
        if ($PSCmdlet.ParameterSetName -eq 'NamedService') {
            $sfParams = @{
                ServiceName = $ServiceName
                Credential  = $Credential
            }
        }
        elseif ($PSCmdlet.ParameterSetName -eq 'NamedServiceTypeName') {
            $sfParams = @{
                ServiceTypeName = $ServiceTypeName
                Credential      = $Credential
            }
        }

        # add NetworkController to hash table for splatting if defined
        if ($NetworkController) {
            [void]$sfParams.Add('NetworkController', $NetworkController)
        }

        # check to determine how many replicas are part of the partition for the service
        # if we only have a single replica, then generate a warning and stop further processing
        # otherwise locate the primary replica
        $service = Get-SdnServiceFabricService @sfParams -ErrorAction Stop
        $serviceFabricReplicas = Get-SdnServiceFabricReplica @sfParams
        if ($serviceFabricReplicas.Count -lt 3) {
            "Moving Service Fabric replica is only supported when running 3 or more instances of Network Controller" | Trace-Output -Level:Warning
            return
        }

        $replicaBefore = $serviceFabricReplicas | Where-Object {$_.ReplicaRole -ieq 'Primary'}

        # regardless if user defined ServiceName or ServiceTypeName, the $service object returned will include the ServiceName property
        # which we will use to perform the move operation with
        if ($NodeName) {
            $sb = {
                Move-ServiceFabricPrimaryReplica -ServiceName $using:service.ServiceName -NodeName $using:NodeName
            }
        }
        else {
            $sb = {
                Move-ServiceFabricPrimaryReplica -ServiceName $using:service.ServiceName
            }
        }

        # no useful information is returned during the move operation, so we will just null the results that are returned back
        if ($NetworkController) {
            $null = Invoke-SdnServiceFabricCommand -NetworkController $NetworkController -ScriptBlock $sb -Credential $Credential -ErrorAction Stop
        }
        else {
            $null = Invoke-SdnServiceFabricCommand -ScriptBlock $sb -Credential $Credential -ErrorAction Stop
        }

        # update the hash table to now define -Primary switch, which will be used to get the service fabric replica primary
        [void]$sfParams.Add('Primary', $true)
        $replicaAfter = Get-SdnServiceFabricReplica @sfParams
        "Replica for {0} has been moved from {1} to {2}" -f $service.ServiceName, $replicaBefore.NodeName, $replicaAfter.NodeName | Trace-Output
    }
    catch {
        $_ | Trace-Exception
    }
}

function New-SdnCertificateRotationConfig {
    <#
    .SYNOPSIS
        Prepare the Network Controller Ceritifcate Rotation Configuration to determine which certificates to be used.
    .PARAMETER NetworkController
        Specifies the name the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> New-SdnCertificateRotationConfig
    .EXAMPLE
        PS> New-SdnCertificateRotationConfig -NetworkController 'NC01' -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) {
            $config = Get-SdnModuleConfiguration -Role 'NetworkController'
            $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
            if (-NOT ($confirmFeatures)) {
                "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning
                return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing
            }
        }

        $NcInfraInfo = Get-SdnNetworkControllerInfoOffline -NetworkController $NetworkController -Credential $Credential

        $CertificateRotationConfig = @{}
        $CertificateRotationConfig["ClusterCredentialType"] = $NcInfraInfo.ClusterCredentialType
        $getNewestCertScript = {
            param(
                [String]
                $certSubject
            )

            # Default to return Node Certificate
            if ([string]::IsNullOrEmpty($certSubject)) {
                $NodeFQDN = (get-ciminstance win32_computersystem).DNSHostName + "." + (get-ciminstance win32_computersystem).Domain
                $certSubject = "CN=$NodeFQDN"
            }

            Write-Verbose "Looking for cert match $certSubject"
            $cert = Get-ChildItem -Path Cert:\LocalMachine\My | ? { $_.Subject -ieq $certSubject } | Sort-Object -Property NotBefore -Descending | Select-Object -First 1
            return $cert.Thumbprint
        }
        $CertificateRotationConfig["NcRestCert"] = Invoke-PSRemoteCommand -ComputerName $NetworkController -ScriptBlock $getNewestCertScript -ArgumentList "CN=$($NcInfraInfo.NcRestName)" -Credential $Credential

        if($NcInfraInfo.ClusterCredentialType -eq "X509"){
            foreach ($ncNode in $($NcInfraInfo.NodeList)) {
                Trace-Output "Looking for Node Cert for Node: $($ncNode.NodeName), IpAddressOrFQDN: $($ncNode.IpAddressOrFQDN)" -Level:Verbose
                $ncNodeCert = Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -ScriptBlock $getNewestCertScript -Credential $Credential
                $CertificateRotationConfig[$ncNode.NodeName.ToLower()] = $ncNodeCert
            }
        }

        return $CertificateRotationConfig
    }
    catch {
        $_ | Trace-Exception
    }
}

function New-SdnNetworkControllerNodeCertificate {
    <#
    .SYNOPSIS
        Generate new Self-Signed Certificate to be used by Network Controller node.
    .PARAMETER NotAfter
        Specifies the date and time, as a DateTime object, that the certificate expires. To obtain a DateTime object, use the Get-Date cmdlet. The default value for this parameter is one year after the certificate was created.
    .PARAMETER CertPassword
        Specifies the password for the exported PFX file in the form of a secure string.
    .PARAMETER Credential
    .EXAMPLE
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [datetime]$NotAfter = (Get-Date).AddYears(1),

        [Parameter(Mandatory = $true)]
        [System.Security.SecureString]$CertPassword,

        [Parameter(Mandatory = $false)]
        [System.String]$Path = "$(Get-WorkingDirectory)\Cert_{0}" -f (Get-FormattedDateTimeUTC),

        [Parameter(Mandatory = $false)]
        [System.Object]$FabricDetails,

        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $config = Get-SdnModuleConfiguration -Role 'NetworkController'
    $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
    if (-NOT ($confirmFeatures)) {
        throw New-Object System.NotSupportedException("The current machine is not a NetworkController, run this on NetworkController.")
    }

    # ensure that the module is running as local administrator
    $elevated = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
    if (-NOT $elevated) {
        throw New-Object System.Exception("This function requires elevated permissions. Run PowerShell as an Administrator and import the module again.")
    }

    try {
        if ($null -eq $FabricDetails) {
            $FabricDetails = [SdnFabricInfrastructure]@{
                NetworkController = (Get-SdnNetworkControllerNode).Server
            }
        }

        if (-NOT (Test-Path -Path $Path -PathType Container)) {
            "Creating directory {0}" -f $Path | Trace-Output
            $CertPath = New-Item -Path $Path -ItemType Directory -Force
        }
        else {
            $CertPath = Get-Item -Path $Path
        }

        $nodeCertSubject = (Get-SdnNetworkControllerNodeCertificate).Subject
        $certificate = New-SdnCertificate -Subject $nodeCertSubject -NotAfter $NotAfter

        # after the certificate has been generated, we want to export the certificate using the $CertPassword provided by the operator
        # and save the file to directory. This allows the rest of the function to pick up these files and perform the steps as normal
        [System.String]$pfxFilePath = "$(Join-Path -Path $CertPath.FullName -ChildPath $nodeCertSubject.ToString().ToLower().Replace('.','_').Replace("=",'_').Trim()).pfx"
        "Exporting pfx certificate to {0}" -f $pfxFilePath | Trace-Output
        $exportedCertificate = Export-PfxCertificate -Cert $certificate -FilePath $pfxFilePath -Password $CertPassword -CryptoAlgorithmOption AES256_SHA256
        $null = Import-SdnCertificate -FilePath $exportedCertificate.FullName -CertStore 'Cert:\LocalMachine\Root' -CertPassword $CertPassword

        Copy-CertificateToFabric -CertFile $exportedCertificate.FullName -CertPassword $CertPassword -FabricDetails $FabricDetails `
            -NetworkControllerNodeCert -Credential $Credential

        return ([PSCustomObject]@{
            Certificate = $certificate
            FileInfo = $exportedCertificate
        })
    }
    catch {
        $_ | Trace-Exception
    }
}

function New-SdnNetworkControllerRestCertificate {
    <#
    .SYNOPSIS
        Generate new Self-Signed Certificate to be used by Network Controller.
    .PARAMETER NotAfter
        Specifies the date and time, as a DateTime object, that the certificate expires. To obtain a DateTime object, use the Get-Date cmdlet. The default value for this parameter is one year after the certificate was created.
    .PARAMETER CertPassword
        Specifies the password for the imported PFX file in the form of a secure string.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [String]$RestName,

        [Parameter(Mandatory = $false)]
        [datetime]$NotAfter = (Get-Date).AddYears(1),

        [Parameter(Mandatory = $true)]
        [System.Security.SecureString]$CertPassword,

        [Parameter(Mandatory = $false)]
        [System.String]$Path = "$(Get-WorkingDirectory)\Cert_{0}" -f (Get-FormattedDateTimeUTC),

        [Parameter(Mandatory = $false)]
        [System.Object]$FabricDetails,

        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    $config = Get-SdnModuleConfiguration -Role 'NetworkController'
    $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
    if (-NOT ($confirmFeatures)) {
        throw New-Object System.NotSupportedException("The current machine is not a NetworkController, run this on NetworkController.")
    }

    # ensure that the module is running as local administrator
    $elevated = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
    if (-NOT $elevated) {
        throw New-Object System.Exception("This function requires elevated permissions. Run PowerShell as an Administrator and import the module again.")
    }

    try {
        if ($FabricDetails) {
            if ($FabricDetails.LoadBalancerMux -or $FabricDetails.Server) {
                $installToSouthboundDevices = $true
            }
            else {
                $installToSouthboundDevices = $false
            }
        }
        else {
            $installToSouthboundDevices = $false

            $FabricDetails = [SdnFabricInfrastructure]@{
                NetworkController = (Get-SdnNetworkControllerNode).Server
            }
        }

        if (-NOT (Test-Path -Path $Path -PathType Container)) {
            "Creating directory {0}" -f $Path | Trace-Output
            $CertPath = New-Item -Path $Path -ItemType Directory -Force
        }
        else {
            $CertPath = Get-Item -Path $Path
        }

        [System.String]$formattedSubject = "CN={0}" -f $RestName.Trim()
        $certificate = New-SdnCertificate -Subject $formattedSubject -NotAfter $NotAfter

        # after the certificate has been generated, we want to export the certificate using the $CertPassword provided by the operator
        # and save the file to directory. This allows the rest of the function to pick up these files and perform the steps as normal
        [System.String]$pfxFilePath = "$(Join-Path -Path $CertPath.FullName -ChildPath $RestName.ToLower().Replace('.','_').Replace('=','_').Trim()).pfx"
        "Exporting pfx certificate to {0}" -f $pfxFilePath | Trace-Output
        $exportedCertificate = Export-PfxCertificate -Cert $certificate -FilePath $pfxFilePath -Password $CertPassword -CryptoAlgorithmOption AES256_SHA256
        $null = Import-SdnCertificate -FilePath $exportedCertificate.FullName -CertStore 'Cert:\LocalMachine\Root' -CertPassword $CertPassword

        Copy-CertificateToFabric -CertFile $exportedCertificate.FullName -CertPassword $CertPassword -FabricDetails $FabricDetails `
            -NetworkControllerRestCertificate -InstallToSouthboundDevices:$installToSouthboundDevices -Credential $Credential

        return ([PSCustomObject]@{
            Certificate = $certificate
            FileInfo = $exportedCertificate
        })
    }
    catch {
        $_ | Trace-Exception
    }
}

function Set-SdnNetworkController {
    <#
    .SYNOPSIS
        Sets network controller application settings.
    .PARAMETER RestIPAddress
        Specifies the IP address on which network controller nodes communicate with the REST clients. This IP address must not be an existing IP address on any of the network controller nodes.
    .PARAMETER RestName
        Switch parameter to configure the network controller application to use the server certificate's subject name as the REST endpoint name.
    .PARAMETER Credential
        Specifies a user credential that has permission to perform this action. The default is the current user.
        Specify this parameter only if you run this cmdlet on a computer that is not part of the network controller cluster.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'RestIPAddress')]
        [ValidateScript({
            $split = $_.split('/')
            if ($split.count -ne 2) { throw "RestIpAddress must be in CIDR format."}
            if (!($split[0] -as [ipaddress] -as [bool])) { throw "Invalid IP address specified in RestIpAddress."}
            if (($split[1] -le 0) -or ($split[1] -gt 32)) { throw "Invalid subnet bits specified in RestIpAddress."}
            return $true
        })]
        [System.String]$RestIpAddress,

        [Parameter(Mandatory = $true, ParameterSetName = 'RestName')]
        [Switch]$RestName,

        [Parameter(Mandatory = $false, ParameterSetName = 'RestIPAddress')]
        [Parameter(Mandatory = $false, ParameterSetName = 'RestName')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential
    )

    $waitDuration = 30 # seconds
    $params = @{}
    if ($Credential -ne [System.Management.Automation.PSCredential]::Empty -and $null -ne $Credential) {
        $params.Add('Credential', $Credential)
    }

    # we do not support this operation being executed from remote session
    # unless we have explicitly specified the Credential parameter
    if ($PSSenderInfo) {
        if ($Credential -eq [System.Management.Automation.PSCredential]::Empty -or $null -eq $Credential) {
            throw New-Object System.NotSupportedException("This operation is not supported in a remote session without supplying -Credential.")
        }
    }

    # ensure that the module is running as local administrator
    $elevated = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
    if (-NOT $elevated) {
        throw New-Object System.Exception("This function requires elevated permissions. Run PowerShell as an Administrator and import the module again.")
    }

    # add disclaimer that this feature is currently under preview
    "This feature is currently under preview. Please report any issues to https://github.com/microsoft/SdnDiagnostics/issues." | Trace-Output -Level:Warning
    $confirm = Confirm-UserInput -Message "Do you want to proceed with operation? [Y/N]:"
    if (-NOT $confirm) {
        "User has opted to abort the operation. Terminating operation" | Trace-Output -Level:Warning
        return
    }

    try {
        $getNetworkController = Get-SdnNetworkController
        if ($null -eq $getNetworkController) {
            throw New-Object System.Exception("Unable to retrieve results from Get-SdnNetworkController.")
        }

        $certSubjectName = $getNetworkController.ServerCertificate.Subject.Split('=')[1].Trim()
        if ($null -eq $certSubjectName) {
            throw New-Object System.Exception("Unable to retrieve current ServerCertificate.Subject property")
        }

        Connect-ServiceFabricCluster | Out-Null
        $param = Get-ServiceFabricApplication -ApplicationName 'fabric:/NetworkController' -ErrorAction Stop
        $version = $param.ApplicationParameters["SDNAPIConfigVersion"].Value
        $client=[System.Fabric.FabricClient]::new()

        switch ($PSCmdlet.ParameterSetName) {
            'RestName' {
                $params.Add('RestName', $certSubjectName)

                if ($getNetworkController.RestName) {
                    "Network Controller is already configured with RestName: {0}" -f $getNetworkController.RestName | Trace-Output -Level:Warning
                    return
                }

                else {
                    # if we changing from RestIPAddress to RestName, then we need to remove the RestIPAddress property and add the RestURL property
                    # once we remove the RestIPAddress property, we can perform a PUT operation to set the RestURL property
                    "Operation will set RestName to {0}." -f $certSubjectName | Trace-Output -Level:Warning
                    $confirm = Confirm-UserInput -Message "Do you want to proceed with operation? [Y/N]:"
                    if ($confirm) {
                        "Deleting the current RestIPAddress property" | Trace-Output
                        $client.PropertyManager.DeletePropertyAsync("fabric:/NetworkController/GlobalConfiguration", "SDNAPI.$version.RestIPAddress")
                        $client.PropertyManager.PutPropertyAsync("fabric:/NetworkController/GlobalConfiguration", "SDNAPI.$version.RestURL", $certSubjectName)
                    }
                    else {
                        "User has opted to abort the operation. Terminating operation" | Trace-Output
                        return
                    }
                }
            }

            'RestIPAddress' {
                $params.Add('RestIPAddress', $RestIpAddress)

                # check to see if the RestIPAddress is already configured, if so, then cross-compare the value currently configured with the new value
                # if we are just changing from one IP to another, then we can just update the value using Set-NetworkController
                if ($getNetworkController.RestIPAddress) {
                    if ($getNetworkController.RestIPAddress -ieq $RestIpAddress) {
                        "RestIPAddress is already set to {0}. Aborting operation." -f $getNetworkController.RestIPAddress | Trace-Output -Level:Warning
                        return
                    }
                    else {
                        "RestIPAddress is currently set to {0}. Operation will set RestIPAddress to {1}." -f $getNetworkController.RestIPAddress, $RestIpAddress | Trace-Output -Level:Warning
                        $confirm = Confirm-UserInput -Message "Do you want to proceed with operation? [Y/N]:"
                        if ($confirm) {
                            # do nothing here directly, since we will be calling Set-NetworkController later on
                        }
                        else {
                            "User has opted to abort the operation. Terminating operation" | Trace-Output
                            return
                        }
                    }
                }

                # if we changing from RestName to RestIPAddress, then we need to remove the RestURL property and add the RestIPAddress property
                # once we remove the RestUrl property, we need to insert a dummy CIDR value to ensure that the Set-NetworkController operation does not fail
                else {
                    "Operation will set RestIPAddress to {0}." -f $RestIpAddress | Trace-Output -Level:Warning
                    $confirm = Confirm-UserInput -Message "Do you want to proceed with operation? [Y/N]:"
                    if ($confirm) {
                        "Deleting the current RestURL property and inserting temporary RestIPAddress" | Trace-Output
                        $client.PropertyManager.DeletePropertyAsync("fabric:/NetworkController/GlobalConfiguration", "SDNAPI.$version.RestURL")
                        $client.PropertyManager.PutPropertyAsync("fabric:/NetworkController/GlobalConfiguration", "SDNAPI.$version.RestIPAddress", "10.65.15.117/27")
                    }
                    else {
                        "User has opted to abort the operation. Terminating operation" | Trace-Output
                        return
                    }
                }
            }
        }

        "Sleeping for {0} seconds" -f $waitDuration | Trace-Output
        Start-Sleep -Seconds $waitDuration # wait for the property to be deleted

        "Calling Set-NetworkController with params: {0}" -f ($params | ConvertTo-Json) | Trace-Output
        Set-NetworkController @params

        return (Get-SdnNetworkController)
    }
    catch {
        $_ | Trace-Exception
    }
    finally {
        if ($client) {
            $client.Dispose()
        }
    }
}

function Set-SdnServiceFabricClusterConfig {
    <#
    .SYNOPSIS
        Gets Service Fabric Cluster Config Properties.
    .PARAMETER NetworkController
        Specifies the name of the network controller node on which this cmdlet operates. Default to local machine.
    .PARAMETER Uri
        The Uri to read properties from ClusterConfiguration, GlobalConfiguration
    .PARAMETER Name
        Property Name to filter the result. If not specified, it will return all properties.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .EXAMPLE
        PS> Set-SdnServiceFabricClusterConfig -NetworkController 'NC01' -Uri "ClusterConfiguration" -Credential (Get-Credential)
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $false)]
        [String]$NetworkController = $(HostName),

        [Parameter(Mandatory = $true)]
        [ValidateSet('GlobalConfiguration', 'ClusterConfiguration')]
        [String]$Uri,

        [Parameter(Mandatory = $true)]
        [String]$Name,

        [Parameter(Mandatory = $true)]
        [System.Object]$Value,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {
        Connect-ServiceFabricCluster | Out-Null
        $client = [System.Fabric.FabricClient]::new()
        $absoluteUri = "fabric:/NetworkController/$Uri"
        $task = $client.PropertyManager.PutPropertyAsync($absoluteUri, $Name, $Value)
        $task.Wait()
    }
    catch {
        $_ | Trace-Exception
    }
}

function Show-SdnVipState {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [IPAddress]$VirtualIPAddress,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $false)]
        [Switch]$Detailed
    )

    try {
        if ($PSSenderInfo) {
            if ($Credential -eq [System.Management.Automation.PSCredential]::Empty -or $null -eq $Credential) {
                throw New-Object System.NotSupportedException("This operation is not supported in a remote session without supplying -Credential.")
            }
        }

        $slbManager = Connect-SlbManager -Credential $Credential -ErrorAction Stop
        if ($slbManager) {
            $consolidatedVipState = $slbManager.GetConsolidatedVipState($VirtualIPAddress, $Detailed)
            return $consolidatedVipState
        }
    }
    catch {
        $_ | Trace-Exception
    }
}

function Start-SdnCertificateRotation {
    <#
    .SYNOPSIS
        Performs a controller certificate rotate operation for Network Controller Northbound API, Southbound communications and Network Controller nodes.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER NcRestCredential
        Specifies a user account that has permission to access the northbound NC API interface. The default is the current user.
    .PARAMETER CertPath
        Path directory where certificate(s) .pfx files are located for use with certificate rotation.
    .PARAMETER GenerateCertificate
        Switch to determine if certificate rotate function should generate self-signed certificates.
    .PARAMETER CertPassword
        SecureString password for accessing the .pfx files, or if using -GenerateCertificate, what the .pfx files will be encrypted with.
    .PARAMETER NotAfter
        Expiration date when using -GenerateCertificate. If ommited, defaults to 3 years.
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    .PARAMETER Force
        Switch to force the rotation without being prompted, when Service Fabric is unhealthy.
    #>

    [CmdletBinding(DefaultParameterSetName = 'GenerateCertificate')]
    param (
        [Parameter(Mandatory = $true, ParameterSetName = 'Pfx')]
        [Parameter(Mandatory = $true, ParameterSetName = 'GenerateCertificate')]
        [Parameter(Mandatory = $true, ParameterSetName = 'CertConfig')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential,

        [Parameter(Mandatory = $false, ParameterSetName = 'Pfx')]
        [Parameter(Mandatory = $false, ParameterSetName = 'GenerateCertificate')]
        [Parameter(Mandatory = $false, ParameterSetName = 'CertConfig')]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $NcRestCredential = [System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory = $true, ParameterSetName = 'Pfx')]
        [System.String]$CertPath,

        [Parameter(Mandatory = $true, ParameterSetName = 'GenerateCertificate')]
        [Switch]$GenerateCertificate,

        [Parameter(Mandatory = $true, ParameterSetName = 'Pfx')]
        [Parameter(Mandatory = $true, ParameterSetName = 'GenerateCertificate')]
        [System.Security.SecureString]$CertPassword,

        [Parameter(Mandatory = $false, ParameterSetName = 'GenerateCertificate')]
        [datetime]$NotAfter = (Get-Date).AddYears(3),

        [Parameter(Mandatory = $true, ParameterSetName = 'CertConfig')]
        [hashtable]$CertRotateConfig,

        [Parameter(Mandatory = $false, ParameterSetName = 'Pfx')]
        [Parameter(Mandatory = $false, ParameterSetName = 'GenerateCertificate')]
        [Parameter(Mandatory = $false, ParameterSetName = 'CertConfig')]
        [switch]$Force
    )

    # ensure that the module is running as local administrator
    $elevated = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
    if (-NOT $elevated) {
        throw New-Object System.Exception("This function requires elevated permissions. Run PowerShell as an Administrator and import the module again.")
    }

    $config = Get-SdnModuleConfiguration -Role 'NetworkController'
    $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature
    if (-NOT ($confirmFeatures)) {
        throw New-Object System.NotSupportedException("The current machine is not a NetworkController, run this on NetworkController.")
    }

    # add disclaimer that this feature is currently under preview
    if (!$Force) {
        "This feature is currently under preview. Please report any issues to https://github.com/microsoft/SdnDiagnostics/issues so we can accurately track any issues and help unblock your cert rotation." | Trace-Output -Level:Warning
        $confirm = Confirm-UserInput -Message "Do you want to proceed with certificate rotation? [Y/N]:"
        if (-NOT $confirm) {
            "User has opted to abort the operation. Terminating operation" | Trace-Output -Level:Warning
            return
        }
    }

    try {
        "Starting certificate rotation" | Trace-Output
        "Retrieving current SDN environment details" | Trace-Output

        if ([String]::IsNullOrEmpty($CertPath)) {
            [System.String]$CertPath = "$(Get-WorkingDirectory)\Cert_{0}" -f (Get-FormattedDateTimeUTC)

            if (-NOT (Test-Path -Path $CertPath -PathType Container)) {
                $null = New-Item -Path $CertPath -ItemType Directory -Force
            }
        }

        [System.IO.FileSystemInfo]$CertPath = Get-Item -Path $CertPath -ErrorAction Stop

        # Get the Network Controller Info Offline (NC Cluster Down case)
        $NcInfraInfo = Get-SdnNetworkControllerInfoOffline -Credential $Credential

        if ($NcInfraInfo.ClusterCredentialType -ieq 'X509') {
            $rotateNCNodeCerts = $true
        }

        # Get the current rest certificate to determine if it is expired scenario or not.
        $currentRestCert = Get-SdnNetworkControllerRestCertificate

        $restCertExpired = (Get-Date) -gt $($currentRestCert.NotAfter)
        $ncHealthy = $true

        if (!$restCertExpired) {
            try {
                $null = Get-NetworkController
            }
            catch {
                $ncHealthy = $false
            }
        }

        if ($restCertExpired -or !$ncHealthy) {
            $postRotateSBRestCert = $true
            if ($restCertExpired) {
                "Network Controller Rest Certificate {0} expired at {1}" -f $currentRestCert.Thumbprint, $currentRestCert.NotAfter | Trace-Output -Level:Warning
            }

            "Network Controller is currently not healthy" | Trace-Output -Level:Warning
            $sdnFabricDetails = [SdnFabricInfrastructure]@{
                NetworkController = $NcInfraInfo.NodeList.IpAddressOrFQDN
            }

            Install-SdnDiagnostics -ComputerName $sdnFabricDetails.NetworkController -Credential $Credential -ErrorAction Stop
        }
        else {
            # determine fabric information and current version settings for network controller
            $sdnFabricDetails = Get-SdnInfrastructureInfo -NetworkController $env:COMPUTERNAME -Credential $Credential -NcRestCredential $NcRestCredential
            $ncClusterSettings = Get-NetworkControllerCluster
            $ncSettings = @{
                NetworkControllerVersion        = (Get-NetworkController).Version
                NetworkControllerClusterVersion = $ncClusterSettings.Version
                ClusterAuthentication           = $ncClusterSettings.ClusterAuthentication
            }

            # before we proceed with anything else, we want to make sure that all the Network Controllers within the SDN fabric are running the current version
            Install-SdnDiagnostics -ComputerName $sdnFabricDetails.NetworkController -ErrorAction Stop

            "Network Controller version: {0}" -f $ncSettings.NetworkControllerVersion | Trace-Output
            "Network Controller cluster version: {0}" -f $ncSettings.NetworkControllerClusterVersion | Trace-Output

            $healthState = Get-SdnServiceFabricClusterHealth -NetworkController $env:COMPUTERNAME
            if ($healthState.AggregatedHealthState -ine 'Ok') {
                "Service Fabric AggregatedHealthState is currently reporting {0}. Please address underlying health before proceeding with certificate rotation" `
                    -f $healthState.AggregatedHealthState | Trace-Output -Level:Error

                if (!$Force) {
                    $confirm = Confirm-UserInput -Message "Do you want to proceed with certificate rotation? Enter N to abort and address the underlying health. Enter Y to force continue:"
                    if (-NOT $confirm) {
                        "User has opted to abort the operation. Terminating operation" | Trace-Output -Level:Warning
                        return
                    }
                }
            }
        }

        #####################################
        #
        # Create Certificate (Optional)
        #
        #####################################

        if ($PSCmdlet.ParameterSetName -ieq 'GenerateCertificate') {
            "== STAGE: CREATE SELF SIGNED CERTIFICATES ==" | Trace-Output

            $newSelfSignedCert = New-SdnNetworkControllerRestCertificate -RestName $NcInfraInfo.NcRestName.ToString() -NotAfter $NotAfter -Path $CertPath.FullName `
            -CertPassword $CertPassword -Credential $Credential -FabricDetails $sdnFabricDetails

            $selfSignedRestCertFile = $newSelfSignedCert.FileInfo

            if ($rotateNCNodeCerts) {
                $null = Invoke-PSRemoteCommand -ComputerName $sdnFabricDetails.NetworkController -Credential $Credential -ScriptBlock {
                    param(
                        [Parameter(Position = 0)][DateTime]$param1,
                        [Parameter(Position = 1)][SecureString]$param2,
                        [Parameter(Position = 2)][PSCredential]$param3,
                        [Parameter(Position = 3)][String]$param4,
                        [Parameter(Position = 4)][System.Object]$param5
                    )

                    New-SdnNetworkControllerNodeCertificate -NotAfter $param1 -CertPassword $param2 -Credential $param3 -Path $param4 -FabricDetails $param5
                } -ArgumentList @($NotAfter, $CertPassword, $Credential, $CertPath.FullName, $sdnFabricDetails)
            }

            $CertRotateConfig = New-SdnCertificateRotationConfig -Credential $Credential
        }

        #####################################
        #
        # PFX Certificates (Optional)
        #
        #####################################

        if ($PSCmdlet.ParameterSetName -ieq 'Pfx') {
            "== STAGE: Install PFX Certificates to Fabric ==" | Trace-Output
            $pfxCertificates = Copy-UserProvidedCertificateToFabric -CertPath $CertPath -CertPassword $CertPassword -FabricDetails $sdnFabricDetails `
            -NetworkControllerHealthy:$ncHealthy -Credential $Credential -RotateNodeCerts:$rotateNCNodeCerts

            $pfxCertificates | ForEach-Object {
                if ($_.CertificateType -ieq 'NetworkControllerRest' ) {
                    if ($_.SelfSigned -ieq $true) {
                        $selfSignedRestCertFile = $_.FileInfo
                    }
                }
            }

            $CertRotateConfig = New-SdnCertificateRotationConfig -Credential $Credential
        }

        #####################################
        #
        # Certificate Configuration
        #
        #####################################

        "== STAGE: DETERMINE CERTIFICATE CONFIG ==" | Trace-Output

        "Validating Certificate Configuration" | Trace-Output
        $certValidated = Test-SdnCertificateRotationConfig -NcNodeList $NcInfraInfo.NodeList -CertRotateConfig $CertRotateConfig -Credential $Credential

        if ($certValidated -ne $true) {
            throw New-Object System.NotSupportedException("Unable to validate certificate configuration")
        }

        $updatedRestCertificate = Get-ChildItem -Path Cert:\LocalMachine\My | Where-Object { $_.Subject -ieq $currentRestCert.Subject } `
        | Sort-Object -Property NotBefore -Descending | Select-Object -First 1

        "Network Controller Rest Certificate {0} will be updated from [Thumbprint:{1} NotAfter:{2}] to [Thumbprint:{3} NotAfter:{4}]" `
            -f $currentRestCert.Subject, $currentRestCert.Thumbprint, $currentRestCert.NotAfter, $CertRotateConfig["NcRestCert"], $updatedRestCertificate.NotAfter `
        | Trace-Output -Level:Warning

        if ($rotateNCNodeCerts) {
            foreach ($node in $NcInfraInfo.NodeList) {
                $nodeCertThumbprint = $certRotateConfig[$node.NodeName.ToLower()]
                $currentNodeCert = Invoke-PSRemoteCommand -ComputerName $node.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                    Get-SdnNetworkControllerNodeCertificate
                }

                $newNodeCert = Invoke-PSRemoteCommand -ComputerName $node.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                    param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                    Get-SdnCertificate -Path $param1 -Thumbprint $param2
                } -ArgumentList @('Cert:\LocalMachine\My', $nodeCertThumbprint)

                "Network Controller Node Certificate {0} will be updated from [Thumbprint:{1} NotAfter:{2}] to [Thumbprint:{3} NotAfter:{4}]" `
                    -f $currentNodeCert.Subject, $currentNodeCert.Thumbprint, $currentNodeCert.NotAfter, `
                    $newNodeCert.Thumbprint, $newNodeCert.NotAfter | Trace-Output -Level:Warning
            }
        }

        if (!$Force) {
            $confirm = Confirm-UserInput
            if (-NOT $confirm) {
                "User has opted to abort the operation. Terminating operation" | Trace-Output -Level:Warning
                return
            }
        }

        #####################################
        #
        # Rotate NC Certificate Expired
        #
        #####################################

        if ($restCertExpired -or !$ncHealthy) {
            # Use this for certificate if either rest cert expired or nc unhealthy, get-networkcontroller failed
            Start-SdnExpiredCertificateRotation -CertRotateConfig $CertRotateConfig -Credential $Credential -NcRestCredential $NcRestCredential
        }

        #####################################
        #
        # Rotate NC Northbound Certificate (REST)
        #
        #####################################

        "== STAGE: ROTATE NC REST CERTIFICATE ==" | Trace-Output

        $null = Invoke-CertRotateCommand -Command 'Set-NetworkController' -Credential $Credential -Thumbprint $CertRotateConfig["NcRestCert"]

        "Waiting for 5 minutes before proceeding to the next step. Script will resume at {0}" -f (Get-Date).AddMinutes(5).ToUniversalTime().ToString() | Trace-Output
        Start-Sleep -Seconds 300

        #####################################
        #
        # Rotate Cluster Certificate
        #
        #####################################

        "== STAGE: ROTATE NC CLUSTER CERTIFICATE ==" | Trace-Output

        $null = Invoke-CertRotateCommand -Command 'Set-NetworkControllerCluster' -Credential $Credential -Thumbprint $CertRotateConfig["NcRestCert"]

        "Waiting for 5 minutes before proceeding to the next step. Script will resume at {0}" -f (Get-Date).AddMinutes(5).ToUniversalTime().ToString() | Trace-Output
        Start-Sleep -Seconds 300

        #####################################
        #
        # Rotate NC Node Certificates
        #
        #####################################

        if ($rotateNCNodeCerts) {
            "== STAGE: ROTATE NC NODE CERTIFICATE ==" | Trace-Output

            foreach ($node in $NcInfraInfo.NodeList) {
                $nodeCertThumbprint = $certRotateConfig[$node.NodeName.ToLower()]
                $null = Invoke-CertRotateCommand -Command 'Set-NetworkControllerNode' -NetworkController $node.IpAddressOrFQDN -Credential $Credential -Thumbprint $nodeCertThumbprint

                "Waiting for 2 minutes before proceeding to the next step. Script will resume at {0}" -f (Get-Date).AddMinutes(5).ToUniversalTime().ToString() | Trace-Output
                Start-Sleep -Seconds 120
            }
        }

        #####################################
        #
        # Rotate NC Southbound Certificates
        #
        #####################################

        "== STAGE: ROTATE SOUTHBOUND CERTIFICATE CREDENTIALS ==" | Trace-Output

        $null = Update-NetworkControllerCredentialResource -NcUri "https://$($NcInfraInfo.NcRestName)" -Credential $NcRestCredential `
            -NewRestCertThumbprint $CertRotateConfig["NcRestCert"] -ErrorAction Stop

        "Southbound certificate rotation completed" | Trace-Output

        #####################################
        #
        # Certificate Seeding (Southbound Nodes)
        #
        #####################################

        # if nc was unhealthy and unable to determine southbound devices in the dataplane earlier
        # we now want to check to see if nc is healthy and if we need to install the rest cert (for self-signed) to southbound devices
        if ($postRotateSBRestCert) {
            if ($selfSignedRestCertFile) {
                $sdnFabricDetails = Get-SdnInfrastructureInfo -Credential $Credential -NcRestCredential $NcRestCredential -Force
                $southBoundNodes = @()
                if ($null -ne $sdnFabricDetails.LoadBalancerMux) {
                    $southBoundNodes += $sdnFabricDetails.LoadBalancerMux
                }
                if ($null -ne $sdnFabricDetails.Server) {
                    $southBoundNodes += $sdnFabricDetails.Server
                }

                if ($southBoundNodes) {
                    "== STAGE: REST SELF-SIGNED CERTIFICATE SEEDING (Southbound Nodes) ==" | Trace-Output

                    # ensure that we have the latest version of sdnDiagnostics module on the southbound devices
                    Install-SdnDiagnostics -ComputerName $southBoundNodes -Credential $Credential -ErrorAction Stop

                    "[REST CERT] Installing self-signed certificate to {0}" -f ($southBoundNodes -join ', ') | Trace-Output
                    [System.String]$remoteFilePath = Join-Path -Path $CertPath.FullName -ChildPath $selfSignedRestCertFile.Name
                    Copy-FileToRemoteComputer -ComputerName $southBoundNodes -Credential $Credential -Path $selfSignedRestCertFile.FullName -Destination $remoteFilePath
                    $null = Invoke-PSRemoteCommand -ComputerName $southBoundNodes -Credential $Credential -ScriptBlock {
                        param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][String]$param2)
                        Import-SdnCertificate -FilePath $param1 -CertStore $param2
                    } -ArgumentList @($remoteFilePath, 'Cert:\LocalMachine\Root') -ErrorAction Stop
                }
            }
        }

        #####################################
        #
        # Restart services
        #
        #####################################

        "== STAGE: RESTART NETWORK CONTROLLER SERVICES ==" | Trace-Output
        # restart the network controller services
        # this will force new TLS connections to be established to southbound devices
        # ensuring that the new certificates are used and we are able to push policies successfully

        # check to determine if we have a multi-node NC cluster and if so, leverage the SF cmdlets to move the replicas
        # otherwise, we will just stop the processes and let SF restart them automatically
        if ($sdnFabricDetails.NetworkController.Count -gt 1) {
            Move-SdnServiceFabricReplica -ServiceTypeName 'SlbManagerService'
            Move-SdnServiceFabricReplica -ServiceTypeName 'VSwitchService'
        }
        else {
            Get-Process -Name 'SDNFW' | Stop-Process -Force -ErrorAction Continue
            Get-Process -Name 'SDNSLBM' | Stop-Process -Force -ErrorAction Continue
        }

        "Certificate rotation has completed" | Trace-Output
    }
    catch {
        $_ | Trace-Exception
    }
}

function Test-SdnCertificateRotationConfig {
    <#
    .SYNOPSIS
        Validate the Cert Rotation Config provided is correct. Ensure certificates specified present on the machine.
    .PARAMETER NcNodeList
        The NcNodeList that retrieved via Get-SdnNetworkControllerInfoOffline.
    .PARAMETER Credential
        Specifies a user account that has permission to perform this action. The default is the current user.
    .PARAMETER CertRotateConfig
        The Config generated by New-SdnCertificateRotationConfig to include NC REST certificate thumbprint and node certificate thumbprint.
    #>

    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject[]]$NcNodeList,

        [Parameter(Mandatory = $true)]
        [hashtable]$CertRotateConfig,

        [Parameter(Mandatory = $false)]
        [System.Management.Automation.PSCredential]
        [System.Management.Automation.Credential()]
        $Credential = [System.Management.Automation.PSCredential]::Empty
    )

    try {

        if ([string]::IsNullOrEmpty($CertRotateConfig["NcRestCert"])) {
            Trace-Output "NcRestCert not specified in CertRotateConfig" -Level:Error
            return $false
        }

        $ncRestCert = $CertRotateConfig["NcRestCert"]
        foreach ($ncNode in $NcNodeList) {
            if ($CertRotateConfig["ClusterCredentialType"] -ieq "X509") {
                $nodeCert = $CertRotateConfig[$ncNode.NodeName.ToLower()]
                if ([string]::IsNullOrEmpty($nodeCert)) {
                    Trace-Output "The ClusterCredentialType is X509 but Node $($ncNode.NodeName) does not have certificate specified" -Level:Error
                    return $false
                }
                else {
                    $certValid = Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                        param([Parameter(Position = 0)][String]$param1)
                        $nodeCertObj = Get-SdnCertificate -Path "Cert:\LocalMachine\My" -Thumbprint $param1
                        if ($null -eq $nodeCertObj) {
                            return $false
                        }
                        else {
                            if ($nodeCertObj.NotAfter -le (Get-Date)) {
                                return $false
                            }
                        }
                        return $true
                    } -ArgumentList $nodeCert

                    if (!$certValid) {
                        Trace-Output "Node $($ncNode.NodeName) does not have validate Node certificate with thumbprint $nodeCert installed" -Level:Error
                        return $false
                    }
                }
            }

            $certValid = Invoke-PSRemoteCommand -ComputerName $ncNode.IpAddressOrFQDN -Credential $Credential -ScriptBlock {
                param([Parameter(Position = 0)][String]$param1)
                $ncRestCertObj = Get-SdnCertificate -Path "Cert:\LocalMachine\My" -Thumbprint $param1
                if ($null -eq $ncRestCertObj) {
                    return $false
                }
                else {
                    if ($ncRestCertObj.NotAfter -le (Get-Date)) {
                        return $false
                    }
                }
                return $true
            } -ArgumentList $ncRestCert

            if (!$certValid) {
                Trace-Output "Node $($ncNode.NodeName) does not have validate NcRest certificate with thumbprint $ncRestCert installed" -Level:Error
                return $false
            }
        }
        return $true
    }
    catch {
        $_ | Trace-Exception
    }
}