NetworkControllerServiceFabricUpdate

0.6

NetworkControllerServiceFabricUpdate.psm1

# SF version to skip
$script:SKIP = "7.0.470.9590"
# SF version to update to after update to 7.0.470.9590 is skipped
$script:SKIPPED = "7.1.409.9590"
$script:VERSION_TO_UPGRADE_TO = "9.1.1993.9590"

function Update-NetworkControllerServiceFabric
{
<#
.SYNOPSIS
Update Service Fabric incrementally to the most recent supported version of Service Fabric.
.PARAMETER NodeName
hostname of NetworkController node to run Service Fabric updates on. Can be omitted if script is invoked from NetworkController node.
.PARAMETER CabLocation
Location of CAB on NetworkController node of $NodeName.
.PARAMETER Credentials
Credentials for NetworkController node of $NodeName. Can be omitted if script is invoked from NetworkController node.
.EXAMPLE
Update-NetworkControllerServiceFabric -NodeName qntpf0jl-1 -CabLocation c:\sf.cab -Credential $credential
^ Script invoked from host
.EXAMPLE
Update-NetworkControllerServiceFabric -CabLocation c:\sf.cab
^ Script invoked from Network Controller node
.Inputs
None.
.Outputs
None.
.NOTES
- Service Fabric version prior to upgrade must be one of the following:
    6.0.240.9494
    6.2.301.9494
    6.4.664.9590
    7.0.470.9590
    7.1.409.9590
    7.1.510.9590
    8.0.536.9590
    9.0.1155.9590
    9.1.1436.9590
- Operating system must be one of the following:
    Windows Server 2022, AZ Stack HCI 21H2, AZ Stack HCI 22H2, AZ Stack HCI 23H2
- If the OS is AZ Stack HCI 23H2, Service Fabric will be updated to V9.1.1993.9590.
    Otherwise it will be updated to V7.1.510.9590.
- If Powershell constrained language mode (CLM) is enabled, the script must be invoked from the Network Controller
    node and the initial Service Fabric version must be at least 7.1.510.9590.
#>


    param (
        [Parameter(Mandatory=$true)]
        [string]$CabLocation,

        [Parameter(Mandatory=$false)]
        [string]$NodeName=[string]::Empty,

        [Parameter(Mandatory=$false)]
        [PSCredential]$Credential=[System.Management.Automation.PSCredential]::Empty,

        [Parameter(Mandatory=$false)]
        [string]$LogLocation="$($env:SystemDrive)\Windows\tracing"
    )
    $Stamp = (Get-Date).toString("yyyy-MM-dd-HH-mm-ss")
    $logFile = $LogLocation + "\UpgradeNCSFVersion_" + $Stamp + ".log"
    try { Stop-Transcript -ErrorAction Ignore | Out-Null } catch { Write-Log "Exception thrown while stopping transcript. Continuing anyway."}
    try { Remove-Item $logFile -ErrorAction Ignore | Out-Null } catch { Write-Log "Exception thrown while removing log file. Continuing anyway."}
    Start-Transcript -Path $logFile -ErrorAction Ignore
    try
    {
        if ($NodeName -eq [string]::Empty)
        {
            $NodeName=hostname
            Write-Log "Parameter NodeName was not supplied. Using hostname: $NodeName instead."
        }
        Write-Log "Validating setup prior to starting upgrades."
        $RemoteSession = New-PSSession -ComputerName $NodeName -Credential $Credential
        if (!$RemoteSession)
        {
            throw "Could not establish remote session with $NodeName. All NC VMs should be healthy and accessible at this time. Exiting."
        }
        # Element at position i of $UpdateSequenceValues is the next version of SF to update to given current SF version (element at position i of $UpdateSequenceKeys)
        $UpdateSequenceKeys = @("6.0.240.9494", "6.2.301.9494", "6.3.205.9494", "6.4.664.9590", "7.0.470.9590", "7.1.409.9590", "7.1.510.9590", "8.0.536.9590", "9.0.1155.9590", "9.1.1436.9590")
        $UpdateSequenceValues = @("6.2.301.9494", "6.4.664.9590", "6.4.664.9590", "7.0.470.9590", "7.1.409.9590", "7.1.510.9590", "8.0.536.9590", "9.0.1155.9590", "9.1.1436.9590", "9.1.1993.9590")
        # What if the NC01 is not reachable, well we should not continue in that case anyway
        $sb = (Get-Item function:ValidateSetup).ScriptBlock
        $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
        Invoke-Command -Session $RemoteSession -ScriptBlock $sb -ArgumentList @($CabLocation, $script:VERSION_TO_UPGRADE_TO, $script:SKIP, $UpdateSequenceKeys, $UpdateSequenceValues)
        Invoke-NCServiceFabricUpdate -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName -CabLocation $CabLocation -UpdateSequenceKeys $UpdateSequenceKeys -UpdateSequenceValues $UpdateSequenceValues
    }
    finally
    {
        if ($RemoteSession)
        {
            $RemoteSession | Remove-PSSession -ErrorAction Ignore | Out-Null
        }
        Stop-Transcript
    }
}

function Write-Log {
    param(
        [Parameter(Mandatory=$True)]
        [string]$Message
    )
    $Stamp = (Get-Date).toString("yyyy/MM/dd HH:mm:ss:fff")
    Write-Output "$Stamp $Message"
}

function ValidateSetup
{
    param(
        [Parameter(Mandatory=$true)]
        [string]$CabLocation,

        [Parameter(Mandatory=$true)]
        [string]$SFVersionToUpgradeTo,

        [Parameter(Mandatory=$true)]
        [string]$skip,

        [Parameter(Mandatory=$true)]
        [string[]] $UpdateSequenceKeys,

        [Parameter(Mandatory=$true)]
        [string[]] $UpdateSequenceValues
    )

    Write-Output "Path to CAB is $CabLocation"
    if (!(Test-Path $CabLocation -PathType leaf) -or !($CabLocation.ToLower() -Like "*.cab"))
    {
        throw "Path to CAB $CabLocation is not valid."
    }
    $parentDir = Split-Path $CabLocation -parent
    cmd.exe /c "expand $CabLocation -F:* $parentDir"

    # Set Fabric.Code path if it is not already set.
    if (!$env:Path.Contains("$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code"))
    {
        $env:Path=$env:Path+";"+"$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code;"
    }

    try
    {
        Connect-ServiceFabricCluster
    }
    catch
    {
        throw "Connect-ServiceFabricCluster failed. Please ensure Service Fabric cluster is installed."
    }
    $sfVersion = Get-ServiceFabricClusterUpgrade
    $currVersionIndex = $UpdateSequenceKeys.IndexOf($sfVersion.TargetCodeVersion)
    Write-Output "Verifying that each required version of Service Fabric is present in extracted CAB."
    foreach ($k in $UpdateSequenceKeys)
    {
        $upgradeTo = $UpdateSequenceValues[$UpdateSequenceKeys.IndexOf($k)]
        if ($k -eq $SFVersionToUpgradeTo)
        {
            break
        }

        if (($UpdateSequenceKeys.IndexOf($k) -lt $currVersionIndex) -or ($upgradeTo -eq $skip))
        {
            continue
        }
        $filename = "MicrosoftServiceFabric." + $upgradeTo + ".cab"
        $fullPath = Join-Path $parentDir $filename -resolve
        if (!$fullPath)
        {
            throw "Missing $filename. $filename must be included in order to upgrade."
        }
    }
    Write-Output "Setup validated succesfully."
}

function Invoke-NCServiceFabricUpdate
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        [Parameter(Mandatory=$true)]
        [string]$NodeName,

        [Parameter(Mandatory=$true)]
        [string]$CabLocation,

        [Parameter(Mandatory=$true)]
        [string[]] $UpdateSequenceKeys,

        [Parameter(Mandatory=$true)]
        [string[]] $UpdateSequenceValues
    )

    try
    {
        Write-Log "Starting Service Fabric updates to update to $script:VERSION_TO_UPGRADE_TO."
        Write-Log "Creating session to NC node - $NodeName"
        $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
        $null = Invoke-Command -Session $RemoteSession -ScriptBlock {Connect-ServiceFabricCluster}
        $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
        $sfVersion = Invoke-Command -Session $RemoteSession -ScriptBlock {Get-ServiceFabricClusterUpgrade}
        Write-Log "sfVersion retrieved is $($sfVersion), with target code version: $($sfVersion.TargetCodeVersion)."

        if (($sfVersion.UpgradeState -like "RollingForwardInProgress") -or ($sfVersion.UpgradeState -like "RollingForwardPending"))
        {
            # A previous SF upgrade is already in progress
            Write-Log "NC Ring is still updating to version $($sfVersion.TargetCodeVersion). Waiting for upgrade to finish."

            #Wait for SF upgrade to finish
            WaitForNCServiceFabricUpgrade -RemoteSession $RemoteSession -NodeName $NodeName -UpgradeState RollingForwardCompleted -UpgradeVersion $sfVersion.TargetCodeVersion -Credential $Credential
            #WaitFor NC cluster to be healthy
            WaitForNCApplicationHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
            #Wait for SF cluster to be healthy
            WaitForSFClusterHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
        }
        elseif (!($sfVersion.TargetCodeVersion -in $UpdateSequenceKeys) -and !($sfVersion.TargetCodeVersion -like $script:VERSION_TO_UPGRADE_TO))
        {
            throw "Upgrades are not supported for $($sfVersion.TargetCodeVersion)."
        }
        else
        {
            Write-Log "NC version is $($sfVersion.TargetCodeVersion), Checking Nodestatus and SF health before proceeding to upgrade SF version."
            #make sure all the NC nodes and SF is healthy before upgrading the servicefabric ring
            $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
            $badNodes = Invoke-Command -Session $RemoteSession -ScriptBlock {Get-ServiceFabricNode | Where-Object { $_.NodeStatus -ne 'Up' }}
            if ($badNodes)
            {
                throw "One or more NC nodes are not healthy: $badNodes. Exiting"
            }

            #if aggregated status is not OK, don't continue
            WaitForNCApplicationHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
            WaitForSFClusterHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
            $updateComplete = $false
            while (!($updateComplete))
            {
                if ($sfVersion.TargetCodeVersion -like $script:VERSION_TO_UPGRADE_TO)
                {
                    Write-Log "Upgrade is complete! Service Fabric version is: $($sfVersion.TargetCodeVersion)."
                    break
                }
                $versionToUpgradeTo = $UpdateSequenceValues[$UpdateSequenceKeys.IndexOf($SFVersion.TargetCodeVersion)]
                if ($versionToUpgradeTo -like $script:SKIP)
                {
                    Write-Log "Fetching the Service Fabric cluster manifest to determine if FileStoreService parameters PrimaryAccountNTLMX509Thumbprint and
                        SecondaryAccountNTLMX509Thumbprint are present. If present, will upgrade directly to $script:SKIPPED. Otherwise, the cluster will be automatically
                        updated to include the parameters."
                    $thumbprintsExist = DoThumbprintsExist -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
                    Write-Log "Thumbprints exist: $thumbprintsExist."
                    if ($thumbprintsExist)
                    {
                        $versionToUpgradeTo = $script:SKIPPED
                    }
                }
                PerformUpdate -RemoteSession $RemoteSession -NodeName $NodeName -NCSFVersion $versionToUpgradeTo -CabLocation $CabLocation -Credential $Credential
                if ($RemoteSession)
                {
                    $RemoteSession | Remove-PSSession -ErrorAction Ignore | Out-Null
                }
                $RemoteSession = New-PSSession -ComputerName $NodeName -Credential $Credential
                if (!$RemoteSession)
                {
                    throw "Could not establish remote session with $NodeName. All NC VMs should be healthy and accessible at this time. Exiting."
                }
                $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
                $sfVersion = Invoke-Command -Session $RemoteSession -ScriptBlock {
                    # Set Fabric.Code path if it is not already set.
                    if (!$env:Path.Contains("$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code"))
                    {
                        $env:Path=$env:Path+";"+"$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code;"
                    }
                    Connect-ServiceFabricCluster | Out-Null; Get-ServiceFabricClusterUpgrade
                }
            }
        }
    }
    catch
    {
        Write-Log "Invoke-NCServiceFabricUpdate failed with exception: $_"
        throw
    }
}

function CheckRemoteSession
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        [Parameter(Mandatory=$true)]
        [string]$NodeName
    )
    if (!$RemoteSession -or $RemoteSession.State -ne "Opened")
    {
        $RemoteSession | Remove-PSSession -ErrorAction Ignore | Out-Null
        $RemoteSession = New-PSSession -ComputerName $NodeName -Credential $Credential
    }
    return $RemoteSession
}

function DoThumbprintsExist
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        [Parameter(Mandatory=$true)]
        [string]$NodeName
    )

    $sb = {
        $connectAttempt = 5
        do
        {
            try
            {
                Connect-ServiceFabricCluster | out-null
                $currentFile = "$env:windir\temp\ClusterManifest_current.xml"
                Remove-Item -Path $currentFile -Force -ErrorAction SilentlyContinue
                Get-ServiceFabricClusterManifest > $currentFile
                $xml = [xml](get-content $currentFile)
                break
            }
            catch
            {
                if($connectAttempt -eq 0)
                {
                    Write-Output "$_.Exception.Message"
                    throw
                }
                else
                {
                    Write-Output "$_.Exception.Message"
                    Start-Sleep -Seconds 5
                }
                $connectAttempt--
            }
        } while($connectAttempt -gt 0)

        $fileStoreSection = $xml.ClusterManifest.FabricSettings.Section | Where-Object { $_.Name -imatch "FileStoreService" }
        if (-Not($fileStoreSection))
        {
            throw "FileStoreService section could not be found in cluster manifest. It must be present to upgrade to next SF version."
        }

        $params = @("PrimaryAccountNTLMX509Thumbprint", "SecondaryAccountNTLMX509Thumbprint")
        $restCert = (Get-NetworkController).ServerCertificate
        foreach ($p in $params)
        {
            $param = $fileStoreSection.Parameter | Where-Object { $_.Name -imatch $p  }
            if (-Not($param) -or !($param.Value -like $restCert.Thumbprint))
            {
                return $false
            }
        }
        return $true
    }
    $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
    $thumbprintsExist = Invoke-Command -Session $RemoteSession -ScriptBlock $sb
    return $thumbprintsExist
}

function UpdateNCServiceFabricVersion
{
    param (
        [Parameter(Mandatory=$true)]
        [string]$NCSFVersion,

        [Parameter(Mandatory=$true)]
        [string]$CabLocation
    )
    Connect-ServiceFabricCluster | out-null

    $SFInstallerCab = "MicrosoftServiceFabric." + $NCSFVersion + ".cab"
    $SF_CAB_DIR = Split-Path -Path $CabLocation
    $ServicefabricInstallerPath = Join-Path $SF_CAB_DIR $SFInstallerCab -resolve
    Write-Output "Path to service fabric cab is: $ServicefabricInstallerPath"
    if (!$ServicefabricInstallerPath)
    {
        Write-Output "Path for service fabric installer is NULL. Exiting."
    }

    Write-Output "Copy package file and register new version with SF Cluster."
    Copy-ServiceFabricClusterPackage -Code -CodePackagePath $ServicefabricInstallerPath -ImageStoreConnectionString "fabric:ImageStore"

    try
    {
        Register-ServiceFabricClusterPackage -Code -CodePackagePath $SFInstallerCab
    }
    catch [System.Fabric.FabricElementAlreadyExistsException]
    {$ServicefabricInstallerPath
        Write-Output "Service Fabric package is already registered. Continuing."
    }

    Write-Output "Start-ServiceFabricClusterUpgrade to SF version $NCSFVersion"
    Start-ServiceFabricClusterUpgrade -Code -CodePackageVersion $NCSFVersion -Monitored -FailureAction Rollback

    $SFUpgradestarted = Get-ServiceFabricClusterUpgrade
    if (!($SFUpgradestarted.TargetCodeVersion -like $NCSFVersion) -or !($SFUpgradestarted.UpgradeState -like "RollingForwardInProgress"))
    {
        throw "ServiceFabricClusterUpgrade did not start. Exiting."
    }
    Write-Output "Successfully started ServiceFabricClusterUpgrade. SF Upgrade is in Progress."
}

function PerformUpdate
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [string]$NodeName,

        [Parameter(Mandatory=$true)]
        [string]$NCSFVersion,

        [Parameter(Mandatory=$true)]
        [string]$CabLocation,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential
    )
    if ($NCSFVersion -like $script:SKIP)
    {
        $sb=(Get-Item function:SetServiceFabricClusterManifest).ScriptBlock
        $a=@()
    }
    else {
        $sb=(Get-Item function:UpdateNCServiceFabricVersion).ScriptBlock
        $a=@($NCSFVersion, $CabLocation)
    }

    $maxRetries = 3
    $retryCount = 0
    while ($retryCount -lt $maxRetries)
    {
        try
        {
            if ($NCSFVersion -like $script:SKIP)
            {
                Write-Log "Running cluster upgrade on $NodeName to add FileStoreService parameters PrimaryAccountNTLMX509Thumbprint and
                    SecondaryAccountNTLMX509Thumbprint. These parameters are necesary to upgrade directly to $script:SKIPPED, retry attempt# $retryCount."
            }
            else
            {
                Write-Log "Running ServiceFabricClusterUpgrade to version $NCSFVersion on $NodeName, retry attempt# $retryCount."
            }
            $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
            Invoke-Command -Session $RemoteSession -ScriptBlock $sb -ArgumentList $a
            Write-Log "Successfully started cluster upgrade on $NodeName. Upgrade is in Progress."
            break
        }
        catch
        {
            Write-Log "PerformUpdate failed with exception $_"
            $retryCount++
            Start-Sleep -Seconds 30
        }
    }

    if ($NCSFVersion -like $script:SKIP)
    {
        $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
        Invoke-Command -Session $RemoteSession -ScriptBlock {
            $currentFile = "$env:windir\temp\ClusterManifest_current.xml"
            $newFile= "$env:windir\temp\ClusterManifest_new.xml"
            Remove-Item -Path $currentFile -Force -ErrorAction SilentlyContinue
            Remove-Item -Path $newFile -Force -ErrorAction SilentlyContinue
        }
    }

    if ($retryCount -eq $maxRetries)
    {
        throw "PerformUpdate failed after maxretries."
    }

    #Wait for SF upgrade to finish
    WaitForNCServiceFabricUpgrade -RemoteSession $RemoteSession -NodeName $NodeName -UpgradeState RollingForwardCompleted -UpgradeVersion $NCSFVersion -Credential $Credential
    #WaitFor NC app cluster to be healthy again
    WaitForNCApplicationHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
    #Wait for SF cluster to be healthy
    WaitForSFClusterHealthState -RemoteSession $RemoteSession -NodeName $NodeName -HealthState Ok -Credential $Credential
}

function WaitForNCServiceFabricUpgrade
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [string]$NodeName,

        [Parameter(Mandatory=$true)]
        [ValidateSet('RollingForwardCompleted', 'RollingForwardInProgress','RollingForwardPending')]
        [string]$UpgradeState,

        [Parameter(Mandatory = $true)]
        [string]$UpgradeVersion,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        # Time out in seconds
        [Int32]$TimeOut = 2500,

        # Retry interval in seconds
        [Int32]$Interval = 30
    )

    Write-Log "Wait for NC ServiceFabricClusterUpgrade to version $UpgradeVersion to finish on $NodeName"
    $waitSuccess = $false
    $responseNC = $false
    $endTime = [DateTime]::Now.AddSeconds($TimeOut)
    while ([DateTime]::Now -lt $endTime)
    {
        try
        {
            Write-Log "Using Invoke-Command to query SF Upgrade Status."
            $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
            $result = Invoke-Command -Session $RemoteSession -ScriptBlock {
                # Set Fabric.Code path if it is not already set.
                if (!$env:Path.Contains("$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code"))
                {
                    $env:Path=$env:Path+";"+"$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code;"
                }
                Connect-ServiceFabricCluster
                Get-ServiceFabricClusterUpgrade
            }
            $responseNC = $true
        }
        catch
        {
            Write-Log "Caught error: $_"
            $responseNC = $false
        }

        if ($responseNC)
        {
            $SFUpgradeState = $result.UpgradeState
            Write-Log "NC ServiceFabric UpgradeState on $NodeName is $SFUpgradeState"

            if ($SFUpgradeState -ilike $UpgradeState)
            {
                Write-Log "NC ServiceFabric UpgradeState on $NodeName matches $UpgradeState. Finish Waiting."
                $waitSuccess = $true
                break
            }
            else
            {
                Write-Log "NC ServiceFabric UpgradeState on $NodeName does not match $UpgradeState. Continue Waiting."
            }
        }

        Start-Sleep -Seconds $Interval
    }

    if (-not $waitSuccess)
    {
        $info = $result | out-string
        Write-Log "NC ServiceFabric UpgradeState: $info"
        throw "Failed waiting for NC ServiceFabric UpgradeState on $NodeName to change to $UpgradeState in $TimeOut seconds."
    }

    if (-Not($UpgradeVersion -like $script:SKIP))
    {
        if ($result.TargetCodeVersion -icontains $UpgradeVersion)
        {
            Write-Log "NC ServiceFabric UpgradeVersion on $NodeName matches $UpgradeVersion. Finish Waiting."
        }
        else
        {
            $info = $result | out-string
            Write-Log "NC ServiceFabric UpgradeVersion: $info"
            throw "NC ServiceFabric UpgradeVersion on $NodeName does not match $UpgradeVersion."
        }
    }
}

# Note: If a replica is down for a while, aggregate health switches to "Error" even with quorum.
# Please check with update team before calling this in NC scale in.
function WaitForNCApplicationHealthState
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [string]$NodeName,

        [Parameter(Mandatory=$true)]
        [ValidateSet('Invalid', 'Ok', 'Warning', 'Error', 'Unknown')]
        [string]$HealthState,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        # Time out in seconds
        [Int32]$TimeOut = 1500,

        # Retry interval in seconds
        [Int32]$Interval = 30
    )

    Write-Log "Wait for NC AggregatedHealthState to turn $HealthState on $NodeName"
    $waitSuccess = $false
    $endTime = [DateTime]::Now.AddSeconds($TimeOut)
    while ([DateTime]::Now -lt $endTime)
    {
        try
        {
            Write-Log "Using Invoke-Command to query NC AggregatedHealthState"
            $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
            $result = Invoke-Command -Session $RemoteSession -ScriptBlock {
                # Set Fabric.Code path if it is not already set.
                if (!$env:Path.Contains("$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code"))
                {
                    $env:Path=$env:Path+";"+"$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code;"
                }
                Connect-ServiceFabricCluster
                Get-ServiceFabricApplicationHealth -ApplicationName "fabric:/NetworkController"
            }
        }
        catch
        {
            Write-Log "Caught error: $_"
        }
        $aggregatedHealthState = $result.AggregatedHealthState
        Write-Log "NC AggregatedHealthState on $NodeName is $aggregatedHealthState"

        if ($aggregatedHealthState -icontains $HealthState)
        {
            Write-Log "NC AggregatedHealthState on $NodeName matches $HealthState. Finish Waiting."
            $waitSuccess = $true
            return
        }
        else
        {
            Write-Log "NC AggregatedHealthState on $NodeName does not match $HealthState. Continue Waiting."
            Start-Sleep -Seconds $Interval
        }
    }

    if (-not $waitSuccess)
    {
        $info = $result | out-string
        Write-Log "Aggregate Information: $info"
        throw "Failed waiting for NC AggregatedHealthState on $NodeName to change to $HealthState in $TimeOut seconds."
    }
}

function WaitForSFClusterHealthState
{
    param (
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        [string]$NodeName,

        [Parameter(Mandatory=$true)]
        [ValidateSet('Invalid', 'Ok', 'Warning', 'Error', 'Unknown')]
        [string]$HealthState,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        # Time out in seconds
        [Int32]$TimeOut = 1500,

        # Retry interval in seconds
        [Int32]$Interval = 30
    )

    Write-Log "Wait for ServiceFabric Cluster Health to turn $HealthState on $NodeName"
    $waitSuccess = $false
    $endTime = [DateTime]::Now.AddSeconds($TimeOut)
    while ([DateTime]::Now -lt $endTime)
    {
        try
        {
            Write-Log "Using Invoke-Command to query ServiceFabric Cluster Health"
            $RemoteSession = CheckRemoteSession -RemoteSession $RemoteSession -Credential $Credential -NodeName $NodeName
            $result = Invoke-Command -Session $RemoteSession -ScriptBlock {
                # Set Fabric.Code path if it is not already set.
                if (!$env:Path.Contains("$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code"))
                {
                    $env:Path=$env:Path+";"+"$env:ProgramFiles\Microsoft Service Fabric\bin\Fabric\Fabric.Code;"
                }
                Connect-ServiceFabricCluster
                Get-ServiceFabricApplicationHealth -ApplicationName "fabric:/System"
            }
        }
        catch
        {
            Write-Log "Caught error: $_"
        }
        $aggregatedHealthState = $result.AggregatedHealthState
        Write-Log "SF cluster AggregatedHealthState on $NodeName is $aggregatedHealthState"

        if ($aggregatedHealthState -icontains $HealthState)
        {
            Write-Log "SF cluster AggregatedHealthState on $NodeName matches $HealthState. Finish Waiting."
            $waitSuccess = $true
            return
        }
        else
        {
            Write-Log "SF cluster AggregatedHealthState on $NodeName does not match $HealthState. Continue Waiting."
            Start-Sleep -Seconds $Interval
        }
    }

    if (-not $waitSuccess)
    {
        $info = $result | out-string
        Write-Log "Aggregate Information: $info"
        throw "Failed waiting for SF cluster AggregatedHealthState on $NodeName to change to $HealthState in $TimeOut seconds."
    }
}

# Creates a new manifest file containing the PrimaryAccountNTLMX509Thumbprint and SecondaryAccountNTLMX509Thumbprint so the SF70CU4 can be skipped.
function SetServiceFabricClusterManifest
{
    $connectAttempt = 5
    do
    {
        try
        {
            Connect-ServiceFabricCluster | out-null
            $currentFile = "$env:windir\temp\ClusterManifest_current.xml"
            $newFile= "$env:windir\temp\ClusterManifest_new.xml"
            Remove-Item -Path $currentFile -Force -ErrorAction SilentlyContinue
            Remove-Item -Path $newFile -Force -ErrorAction SilentlyContinue
            Get-ServiceFabricClusterManifest > $currentFile
            $xml = [xml](get-content $currentFile)
            $newVersion=""
            break
        }
        catch
        {
            if($connectAttempt -eq 0)
            {
                Write-Output "$_.Exception.Message"
                throw
            }
            else
            {
                Write-Output "$_.Exception.Message"
                Start-Sleep -Seconds 5
            }
            $connectAttempt--
        }
    }while($connectAttempt -gt 0)

    $splitVersion = $($xml.ClusterManifest.Version).Split('.')
    $splitVersion[$splitVersion.Count - 1] = [int]$($splitVersion[$splitVersion.Count - 1]) + 1
    $newVersion = $splitVersion -join '.'
    Write-Output "New SF cluster manifest version: $newVersion old version: $($xml.ClusterManifest.Version)"
    $xml.ClusterManifest.Version = $newVersion

    $fileStoreSection = $xml.ClusterManifest.FabricSettings.Section | Where-Object { $_.Name -imatch "FileStoreService" }
    if (-Not($fileStoreSection))
    {
        throw "FileStoreService section could not be found. It must be present."
    }

    $params = @("PrimaryAccountNTLMX509Thumbprint", "SecondaryAccountNTLMX509Thumbprint")
    $restCert= (Get-NetworkController).ServerCertificate
    foreach ($p in $params)
    {
        $param = $fileStoreSection.Parameter | Where-Object { $_.Name -imatch $p }
        if (-Not($param) -or !($param.Value -like $restCert.Thumbprint))
        {
            $paramNode = $xml.CreateElement("Parameter")
            $paramNode.SetAttribute("Name", $p)
            $paramNode.SetAttribute("Value", $restCert.Thumbprint)
            $fileStoreSection.appendchild($paramNode)
        }
        else
        {
            $param.SetAttribute("Value", $restCert.Thumbprint)
        }
    }
    Write-Output "Save NC service fabric cluster manifest"
    $xml = [xml]$($xml.OuterXml.Replace('xmlns=""',""))
    $xml.save($newFile)

    Write-Output "Calling Get-ServiceFabricRegisteredClusterConfigVersion to check for duplicated versions."
    $config = Get-ServiceFabricRegisteredClusterConfigVersion -ConfigVersion $newVersion
    if($null -ne $config)
    {
        Write-Output "Unregister-ServiceFabricClusterPackage $($newVersion) from previous attempts."
        Unregister-ServiceFabricClusterPackage -Config -ClusterManifestVersion $newVersion -Force
    }
    Write-Output "Calling Test-ServiceFabricClusterManifest to test manifest changes."

    # Test new SF cluster manifest file and then apply it
    if(Test-ServiceFabricClusterManifest -ClusterManifestPath $newFile -OldClusterManifestPath $currentFile -ErrorAction Stop)
    {
        Write-Output "Tested ServiceFabricClusterManifest succesfully. Calling Copy and Register clusterpackage."
        try
        {
            Copy-ServiceFabricClusterPackage -Config -ImageStoreConnectionString "fabric:ImageStore" -ClusterManifestPath $newFile -ClusterManifestPathInImageStore "ClusterManifest.xml"
            Register-ServiceFabricClusterPackage -Config -ClusterManifestPath "ClusterManifest.xml"
        }
        catch [Exception]
        {
            $message = "Exception: $($_), $($_.Exception), $($_.ScriptStackTrace), $($_.ErrorDetails), $($_.Exception.Message), $($_.Exception.StackTrace), $($_.Exception.InnerException)"
            throw $message
        }

        Write-Output "Calling Start-ServiceFabricClusterUpgrade."
        Start-ServiceFabricClusterUpgrade -ClusterManifestVersion "$newVersion" -Config -FailureAction Rollback -Monitored
        $SFUpgradestarted = Get-ServiceFabricClusterUpgrade
        if (!($SFUpgradestarted.UpgradeState -like "RollingForwardInProgress"))
        {
            throw "ServiceFabricClusterUpgrade did not start. Exiting."
        }
        Write-Output "Successfully started ServiceFabricClusterUpgrade. SF Upgrade is in Progress."
    }
    else
    {
        throw "Testing NC service fabric cluster manifest failed"
    }
}