AzStackHciStandaloneObservability/package/bin/ObsDep/content/Powershell/Roles/Common/RoleHelpers.psm1
|
<###################################################
# # # Copyright (c) Microsoft. All rights reserved. # # # ##################################################> Import-Module $PSScriptRoot\JustEnoughAdministrationHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module NetworkControllerRESTWrappers -DisableNameChecking -Verbose:$false | Out-Null Import-Module $PSScriptRoot\NetworkControllerWorkloadHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module $PSScriptRoot\..\..\Common\Helpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module $PSScriptRoot\..\..\Common\NetworkHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module $PSScriptRoot\..\..\Common\StorageHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module $PSScriptRoot\..\..\Common\ClusterHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null Import-Module PSDiagnostics -DisableNameChecking -Verbose:$false | Out-Null Import-LocalizedData LocalizedStrings -FileName Roles.Strings.psd1 -ErrorAction SilentlyContinue Import-LocalizedData LocalizedNetworkData -FileName Network.Strings.psd1 -ErrorAction SilentlyContinue $GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES = 10 #Checks if the value is null or empty and throws the exception function Throw-IfNullOrEmpty { Param ( [object] $DataToCheck, [string] $ErrorMessage ) if ($DataToCheck) { if (([string]::IsNullOrEmpty($DataToCheck)) -or ($DataToCheck.Length -eq 0)) { throw $ErrorMessage } } else { throw $ErrorMessage } } #Gets the CA certificate password function Get-CACertPassword { Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration Throw-IfNullOrEmpty -DataToCheck $cloudRole -ErrorMessage "Unable to get cloudRole information from ECE" $securityInfo = $cloudRole.PublicInfo.SecurityInfo Throw-IfNullOrEmpty -DataToCheck $securityInfo -ErrorMessage "Unable to get securityInfo information from ECE" $caCertUser = $securityInfo.CACertUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.CACertAccountID Throw-IfNullOrEmpty -DataToCheck $caCertUser -ErrorMessage "Unable to get caCertUser information from ECE" $secureCertPwd = $Parameters.GetCredential($caCertUser.Credential).Password Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Unable to get secureCertPwd information from ECE" return $secureCertPwd } #Gets the External Domain FQDN function Get-ExternalDomainFQDN { Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainRole = $Parameters.Roles["Domain"].PublicConfiguration Throw-IfNullOrEmpty -DataToCheck $domainRole -ErrorMessage "Unable to get domainRole information from ECE" $externalDomainFqdn = $domainRole.PublicInfo.DomainConfiguration.EXTERNALFQDN Throw-IfNullOrEmpty -DataToCheck $externalDomainFqdn -ErrorMessage "Unable to get External DomainFQDN information from ECE" return $externalDomainFqdn } #Gets the Internal Domain FQDN function Get-InternalDomainFQDN { Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainRole = $Parameters.Roles["Domain"].PublicConfiguration Throw-IfNullOrEmpty -DataToCheck $domainRole -ErrorMessage "Unable to get domainRole information from ECE" $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN Throw-IfNullOrEmpty -DataToCheck $domainFqdn -ErrorMessage "Unable to get DomainFQDN information from ECE" return $domainFqdn } function Test-PSSession { Param( [Parameter(Mandatory=$true)] $RemoteSession, [Parameter(Mandatory=$true)] $SessionCredential = $true, [Parameter(Mandatory=$true)] $SessionComputer = $true ) if (($RemoteSession -eq $null) -or ($RemoteSession.State -ne "Opened")) { Trace-Execution "Session not ready, state: $($RemoteSession.State)" Trace-Execution "Removing the session" $RemoteSession | Remove-PSSession Trace-Execution "Regenerating the session" $RemoteSession = New-PSSession -ComputerName $SessionComputer -Credential $SessionCredential Initialize-ECESession $RemoteSession Trace-Execution "State after regenerate = $($RemoteSession.State)" } else { Trace-Execution "Session Opened" } return $RemoteSession } function Test-SFRingHealth { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) Trace-Execution "Starting HealthCheck for $($Parameters.Configuration.Role.ID)." $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric" $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\VerifyClusterHealth.psm1" import-module $verifyClusterHealth try { Trace-Execution "Checking health of Service Fabric cluster..." VerifyClusterConnection -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -CheckAggregateHealthState -RetryTimes 10 -RetryIntervalSec 15 -ConnectionTimeoutSec 30 -Verbose } catch { Trace-Execution "Check health of Service Fabric cluster failed." Trace-Warning $_ Trace-Execution "Attempting to repair Service Fabric cluster..." Connect-AzureStackServiceFabricCluster -ConnectionEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterSpn $repairServiceFabricClusterScript = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\RepairServiceFabricCluster.psm1" Import-Module $repairServiceFabricClusterScript -DisableNameChecking -Verbose:$false | Out-Null Repair-AzureStackServiceFabricCluster -Verbose Trace-Execution "Rechecking health of Service Fabric cluster..." VerifyClusterConnection -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -CheckAggregateHealthState -RetryTimes 10 -RetryIntervalSec 15 -ConnectionTimeoutSec 30 -Verbose } } function Test-PrecheckClusterNodeHealth { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $node = Get-ExecutionContextNodeName $Parameters Trace-Execution "Starting Node Health Check for $node on $($Parameters.Configuration.Role.ID)." $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric" $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\VerifyClusterHealth.psm1" Import-Module $verifyClusterHealth Trace-Execution "Checking Node health from Service fabric..." PrecheckClusterNodeHealth -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -NodeExemptIP $node -ConnectionTimeoutSec 30 -IncludeDisabled $true -Verbose } function Test-PrecheckNCClusterNodeHealth { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $roleDefinition = $Parameters.Configuration.Role [string] $TimeString = Get-Date -Format "yyyyMMdd-HHmmss" $RemoteLOGFILE = "$env:systemdrive\MASLogs\$($RoleDefinition.Id)_$($MyInvocation.MyCommand.Name)_$TimeString.log" $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo #Setup Account Info $setupAccountUser = $securityInfo.DomainUsers.User | ? Role -EQ $roleDefinition.PrivateInfo.Accounts.RunAsAccountID $fabricAdminCredential = $Parameters.GetCredential($setupAccountUser.Credential) if (-not $("$($fabricAdminCredential.UserName)" -like "*\*")) { $fabricAdminCredential = New-Credential -UserName "$domainName\$($fabricAdminCredential.UserName)" -Password $fabricAdminCredential.GetNetworkCredential().Password } Trace-Execution "Using $($fabricAdminCredential.UserName) to communicate with NC" $node = Get-ExecutionContextNodeName $Parameters # Get VM names [string[]] $vms = $roleDefinition.Nodes.Node | ? { $_.Name -ne $node } | % Name Trace-Execution "Starting Node Health Check with context $node on $($roleDefinition.ID)." $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters $scriptBlock = { param($clusterConnInfo, $node) Start-Transcript -Append -Path $using:RemoteLOGFILE try{ $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric" $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\NC\VerifyClusterHealth.psm1" Import-Module $verifyClusterHealth Trace-Execution "Checking Node health from Service fabric..." PrecheckClusterNodeHealth -ClusterSpn $clusterConnInfo.clusterSpn -NodeExemptIP $node -ConnectionTimeoutSec 30 -IncludeDisabled $true -Verbose }finally{ Stop-Transcript -ErrorAction Ignore } } [System.Management.Automation.Runspaces.PSSession[]] $remoteSession = $null try { $remoteSession = New-PSSession -ComputerName $vms[0] -Credential $fabricAdminCredential -Authentication Credssp $null = Invoke-Command -Session $remoteSession -ScriptBlock $ScriptBlock -ArgumentList $clusterConnInfo, $node } finally { $remoteSession | Remove-Pssession -ErrorAction Ignore | Out-Null } } function Get-SFClusterConnectionInfo { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN $clusterSpn = $Parameters.Configuration.Role.PublicInfo.ClusterSpn.Name [string[]] $sfClusterEndpoints = $Parameters.Configuration.Role.Nodes.Node.Name | ForEach-Object { $_ + "." + $domainFqdn + ":19000" } return @{ClusterSPN=$clusterSpn SFClusterEndpoints=$sfClusterEndpoints} } # This function creates the role VM clusters as described in the manifest. function Add-GuestCluster { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [String] $RunAsUserID ) $ErrorActionPreference = 'Stop' $vmRole = $Parameters.Configuration.Role # Account info $secInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo #Setup Account Info $setupAccountUser = $secInfo.DomainUsers.User | ? Role -EQ $RunAsUserID $setupAccountCredential = $Parameters.GetCredential($setupAccountUser.Credential) $domainFqdn = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo.DomainConfiguration.FQDN $null = ipconfig /flushdns [array] $vms = $vmRole.Nodes.Node | % Name | Foreach-Object {"$_`.$domainFqdn"} $clusterName = $vmRole.PublicInfo.Cluster.Name $clusterIPAddress = $vmRole.PublicInfo.Cluster.IPv4Address [array] $VMsToAdd = @() $Cluster = $null $createNewCluster = $true try { $Cluster = Get-Cluster -Name $clusterName } catch { } if($Cluster -ne $null) { Trace-Warning "The cluster by the name '$clusterName' already exists." $createNewCluster = $false [array] $preClusteredVMs = Get-Cluster -Name $clusterName | Get-ClusterNode | % Name | Foreach-Object {"$_`.$domainFqdn"} [array] $newVMsToAdd = @() foreach($vm in $vms) { if($preClusteredVMs -contains $vm) { $result = $true } else { $newVMsToAdd += $vm } } if($newVMsToAdd.Count -eq 0) { # In the event all nodes are part of the cluster, we should still run cluster validation to rule out failures from previous attempts to add-clusternode Trace-Execution "Skipping: Nodes '$($preClusteredVMs -join ',')' are already part of Cluster '$clusterName'." Trace-Execution "Testing cluster '$clusterName'" $null = Test-Cluster -Cluster $clusterName -Verbose -ErrorAction Stop return } else { $VMsToAdd = $newVMsToAdd } } else { $VMsToAdd = $vms } Trace-Execution -Message "Prepare nodes for clustering: '$($VMsToAdd -join ',')'" $vmsPendingFeatureInstall = $VMsToAdd | % { if (-not (Get-WindowsFeature -Name Failover-Clustering -ComputerName "$_" -Credential $setupAccountCredential).Installed) { $_ } } if ($vmsPendingFeatureInstall) { Trace-Execution "Installing Failover-Clustering feature on the following VMs: '$($vmsPendingFeatureInstall -join ',')'." $null = $vmsPendingFeatureInstall | % { Invoke-Command -ComputerName "$_" -Credential $setupAccountCredential -ErrorAction Stop -ScriptBlock { Add-WindowsFeature -Name Failover-Clustering -IncludeAllSubFeature -IncludeManagementTools } } Trace-Execution "Rebooting the following machines: '$($vmsPendingFeatureInstall -join ',')'." Restart-Machine -ComputerName $vmsPendingFeatureInstall -Credential $setupAccountCredential Trace-Execution "Waiting for 1 minute to allow for post-installation configuration." Start-Sleep -Seconds 60 Trace-Execution "Waiting for clustering features to become available on following VMs after reboot: '$($vmsPendingFeatureInstall -join ',')'." foreach ($vm in $vmsPendingFeatureInstall) { Trace-Execution "Waiting for clustering features on '$vm'." $waitScript = { $null = ipconfig /flushdns Invoke-Command -ComputerName $vm -Credential $setupAccountCredential { Get-Command Get-Cluster -ErrorAction Stop } } if (Wait-Result -ValidationScript $waitScript -TimeOut ($GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES * 60) -Interval 10) { Trace-Execution "Clustering feature is now available on '$vm'." } else { Trace-Error "The VM '$vm' didn't reboot in $GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES minutes or the clustering features could not be installed." } } } foreach($vm in $VMsToAdd){ Trace-Execution "Enabling client/diagnostic event log on $vm" Invoke-Command -ComputerName $vm -Credential $setupAccountCredential -Scriptblock { $logName = 'Microsoft-Windows-FailoverClustering-Client/Diagnostic' $log = New-Object System.Diagnostics.Eventing.Reader.EventLogConfiguration $logName if($log.IsEnabled -ne $true){ $log.IsEnabled = $true $log.SaveChanges() } } } try { if($createNewCluster) { $domainCredential = Get-DomainCredential -Parameters $Parameters Remove-ComputerAndDnsRecord -Name $clusterName -DomainFQDN $domainFqdn -Credential $domainCredential Trace-Execution "Creating the cluster '$clusterName'." Invoke-Command -ComputerName $env:COMPUTERNAME -Credential $domainCredential -Authentication Credssp -Scriptblock { $null = New-Cluster -Name $using:clusterName -Node $using:VMsToAdd -StaticAddress $using:clusterIPAddress -NoStorage -ErrorAction Stop } -ErrorAction Stop } else { # We need new cluster reports if this fails Trace-Execution -Message "Adding the New VMs: '$($VMsToAdd -join ',')' to Pre-existing Cluster $clusterName" try { Add-ClusterNode -Name $VMsToAdd -NoStorage -Cluster $clusterName -Verbose -ErrorAction Stop } finally { Trace-Execution "Testing cluster '$clusterName'" $null = Test-Cluster -Cluster $clusterName -Verbose -ErrorAction Stop } } # We should validate that the cluster is in an expected state Trace-Execution "Wait until new cluster nodes can be enumerated on the name '$clusterName'." $timeOut = 30 # 30 minutes $waitResult = Wait-Result { $null = ipconfig /flushdns $cluster = Get-Cluster -Domain $domainFqdn | ? Name -eq $clusterName $clusterNodes = $cluster | Get-ClusterNode -WarningAction SilentlyContinue | ? { % {[system.net.dns]::GetHostByName($_).HostName -in $VMsToAdd } } if(-not $clusterNodes){ Trace-Warning "No VMsToAdd were discovered. Waiting for: '$VMsToAdd'" return $false } $offlineNodes = $clusterNodes | ? { $_.State -ine "Up" } # We round-robin VM update, so by this point no nodes should be offline $clusterNodesFQDN = $clusterNodes | % {[system.net.dns]::GetHostByName($_).HostName} $allEnumerated = Compare-Object -ReferenceObject $VMsToAdd -DifferenceObject $clusterNodesFQDN if($offlineNodes) { Trace-Execution "Not all nodes are online: '$($offlineNodes.Name)'" return $false } elseif($allEnumerated) # clusterNodes should be the complete list of VMsToAdd { Trace-Execution "Not all nodes were enumerated" Trace-Execution "Expected nodes: '$VMsToAdd'" Trace-Execution "Found nodes: '$($clusterNodes.Name)'" return $false } else { return $true } } -TimeOut ($timeOut*60) -Interval 5 if (-not $waitResult) { $cluster = Get-Cluster -Domain $domainFqdn | ? Name -eq $clusterName $clusterNodes = $cluster | Get-ClusterNode -WarningAction SilentlyContinue $clusterResources = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock {$cluster | Get-ClusterResource} $clusterNodeStates = "" $clusterResourceStates = "" foreach($node in $clusterNodes){ $clusterNodeStates += "'$($node.Name)' is '$($node.State)'; " } foreach ($resource in $clusterResources){ $clusterResourceStates += "'$($resource.Name)' is '$($resource.State)'; " } Trace-Execution "VMs to add to cluster '$clusterName' included: '$VMsToAdd'" Trace-Execution "Nodes of the cluster '$clusterName' are in state: '$clusterNodeStates'" Trace-Execution "Cluster resources are in state: '$clusterResourceStates'" Trace-Error "Nodes of the cluster '$clusterName' did not come online in '$timeOut' minutes." } else { Trace-Execution "Cluster '$clusterName' is now online." } } catch { Trace-Error $_ } finally { foreach($vm in $VMsToAdd) { Trace-Execution "Collecting client/diagnostic event log on $vm" # Have to disable and collect winevent, and eventlog does not have the client/diagnostic information # To avoid breaking engine execution, we'll print any failures to the log directory. Failure to collect logs should not break procedures. try { Invoke-Command -ComputerName $vm -Credential $setupAccountCredential -Scriptblock { $writePath = "$($env:systemDrive)\AzSLogs" if(!(Test-Path $writePath)) { mkdir $writePath } $logName = 'Microsoft-Windows-FailoverClustering-Client/Diagnostic' $log = New-Object System.Diagnostics.Eventing.Reader.EventLogConfiguration $logName if($log.IsEnabled -ne $false){ $log.IsEnabled = $false $log.SaveChanges() } Get-ClusterLog -Cluster $using:clusterName -Node $using:vm -Destination "$writePath\$(Get-Date -f yyyyMMdd-hhmmss)" Get-WinEvent -LogName $logName -Oldest | Format-Table TimeCreated, Id, LevelDisplayName, Message -wrap -autosize | Out-File "$writePath\Cluster_Client_Diagnostics_$(Get-Date -f yyyyMMdd-hhmmss).txt" } } catch { Trace-Warning "Issue with collecting dlient/diagnostic event log: $_" } } } } # This function gets the machine name of the primary server handling external DNS zones. function Get-ExternalDnsMachineName { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $false)] [bool] $ErrorOnFail = $true ) $ErrorActionPreference = 'Stop' $waspRole = $Parameters.Roles["WASPUBLIC"].PublicConfiguration $externalDnsRole = $Parameters.Roles["ExternalDNS"].PublicConfiguration $DnsServerNamesList = $waspRole.Nodes.Node | Foreach-Object { $_.Name } $ExternalZoneFqdn = $externalDnsRole.PublicInfo.DnsConfiguration.ExternalZoneFqdn $domainAdminCredential = Get-DomainCredential -Parameters $Parameters $domainAdminCimSession = New-CimSession -Credential $domainAdminCredential Trace-Execution "Finding primary DNS server for external zone '$ExternalZoneFqdn' in list '$DnsServerNamesList'." $PrimaryDnsServerNames = @() foreach($DnsServer in $DnsServerNamesList) { try { $zone = Get-DnsServerZone -Name $ExternalZoneFqdn -ComputerName $DnsServer -CimSession $domainAdminCimSession if ($zone.ZoneType -eq "Primary") { Trace-Execution "Found primary DNS server '$DnsServer'." $PrimaryDnsServerNames += $DnsServer } } catch { Write-Verbose "Get-ExternalDnsMachineName failed to get zone from $DnsServer. Suppressing Exception $_" } } if ($PrimaryDnsServerNames.Count -eq 0) { if ($ErrorOnFail) { Trace-Error "Failed to find the primary DNS server for the zone '$ExternalZoneFqdn'." } else { Trace-Warning "Failed to find the primary DNS server for the zone '$ExternalZoneFqdn'." } } return $PrimaryDnsServerNames } # This function configures all the VIPs as described in the manifest for all roles by default. # If ThisRoleOnly parameter is set to one particular role's ID (for example, WAS), only VIPs for # the role will be configured. function Add-AllVIPs { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $false)] [bool] $EnableLoadBalancerProbe = $false, [Parameter(Mandatory = $false)] [bool] $AddDnsRecords = $true, [Parameter(Mandatory = $false)] [string] $ThisRoleOnly = $null, [Parameter(Mandatory = $false)] [string] $ThisVipOnly = $null ) $ErrorActionPreference = 'Stop' $VerbosePreference = "Continue" Trace-ECESCript "Add All VIPS" { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $infraRole = $Parameters.Roles["Infrastructure"].PublicConfiguration $roleDefinition = $Parameters.Configuration.Role $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $ncRoleDefinition = $Parameters.Roles["NC"].PublicConfiguration if ( $cloudRole.PublicInfo.Setup.CloudType -eq 'HyperConverged') { $computeRoleDefinition = $Parameters.Roles["Storage"].PublicConfiguration } else { $computeRoleDefinition = $Parameters.Roles["Compute"].PublicConfiguration } # Get the account credentials to be used. $securityInfo = $cloudRole.PublicInfo.SecurityInfo $runAsAccountId = $roleDefinition.PrivateInfo.Accounts.RunAsAccountID # Update case context needs to use VirtualMachines public account info if ($runasAccountId -eq $null) { $runAsAccountId = $Parameters.Roles["VirtualMachines"].PublicConfiguration.PublicInfo.Accounts.RunAsAccountId } $setupAccountUser = $securityInfo.DomainUsers.User | ? Role -EQ $runAsAccountId $setupAccountCredential = $Parameters.GetCredential($setupAccountUser.Credential) $networkControllerRestName = $ncRoleDefinition.PublicInfo.NetworkControllerRestIP.Name $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters $domainAdminUser = $securityInfo.DomainUsers.User | ? {$_.Role -eq "DomainAdmin"} $domainAdminCredential = $Parameters.GetCredential($domainAdminUser.Credential) $isOneNode = @($Parameters.Roles["Storage"].PublicConfiguration.Nodes.Node).Count -eq 1 $RestoreContext = Get-RestoreParameters $Parameters if ($isOneNode -and $RestoreContext.RestoreInprogress) { [string] $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential } else { [string] $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential } $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN $externalDomainFqdn = $domainRole.PublicInfo.DomainConfiguration.EXTERNALFQDN $externalDnsMachineNames = Get-ExternalDnsMachineName -Parameters $Parameters $allHostNames = $computeRoleDefinition.Nodes.Node | % Name $clusterName = Get-ManagementClusterName $Parameters $allHostNames = Get-ActiveClusterNodes -Parameters $Parameters -ClusterName $clusterName Trace-Execution "Connecting to the network controller with REST name '$networkControllerRestName'." $null = Set-NCConnection -RestIP $networkControllerRestName -credential $setupAccountCredential $credentialParameterHashTable = @{} $result = $allHostNames |where { $_ -contains $env:COMPUTERNAME } if(!$result) { $credentialParameterHashTable = @{Credential=$setupAccountCredential} } $allVMs = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock { $allHostNames = Get-ActiveClusterNodes -Parameters $Parameters -ClusterName $clusterName if($credentialParameterHashTable) { Get-VM -ComputerName $allHostNames @credentialParameterHashTable } else { Get-VM -ComputerName $allHostNames } } $allLBs = Get-NCLoadBalancer $allPublicIps = Get-NCPublicIpAddress $allKeys = $Parameters.Roles.Keys foreach($key in $allKeys) { if (((-not $ThisRoleOnly) -or ($ThisRoleOnly -eq $key) ) -and $Parameters.Roles[$key].PublicConfiguration.PublicInfo.VIPs -ne $null) { Trace-Execution "Adding all VIPs for role '$key'." $roleToProcess = $Parameters.Roles[$key].PublicConfiguration $singleVmIndexes = @{} try { Trace-ECEScript "Configuring VIPs for role '$key' containing VMs: '$($constituentVMs.Name -join '', '')'." { $vips = $roleToProcess.PublicInfo.VIPs.VIP if ($ThisVipOnly) { $vips = $vips | Where-Object { $_.Id -eq $ThisVipOnly } } foreach ($vip in $vips) { $vipId = $vip.Id $vipResourceId = $null $vipIPv4Address = $vip.IPv4Address.Split('/')[0] $VMNicName = $vip.InterfaceName $targetLb = $null $useStaticIp = $true [string[]] $constituentVMNames = $roleToProcess.Nodes.Node | % Name $constituentVMs = $allVMs | Where { $constituentVMNames.Contains($_.Name) } # Support a 1-to-1 mapping of VIP to DIP. # Uses the specified tag to allocate a single VM on the backend. if ($vip.MapToSingleVMTag) { Trace-Execution "Detected MapToSingleVMTag '$($vip.MapToSingleVMTag)' on VIP." $tag = $vip.MapToSingleVMTag if($singleVmIndexes.ContainsKey($tag)) { $singleVmIndexes[$tag] += 1 } else { $singleVmIndexes.Add($tag, 0) } $vmIndex = $singleVmIndexes[$tag] if($vmIndex -ge $constituentVMs.Count) { Trace-Error "There are not enough VMs in the role '$key' to support the VIP configuration '$($vip.Id)' for tag '$tag'." } Trace-Execution "Pairing VIP '$($vip.Id)' with the VM index '$($vmIndex)'." $constituentVMs = $constituentVMs[$vmIndex] } # check if the LB already exists by comparing the static IPs and publicIP resources against the provided VIP $targetPublicIpObject = $allPublicIps | where-object {$_.properties.ipaddress -eq $vipIPv4Address} if(-not (Get-member -Name "nextLink" -InputObject $allLBs)) { $targetLb = $allLBs | where-object { ($_.properties.frontendipconfigurations[0].properties.privateIpaddress -eq $vipIPv4Address) -or ($targetPublicIpObject -and ($targetPublicIpObject.resourceRef -eq $_.properties.frontendipconfigurations[0].properties.publicIPAddress.resourceRef)) } } if ($targetLb) { $vipResourceId = $targetLb.resourceid Trace-Execution "Found LoadBalancer using the VIP $vipIPv4Address. Using its resourceId $vipResourceId" } else { $vipResourceId = [System.Guid]::NewGuid().ToString() Trace-Execution "LoadBalancer using the VIP $vipIPv4Address is not found. Use a new random GUID as new LB resourceId $vipResourceId." } if($targetPublicIpObject) { $useStaticIp = $false } Trace-Execution "Use static IP for the load balancer: $useStaticIp." $vipName = $vip.Name $isEnableOutboundNat = $true if ([bool]::TryParse($vip.EnableOutboundNat, [ref] $isEnableOutboundNat) -ne $true) { $isEnableOutboundNat = $false } Trace-Execution "Processing VIP ID '$vipId' named '$vipName' with IPv4 Address '$vipIPv4Address' and EnableOutboundNat set to '$isEnableOutboundNat'." if ($AddDnsRecords) { foreach ($endpoint in $vip.DnsEndpoints.Endpoint) { $externalNetworkName = Get-NetworkNameForCluster -ClusterName "s-cluster" -NetworkName "External" if($vip.NetworkId -eq $externalNetworkName) { foreach ($externalDnsMachineName in $externalDnsMachineNames) { Add-DnsRecord -DnsName $endpoint.Path -DnsZoneName $externalDomainFqdn -IPv4Address $vipIPv4Address -DnsServerName $externalDnsMachineName -RemoteCredential $domainAdminCredential } } else { Add-DnsRecord -DnsName $endpoint.Path -DnsZoneName $domainFqdn -IPv4Address $vipIPv4Address -DnsServerName $primaryDomainController -RemoteCredential $domainAdminCredential } } } foreach ($portMapping in $vip.PortMapping.Mapping) { $mappingName = $portMapping.Name $frontEndPort = $portMapping.FrontEndPort $backEndPort = $portMapping.BackEndPort $protocol = $portMapping.Protocol $probes = @() if ($portMapping.Probe -ne $null -and $EnableLoadBalancerProbe) { foreach ($probe in $portMapping.Probe) { $probeProtocol = $probe.Protocol $probePort = $probe.Port $probeRequestPath = $probe.RequestPath $probeIntervalInSeconds = $probe.IntervalInSeconds $probeNumberOfProbes = $probe.NumberOfProbes $probeResourceId = [system.guid]::NewGuid().ToString() # Check if the probe already exists in the LB's Probes, if so, just reuse that probe's resourceId if ($targetLb -ne $null -and $targetLb.Properties.Probes -ne $null) { foreach ($existedProbe in $targetLb.Properties.Probes) { if( $existedProbe.properties.protocol -eq $probeProtocol -and $existedProbe.properties.port -eq $probePort -and $existedProbe.properties.requestPath -eq $probeRequestPath -and $existedProbe.properties.intervalInSeconds -eq $probeIntervalInSeconds -and $existedProbe.properties.numberOfProbes -eq $probeNumberOfProbes) { $probeResourceId = $existedProbe.resourceId break } } } Trace-Execution "Creating in-memory LB Probe object for port mapping named '$mappingName': Protocol: $probeProtocol; Port: $probePort; RequestPath: $probeRequestPath; IntervalInSeconds: $probeIntervalInSeconds; NumberOfProbes: $probeNumberOfProbes" $probes += New-NCLoadBalancerProbeObject -resourceID $probeResourceId ` -protocol $probeProtocol ` -port $probePort ` -requestPath $probeRequestPath ` -intervalInSeconds $probeIntervalInSeconds ` -numberOfProbes $probeNumberOfProbes } } if ($probes.Count -eq 0) { $probes = $null } Trace-Execution "Configuring port mapping named '$mappingName' with ports: '$frontEndPort' and '$backEndPort'; protocol: '$protocol'; and VIP IPv4 address '$vipIPv4Address'." $params = @{ VMPool = $constituentVMs; protocol = $protocol; frontendPort = $frontEndPort; backendPort = $backEndPort; EnableOutboundNat = $isEnableOutboundNat; Probe = $probes; LoadBalancerResourceID = $vipResourceId; VMNicName = $VMNicName; } if ($useStaticIp) { $params.Add('Vip', $vipIpv4Address) } else { $params.Add('PublicIpResourceRef', $targetPublicIpObject.resourceRef) } $null = New-LoadBalancerVIP @params } # A LB VIP can be created with no inbound port mappings and only OB rules if ($vip.PortMapping -eq $null -and $isEnableOutboundNat) { Trace-Execution "Configuring outbound NAT rule with VIP IPv4 address '$vipIPv4Address', VMNicName '$VMNicName', and LoadBalancerResourceID '$vipResourceId'." $params = @{ VMPool = $constituentVMs; EnableOutboundNat = $isEnableOutboundNat; LoadBalancerResourceID = $vipResourceId; VMNicName = $VMNicName } if ($useStaticIp) { $params.Add('Vip', $vipIpv4Address) } else { $params.Add('PublicIpResourceRef', $targetPublicIpObject.resourceRef) } $null = New-LoadBalancerVIP @params } } } } catch { Write-Verbose "Failed in Add-AllVIPs. Exception $_" throw } } } } } $global:jobArguments = @{} $global:jobImports = @{} $global:jobCredentials = @{} $global:jobConfigurationNames = @{} $global:jobAuthenticationMechanisms = @{} function Start-ParallelWork{ [CmdletBinding()] param( [Parameter(Mandatory = $true, ParameterSetName="Local")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [ScriptBlock] $ScriptBlock, [Parameter(Mandatory = $false, ParameterSetName="Local")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [Object[]] $ArgumentList, [Parameter(Mandatory = $false, ParameterSetName="Local")] [bool] $ImportModulesInSession = $false, [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [string[]] $ComputerName, [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [string] $ConfigurationName, [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [PSCredential] $Credential, [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [System.Management.Automation.Runspaces.AuthenticationMechanism] $Authentication ) $jobs = New-Object System.Collections.ArrayList if($PsCmdlet.ParameterSetName -eq "Local"){ $jobs.Add((Create-LocalJob -ImportModulesInSession $ImportModulesInSession -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList)) | out-null } else{ foreach($name in $ComputerName){ $jobs.Add((Create-RemoteJob -ComputerName $name -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -Authentication $Authentication)) | out-null } } foreach($job in $jobs){ $global:jobArguments[$job.InstanceId] = $ArgumentList $global:jobImports[$job.InstanceId] = $ImportModulesInSession $global:jobCredentials[$job.InstanceId] = $Credential $global:jobConfigurationNames[$job.InstanceId] = $ConfigurationName $global:jobAuthenticationMechanisms[$job.InstanceId] = $Authentication } return ,$jobs } function Create-LocalJob{ [CmdletBinding()] param( [bool] $ImportModulesInSession, [ScriptBlock] $ScriptBlock, [Object[]] $ArgumentList ) if($ImportModulesInSession){ $currentFilePath = $PSCommandPath $currentFilePathArray = ,($currentFilePath) $initializationScriptContent += Create-ImportModuleString -ModulePaths $currentFilePathArray $modulePaths = (Get-Module).Path | where {$_ -notlike "*WindowsPowerShell*Modules*"} $initializationScriptContent += Create-ImportModuleString -ModulePaths $modulePaths } $initializationScript = [Scriptblock]::Create($initializationScriptContent) $job = Start-Job -InitializationScript $initializationScript -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList return $job } function Create-ImportModuleString { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [Object[]] $ModulePaths ) foreach($modulePath in $modulePaths){ $importModuleString += "Import-Module $modulePath -DisableNameChecking;" } return $importModuleString } function Create-RemoteJob{ [CmdletBinding()] param( [string] $ComputerName, [ScriptBlock] $ScriptBlock, [Object[]] $ArgumentList, [string] $ConfigurationName, [PSCredential] $Credential, [System.Management.Automation.Runspaces.AuthenticationMechanism] $Authentication ) if($ConfigurationName){ if($Authentication){ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -Authentication $Authentication -AsJob) } else{ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -AsJob) } }else{ if($Credential){ if($Authentication){ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -Authentication $Authentication -AsJob) }else{ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -AsJob) } }else{ if($Authentication){ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Authentication $Authentication -AsJob) }else{ $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -AsJob) } } } return $job } function Wait-ParallelWork{ param ( [Parameter(Mandatory=$true)] [System.Collections.ArrayList] $Jobs, [Parameter(Mandatory=$false)] [int] $RetryCount = 0, [Parameter(Mandatory = $false)] [int] $RetrySleepTimeInSeconds = 3 ) $retryMap = @{} while($Jobs.Count -ne 0){ $finishedJob = Wait-Job -Job $Jobs -Any $location = $finishedJob.Location $verboseBufferFromJob = $finishedJob.ChildJobs[0].Verbose if($verboseBufferFromJob){ $verboseBufferFromJob | % { Trace-Execution "$location : $_" } } $warningBufferFromJob = $finishedJob.ChildJobs[0].Warning if($warningBufferFromJob){ $warningBufferFromJob | % { Trace-Warning "$location : $_" } } $errorBufferFromJob = $finishedJob.ChildJobs[0].Error if($errorBufferFromJob){ $errorBufferFromJob | % { Trace-Error "$location : $_" } } $informationBufferFromJob = $finishedJob.ChildJobs[0].Information if($informationBufferFromJob){ $informationBufferFromJob | % { Write-Information "$location : $_" } } if($finishedJob.State -ne 'Completed'){ $instanceId = $finishedJob.InstanceId if(!$retryMap.ContainsKey($instanceId)){ $retryMap[$instanceId] = 0 } $currentRetryNumber = $retryMap[$instanceId] if($currentRetryNumber -ge $RetryCount){ foreach($job in $Jobs){ $global:jobArguments.Remove($job.InstanceId) $global:jobImports.Remove($job.InstanceId) $global:jobCredentials.Remove($job.InstanceId) $global:jobConfigurationNames.Remove($job.InstanceId) $global:jobAuthenticationMechanisms.Remove($job.InstanceId) } $Jobs | Stop-Job $Jobs | Remove-Job $failedReason = $finishedJob.ChildJobs[0].JobStateInfo.Reason throw "The job running on $location failed due to: $failedReason after retrying $RetryCount times" } else{ $nextRetryNumber = $currentRetryNumber + 1 Trace-Execution "Retrying parallel work on $location for the $nextRetryNumber time" Start-Sleep -s $RetrySleepTimeInSeconds $jobScriptBlock = [Scriptblock]::Create($finishedJob.Command) $jobArgumentList = $global:jobArguments[$instanceId] $jobImports = $global:jobImports[$instanceId] $jobConfiguration = $global:jobConfigurationNames[$instanceId] $jobCredential = $global:jobCredentials[$instanceId] $jobAuthenticationMechanism = $global:jobAuthenticationMechanisms[$instanceId] if($finishedJob.Location -eq "localhost"){ $job = Create-LocalJob -Scriptblock $jobScriptBlock -ArgumentList $jobArgumentList -ImportModulesInSession $jobImports } else{ $job = Create-RemoteJob -ComputerName $location -ScriptBlock $jobScriptBlock -ArgumentList $jobArgumentList -ConfigurationName $jobConfiguration -Credential $jobCredential -Authentication $jobAuthenticationMechanism } $newInstanceId = $job.InstanceId $global:jobArguments[$newInstanceId] = $jobArgumentList $global:jobArguments.Remove($instanceId) $global:jobImports[$newInstanceId] = $jobImports $global:jobImports.Remove($instanceId) $global:jobConfigurationNames[$newInstanceId] = $jobConfiguration $global:jobConfigurationNames.Remove($instanceId) $global:jobCredentials[$newInstanceId] = $jobCredential $global:jobCredentials.Remove($instanceId) $global:jobAuthenticationMechanisms[$newInstanceId] = $jobAuthenticationMechanism $global:jobAuthenticationMechanisms.Remove($instanceId) $retryMap[$newInstanceId] = $nextRetryNumber $retryMap.Remove($instanceId) $Jobs.Add($job) | out-null } } $finishedJob | Remove-Job $Jobs.Remove($finishedJob) } } function Start-ParallelWorkAndWait{ [CmdletBinding()] param( [Parameter(Mandatory = $true, ParameterSetName="Local")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [ScriptBlock] $ScriptBlock, [Parameter(Mandatory = $false, ParameterSetName="Local")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [Object[]] $ArgumentList, [Parameter(Mandatory = $false, ParameterSetName="Local")] [bool] $ImportModulesInSession = $false, [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [string[]] $ComputerName, [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [string] $ConfigurationName, [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")] [PSCredential] $Credential, [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [System.Management.Automation.Runspaces.AuthenticationMechanism] $Authentication = "Default", [Parameter(Mandatory = $false, ParameterSetName="Local")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [int] $RetryCount = 0, [Parameter(Mandatory = $false, ParameterSetName="Local")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")] [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")] [int] $RetrySleepTimeInSeconds = 3 ) if($PsCmdlet.ParameterSetName -eq "Local"){ $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ImportModulesInSession $ImportModulesInSession }elseIf($PsCmdlet.ParameterSetName -eq "RemoteWithJEA"){ $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -ConfigurationName $ConfigurationName -Credential $Credential -Authentication $Authentication }else{ if($Credential){ $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -Credential $Credential -Authentication $Authentication }else{ $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -Authentication $Authentication } } Wait-ParallelWork -Jobs $jobs -RetryCount $RetryCount -RetrySleepTimeInSeconds $RetrySleepTimeInSeconds } # Make sure the MAC addresses and IPv4Address are loaded into Parameters object from the manifest. function Set-MacAndIPAddressSingleNode { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory=$true)] [PSObject] $Node, [ValidateSet('PXE','Management')] [string] $IPv4AddressSource = 'PXE' ) if (-not $node.MacAddress) { Trace-Execution "Failed to find MAC address for node '$($node.Name)' in the manifest. Skip this" } else { $macAddress = $node.MacAddress Trace-Execution "Normalizing MAC address '$macAddress'." # Normalize MAC address to .NET canonical form, e.g., E4-1D-2D-1D-25-30. $node.MacAddress = $node.MacAddress.ToUpper().Replace(':', '-') } $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters $ipv4Address = $allHostNodes[$node.Name] Trace-Execution "Setting IPv4 address $ipv4Address for node $($node.Name) ." if(-not $node.IPv4Address) { $node | Add-Member -MemberType NoteProperty -Name "IPv4Address" -value $Ipv4Address } else { $node.IPv4Address = $ipv4Address } } function Set-MacAndIPAddress { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [System.Xml.XmlElement] $PhysicalMachinesRole, [ValidateSet('PXE','Management')] [string] $IPv4AddressSource = 'PXE', [Parameter(Mandatory=$false)] [System.Xml.XmlElement] $Node = $null ) Trace-ECEScript "Setting IP addresses." { if ($Node) { Set-MacAndIPAddressSingleNode -Parameters $Parameters -Node $node -IPv4AddressSource $IPv4AddressSource } else { foreach ($node in $PhysicalMachinesRole.Nodes.Node) { Set-MacAndIPAddressSingleNode -Parameters $Parameters -Node $node -IPv4AddressSource $IPv4AddressSource } } } } # This function removes a specific DNS record function Remove-DnsRecord { Param ( [Parameter(Mandatory=$true)] [string] $DnsName, [Parameter(Mandatory=$true)] [string] $DnsZoneName, [Parameter(Mandatory=$false)] [bool] $IPv4Address = $false, [Parameter(Mandatory=$false)] [bool] $IPv6Address = $false, [Parameter(Mandatory=$false)] [bool] $AliasCName = $false, [Parameter(Mandatory=$true)] [string] $DnsServerName, [Parameter(Mandatory=$false)] [PSCredential] $Credential = $null ) $ErrorActionPreference = 'Stop' return if($Credential) { $CimSession = New-CimSession -Credential $Credential } else { $CimSession = New-CimSession } if ($IPv4Address -eq $true) { Trace-Execution "Removing DNS resource record for name '$DnsName' with IP '$IPv4Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -Force } else { Trace-Execution "Record for '$IPv4Address' (A) not present." } } if ($IPv6Address -eq $true) { Trace-Execution "Removing DNS resource record for name '$DnsName' with IP '$IPv6Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -Force } else { Trace-Execution "Record for '$IPv6Address' (AAAA) not present." } } if ($AliasCName -eq $true) { Trace-Execution "Removing DNS resource record for name '$DnsName' with alias '$AliasCName' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -Force } else { Trace-Execution "Record for '$AliasCName' (CName) not present." } } } function Get-DnsServerResourceRecordRemoteOrLocal { Param ( [Parameter(Mandatory=$true)] [string] $ZoneName, [Parameter(Mandatory=$true)] [string] $RRtype, [Parameter(Mandatory=$true)] [string] $Name, [Parameter(Mandatory=$false)] [string] $ComputerName, [Parameter(Mandatory=$false)] [PSCredential] $RemoteCredential=$null ) $ErrorActionPreference = 'Stop' try { if ($RemoteCredential) { $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential Invoke-Command -Session $computerSession -ScriptBlock { Get-DnsServerResourceRecord -ZoneName $using:ZoneName -RRtype $using:RRtype -Name $using:Name -ComputerName $using:ComputerName -ErrorAction Ignore } } else { Get-DnsServerResourceRecord -ZoneName $ZoneName -RRtype $RRtype -Name $Name -ComputerName $ComputerName -ErrorAction Ignore } } finally { if ($computerSession) { Remove-PSSession $computerSession } } } function Remove-DnsServerResourceRecordRemoteOrLocal { Param ( [Parameter(Mandatory=$true)] [string] $ZoneName, [Parameter(Mandatory=$true)] [string] $RRtype, [Parameter(Mandatory=$true)] [string] $Name, [Parameter(Mandatory=$false)] [string] $ComputerName, [Parameter(Mandatory=$false)] [PSCredential] $RemoteCredential=$null ) $ErrorActionPreference = 'Stop' try { if ($RemoteCredential) { $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential Invoke-Command -Session $computerSession -ScriptBlock { Remove-DnsServerResourceRecord -ZoneName $using:ZoneName -RRtype $using:RRtype -Name $using:Name -ComputerName $using:ComputerName -Force } } else { Remove-DnsServerResourceRecord -ZoneName $ZoneName -RRtype $RRtype -Name $Name -ComputerName $ComputerName -Force } } finally { if ($computerSession) { Remove-PSSession $computerSession } } } function Add-DnsServerResourceRecordCNameRemoteOrLocal { Param ( [Parameter(Mandatory=$true)] [string] $ZoneName, [Parameter(Mandatory=$true)] [string] $Name, [Parameter(Mandatory=$true)] [string] $HostNameAlias, [Parameter(Mandatory=$false)] [string] $ComputerName, [Parameter(Mandatory=$false)] [PSCredential] $RemoteCredential=$null ) $ErrorActionPreference = 'Stop' try { if ($RemoteCredential) { $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecordCName -ZoneName $using:ZoneName -Name $using:Name -HostNameAlias $using:HostNameAlias -ComputerName $using:ComputerName } } else { Add-DnsServerResourceRecordCName -ZoneName $ZoneName -Name $Name -HostNameAlias $HostNameAlias -ComputerName $DnsServerName } } finally { if ($computerSession) { Remove-PSSession $computerSession } } } function Add-DnsServerResourceRecordRemoteOrLocal { Param ( [Parameter(Mandatory=$true)] [string] $ZoneName, [Parameter(Mandatory=$true)] [string] $Name, [Parameter(Mandatory=$false)] [switch] $IsIPv6, [Parameter(Mandatory=$true)] [string] $IPAddress, [Parameter(Mandatory=$false)] [string] $ComputerName, [Parameter(Mandatory=$false)] [PSCredential] $RemoteCredential=$null ) $ErrorActionPreference = 'Stop' try { if ($RemoteCredential) { $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential if ($IsIPv6) { Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecord -ZoneName $using:ZoneName -AAAA -Name $using:Name -IPv6Address $using:IPAddress -ComputerName $using:ComputerName } } else { Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecord -ZoneName $using:ZoneName -A -Name $using:Name -IPv4Address $using:IPAddress -ComputerName $using:ComputerName } } } else { if ($IsIPv6) { Add-DnsServerResourceRecord -ZoneName $ZoneName -AAAA -Name $Name -IPv6Address $IPAddress -ComputerName $ComputerName } else { Add-DnsServerResourceRecord -ZoneName $ZoneName -A -Name $Name -IPv4Address $IPAddress -ComputerName $ComputerName } } } finally { if ($computerSession) { Remove-PSSession $computerSession } } } # This function creates the DNS record for the given name. function Add-DnsRecord { Param ( [Parameter(Mandatory=$true)] [string] $DnsName, [Parameter(Mandatory=$true)] [string] $DnsZoneName, [Parameter(Mandatory=$false)] [string] $IPv4Address, [Parameter(Mandatory=$false)] [string] $IPv6Address, [Parameter(Mandatory=$false)] [string] $AliasCName, [Parameter(Mandatory=$true)] [string] $DnsServerName, [Parameter(Mandatory=$false)] [bool] $RemovePreviousRecord = $true, [Parameter(Mandatory=$false)] [PSCredential] $RemoteCredential=$null ) $ErrorActionPreference = 'Stop' Trace-ECEScript "Adding DNS record $DnsName" { if ($IPv4Address) { Trace-Execution "Adding DNS resource record for name '$DnsName' with IP '$IPv4Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential) { if ($RemovePreviousRecord -eq $true) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential } } Trace-Execution "Adding record for '$DnsName'." $RetryTimes = 3 for ($retry = 0; $retry -lt $RetryTimes; $retry ++) { try { Add-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -IPAddress $IpV4Address -ComputerName $DnsServerName -RemoteCredential $RemoteCredential break; } catch { if ($retry -lt ($RetryTimes - 1)) { continue } else { throw } } } } if ($IPv6Address) { Trace-Execution "Adding DNS resource record for name '$DnsName' with IP '$IPv6Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential) { if ($RemovePreviousRecord -eq $true) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential } } Trace-Execution "Adding record for '$DnsName'." Add-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -IsIPv6 -IPAddress $IPv6Address -ComputerName $DnsServerName -RemoteCredential $RemoteCredential } if ($AliasCName) { Trace-Execution "Adding DNS resource record for name '$DnsName' with alias '$AliasCName' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'." if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential) { if ($RemovePreviousRecord -eq $true) { Trace-Execution "Removing existing record for '$DnsName'." Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential } } Trace-Execution "Adding record for '$DnsName'." Add-DnsServerResourceRecordCNameRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -HostNameAlias $AliasCName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential } } } # adds PTR records for all a-type records in external zone FQDN function Add-DnsExternalPtrRecords { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $primaryDnsNames = Get-ExternalDnsMachineName -Parameters $Parameters -ErrorOnFail $false if ($primaryDnsNames.Count -eq 0) { Trace-Error "No Primary DNS server found. Selecting default all WASP machines" $waspRole = $Parameters.Roles["WASPUBLIC"].PublicConfiguration $primaryDnsNames = [string[]]$waspRole.Nodes.Node.Name # <VM Name="[PREFIX]-WASP##" /> } foreach($primaryDnsName in $primaryDnsNames) { $DnsExternalZoneFqdn = $Parameters.Roles["ExternalDNS"].PublicConfiguration.PublicInfo.DnsConfiguration.ExternalZoneFqdn $DnsResourceRecords = @(Get-DnsServerResourceRecord -ZoneName $DnsExternalZoneFqdn -computerName $primaryDnsName) $ExternalZoneTTLInHours = $Parameters.Roles["ExternalDNS"].PublicConfiguration.PublicInfo.DnsConfiguration.ExternalZoneTTLInHours $timeToLive = (New-TimeSpan -Hours $ExternalZoneTTLInHours) Trace-Execution "Getting Dns Resource Records for zone $DnsExternalZoneFqdn" foreach ($DnsResourceRecord in $DnsResourceRecords | where {$_.RecordType -eq 'A'}) { Trace-Execution "Processing A record $($DnsResourceRecord.RecordData)" $IPv4 = $DnsResourceRecord.RecordData.IPv4address.tostring().split('.') $zoneName = $IPv4[0] + ".in-addr.arpa" $PtrName = $IPv4[3..1] -join "." $ptrDomainName = $DnsResourceRecord.HostName + '.' + $DnsExternalZoneFqdn Invoke-ScriptBlockWithRetries -RetryTimes 3 -RetrySleepTimeInSeconds 1 -ScriptBlock { $serverZone = Get-DnsServerZone -Name $zoneName -computerName $primaryDnsName -ErrorAction SilentlyContinue if(-not $serverZone) { Throw "Failed to add PTR record $PtrName, could not find zoneName: $zoneName" } $ptrRecord = Get-DnsServerResourceRecord -ZoneName $zoneName -name $PtrName -computerName $primaryDnsName -ErrorAction SilentlyContinue if (-not $ptrRecord) { Trace-Execution "Adding PtrDomain $ptrDomainName, PtrRecordName $PtrName for zone $zoneName" try { Add-DnsServerResourceRecordPtr -computerName $primaryDnsName -ZoneName $zoneName -PtrDomainName $ptrDomainName -name $PtrName -TimeToLive $timeToLive } catch { Trace-Warning "Trying to handle Exception: $_" $ptrZoneResourceRecord = @(Get-DnsServerResourceRecord -ZoneName $zoneName -computerName $primaryDnsName) # check if there is a delegate zone blocking if ($ptrZoneResourceRecord | where {$_.RecordType -eq 'NS' -and $PtrName -match $_.HostName.replace(".", "\.")}) { Trace-Execution "Did not add $PtrName to zone $zoneName because a delegate zone $ptrZoneNsRecord exists" continue } Trace-Warning "Failed to add ptrRecord $PtrName to zone $zoneName. No NS zone blocking ptrRecord found." } } # Update the TTL for PTR record $oldRecordPTR = Get-DnsServerResourceRecord -ZoneName $zoneName -name $PtrName -computerName $primaryDnsName -ErrorAction SilentlyContinue if ($oldRecordPTR -and $oldRecordPTR.TimeToLive -ne $timeToLive) { # Calling Add on an existing record is safe. It overwrites the existing record if there is one. Trace-Execution "Updating the TTL of $PtrName on zone $zoneName to $timeToLive." Add-DnsServerResourceRecordPtr -ZoneName $zoneName -PtrDomainName $ptrDomainName -name $PtrName -TimeToLive $timeToLive -ComputerName $primaryDnsName } # Update the TTL for A record $oldRecordA = Get-DnsServerResourceRecord -ZoneName $DnsExternalZoneFqdn -name $DnsResourceRecord.HostName -ComputerName $primaryDnsName -ErrorAction SilentlyContinue if ($oldRecordA -and $oldRecordA.TimeToLive -ne $timeToLive) { # Calling Add on an existing record is safe. It overwrites the existing record if there is one. Trace-Execution "Updating the TTL of $($DnsResourceRecord.HostName) on zone $DnsExternalZoneFqdn to $timeToLive." Add-DnsServerResourceRecordA -ZoneName $DnsExternalZoneFqdn -name $DnsResourceRecord.HostName -TimeToLive $timeToLive -IPv4Address $DnsResourceRecord.RecordData.IPv4Address -ComputerName $primaryDnsName } } } } } # Returns the offline domain join blob used for unattended domain join. function Get-OfflineDjoinBlob { Param ( [Parameter(Mandatory=$true)] [string[]] $ComputerName, [Parameter(Mandatory=$true)] [string] $OuPath, [string[]] $DomainFqdn = $env:USERDNSDOMAIN, [Parameter(Mandatory=$false)] [PSCredential] $DomainJoinCredential=$null ) foreach ($eachComputer in $ComputerName) { $tempFilePath = [IO.Path]::GetTempFileName() Trace-Execution "Retrieving odj blob for host '$eachComputer' using the temporary file '$tempFilePath'." function Join-DomainOffline { param($DomainFqdn, $ComputerName, $tempFilePath, $OuPath) # Entirely arbitrary number of retries until we understand the occasional failure. $counter = 6 do { djoin.exe /provision /domain $DomainFqdn /machine $ComputerName /machineOU $OuPath /savefile $tempFilePath /reuse if (0 -eq $LASTEXITCODE) { break } else { Write-Warning "Failed to execute DJOIN.exe, retrying." Start-Sleep -Seconds 5 } $counter-- } while ($counter -ge 0) } if ($DomainJoinCredential) { $pssession = New-PSSession -Credential $DomainJoinCredential -Authentication Credssp try { $cmdOutput = Invoke-Command ` -ArgumentList $DomainFqdn, $eachComputer, $tempFilePath, $OuPath ` -ScriptBlock ${function:Join-DomainOffline} ` -Session $pssession } finally { $pssession | Remove-PsSession } } else { $cmdOutput = Join-DomainOffline -DomainFqdn $DomainFqdn -ComputerName $eachComputer -tempFilePath $tempFilePath -OuPath $OuPath } [string] $eachOdjBlob = Get-Content $tempFilePath -Raw if (-not $eachOdjBlob) { Trace-Error "Failed to retrieve the blob. Ouput of djoin.exe: '$cmdOutput'." } # Return the blob for each input. $eachOdjBlob = ($eachOdjBlob.Split(0))[0] # Remove the end of line null character (results in XML parse errors). Write-Output $eachOdjBlob } # Force a synchronization of domain controllers # (Get-ADDomainController -Filter *).Name | Foreach-Object {repadmin /syncall $_ (Get-ADDomain).DistinguishedName /e /A | Out-Null} } function Get-BareMetalCredential { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $cloudRole = $Parameters.Roles['Cloud'].PublicConfiguration # Account info $securityInfo = $cloudRole.PublicInfo.SecurityInfo $bareMetalUser = $securityInfo.HardwareUsers.User | Where-Object -Property Role -EQ 'BareMetalAdmin' $bareMetalCredential = $Parameters.GetCredential($bareMetalUser.Credential) return $bareMetalCredential } # Returns the parent action plan instance ID. function Get-ActionPlanInstanceID { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] if ($runtimeParameters -and $runtimeParameters.ContainsKey('ParentActionPlanInstanceIDToken')) { [Guid]$actionPlanInstanceId = $runtimeParameters['ParentActionPlanInstanceIDToken'] } else { $actionPlanInstanceId = [Guid]::Empty } return $actionPlanInstanceId } # Returns a common location to place test logs. # If the value found is $null, return $null. Otherwise, make sure the folder exist. function Get-TestLogsPath { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [switch] $RemotePath ) if ($RemotePath) { $testLogsPath = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.TestInfo.TestLogs.RemotePath } else { $testLogsPath = $ExecutionContext.InvokeCommand.ExpandString($Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.TestInfo.TestLogs.LocalPath) if (-not $testLogsPath) { # Not localizing as this would be a CustomerConfig template error Trace-Error "Path missing in CustomerConfig: Cloud\PublicInfo\TestInfo\TestLogs\LocalPath" } } if (-not $testLogsPath) { # This is a Deployment and a share is not available yet, so keep the logs on their local path return $null } $testLogsPath = $ExecutionContext.InvokeCommand.ExpandString($testLogsPath) if (-not (Test-Path -Path $testLogsPath -PathType Container)) { Trace-Execution "Creating folder $testLogsPath" New-Item -Path $testLogsPath -ItemType Directory | Out-Null } return $testLogsPath } # This function runs the specified test (local or remote) using Pester framework. function Start-Test { Param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, # Specify the Script parameter to be passed to Pester. # For Example, @{'Path' = "d:\Tests"; 'Parameters' = @{'ComputerName' = "r1s8f33-SUS01"} # where 'Path' is the path to the test files, and 'Parameters' is a hashtable of parameters to be passed to the test. [Parameter(Mandatory=$true)] [hashtable] $Script, [string[]] $ComputerName = $env:computername, [PSCredential] $Credential, [string] $TestName = "Validate-Role" ) $ErrorActionPreference = 'Stop' $testPath = $Script.Get_Item("Path") if (-not $testPath) { Trace-Error ($LocalizedStrings.KeyValueNotProvided -f "Script","Path") } $roleName = $Parameters.Configuration.Role.Id $logPath = Get-TestLogsPath -Parameters $Parameters foreach ($computer in $ComputerName) { # Run test and create log locally on the machine Trace-Execution ($LocalizedStrings.ValidationTestStarting -f $roleName,$computer) # To avoid clash with parallel running test generate unique Guid $uniqueSubString = [Guid]::NewGuid().toString().subString(0,4) $outputFileName = "TestResults_$($computer)_$($roleName)_$($uniqueSubString)_$((Get-Date).ToString("yyyy-MM-dd-HH-mm-ss")).xml" $outputFile = "$logPath\$outputFileName" $pesterParameters = @{Script=$Script;TestName=$TestName;OutputFile=$outputFile;OutputFormat="NUnitXml";PassThru=$true;Quiet=$true} $remoteScript = { param( $LogPath, $PesterParameters ) Import-Module pester -RequiredVersion "3.4.0" if (-not (Test-Path -Path $LogPath -PathType Container)) { New-Item -Path $LogPath -ItemType Directory | Out-Null } Invoke-Pester @PesterParameters } if ($Credential) { $result = Invoke-Command -ScriptBlock $remoteScript -ArgumentList $logPath,$pesterParameters -ComputerName $computer -Credential $Credential } elseif ($computer -eq $env:computername) { Import-Module pester -RequiredVersion "3.4.0" $result = Invoke-Pester @pesterParameters } else { $result = Invoke-Command -ScriptBlock $remoteScript -ArgumentList $logPath,$pesterParameters -ComputerName $computer } # Copy log to remote location $remoteLogPath = Get-TestLogsPath -Parameters $Parameters -RemotePath $remoteOutputFile = "\\$computer\$($outputFile.Replace(':','$'))" # For Deployment action, $remoteLogPath will be $null. # In this case, don't move the test logs, and leave them on the test machine. if ($remoteLogPath) { # copy the pester xml result file to a remote share (or some other local path) $copyResult = Copy-Item -Path $remoteOutputFile -Destination $remoteLogPath -Force -PassThru if ($copyResult) { $fileToRemove = $remoteOutputFile $remoteOutputFile = "$remoteLogPath\$outputFileName" # remove the original pester output file to avoid wasting space Remove-Item -Path $fileToRemove -Force -ErrorAction SilentlyContinue | Out-Null } } # Print passing tests foreach ($testResult in ($result.TestResult | Where-Object {$_.Passed})) { Trace-Execution "Test $($testResult.Result): $($testResult.Name)" } if ($result.FailedCount) { $message = $LocalizedStrings.ValidationTestFailed -f $roleName,$computer $message += "`r`n$($LocalizedStrings.TestResultsLocation -f $remoteOutputFile)" # Include failed tests in the message foreach ($testResult in ($result.TestResult | Where-Object {$_.Passed -ne $true})) { $message += "`r`nTest $($testResult.Result): $($testResult.Name)" } Trace-Error $message } elseif ($result.PassedCount -eq 0) { $message = $LocalizedStrings.ValidationTestNotAvailable -f $roleName,$computer $message += "`r`n$($LocalizedStrings.TestResultsLocation -f $remoteOutputFile)" Trace-Error $message } else { Trace-Execution ($LocalizedStrings.ValidationTestFinished -f $roleName,$computer) Trace-Execution ($LocalizedStrings.TestResultsLocation -f $remoteOutputFile) } } } function Get-AvailableServer { [CmdletBinding()] Param ( [Parameter(Mandatory=$true)] [string[]] $Server ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $Server | % { $null = Test-WSMan -ComputerName $_ -ErrorAction SilentlyContinue -ErrorVariable TestWSMANError if(-not $testWsmanError) { $availableServer += @($_) } else { Write-Warning ($LocalizedStrings.CouldNotConnectToServerWarning -f $_) } } return $availableServer } function Start-CloudCluster { [CmdletBinding()] Param ( [Parameter(Mandatory=$true)] [string] $ClusterName, [Parameter(Mandatory=$true)] [string[]] $ClusterNodeNames, [Parameter(Mandatory=$true)] [string[]] $ResourceName, [int] $StartClusterTimeoutSec = 180, [Parameter(Mandatory=$true)] [PSCredential] $Credential ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop # Set Cluster service to automatic if its not already. Trace-Execution ($LocalizedStrings.SetClusterServiceToAutomatic -f @($ClusterName, ($ClusterNodeNames -join ', '))) Invoke-Command -ComputerName $ClusterNodeNames -Credential $Credential { Set-Service -Name ClusSvc -StartupType Automatic -ErrorAction Stop } # Check if Cluster is already up. Trace-Execution ($LocalizedStrings.VerifyingCluster -f @($ClusterName, ($ResourceName -join ', '))) $clusterAlreadyAvailable = $true $clusterObject = $null $clusterObject = Get-Cluster -Name $ClusterName -ErrorAction SilentlyContinue -Verbose:$false if(-not $clusterObject) { $clusterAlreadyAvailable = $false } else { $clusterAlreadyAvailable = Test-ClusterResourceExist -ClusterObject $clusterObject -ResourceName $ResourceName } if(-not $clusterAlreadyAvailable) { Trace-Execution ($LocalizedStrings.ClusterNotReadyRestartingService -f @($ClusterName, ($ClusterNodeNames -join ', '))) # Starting all the cluster services starts the cluster when quorum is reached. Invoke-Command -ComputerName $ClusterNodeNames -Credential $Credential { Restart-Service -Name ClusSvc -ErrorAction Stop } Trace-Execution ($LocalizedStrings.CheckingClusterResources -f @($ClusterName, ($ResourceName -join ', '))) $result = Wait-Result -TimeOut $startClusterTimeoutSec -Interval 10 -ValidationScript { Get-Cluster -Name $ClusterName } if(-not $result) { throw ($LocalizedStrings.UnableToGetClusterTimeout -f @($ClusterName, $startClusterTimeoutSec)) } $clusterObject = Get-Cluster -Name $ClusterName $result = Wait-Result -TimeOut $startClusterTimeoutSec -Interval 10 ` -ValidationScript { Test-ClusterResourceExist -ClusterObject $clusterObject -ResourceName $ResourceName } if(-not $result) { throw ($LocalizedStrings.ClusterResourcesTimeout -f @($($ResourceName -join ', '), $StartClusterTimeoutSec)) } Trace-Execution ($LocalizedStrings.ClusterStartedSuccessfully -f $ClusterName) } else { Trace-Execution ($LocalizedStrings.ClusterAlreadyStarted -f $ClusterName) } } function Test-ClusterResourceExist { [CmdletBinding()] [OutputType([System.Boolean])] Param ( [Parameter(Mandatory=$true)] [Object] $ClusterObject, [Parameter(Mandatory=$true)] [string[]] $ResourceName ) $clusterResources = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock {Get-ClusterResource -Cluster $ClusterObject} $resourcesresult = $clusterResources | ? Name -In $ResourceName # Resources not found yet if(-not $resourcesresult) { return $false } else { $onlineResult = $resourcesresult | ? State -NE 'Online' # One or more Resources not online yet if($onlineResult) { return $false } } return $true } function Stop-CloudCluster { [CmdletBinding()] Param ( [Parameter(Mandatory=$true)] [string] $ClusterName, [Parameter(Mandatory=$true)] [string[]] $ClusterNodeNames, [Parameter(Mandatory=$true)] [PSCredential] $Credential ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $availableServer = Get-AvailableServer -Server $ClusterNodeNames if($availableServer) { Trace-Execution ($LocalizedStrings.GettingCluster -f $ClusterName) $clusterObject = $null $clusterObject = Get-Cluster -Name $ClusterName -ErrorAction SilentlyContinue -ErrorVariable clusterError -Verbose:$false if($clusterObject) { Trace-Execution ($LocalizedStrings.StoppingCluster -f $ClusterName) Stop-Cluster -Cluster $clusterObject -Force } else { Trace-Execution $clusterError Trace-Warning ($LocalizedStrings.UnableToGetCluster -f $ClusterName) } } } Function New-CimSessionVerify { param( [string]$ComputerName, [PSCredential] $Credential, [Int]$Retries = 20 ) $tracingStarted = $false do { $cimSession = New-CimSession -ComputerName $computerName -Credential $Credential -ErrorAction Ignore if ($cimSession -ne $null) { Trace-Execution "CimSession established" break } $traceName = "$env:systemdrive\CimSessionTrace$Retries.etl" if ($tracingStarted -eq $true) { Trace-Execution "Stopping CimSession tracing Iteration $Retries ($traceName)" # stop any previous tracing $log = Invoke-Command -ScriptBlock {netsh trace stop} Trace-Execution "Log $log" $tracingStarted = $false } $Retries-- # stop any previous tracing $log = Invoke-Command -ScriptBlock {netsh trace stop} Trace-Execution "Log $log" # start further tracing Trace-Execution "Start Tracing for CimSession, iteration $Retries ($traceName)" $log = Invoke-Command -ScriptBlock { ` netsh trace start scenario=virtualization provider=microsoft-windows-tcpip level=5 provider=microsoft-windows-dns-client capture=yes capturetype=both report=disabled tracefile=$traceName ov=yes } Trace-Execution "Log $log" $tracingStarted = $true #verify we have DNS resolution Trace-Execution "Verify DNS resolution to $computerName" $dns = Resolve-DnsName $computerName -ErrorAction Ignore if ($dns) { Trace-Execution "Resolve-DnsName returned" Trace-Execution ($dns | out-string) } # verify that we have connectivity to DC Trace-Execution "Verify IP connectivity to $computerName" $tnc = Test-NetConnection -ComputerName $computerName -Verbose -ErrorAction Ignore if ($tnc) { Trace-Execution "Test-NetConnection returned" Trace-Execution ($tnc | out-string) } # verify that we have WASMAN connectivity to DC Trace-Execution "Verify WSMAN connectivity to $computerName" $wsman = Test-WSMan -ComputerName $computerName -Verbose -ErrorAction Ignore if ($wsman) { Trace-Execution "Test-WSMan returned" Trace-Execution ($wsman | out-string) } Trace-Execution "Waiting 15 secs" Sleep 15 } while ($Retries -gt 0) if ($tracingStarted -eq $true) { Trace-Execution "Stopping CimSession tracing Iteration $Retries ($traceName)" # stop any previous tracing $log = Invoke-Command -ScriptBlock {netsh trace stop} Trace-Execution "Log $log" $tracingStarted = $false } if ($cimSession -eq $null) { Trace-Error "No CIM Session available on $computerName" throw } return $cimSession } Function Set-DNSForwarder { [CmdletBinding()] param( [String[]]$IPAddress, [String] $ComputerName, [PSCredential] $Credential ) # Setting IP DNS forwarders Trace-Execution "Setting IP DNS forwarders on $computerName to '$IPAddress'" $cimSession = New-CimSessionVerify -ComputerName $computerName -Credential $Credential if ($cimSession) { foreach($dnsIP in $IPAddress) { Trace-Execution "Start: Set the IP DNS forwarder '$dnsIP'." $ip = [System.Net.IPAddress]::Parse($dnsIP) $addressExist = (Get-DnsServerForwarder -CimSession $cimSession).IPAddress | where {$_.Address -eq $ip.Address} if ($addressExist) { Trace-Execution "Address $dnsIP existed, removing" Remove-DnsServerForwarder -IPAddress $dnsIP -CimSession $cimSession -Force } Trace-Execution "Adding address $dnsIP" Add-DnsServerForwarder -IPAddress $dnsIP -CimSession $cimSession Trace-Execution "End: Set the IP DNS forwarder '$dnsIP'." } Remove-CimSession $cimSession } else { Trace-Error "No CIM Session available on $computerName" } } # Creates users, groups, and group membership in the AD as defined in the SecurityInfo section. function Add-AccountMemberships { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [String] $PrimaryDomainControllerName, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters $primaryDomainControllerIP = $domainIPMapping[$PrimaryDomainControllerName] [bool] $isHyperConverged = $cloudRole.PublicInfo.Setup.CloudType -eq "HyperConverged" $SecurityInfoRoles = @() $domainGroups = @() $domainUsers = @() $serviceAccounts = @() $SecurityInfoRoles = $Parameters.Roles.GetEnumerator() | ? { $Parameters.Roles[$_.Name].PublicConfiguration.PublicInfo.SecurityInfo } | % { $_.Name } foreach ($role in $SecurityInfoRoles) { $securityInfo = $Parameters.Roles[$role].PublicConfiguration.PublicInfo.SecurityInfo $domainGroups += $securityInfo.DomainGroups.Group $domainUsers += $securityInfo.DomainUsers.User $serviceAccounts += $securityInfo.ServiceAccounts.Account } #Removing empty lines from the array $domainGroups = $domainGroups | ? {$_} $domainUsers = $domainUsers | ? {$_} $serviceAccounts = $serviceAccounts | ? {$_} # Domain Admin Credentials $domainAdminUser = $cloudRole.PublicInfo.SecurityInfo.DomainUsers.User | ? Role -EQ "DomainAdmin" $domainCredential = $Parameters.GetCredential($domainAdminUser.Credential) $primaryDomainControllerSession = New-PSSession -ComputerName $primaryDomainControllerIP -Credential $domainCredential -Authentication Credssp $secretAgreementParam = Get-Content -Encoding Byte "$PSScriptRoot\param3072" # Get update version in case it is an update # No need to change KDS key len for update $updateVersion = Get-InProgressUpdateVersion -Parameters $Parameters if($updateVersion -eq $null) { Trace-Execution "Updating KDS..." Invoke-Command -Session $primaryDomainControllerSession -ScriptBlock { Trace-Execution "Retrieving the current KDS config." $kdsConfig = Get-KdsConfiguration if (($kdsConfig.SecretAgreementPublicKeyLength -eq 3072) -and ($kdsConfig.SecretAgreementAlgorithm -eq 'DH')) { Trace-Execution "KDS key Length is already 3072: no need to take further actions." return } Trace-Execution "Changing KDS key length to 3072." $kdsConfig = Set-KdsConfiguration -SecretAgreementPublicKeyLength 3072 -SecretAgreementParameters $using:secretAgreementParam -SecretAgreementAlgorithm DH if (($kdsConfig.SecretAgreementPublicKeyLength -ne 3072) -or ($kdsConfig.SecretAgreementAlgorithm -ne 'DH')) { Trace-Error "Failed to update KDS: SecretAgreementPublicKeyLength $($kdsConfig.SecretAgreementPublicKeyLength), SecretAgreementAlgorithm $($kdsConfig.SecretAgreementAlgorithm)" } # Added so that domain service accounts can be created from all domain controllers (executed as domain admin). Trace-Execution "Changed KDS key length to 3072, now adding KDS root key." $rootKey = Add-KdsRootKey -EffectiveTime ((Get-Date).AddHours(-10)) Trace-Execution "New KDS root key GUID is $($rootKey.Guid)" } } Trace-Execution "Processing domain groups defined in the manifest." # Decoupling membership and new domian group creation #creating domain grous. Trace-Execution "Adding new user groups" foreach ($domainGroup in $domainGroups) { if($isHyperConverged -and $domainGroup.Id -eq "ComputeNodes") { Trace-Execution "Skipping Group $($domainGroup.Id) as its a HyperConverged topology." } else { if ($domainGroup.SkipCreation -and $domainGroup.SkipCreation -eq "True") { Trace-Execution "Skipping Group $($domainGroup.Id) creation." } else { Trace-Execution "Adding Group $($domainGroup.Id)" $null = New-UserGroup -ADRemoteSession $primaryDomainControllerSession -Groups $domainGroup -IsIdempotentRun $IsIdempotentRun } } } #Adding membership to the above created domain groups. Trace-Execution "Adding group memberships" foreach ($domainGroup in $domainGroups) { if($isHyperConverged -and $domainGroup.Id -eq "ComputeNodes") { Trace-Execution "Skipping Group $($domainGroup.Id) as its a HyperConverged topology." } else { Trace-Execution "Adding memberships for Group $($domainGroup.Id)" foreach ($membership in $domainGroup.Membership.MemberOf) { $null = Add-GroupMembership -ADRemoteSession $primaryDomainControllerSession -GroupName $domainGroup.Name -ParentGroupName $membership.GroupName -IsIdempotentRun $IsIdempotentRun } } } Trace-Execution "Processing domain users defined in the manifest." #creating domain users $resetPassword = (Get-RestoreParameters $Parameters).RestoreInProgress foreach ($domainUser in $domainUsers) { # by default all accounts are enabled $accountEnabled = $true # disable account if requested by configuration (Disabled="true" attribute) if ($domainUser.Disabled -and [Boolean]::Parse($domainUser.Disabled)) { $accountEnabled = $false } $domainUserCredential = $Parameters.GetCredential($domainUser.Credential) $null = New-UserAccount -ADRemoteSession $primaryDomainControllerSession -UserAccounts $domainUserCredential -DomainFqdn $DomainFQDN -Password $domainUserCredential.Password -ResetPassword:$resetPassword -Enabled $accountEnabled -IsIdempotentRun $IsIdempotentRun foreach ($acl in $domainUser.Acls.Acl) { $null = Add-AccountAcls -ADRemoteSession $primaryDomainControllerSession -UserAccount $domainUserCredential.UserName -ObjectName $acl.ObjectName -Permission $acl.Permission -DomainFqdn $domainFqdn } } # adding membership to the above users foreach ($domainUser in $domainUsers) { $domainUserCredential = $Parameters.GetCredential($domainUser.Credential) foreach($membership in $domainUser.Membership.MemberOf) { $null = Add-UserMembership -ADRemoteSession $primaryDomainControllerSession -ParentGroupName $membership.GroupName -UserIdentityName $domainUserCredential.GetNetworkCredential().UserName -IsIdempotentRun $IsIdempotentRun } } Trace-Execution "Processing service accounts defined in the manifest." # creating service accounts foreach ($serviceAccount in $serviceAccounts) { $null = New-ServiceAccount -ADRemoteSession $primaryDomainControllerSession -ServiceAccounts $serviceAccount -DomainFQDN $DomainFQDN -IsIdempotentRun $IsIdempotentRun } # adding membership foreach ($serviceAccount in $serviceAccounts) { foreach($membership in $serviceAccount.Membership.MemberOf) { $null = Add-ServiceAccountMembership -ServiceAccountIdentityName $serviceAccount.Name ` -ParentGroupName $membership.GroupName ` -ADRemoteSession $primaryDomainControllerSession ` -IsIdempotentRun $IsIdempotentRun } } # add SPNs for service accounts foreach ($serviceAccount in $serviceAccounts) { foreach ($spn in $serviceAccount.ServicePrincipalNames.ServicePrincipalName) { $null = Add-ServiceAccountSpn -ServiceAccountIdentityName $serviceAccount.Name ` -ServicePrincipalName $spn.Name ` -ADRemoteSession $primaryDomainControllerSession ` -IsIdempotentRun $IsIdempotentRun } } } # Creates a new local user. function New-LocalUserWrapper { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [PSCredential] $Credential, [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String] $Username, [Parameter(Mandatory = $true)] [SecureString] $Password ) Invoke-Command -ComputerName $ComputerName -Credential $Credential { $userAlreadyExists = Get-LocalUser -Name $Using:Username -ErrorAction SilentlyContinue if (-not $userAlreadyExists) { $null = New-LocalUser -Name $Using:Username -Password $Using:Password } } } # Adds user account to the local administrators group. function Add-LocalAdministrators { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [PSCredential] $Credential, [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String] $Username ) Invoke-Command -ComputerName $ComputerName -Credential $Credential { $ADMINISTRATORS_SID = 'S-1-5-32-544' $localAdministratorsGroupName = (Get-WMIObject Win32_Group -filter "LocalAccount=True AND SID='$ADMINISTRATORS_SID'").Name if($localAdministratorsGroupName) { [string[]] $currentLocalAdministrators = net localgroup $localAdministratorsGroupName if ($currentLocalAdministrators -notcontains $Using:Username) { $null = net localgroup $localAdministratorsGroupName /add $Using:Username } } } } # Gets the current active Domain Controller computer name. function Get-AvailableADComputerName { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [String[]] $AllADComputerName, [Parameter(Mandatory = $false)] [PSCredential] $RemoteServiceCredentials = $null ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $domainVM = hostname return $domainVM Trace-ECEScript "Getting the current active Domain Controller computer name." { # Proactively look for AD running on local machine (in case of multi-node, this will return the DVM). $service = Get-Service -Name "NTDS" -ErrorAction SilentlyContinue if($service -and $service.Status -eq "Running") { $activeADComputer = $env:COMPUTERNAME } else { # In case of one Node or post deployment, this will return DC01. foreach($adComputer in $AllADComputerName) { if ($RemoteServiceCredentials) { try { $psSession = New-PsSession -Credential $RemoteServiceCredentials -ComputerName $adComputer -ErrorAction SilentlyContinue if($psSession) { $service = Invoke-Command -Session $psSession -ScriptBlock { Get-Service -Name "NTDS" -ErrorAction SilentlyContinue } } } finally { if ($psSession) { Remove-PsSession $psSession } } } else { $service = Invoke-Command -ComputerName $adComputer -ScriptBlock { Get-Service -Name "NTDS" -ErrorAction SilentlyContinue } -ErrorAction SilentlyContinue } if ($service -and $service.Status -eq "Running") { $activeADComputer = $adComputer break } } } } if($activeADComputer) { Trace-Execution "Found AD running on $activeADComputer" return $activeADComputer } else { Trace-Error "No Available AD computer found." } } function New-ServiceAccount { Param ( [Parameter(Mandatory=$true)] [PSObject[]] $ServiceAccounts, [Parameter(Mandatory=$true)] [string] $DomainFQDN, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop foreach ($serviceAccount in $ServiceAccounts) { $serviceAccountName = $serviceAccount.Name $vmsGroupName = $serviceAccount.PrincipalsAllowedToRetrieveManagedPassword $managedPasswordIntervalInDays = $serviceAccount.ManagedPasswordIntervalInDays $ServicePrincipalName = $serviceAccount.ServicePrincipalName Trace-Execution "Check AD group $vmsGroupName" $adGroup = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:vmsGroupName | Tee-Object -Variable adGroup } if ($adGroup) { if($IsIdempotentRun) { Trace-Execution "Skipping: Group '$vmsGroupName' already exists in the AD." } else { Trace-Execution "Removing the pre-existing group '$vmsGroupName'." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADGroup -Identity $adGroup -Confirm:$false } } } if(-not $IsIdempotentRun) { Trace-Execution "Adding new AD group '$vmsGroupName'." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADGroup -Name $using:vmsGroupName -DisplayName $using:vmsGroupName -Description $using:vmsGroupName -GroupCategory Security -GroupScope Global } } Trace-Execution "Check AD service account $serviceAccountName" $adUser = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADServiceAccount -Filter {Name -like '*'} | ? Name -eq $using:serviceAccountName | Tee-Object -Variable adUser } -ErrorAction Stop if ($adUser) { if(-not $IsIdempotentRun) { Trace-Execution "Removing the pre-existing user '$serviceAccountName'." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADServiceAccount -Identity $adUser -Confirm:$false } Trace-Execution "Setting service account '$serviceAccountName' with VM group name '$vmsGroupName' and managed password interval of '$managedPasswordIntervalInDays' days." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADServiceAccount -Name $using:serviceAccountName -DNSHostName ($using:serviceAccountName + "." + $using:DomainFQDN) -PrincipalsAllowedToRetrieveManagedPassword $using:vmsGroupName -ManagedPasswordIntervalInDays $using:managedPasswordIntervalInDays -KerberosEncryptionType AES256 } } } else { Trace-Execution "Setting service account '$serviceAccountName' with VM group name '$vmsGroupName' and managed password interval of '$managedPasswordIntervalInDays' days." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADServiceAccount -Name $using:serviceAccountName -DNSHostName ($using:serviceAccountName + "." + $using:DomainFQDN) -PrincipalsAllowedToRetrieveManagedPassword $using:vmsGroupName -ManagedPasswordIntervalInDays $using:managedPasswordIntervalInDays -KerberosEncryptionType AES256 } } if($ServicePrincipalName -ne $null) { Trace-Execution "Adding service account '$serviceAccountName' with ServicePrincipalName '$ServicePrincipalName'." $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADServiceAccount $using:serviceAccountName | Set-AdServiceAccount -ServicePrincipalNames @{Add="$Using:ServicePrincipalName"} -Confirm:$false } } } } function New-UserGroup { Param ( [Parameter(Mandatory=$true)] [PSObject[]] $Groups, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Processing new user groups" foreach ($group in $Groups) { $groupName = $group.Name $groupScope = $group.Scope Trace-Execution "Check AD group $groupName" $adGroup = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:groupName | Tee-Object -Variable adGroup} if ($adGroup) { if($IsIdempotentRun) { Trace-Execution "Skipping: the group '$groupName' already exists in AD." continue } Trace-Execution "Removing the pre-existing group '$groupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADGroup -Identity $adGroup -Confirm:$false } } Trace-Execution "Adding new AD group '$groupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADGroup -Name $using:groupName -DisplayName $using:groupName -Description $using:groupName -GroupCategory Security -GroupScope $using:groupScope } } Trace-Execution "Finished processing new user groups" } function Add-GroupMembership { param ( [Parameter(Mandatory=$true)] [string] $GroupName, [Parameter(Mandatory=$true)] [string] $ParentGroupName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Check AD group $GroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $childGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:GroupName } Trace-Execution "Check AD group $ParentGroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName } Trace-Execution "Check membership of $GroupName in $ParentGroupName" $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:GroupName } if($IsIdempotentRun -and $member) { Trace-Execution "Skipping: Group '$GroupName' is already a member of the group '$ParentGroupName'." return } Trace-Execution "Adding group '$GroupName' as a member of the group '$ParentGroupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock {Add-ADGroupMember -Identity $parentGroup -Members $childGroup } Trace-Execution "Finished adding group '$GroupName' as a member of the group '$ParentGroupName'." } function Add-MachineMembership { param ( [Parameter(Mandatory=$true)] [string] $MachineName, [Parameter(Mandatory=$true)] [string] $ParentGroupName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Check AD computer $MachineName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $machineObject = Get-ADComputer -Filter {Name -like '*'} | ? Name -eq $using:MachineName } Trace-Execution "Check AD group $ParentGroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $groupObject = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName } Trace-Execution "Check membership of $MachineName in $ParentGroupName" $isMember = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $groupObject |? Name -eq $using:MachineName } if ($isMember) { Trace-Execution "Skipping: Machine '$MachineName' is already a member of the group '$ParentGroupName'." return } Trace-Execution "Adding machine '$MachineName' as a member of the group '$ParentGroupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Add-ADGroupMember -Identity $groupObject -Members $machineObject } Trace-Execution "Finished adding machine '$MachineName' as a member of the group '$ParentGroupName'." } function Add-UserMembership { param ( [Parameter(Mandatory=$true)] [string] $UserIdentityName, [Parameter(Mandatory=$true)] [string] $ParentGroupName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Check AD user $UserIdentityName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:UserIdentityName } Trace-Execution "Check AD group $ParentGroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName } Trace-Execution "Check membership of $UserIdentityName in $ParentGroupName" $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:UserIdentityName } if($IsIdempotentRun -and $member) { Trace-Execution "Skipping: User '$UserIdentityName' is already a member of the group '$ParentGroupName'." return } Trace-Execution "Adding user '$UserIdentityName' as a member of the group '$ParentGroupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Add-ADGroupMember -Identity $parentGroup -Members $userIdentity } Trace-Execution "Finished adding user '$UserIdentityName' as a member of the group '$ParentGroupName'." } function Remove-UserMembership { param ( [Parameter(Mandatory=$true)] [string] $UserIdentityName, [Parameter(Mandatory=$true)] [string] $ParentGroupName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Check AD user $UserIdentityName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:UserIdentityName } Trace-Execution "Check AD group $ParentGroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName } Trace-Execution "Check membership of $UserIdentityName in $ParentGroupName" $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:UserIdentityName } if($member -or $IsIdempotentRun) { Trace-Execution "Removing user '$UserIdentityName' as a member of the group '$ParentGroupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADGroupMember -Identity $parentGroup -Members $userIdentity -Confirm:$false } } else { Trace-Execution "Skipping: User '$UserIdentityName' is already removed from the group '$ParentGroupName'." } Trace-Execution "Finished removing user '$UserIdentityName' from the group '$ParentGroupName'." return } function Add-ServiceAccountMembership { param ( [Parameter(Mandatory=$true)] [string] $ServiceAccountIdentityName, [Parameter(Mandatory=$true)] [string] $ParentGroupName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Check AD service account $ServiceAccountIdentityName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADServiceAccount -Filter {Name -like '*'} | ? Name -eq $using:ServiceAccountIdentityName } Trace-Execution "Check AD group $ParentGroupName" Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName } Trace-Execution "Check membership of $ServiceAccountIdentityName in $ParentGroupName" $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:ServiceAccountIdentityName } if($IsIdempotentRun -and $member) { Trace-Execution "Skipping: User '$ServiceAccountIdentityName' is already a member of the group '$ParentGroupName'." return } Trace-Execution "Adding user '$ServiceAccountIdentityName' as a member of the group '$ParentGroupName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Add-ADGroupMember -Identity $parentGroup -Members $userIdentity } Trace-Execution "Finished adding user '$ServiceAccountIdentityName' as a member of the group '$ParentGroupName'." } function Add-ServiceAccountSpn { param ( [Parameter(Mandatory=$true)] [string] $ServiceAccountIdentityName, [Parameter(Mandatory=$true)] [string] $ServicePrincipalName, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $existingSpnNames = Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADServiceAccount -Filter {Name -like '*'} -Properties ServicePrincipalNames | ? Name -eq $using:ServiceAccountIdentityName $userIdentity.ServicePrincipalNames } if ($IsIdempotentRun -and $ServicePrincipalName -in $existingSpnNames) { Trace-Execution "Skipping: SPN '$ServicePrincipalName' is already created for account '$ServiceAccountIdentityName'." return } Trace-Execution "Adding SPN '$ServicePrincipalName' to user account '$ServiceAccountIdentityName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADServiceAccount -Identity $userIdentity -ServicePrincipalNames @{Add=$using:ServicePrincipalName}} } function New-UserAccount { Param ( [Parameter(Mandatory=$true)] [PSCredential[]] $UserAccounts, [Parameter(Mandatory=$true)] [string] $DomainFqdn, [Parameter(Mandatory=$true)] [SecureString] $Password, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession, [Switch] $ResetPassword, [Bool] $Enabled=$true, [Bool] $IsIdempotentRun=$false ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop foreach ($account in $UserAccounts) { $userName = $account.GetNetworkCredential().UserName Trace-Execution "Check AD user $userName" $adUser = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:userName | Tee-Object -Variable adUser } -ErrorAction Stop if ($adUser) { if($IsIdempotentRun) { # Enable/disable account if detected a change in state if ($adUser.Enabled -ne $Enabled) { $stateName = if ($Enabled) { "Enabling" } else { "Disabling" } Trace-Execution "$stateName the user '$userName' in AD." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADUser -Identity $adUser -Enabled $using:Enabled -Confirm:$false } } if ($ResetPassword) { Trace-Execution "Resetting the password for the user '$userName', which already exists in AD." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADAccountPassword -Identity $adUser -NewPassword $using:Password -Confirm:$false } } else { Trace-Execution "Skipping: the user '$userName' already exists in AD." } continue } Trace-Execution "Removing the pre-existing user '$userName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADUser -Identity $adUser -Confirm:$false } } #BUG 9716605: We should be taking expiration period from customer config file. Trace-Execution "Adding new AD user '$userName'." Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADUser -Name $using:userName -GivenName $using:userName -AccountPassword $using:Password -UserPrincipalName "$using:userName@$using:DomainFqdn" -Enabled $using:Enabled -PasswordNeverExpires $true -KerberosEncryptionType AES256 } Trace-Execution "Finished adding new AD user '$userName'." } } function Add-AccountAcls { Param ( [Parameter(Mandatory=$true)] [string] $UserAccount, [Parameter(Mandatory=$true)] [string] $ObjectName, [Parameter(Mandatory=$true)] [string] $Permission, [Parameter(Mandatory=$true)] [string] $DomainFqdn, [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $ADRemoteSession ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-Execution "Setting ACL permission '$Permission' for account '$UserAccount' and object '$ObjectName'." try { $domainNames = $DomainFqdn.Split('.') foreach ($domainName in $domainNames) { $objectName += ",DC=$domainName" } $dsaclsCmdline = "DSACLS `"$objectName`" /G `"$DomainFqdn\$UserAccount`:$Permission`"" Trace-Execution "Executing the following command on '$($ADRemoteSession.ComputerName)': '$dsaclsCmdline'." [string] $dsaclsOutput = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Invoke-Expression "$using:dsaclsCmdline" } } catch { Trace-Error "Could not set ACL permissions for account '$UserAccount'. Reason: $_" } } # Virtual Network Common function Test-NetworkMap { param ( [Parameter(Mandatory=$true)] $NetworkMap, [Parameter(Mandatory=$false)] [UInt32] $RetryCount = 0, [Parameter(Mandatory=$false)] [UInt32] $RetryIntervalInSeconds = 60, [Parameter(Mandatory=$false)] [string[]] $NodeNames ) Trace-ECEScript $LocalizedNetworkData.CheckNICConnectivity { $nodesToCheck = $NetworkMap.Keys if ($NodeNames) { $nodesToCheck = $nodesToCheck | Where-Object {$_ -in $NodeNames} } $retry = 0 do { try { # Test nic connectivity for each machine foreach ($nodeName in ($nodesToCheck | Sort-Object)) { $node = $NetworkMap.$nodeName $hasPingableConnections = $false foreach ($nicId in ($node.Keys | Sort-Object)) { $nic = $NetworkMap.$nodeName.$nicId try { # This cast will fail if the IP address isn't well-formed $ipAddress = [ipaddress]$nic.IPv4Address.Split('/')[0] $canPing = Test-IPConnection $ipAddress.IPAddressToString } catch { $canPing = $false } if ($canPing) { $hasPingableConnections = $true } $nic | Add-Member -MemberType NoteProperty -Name CanPing -Value $canPing Trace-Execution "$(if ($canPing) {'+'} else {'-'}) $nodeName | $($nic.Name)" } if (-not $hasPingableConnections) { throw ($LocalizedNetworkData.NoPingableConnections -f $nodeName) } } break } catch { if($retry -eq $RetryCount) { throw } Trace-Execution "Test-NetworkMap failed. Will retry in $RetryIntervalInSeconds seconds. Retry attempted: $retry." Start-Sleep -Seconds $RetryIntervalInSeconds } $retry++ } while ($retry -le $RetryCount) } } <# .Synopsis Function to clear a potentially remote DNS server cache from any context. .Parameter DNSServerName The DNS server whose cache should be cleared defaulting to the local machine. .Parameter DNSServerCredential Credentials to use when resetting the cache, only used when the server name is also specified. .Example Clear-RemoteDNSServerCache This will clear the local DNS server's cache using the current credentials #> function Clear-RemoteDNSServerCache { param ( [Parameter(Mandatory=$false)] [string] $DNSServerName = $null, [Parameter(Mandatory=$false)] [PSCredential] $DNSServerCredential = $null ) Trace-Execution "Clear DNS server cache." try { if ($DNSServerName) { Trace-Execution "Remote DNS server $DNSServerName used." $dnsSession = New-PsSession -ComputerName $DNSServerName -Credential $DNSServerCredential } else { $dnsSession = New-PsSession } $null = Invoke-Command -Session $dnsSession -ScriptBlock { Clear-DnsServerCache -Force -ErrorAction SilentlyContinue } } finally { $dnsSession | Remove-PsSession } Trace-Execution "DNS server cache cleared." } function Get-IsVirtualNetworkAlreadyConfigured { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] $NetworkMap, [Parameter(Mandatory=$true)] $IsOneNode ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop # Check that all non-Deployment vNICs can be pinged. $notPingableAdapters = $NetworkMap.Values.Values | ? CanPing -ne $true if (-not $notPingableAdapters) { Trace-Execution $LocalizedNetworkData.NetworkingAlreadyConfigured return $true } return $false } # BUGBUG: This function may not needed any more in ASZ. Leave it here to not break existing code. Need refactoring. function Get-NetworkMap { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $PhysicalMachinesRole ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $nodeNames = $PhysicalMachinesRole.Nodes.Node | % Name Trace-ECESCript "Get Network Map for nodes $nodeNames" { # If multiple nodes are being added at the same time, only test those that are being deployed by the current action plan instance $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName $hostsList = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name if ($executionRoleName -ieq "Cluster") { $hostsList = $null $hostsList = [array]($Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.PhysicalNodes.PhysicalNode.Name) } $networkmap = @{} $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters foreach ($node in $PhysicalMachinesRole.Nodes.Node) { if (-not($node.OperationStatus -eq "RequiresRemediation") -and (-not($node.OperationStatus -eq "Adding") -or ($hostsList -contains $node.Name))) { # there would be only 1 IP on each host that may need checking. $nodeNics = @{} $nicIPv4 = $allHostNodes[$node.Name] $nicName = $node.Name if (-not (Test-NetworkIPv4Address -IPv4Address $nicIPv4)) { Trace-Execution "[Get-NetworkMap]: IPv4 Address for node $($node.Name) $nicIPv4 is not with expected format, skip." continue } $nicData = New-Object PSObject -Property @{ Name = $nicName IPv4Address = $nicIPv4 } $nodeNics.$nicName = $nicData $networkMap.($node.Name) = $nodeNics } } } return $networkMap } function Wait-VirtualNetwork { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $PhysicalMachinesRole, [Boolean] $IsIdempotentRun = $true, [Parameter(Mandatory=$false)] [string] $UpdateNodeName ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-ECEScript "Waiting for Virtual Network to be configured on the host." { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $localAdminUser = $securityInfo.LocalUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID $credential = $Parameters.GetCredential($localAdminUser.Credential) Set-MacAndIPAddress -Parameters $Parameters -PhysicalMachinesRole $PhysicalMachinesRole -IPv4AddressSource 'Management' # in the case of Bare Metal the hosts, it is possible that the DSC hasn't completed the configuration for the hosts at this point, # so add some retry logic to test the network connectivity $networkMap = Get-NetworkMap -Parameters $Parameters -PhysicalMachinesRole $PhysicalMachinesRole if ($UpdateNodeName) { Test-NetworkMap -NetworkMap $networkMap -RetryCount 10 -NodeNames @($UpdateNodeName) } else { Test-NetworkMap -NetworkMap $networkMap -RetryCount 10 } $nodes = $PhysicalMachinesRole.Nodes.Node $nodeNames = $nodes | ForEach-Object Name $jobs = @() $isOneNode = $false if ($UpdateNodeName) { # For update, only one host is service at a time and this should not get confused with One Node MAS environment $nodes = $nodes | Where-Object Name -eq $UpdateNodeName } elseif ($nodeNames.Count -eq 1 ) { $isOneNode = $true # MASD One Node will always be run locally $nodeNames = 'LocalHost' } if ($IsIdempotentRun) { if (Get-IsVirtualNetworkAlreadyConfigured -NetworkMap $networkMap -IsOneNode $isOneNode) { return } } $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters if ($env:OSImageType -ne "ISO") { Trace-ECEScript "Start PowerShell jobs to retrieve and set the DSC state on all nodes $nodeNames." { foreach ($node in $nodes) { # Collect information about the state of the recently deployed physical host. $managementIpAddress = $allHostNodes[$node.Name] $nodeName = $node.Name Trace-Execution "Sending DSC query to $managementIpAddress for $nodeName" $jobs += Start-Job -ScriptBlock { $retVal = $false $logText = "DSC report for node $(($using:Node).Name)`r`n" # Wait as much as ten minutes for the exiting DSC configuration to complete. This is just a best effort. for ($i = 0; $i -lt 40; $i++) { $cimSession = $null $result = $null try { # This call may fail if it can't connect to the remote host when remote host is configuring vmswitch and host vnic. $cimSession = New-CimSession -ComputerName $Using:nodeName -Credential $using:credential -ErrorAction SilentlyContinue if ($null -eq $cimSession) { $cimSession = New-CimSession -ComputerName $using:managementIpAddress -Credential $using:credential -ErrorAction Stop } # This call will throw exception if there is currently a DSC config running on the host. # we use it as a way as an indication. $logText += "Calling Get-DscConfigurationStatus at attempt #$i. `r`n" $result = Get-DscConfigurationStatus -CimSession $cimSession -ErrorAction Stop break } catch { Start-Sleep -Seconds 15 $logText += "Get-DscConfigurationStatus failed with exception ($($_.Exception.Message)) at attempt #$i `r`n" } finally { if ($cimSession) { Remove-CimSession $cimSession } } } $logText += "Trying to make sure host DSC configuration is completed. `r`n" for ($i = 0; $i -lt 5; $i++) { $cimSession = $null try { $cimSession = New-CimSession -ComputerName $Using:nodeName -Credential $using:credential -ErrorAction SilentlyContinue if ($null -eq $cimSession) { $cimSession = New-CimSession -ComputerName $using:managementIpAddress -Credential $using:credential -ErrorAction Stop } $logText += "Calling Start-DscConfiguration at attempt #$i `r`n" Start-DscConfiguration -UseExisting -Wait -CimSession $cimSession -Verbose -Force -ErrorAction Stop $logText += "Calling Get-DscConfigurationStatus at attempt #$i. `r`n" $result = Get-DscConfigurationStatus -CimSession $cimSession -ErrorAction Stop $logText += "Calling Get-DscConfiguration at attempt #$i. `r`n" $logText += (Get-DscConfiguration -CimSession $cimSession -Verbose -ErrorAction Stop | out-string) } catch { # While DSC configuration on the host is running, it may cause the network connectivity to be disrupted. # If that disruption last too long, say more than 3 or 4 minutes, the above calls may fail with exception. # So catch the exception here but do nothing. $logText += "Attempt #$i : failed with exception ($($_.Exception.Message)). `r`n" } finally { if ($result.status -eq 'Success') { $retVal = $true } if ($cimSession) { Remove-CimSession $cimSession } } if ($retVal) { break } } $retObj = @{RetVal = $retVal; LogText = $logText; HostIp = $using:managementIpAddress} $retObj } } } # Wait for all the nodes to either converge or fail to converge. If they failed to # converge, or if we failed to send the WSManAction to them, then use the more stateful # networking methods to figure out why, later. Trace-ECEScript "Waiting for DSC query to complete" { $results = Receive-Job -Wait -Job $jobs $results | ForEach-Object { Trace-Execution $_.LogText if ($_.RetVal -ne $true) { Trace-Error "DSC failed to converge on a physical host $($_.HostIp)." } } } $jobs | Remove-Job -ErrorAction SilentlyContinue } else { Trace-Execution "Skipping wait for DSC config as this is an ISO-based deployment." } } } function Get-DomainIPMapping { param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainIPMapping = @{} $deploymentMachineRoleNode = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node if($deploymentMachineRoleNode -ne $null) { $domainIPMapping.Add($deploymentMachineRoleNode.Name, $deploymentMachineRoleNode.IPv4Address) } return $domainIPMapping } # Get the node names from the ExecutionContext in the $Parameters variable # if available; otherwise, return $null function Get-ExecutionContextNodeName { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $false)] [switch] $EnsureSingle ) $nodeName = $null $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName $nodeName = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name if ($null -ne $executionRoleName -and $null -eq $nodeName) { # If node names are not provided, get all nodes associated to the $executionRoleName $nodeName = ($Parameters.Roles[$executionRoleName].PublicConfiguration.Nodes.Node.Name) | Select-Object -Unique } Trace-Execution "Received ExecutionContext with role name $executionRoleName and node(s): $($nodeName -join ', ')" if ($EnsureSingle -and (!$nodeName -or ($nodeName.Count -ne 1))) { throw "Expected single node in ExecutionContext but found node(s): $($nodeName -join ', ')" } return $nodeName } # Get the cluster name from the ExecutionContext in the $Parameters variable # if available; otherwise, return $null function Get-ExecutionContextClusterName { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName $clusterName = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.ClusterName Trace-Execution "Received ExecutionContext with role name $executionRoleName and clusterName : $clusterName" return $clusterName } # Gets the names of the roles specified in the execution context. function Get-ExecutionContextRoleName { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) # ExecutionContext can specify RolePath and/or RoleName attributes. Since we want just the names # we truncate the role paths if they are the only attribute provided, and return a list of the unique role names. $executionRoleName = @( $Parameters.Context.ExecutionContext.Roles.Role.RoleName ) $executionRolePath = @( $Parameters.Context.ExecutionContext.Roles.Role.RolePath ) $executionRoleName += $executionRolePath | Split-Path -Leaf return ($executionRoleName | Select-Object -Unique) } # Determines whether the environment is one-node from the interface parameters. function IsOneNode { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) return @($Parameters.Roles["Storage"].PublicConfiguration.Nodes.Node).Count -eq 1 } # Determines whether the environment is virtual from the interface parameters. function IsVirtualAzureStack { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $oemModel = $Parameters.Roles["OEM"].PublicConfiguration.PublicInfo.UpdatePackageManifest.UpdateInfo.Model return $oemModel -eq "Hyper-V" } # This function checks whether the deployment is running on Virtual Machine by system prams function IsOnVirtualMachine { try { # is on VM $computerSystem = Get-WmiObject win32_computersystem $isOnVM = (('Microsoft Corporation' -eq $computerSystem.Manufacturer) -and ('Virtual Machine' -eq $computerSystem.Model)) } catch { Trace-Error "Error while checking system info for Virtual Machine deployment" return $false } return $isOnVM } # This function starts each service in serial if not running, and waits for each service to be Running function Assert-Service { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [System.String[]] $Name, [Parameter(Mandatory = $true)] [System.String] $ComputerName, [Parameter(Mandatory = $true)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [switch] $WaitOnly, [Parameter(Mandatory = $false)] [ScriptBlock] $WaitScript ) $ErrorActionPreference = 'Stop' $timeoutInSeconds = 120 Trace-Execution "Ensure services '$Name' on $ComputerName machine are running." $psSession = New-PSSession -ComputerName $ComputerName -Credential $Credential $services = Invoke-Command $psSession {Get-Service -Name $using:Name -ErrorAction Stop} # Normally all services should start automatically when machine is turned on foreach ($serviceController in $services) { $serviceName = $serviceController.Name if ($serviceController.Status -ne 'Running') { Trace-Execution "Service '$serviceName' is not running." if (-not $WaitOnly) { Trace-Execution "Starting the service '$serviceName'." Invoke-Command $psSession { Start-Service -Name $using:serviceName -ErrorAction Stop } } $waitServiceScript = { $service = Invoke-Command $psSession {Get-Service -Name $using:serviceName -ErrorAction Stop}; ($service.Status -eq 'Running') } if (Wait-Result -ValidationScript $waitServiceScript -TimeOut ($timeoutInSeconds) -Interval 10) { Trace-Execution "Service '$serviceName' has successfully started." } else { Trace-Error ($LocalizedStrings.ServiceDidNotStartup -f $serviceName,$ComputerName,$timeoutInSeconds) } } else { Trace-Execution "Service '$serviceName' is already running." } if($waitScript) { Trace-Execution "Running wait script to ensure service can receive requests." if (Wait-Result -ValidationScript $waitScript -TimeOut ($timeoutInSeconds) -Interval 10) { Trace-Execution "Wait script completed successfully." } else { Trace-Error ($LocalizedStrings.ServiceDidNotStartup -f $serviceName, $ComputerName, $timeoutInSeconds) } } } } # Checks the RunInformation for UpdateVersion. Returns the value if found else returns null function Get-InProgressUpdateVersion { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $version = $null Trace-Execution "Getting in progress Update Version." # In case of update, run time information will be populated if($Parameters.RunInformation -ne $null) { $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] if( $runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('UpdateVersion') ) { $version = $runtimeParameters["UpdateVersion"] Trace-Execution "Version of the update being applied : $version" return $version } } # If it is a one node or bootstrap VM return null as an update will not run on it. $buildLocally = $Parameters.Context.ExecutionContext.BuildVhdLocally -and [bool]::Parse($Parameters.Context.ExecutionContext.BuildVhdLocally) $RoleName = $Parameters.Configuration.Role.Id $isOneNode = @($Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node).Count -eq 1 if(($isOneNode -eq $true) -or ($buildLocally -and ($RoleName -eq "Domain"))) { return $version } # If the version is set to null, check if share has update version info file. # This file only exists while a host is being updated. if($version -eq $null) { $bareMetalRole = $Parameters.Roles["BareMetal"].PublicConfiguration $storageClusterName = Get-ManagementClusterName $Parameters if($bareMetalRole.PublicInfo.UpdateInformation -ne $null) { $updateInfoFolderPath = Get-SharePath $Parameters $bareMetalRole.PublicInfo.UpdateInformation.UpdateVersionFile.FolderName $storageClusterName $updateInfoFileName = $bareMetalRole.PublicInfo.UpdateInformation.UpdateVersionFile.FileName $filePath = Join-Path -Path $updateInfoFolderPath -ChildPath $updateInfoFileName Trace-Execution "Check if version file $filePath exists" if(Test-Path -Path $filePath) { Trace-Execution "The version file is present. Extracting version number and override status from it" $versionFileContent = [xml] ( Get-Content -Path $filePath ) if($versionFileContent) { $versionValue = $versionFileContent.InProgressUpdateVersion.Version $useThisVersion = $versionFileContent.InProgressUpdateVersion.UseThisVersion if($useThisVersion -eq "True") { $version = $versionValue } else { Trace-Error "The version file exists but is not being used as UseThisVersion flag is set to False. This flag should be set to True if FRU is being used to recover from an update failure" } } else { Trace-Error "Version file found at $filePath but has invalid content" } } } } return $version } # Gets the file share directory containing expanded contents of a currently in-progress update package. function Get-InProgressUpdatePackagePath { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] $packagePath = $runtimeParameters["UpdatePackagePath"] if (-not $packagePath) { Trace-Execution "The UpdatePackagePath runtime parameter is not present." return $null } Trace-Execution "Upgrade package @: [$packagePath]" # The package path is expected to be a full path to an update package file (e.g. a self- # extracting .exe, or a metadata.xml file). The path where the contents are extracted is # the directory containing this file. return (Split-Path $packagePath) } # Gets the location of the folder containing nugets in the currently in-progress update package. function Get-InProgressUpdateNugetStore { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $updatePackagePath = Get-InProgressUpdatePackagePath -Parameters $Parameters if ($updatePackagePath) { $urpRole = $Parameters.Roles["URP"].PublicConfiguration $nugetSourceFolderName = $urpRole.PublicInfo.UpdatePackagePaths.NugetFolder.Name $nugetStorePath = Join-Path $updatePackagePath $nugetSourceFolderName if (Test-Path $nugetStorePath) { return $nugetStorePath } } else { Trace-Execution "No update is currently in progress." } return $null } <# .SYNOPSIS Expands the deployment artifacts to the destination location. .DESCRIPTION Expands all deployment artifacts into the destination root path. .EXAMPLE Expand-DeploymentArtifacts -DeploymentContentNode $Parameters.Configuration.Role.PrivateInfo.DeploymentContent -DestinationRootPath "C:" .PARAMETER DeploymentContentNode The xml node containing the deployment content to deliver to the destination. .PARAMETER DestinationRootPath The root path under which to deliver the nuget content. #> function Expand-DeploymentArtifacts { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateNotNull()] [System.Xml.XmlElement] $DeploymentContentNode, [Parameter(Mandatory=$true)] [string] $DestinationRootPath, [Parameter(Mandatory=$false)] [string] $NugetStorePath = "$env:SystemDrive\CloudDeployment\NuGetStore" ) $nugetDeploymentContentItems = $DeploymentContentNode.NugetArtifact $installRolePackages = $false $rolePackageDestination = Join-Path $DestinationRootPath 'NugetStore' foreach( $deployContent in $nugetDeploymentContentItems ) { Write-Verbose -Verbose "Copying content from nuget $($deployContent.Name) into $DestinationRootPath" $destination = Join-Path $DestinationRootPath $deployContent.DestinationPath $sourceIsFile = [bool]$deployContent.SourceIsFile $isNugetInstall = [bool]$deployContent.NugetInstall if($isNugetInstall) { $installRolePackages = $true } Write-Verbose -Verbose "$($deployContent.SourcePath) will be copied to $destination" Write-Verbose -Verbose "IsNugetInstall : $isNugetInstall" Expand-NugetContent -NugetName $deployContent.Name -SourcePath $deployContent.SourcePath -DestinationPath $destination -SourceIsFile:$sourceIsFile -Verbose:$VerbosePreference -NugetStorePath $NugetStorePath -IsNugetInstall:$isNugetInstall Write-Verbose -Verbose "Nuget $($deployContent.Name) with source [$($deployContent.SourcePath)] copied to $destination" } if($installRolePackages) { Write-Verbose -Verbose "Install Role Nuget packages to NugetStore" Install-RoleNugetPackages -NugetStorePath $NugetStorePath -Destination $rolePackageDestination } } function Expand-UpdateContent { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory=$false)] [string[]] $ExecutionRoleName, [Parameter(Mandatory=$false)] [switch] $DSCOnly, [Parameter(Mandatory=$false)] [switch] $EngineUpdateOnly, [Parameter(Mandatory=$false)] [string[]] $NodeName ) $domainCredential = Get-DomainCredential -Parameters $Parameters $currentContext = $Parameters.Context.ExecutionContext if(-not $ExecutionRoleName) { $ExecutionRoleName = $currentContext.Roles.Role.RoleName } $vmRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration $localExecutionHost = $env:COMPUTERNAME $clusterName = Get-ManagementClusterName $Parameters $allManagementHosts = Get-ManagementClusterNodes $Parameters $uncLibraryShareNugetStorePath = Get-SharePath $Parameters $vmRole.PublicInfo.LibraryShareNugetStoreFolder.Path $clusterName # During AsZ deployment expand nugets on all the nodes part of cluster # Cluster path at this time doesn't exist so use the absolute path if (!(Test-Path -Path $uncLibraryShareNugetStorePath)) { Trace-Execution "AsZ deployment scenario so use absolute path on SeedNode" $uncLibraryShareNugetStorePath = "$env:SystemDrive\CloudDeployment\NuGetStore" } $localLibraryShareNugetStorePath = $null # Bug 15111454: Waiting for fix it should not block deployment # if(Test-LocalHostHasClusterStoragePath -Parameters $Parameters) # { # Trace-Execution "Getting local CSV path for $uncLibraryShareNugetStorePath" # $localLibraryShareNugetStorePath = Get-LocalCsvPathFromSharePath -Parameters $Parameters -UNCSharePath $uncLibraryShareNugetStorePath # } Trace-Execution "uncLibraryShareNugetStorePath: $uncLibraryShareNugetStorePath" Trace-Execution "localLibraryShareNugetStorePath: $localLibraryShareNugetStorePath" $systemDrive = $env:SystemDrive -Replace (':','$') $healthyNodes = $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes.Node | where { $_.ProvisioningStatus -eq "Complete" } | % Name $healthyNodes += $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node | where { $_.ProvisioningStatus -eq "Complete" } | % Name foreach ($roleName in $ExecutionRoleName) { Trace-Execution "Expand-UpdateContent adding update contents for role: $roleName" $roleConfiguration = $Parameters.Roles[$roleName].PublicConfiguration if ($NodeName) { $nodes = $NodeName } else { $nodes = $roleConfiguration.Nodes.Node.Name |where { $healthyNodes.Contains($_)} } Trace-Execution "Expand-UpdateContent for Nodes : $($nodes)" # Skipping nodes which are provided in runtime parameters $nodes = Skip-NodesProvidedInRuntimeParameter -Parameters $Parameters -Nodes $nodes Trace-Execution "Expand-UpdateContent for Nodes (after filtering) : $($nodes)" # Get update content for the role $nugetUpdateContentItems = $roleConfiguration.PublicInfo.DeploymentContent.NugetArtifact | where {$_.Update -eq "True" -or $_.EngineUpdate -eq "True"} # Copy the update content if ($nugetUpdateContentItems) { foreach( $updateContent in $nugetUpdateContentItems ) { if($DSCOnly -and ($updateContent.DSCContent -ne "True")) { Trace-Execution "$($updateContent.Name) is not marked as nuget with DSC content. Skipping update on the target VM(s) " continue } elseif ($EngineUpdateOnly -and ($updateContent.EngineUpdate -ne "True")) { Trace-Execution "$($updateContent.Name) is not marked as nuget with engine content. Skipping update on the target VM(s)." continue } foreach ($node in $nodes) { $libraryShareNugetStorePath = $null if (($node -in $allManagementHosts) -and ($null -ne $localLibraryShareNugetStorePath)) { Trace-Execution "For node $node - member the management cluster - will use local nuget store path $localLibraryShareNugetStorePath" $libraryShareNugetStorePath = $localLibraryShareNugetStorePath } else { Trace-Execution "For node $node will use UNC nuget store path $uncLibraryShareNugetStorePath" $libraryShareNugetStorePath = $uncLibraryShareNugetStorePath } Trace-Execution "Start process to expand nuget content:" Trace-Execution "Local execution host : $localExecutionHost" Trace-Execution "Nuget store Path : $libraryShareNugetStorePath" Trace-Execution "Processing node : $node" $DestinationRootPath = "\\$node\$systemDrive\" if ($updateContent.UpdatePath) { $destination = Join-Path $DestinationRootPath $updateContent.UpdatePath } else { $destination = Join-Path $DestinationRootPath $updateContent.DestinationPath } $isNugetInstall = $null [bool]::TryParse($updateContent.NugetInstall, [ref]$isNugetInstall) $retry = 0 $maxRetry = 3 while ($retry -lt $maxRetry) { $retry++ try { Expand-NugetContent -NugetName $updateContent.Name -SourcePath $updateContent.SourcePath -DestinationPath $destination -IsUnc -NugetStorePath $libraryShareNugetStorePath -IsNugetInstall:$isNugetInstall -Credential $domainCredential Write-Verbose -Verbose "$($updateContent.SourcePath) copied to $destination" break } catch { # Repair item for ICM 127772311 if($retry -lt $maxRetry -and $roleName -in ('BareMetal', 'ACS') -and "$_" -match 'being used by another process') { Trace-Execution "Caught exception: $_, try apply remediation..." Import-Module -Name "$PSScriptRoot\..\ACS\ACSRoot.psm1" -DisableNameChecking -Force Stop-DanglingAcsDeploymentPSProcess -Parameters $Parameters -NodeToFix $node } elseif($retry -lt $maxRetry -and $roleName -in ('SeedRingServices')) { Trace-Execution "Caught exception: $_, try apply remediation to close PEP session..." Import-Module $PSScriptRoot\..\SeedRingServices\SeedRingServices.psm1 -DisableNameChecking -Force Close-PEPSessions -Parameters $Parameters } else { throw } } } } } } } } #execute to a script block over a VMGroup using PSDirect instead of WinRM function Invoke-PSDirectOnVM { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [String[]]$VMNames, [Parameter(Mandatory = $true)] [Object]$VMCredential, [Parameter(Mandatory = $true)] [ScriptBlock]$ScriptBlock, [Object[]]$ArgumentList = $null ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop #retrieve the environment info $physicalMachineRole = $Parameters.Roles["BareMetal"].PublicConfiguration $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $allPhysicalMachines = Get-ActiveClusterNodes $Parameters # Account info $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo # Domain Info $domainAdminCredential = Get-DomainCredential -Parameters $Parameters # for each physical host, iterate over all the infra VMs that are part of it $hostTrace = @() foreach ($phyHost in $allPhysicalMachines) { Trace-Execution "For Host $($phyHost)" # test if the target host is reachable $output = $false $output = Test-WSMan -ComputerName $phyHost -ErrorAction:Ignore if (-not $output) { Write-Host "Host $($phyHost) not responding..." continue } $localHostTrace = Invoke-Command -ComputerName $phyHost -Credential $domainAdminCredential -ScriptBlock { $vmTrace = @() $thrownExceptions = @() # get the infra VMs in this physical node $retries = 5 for($retry = 0; $retry -lt $retries; ++$retry) { try { # Unfiltered calls to Get-VM may fail if a VM is being deleted from that host $localVMs = (Get-VM | ? Name -In $using:VMNames).Name break } catch { if ($retry -eq $retries - 1) { throw } } } if($localVMs) { Trace-Execution "Executing on local VMs $($localVMs -join ',')" try { # change PS language mode to FULL to allow script block invocation if (!$using:ArgumentList) { $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ScriptBlock ([ScriptBlock]::Create($("Import-Module OpenUpSession; Set-FullLanguage; `r`n" + $using:ScriptBlock))) } # check if the scriptblock declares Param because special handling is needed elseif ($using:ScriptBlock -match "param[ ]*\(|param[ ]*\r\n") { $sb = {param($ScriptBlock, [Object[]]$ArgumentList) Import-Module OpenUpSession; Set-FullLanguage; [scriptBlock]::Create($ScriptBlock).invoke($ArgumentList)} $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ScriptBlock $sb -ArgumentList $using:ScriptBlock, $using:ArgumentList } else { $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ArgumentList $using:ArgumentList -ScriptBlock ([ScriptBlock]::Create($("Import-Module OpenUpSession; Set-FullLanguage; `r`n" + $using:ScriptBlock))) } } catch { Write-Host "Local exception: $($_)" $thrownExceptions += $_ } } if ($vmTrace) { Write-Host "VM(s) returned: $vmTrace" } if ($thrownExceptions.Count -gt 0) { throw $thrownExceptions } return $vmTrace } if ($localHostTrace) { Trace-Execution "Local Host Output: $($localHostTrace)" } $hostTrace += $localHostTrace } return $hostTrace } function ReliableGetService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $sbGetService = [ScriptBlock]::Create("Get-Service -Name $($ServiceName)") Trace-Execution "Retrieving service $ServiceName" try { if ($PSCmdlet.ParameterSetName -eq "Local") { $service = Invoke-Command -ScriptBlock $sbGetService } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $service = Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbGetService } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $service = Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbGetService } } catch { Trace-Error "Service $ServiceName throw $($_)" throw } if ($service) { Trace-Execution "For service $($ServiceName), current status: $($service.Status); start type: $($service.StartType)" } return $service } function ReliableSetService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [ValidateSet("Boot","System","Automatic","Manual","Disabled")] [String] $StartupType, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [ValidateSet("Running","Stopped","Pause")] [String] $Status, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int32] $MaxRetries = 10, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int] $Delay = 10, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $iteration = 0; do { Trace-Execution "$StartupType service $ServiceName" if ($Status -eq "") { $sbActionService = [ScriptBlock]::Create("Set-Service -Name $($ServiceName) -StartupType $($StartupType)") } else { $sbActionService = [ScriptBlock]::Create("Set-Service -Name $($ServiceName) -StartupType $($StartupType) -Status $($Status)") } try { if ($PSCmdlet.ParameterSetName -eq "Local") { Invoke-Command -ScriptBlock $sbActionService } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbActionService } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbActionService } } catch { Trace-Error "Trying to set Service $ServiceName to $($StartupType) throw $($_)" throw } # check service status if ($PSCmdlet.ParameterSetName -eq "Local") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -Local } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters } if ($Status -eq "") { if ($serviceVerify.StartType -eq $StartupType) { Trace-Execution "Service $($ServiceName) is $($StartupType)" break } } else { if (($serviceVerify.StartType -eq $StartupType) -and ($serviceVerify.Status -eq $Status)) { Trace-Execution "Service $($ServiceName) is $($StartupType) and $($Status)" break } } $iteration++ if ($iteration -eq $MaxRetries) { if ($Status -eq "") { Trace-Error "Couldn't change the Type to $StartupType" } else { Trace-Error "Couldn't change the Type to $StartupType and $Status" } throw } Start-Sleep $Delay Trace-Execution "Retrying, iteration $iteration" } while ($true); } function ReliableActionService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [ValidateSet("Start","Stop", "Restart")] [String] $Action, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Boolean] $Force, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int32] $MaxRetries = 10, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int] $Delay = 10, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop switch ($Action) { "Start" { $verifyAction = "Running" } "Stop" { $verifyAction = "Stopped" } "Restart" { $verifyAction = "Running" } } $iteration = 0; do { Trace-Execution "$Action service $ServiceName" $psToExecute = $Action+"-Service -Name "+$ServiceName if ($Force -eq $true) { $psToExecute += " -Force" } $sbActionService = [ScriptBlock]::Create($psToExecute) # check the service before attempting to control it if ($PSCmdlet.ParameterSetName -eq "Local") { $service = ReliableGetService -ServiceName $ServiceName -Local } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters } # validate the state if ($service.Status -eq $verifyAction) { Trace-Execution "Service $($ServiceName) is already $($service.Status)" return } $service.Status if ($service.Status -imatch "Pending") { Trace-Execution "Service $($ServiceName) is in pending state: $($service.Status). Will retry." } else { try { #Perform the action if ($PSCmdlet.ParameterSetName -eq "Local") { $service = Invoke-Command -ScriptBlock $sbActionService } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $service = Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbActionService } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $service = Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbActionService } } catch { Trace-Error "Trying to $Action $ServiceName throw $($_)" throw } # check service status if ($PSCmdlet.ParameterSetName -eq "Local") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -Local } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $serviceVerify = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters } # validate the state if ($serviceVerify.Status -eq $verifyAction) { Trace-Execution "Service $($ServiceName) is in desired state: $($verifyAction)" break } } $iteration++ if ($iteration -eq $MaxRetries) { Trace-Error "Couldn't $Action the service" throw } Start-Sleep $Delay Trace-Execution "Retrying, iteration $iteration" } while ($true); } function ReliableStopService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Boolean] $Force = $false, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Boolean] $DisableService = $false, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int32] $MaxRetries = 10, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int] $Delay = 10, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) if ($PSCmdlet.ParameterSetName -eq "Local") { $service = ReliableGetService -ServiceName $ServiceName -Local } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters } # check if we need to disable the service if (($DisableService -eq $true) -and ($service.StartType -ne "Disabled")) { if ($PSCmdlet.ParameterSetName -eq "Local") { ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -Local -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay } } # ReliableActionService will handle if the service is already in the requested state. if ($PSCmdlet.ParameterSetName -eq "Local") { ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -Local -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay } } function ReliableRestartService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Boolean] $Force = $false, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int32] $MaxRetries = 10, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int] $Delay = 10, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) if ($PSCmdlet.ParameterSetName -eq "Local") { ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -Local -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay } } function ReliableStartService{ param ( [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Parameter(Mandatory = $true)] [String] $ServiceName, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Boolean] $AutomaticService = $false, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int32] $MaxRetries = 10, [Parameter(ParameterSetName="Local")] [Parameter(ParameterSetName="Remote")] [Parameter(ParameterSetName="VMRemote")] [Int] $Delay = 10, [Parameter(ParameterSetName="Local")] [Switch] $Local, [Parameter(ParameterSetName="Remote")] [String[]]$Hostnames, [Parameter(ParameterSetName="Remote")] [PSCredential]$Credential, [Parameter(ParameterSetName="VMRemote")] [String[]]$VMNames, [Parameter(ParameterSetName="VMRemote")] [PSCredential]$VMCredential, [Parameter(ParameterSetName="VMRemote")] [CloudEngine.Configurations.EceInterfaceParameters]$Parameters ) if ($PSCmdlet.ParameterSetName -eq "Local") { $service = ReliableGetService -ServiceName $ServiceName -Local } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters } # check if we need to change to Automatic service if (($AutomaticService -eq $true) -and ($service.StartType -ne "Automatic")) { if ($PSCmdlet.ParameterSetName -eq "Local") { ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -Local -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay } } # ReliableActionService will handle if the service is already in the requested state. if ($PSCmdlet.ParameterSetName -eq "Local") { ReliableActionService -ServiceName $ServiceName -Action "Start" -Local -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "Remote") { ReliableActionService -ServiceName $ServiceName -Action "Start" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") { ReliableActionService -ServiceName $ServiceName -Action "Start" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay } } function Invoke-ScriptBlockWithRetries { [CmdletBinding(DefaultParameterSetName="All")] param( [ScriptBlock] [parameter(Mandatory = $true)] $ScriptBlock, [Int32] [parameter(Mandatory = $false)] $RetryTimes = 10, [Int32] [parameter(Mandatory = $false)] $RetrySleepTimeInSeconds = 10, [Switch] [parameter(Mandatory = $false)] $InDisconnectedSession=$false, [string] [parameter(Mandatory = $false, ParameterSetName="ByComputerName")] $ComputerName = $null, [PSCredential] [parameter(Mandatory = $false)] $Credential = $null, [parameter(Mandatory = $false, ParameterSetName="BySession")] $Session = $null, [parameter(Mandatory = $false)] $ArgumentList = $null ) $ErrorActionPreference = "Stop" for($retry = 0; $retry -lt $RetryTimes; ++$retry) { try { Write-Verbose "Invoke-ScriptBlockWithRetries: retry #$retry of $RetryTimes, retry sleep time is $RetrySleepTimeInSeconds seconds." -Verbose $invokeParams = @{ } $noNewScope = $true if ($ComputerName) { $invokeParams.ComputerName = $ComputerName $noNewScope = $false } elseif ($Session) { $invokeParams.Session = $Session $noNewScope = $false } if ($Credential) { $invokeParams.Credential = $Credential $noNewScope = $false } if ($noNewScope) { $invokeParams.NoNewScope = $true } if ($InDisconnectedSession) { $invokeParams.InDisconnectedSession = $true } if ($ArgumentList) { $invokeParams.ArgumentList = $ArgumentList } $invokeParams.ScriptBlock = $ScriptBlock return Invoke-Command @invokeParams } catch { Write-Verbose "Invoke-ScriptBlockWithRetries failed with exception: $_" -Verbose if ($retry -lt ($RetryTimes - 1)) { Start-Sleep -Seconds $RetrySleepTimeInSeconds } else { throw } } } } function Invoke-ScriptBlockInParallel { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string[]] $ComputerNames, [Parameter(Mandatory=$false)] [PSCredential] $Credential, [ScriptBlock] [parameter(Mandatory = $true)] [ValidateScript({$_ -NotLike '*$Using:*'})] $ScriptBlock, [Object[]] [parameter(Mandatory = $false)] $ArgumentList = $null, [Int32] [parameter(Mandatory = $false)] $RetryTimes = 2, [Int32] [parameter(Mandatory = $false)] $RetrySleepTimeInSeconds = 5, [Parameter(Mandatory = $false)] [string] $Authorization = 'Default' ) $ErrorActionPreference = 'Stop' $computerNames = $ComputerNames $jobs = @() $sessions =@() $errorMessages = @() # Just retry for kicking off the ScriptBlock in Parallel for ($retry = 0; $retry -lt $RetryTimes; ++$retry) { $retryComputers = @() $session = $null $job = $null Trace-Execution "Invoke-ScriptBlockInParallel: retry #$retry of $RetryTimes, retry sleep time is $RetrySleepTimeInSeconds seconds, machines are $computerNames." foreach ($computerName in $computerNames) { try { if($Credential) { $session = New-PSSession -ComputerName $computerName -Credential $Credential -Authentication $Authorization } else { $session = New-PSSession -ComputerName $computerName } Initialize-ECESession -Session $session $job = Invoke-Command -ErrorAction Stop -Session $session -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -AsJob $sessions += $session $jobs += $job } catch { $retryComputers += $computerName $errorMessage = "Machine $computerName failed with error: $($_.Exception.Message)" $errorMessages += $errorMessage if ($session -ne $null) { $session | Remove-PSSession -ErrorAction Ignore } if ($job -ne $null) { $job | Remove-Job –ErrorAction Ignore } } } if (($retryComputers.Count -eq 0) -or ($retry -ge ($RetryTimes -1))) { Trace-Execution "Invoke-ScriptBlockInParallel exits from retries for kicking off jobs in $retry retries" break } $computerNames = $retryComputers Start-Sleep -Second $RetrySleepTimeInSeconds $errorMessages = @() } # Waiting for all jobs to finish try { if ($jobs.Count -eq 0) { $errorMessage = "Invoke-ScriptBlockInParallel failed: Script could not be invoked after $RetryTimes retries" $errorMessages += $errorMessage throw "Invoke-ScriptBlockInParallel failed: $errorMessages" } Trace-Execution "Invoke-ScriptBlockInParallel: Waiting for jobs" $jobs | Wait-Job | Receive-Job Trace-Execution "Invoke-ScriptBlockInParallel: finished jobs" $failedJobs = $jobs | ? { $_.JobStateInfo.State -ne "Completed" } foreach ($failedJob in $failedJobs) { $newErrorMessage = $null try { $newErrorMessage = $failedJob.ChildJobs[0].JobStateInfo.Reason.Message } catch { } $errorMessages += "Machine $($failedJob.Location): Failed with error: $newErrorMessage" } if ($errorMessages.Count -gt 0) { throw "Invoke-ScriptBlockInParallel failed: $errorMessages" } } finally { $sessions | Remove-PSSession -ErrorAction Ignore $jobs | Remove-Job –ErrorAction Ignore } } # Deprecated as iDNS is no longer used function GetiDNSServersSettings { Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) [string[]] $iDnsServersIPAddress = @() $iDnsServersIPAddress } <# .SYNOPSIS Create a folder on management file share for host update .DESCRIPTION This function creates a folder on management file share for host update .PARAMETER Parameters This object is based on the customer manifest. It contains the private information and public information of cloud role. This Parameters object is passed down by the deployment engine. #> function Get-HostUpdateShare { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Trace-ECEScript "Getting the host update share." { $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration $clusterName = Get-ManagementClusterName $Parameters $managementLibraryPath = Get-SharePath $Parameters $physicalMachinesRole.PublicInfo.WindowsUpdateStagingFolder.Path $clusterName $managementLibraryPath = Split-Path -Path $managementLibraryPath $hostUpdateLogShare = Join-Path $managementLibraryPath 'Logs\HostUpdate' if(-not (Test-Path $hostUpdateLogShare)) { Trace-Execution " Create a folder on management file share $hostUpdateLogShare " try { New-Item $hostUpdateLogShare -ItemType Directory -Force | Out-Null } catch { throw "Failed to create folder on management file share $hostUpdateLogShare. Error $_" } } } return $hostUpdateLogShare } # Returns the list of orchestrators that are currently available. function Get-CurrentOrchestrators { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) # Check if the code is running on DVM if($Parameters.Roles["DeploymentMachine"]) { $dvmName = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node | % Name if($env:ComputerName -eq $dvmName) { $dvmIpAddress = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node | % IPv4Address $orchestrator = New-Object -TypeName psobject -Property @{ Name = $dvmName; IPAddress = $dvmIpAddress } return @($orchestrator) } } else { # Check if we only have one host. In that case the code should be running on the same host or the DVM (if performing bare metal POC) $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration $allPhysicalHosts = @($physicalMachinesRole.Nodes.Node | % Name) if( ($allPhysicalHosts.Count -eq 1) -and ($allPhysicalHosts -eq $env:ComputerName) ) { $orchestrator = New-Object -TypeName psobject -Property @{ Name = $env:ComputerName } return @( $orchestrator) } } # If the orchestrator is not the DVM or host, then return ERCS VM information $orchestrators = @() $virtualMachineNodes = $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes $ercsNodes = $virtualMachineNodes.Node | where { $_.Role -eq "SeedRingServices"} foreach($ercsNode in $ercsNodes) { $name = $ercsNode.Name $ercsIpAddress = ($ercsNode.NICs.NIC.IPv4Address -split "/")[0] $orchestrator = New-Object -TypeName psobject -Property @{ Name = $name; IPAddress = $ercsIpAddress} $orchestrators += $orchestrator } return @($orchestrators) } # Clean up the AD computer object and its DNS record, if it exists function Remove-ComputerAndDnsRecord { [CmdletBinding()] param ( [string]$Name, [string]$DomainFQDN, [string]$DNSServerName, [PSCredential]$Credential = $null ) if (!$DNSServerName) { $DNSServerName = (Get-ADDomainController -Discover -DomainName $DomainFQDN).Name } Trace-Execution "Clean up the AD computer object $Name and its DNS record" $computer = Get-ADComputer -Filter {Name -eq $Name} if ($computer) { if (Get-ADObject -SearchBase $computer -SearchScope OneLevel -filter *) { Remove-ADObject -Identity $computer -Recursive -Confirm:$false } else { $computer | Remove-ADComputer -Confirm:$false } } Remove-DnsRecord -DnsZoneName $DomainFQDN -DnsName $Name -DnsServerName $DNSServerName -IPv4Address $true -IPv6Address $true -Credential $Credential } #return an list of node which can be connected by PSsession function Get-AvailableNode { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string[]]$NodeNames, [Parameter(Mandatory=$true)] [PSCredential]$Credential, [Parameter(Mandatory=$false)] [ScriptBlock]$ScriptBlock, #script to validate the current node is available or not with additional check. return $true if it is available. [Switch]$Credssp #Whether to use Credssp Authentication for session connected to nodes. ) $ErrorActionPreference = 'Stop' $reachableNodes = @() foreach($NodeName in $NodeNames) { try { if($Credssp.IsPresent) { $AdditionalParam = @{"Authentication" = "CredSsp"} } else { $AdditionalParam = @{} } $session = New-PSSession -ComputerName $nodeName -Credential $Credential @AdditionalParam if($session -eq $null) { throw "Cannot create session into computer $nodename" } $result = $true if($ScriptBlock -ne $null) { Invoke-Command -Session $session -ScriptBlock {$ErrorActionPreference = 'Stop'} $result = $false $result = Invoke-Command -Session $session -ScriptBlock $ScriptBlock } if($result) { $reachableNodes += $nodename } } catch { Trace-Execution "Node $nodename availability validation failed with exception $_" } finally { $session | Remove-PSSession -Confirm:$false -ErrorAction Ignore } } return $reachableNodes } #Test whether a role can maintain its availability #It checks a role's available nodes is able to maintain its functionality #ExcludedNodeNames is the node name for physical machine/the host name for virtual machine. These nodes will be taken as not available function Test-RoleNodeGroupAvailability { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory=$true)] [string]$RoleName, [Parameter(Mandatory=$true)] [PSCredential]$Credential, [Parameter(Mandatory=$false)] [ScriptBlock]$ScriptBlock, #script to validate the current node is available or not with additional check. return $true if it is available. [Switch]$Credssp,#Whether to use Credssp Authentication for session connected to nodes. [Parameter(Mandatory=$false)] [string[]]$ExcludedNodeNames, [Switch] $IsHost, [Switch] $SingleNodeQuorum ) Trace-Execution "Verifying node ring for role $RoleName" [Array]$Nodes = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node $Nodes += $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes.Node [Array]$RoleNodes = $Nodes | where { ($_.Role -eq $RoleName) } [Array]$RoleProvisionedNodes = $RoleNodes | where { ($_.ProvisioningStatus -eq "Complete")} if($IsHost.IsPresent -eq $false) { $ExcludedNodeNames = $($RoleNodes | where {(-not [String]::IsNullOrEmpty($_.RefNodeId)) -and ($ExcludedNodeNames -contains $_.RefNodeId) }) |%{$_.Name} } Trace-Execution "Exclude node $($ExcludedNodeNames -join ',')" [Array]$ToVerifiedNodeNames = $($RoleProvisionedNodes.Name | where {$ExcludedNodeNames -notcontains $_}) Trace-Execution "Verifying node connectivity $($ToVerifiedNodeNames -join ',')" if($ToVerifiedNodeNames -and $ToVerifiedNodeNames.Count -ne 0) { $reachableNodes = Get-AvailableNode -NodeNames $ToVerifiedNodeNames -Credential $Credential -ScriptBlock $ScriptBlock -Credssp:($Credssp.IsPresent) } else { $reachableNodes = @() } Trace-Execution "Nodes $($reachableNodes -join ',') are reachable" if($reachableNodes -and $SingleNodeQuorum.IsPresent -and $reachableNodes.Count -gt 0) { Trace-Execution "After reducing capacity, there still is at least one node running." return } if($reachableNodes -and (-not $SingleNodeQuorum.IsPresent) -and ($reachableNodes.Count * 2 -gt $RoleNodes.Count)) { Trace-Execution "Reducing capacity will maintain a majority of role nodes running." return } throw "Reducing capacity with fewer than or equal to half of initially deployed nodes is not supported for role $RoleName" } function Get-CredentialFromStore { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] $SecurityInfoUsers, [Parameter(Mandatory = $false)] [String] $AccountType, [Parameter(Mandatory = $false)] [String] $DomainName ) $VerbosePreference="Continue" $ErrorActionPreference = "stop" if($AccountType -ne $null) { $AccountUser = $SecurityInfoUsers | ? Role -EQ $AccountType if($AccountUser -ne $null) { $Credential = $Parameters.GetCredential($AccountUser.Credential) if(-not $("$($Credential.UserName)" -like "*\*") -and [String]::IsNullOrEmpty($DomainName) -eq $false) { $Credential = New-Credential -UserName "$domainName\$($Credential.UserName)" -Password $Credential.GetNetworkCredential().Password } } } Write-Output $Credential } # Note: This is a temporary implementation to unblock Image Based update function Update-JEAEndpointsForUpdate { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) # Update PXE endpoint - Remove this once PXE VM has an updated endpoint. # PXE Role Trace-Execution "Start - Update module needed by PXEEndpoint on PXE VM." $pxeRole = $Parameters.Roles["PXE"].PublicConfiguration $pxeServerName = $pxeRole.Nodes.Node.Name # PhysicalMachine Role $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration $allPhysicalHosts = $physicalMachinesRole.Nodes.Node | % Name # Skipping nodes which are provided in runtime parameters $allPhysicalHosts = Skip-NodesProvidedInRuntimeParameter -Parameters $Parameters -Nodes $allPhysicalHosts # VirtualMachine Role $virtualMachinesRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration $clusterName = Get-ManagementClusterName $Parameters $libraryShareNugetStorePath = Get-SharePath $Parameters $virtualMachinesRole.PublicInfo.LibraryShareNugetStoreFolder.Path $clusterName $tempNugetDir = Join-Path $env:temp 'AzureStackPXE.Deployment' Trace-Execution "Expanding Nuget Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment at $tempNugetDir" Expand-NugetContent -NugetName 'Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment' ` -NugetStorePath $libraryShareNugetStorePath ` -SourcePath 'content\Scripts' ` -DestinationPath $tempNugetDir $sourcePath = "$tempNugetDir\*" $destinationPath = "\\$pxeServerName\c$\Program Files\WindowsPowerShell\Modules\Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment\Scripts" Trace-Execution "Copying $sourcePath to $destinationPath" New-Item -Path $destinationPath -ItemType Directory -Force -ErrorAction SilentlyContinue Copy-Item -Path $sourcePath -Destination $destinationPath -Force Remove-Item -Force -Path $tempNugetDir -Recurse Trace-Execution "Completed - Update module needed by PXEEndpoint on PXE VM." Trace-Execution "Start - Update module needed by NewManagementVMEndpoint on all the hosts" $tempNugetDir = Join-Path $env:temp 'NodeUtils_NewManagementVM' Expand-NugetContent -NugetName 'Microsoft.AzureStack.Solution.Deploy.Common.NodeUtils' ` -NugetStorePath $libraryShareNugetStorePath ` -SourcePath 'content\NewManagementVM' ` -DestinationPath $tempNugetDir foreach ($nodeName in $allPhysicalHosts) { $sourcePath = "$tempNugetDir\*" $destinationPath = "\\$nodeName\c$\Program Files\WindowsPowerShell\Modules\NewManagementVM" Trace-Execution "Copying $sourcePath to $destinationPath" New-Item -Path $destinationPath -ItemType Directory -Force -ErrorAction SilentlyContinue Copy-Item -Path $sourcePath -Destination $destinationPath -Force } Remove-Item -Force -Path $tempNugetDir -Recurse Trace-Execution "Completed - Update module needed by NewManagementVMEndpoint on all the hosts." } # Returns credentials needed for various operations during deployment and update. function Get-DomainCredential { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $domainAdminUser = $securityInfo.DomainUsers.User | ? Role -EQ "DomainAdmin" $domainAdminCredential = $Parameters.GetCredential($domainAdminUser.Credential) $domainAdminName = $domainAdminCredential.GetNetworkCredential().UserName $domainAdminPassword = $domainAdminCredential.GetNetworkCredential().Password $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN $domainCredential = New-Credential "$domainFqdn\$domainAdminName" $domainAdminPassword return $domainCredential } # Returns Temporary Domain Admin credentials. function Get-TemporaryDomainCredential { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainAdminCredential = Get-UserAccountCredential -Parameters $Parameters return $domainAdminCredential } function Get-UserAccountCredential { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory=$false)] [string] $UserRole = "DomainAdmin" ) $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $UserAccount = $securityInfo.DomainUsers.User | ? Role -EQ $UserRole $UserAccountCredential = $Parameters.GetCredential($UserAccount.Credential) $domainName = $domainRole.PublicInfo.DomainConfiguration.DomainName if(-not $("$($UserAccountCredential.UserName)" -like "*\*") -and [String]::IsNullOrEmpty($DomainName) -eq $false) { $UserAccountCredential = New-Credential -UserName "$domainName\$($UserAccountCredential.UserName)" -Password $UserAccountCredential.GetNetworkCredential().Password } return $UserAccountCredential } # Returns the computer name of the primary DC. function Get-PrimaryDomainController { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $isOneNode = IsOneNode -Parameters $Parameters $restoreContext = Get-RestoreParameters -Parameters $Parameters $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters $domainAdminCredential = Get-DomainCredential -Parameters $Parameters if ($isOneNode -and $restoreContext.RestoreInprogress) { $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential } else { $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential } return $primaryDomainController } # This function retrieves the DNS Servers. function Get-DNSServers { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $domainRole = $Parameters.Roles["Domain"].PublicConfiguration $virtualMachinesRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration $domainControllerName = @($domainRole.Nodes.Node.Name) $domainControllers = $virtualMachinesRole.Nodes.Node | ? Name -in $domainControllerName $dnsServers = @() foreach($nic in $domainControllers.NICs.NIC) { # IP address is in IP/netmask format, get just the IP address $dnsServers += $nic.Ipv4Address.Split('/')[0] } return $dnsServers } # If $Parameters is passed then retry logic will kick in for the exception case # This is done to handle SF app failover after we fetch the endpoints from the primary # and before we are done with the SSL binding validation # the window is very-very small so chances of this are very rare, but do not want to take any chance and handling it function ValidateCertBinding { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string] $SSLHost, [Parameter(Mandatory=$true)] [int] $Port, [Parameter(Mandatory=$true)] [string] $ExpectedCertThumbprint, [Parameter(Mandatory=$false)] [string] $VMName, [Parameter(Mandatory = $false)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $false)] $retryIteration ) $ErrorActionPreference = 'Stop' # As the path of the dll gets too long if we use it from CloudDeployment\Managed folder, copying it to PS Modules folder until ECE fixes the issue $secretManagementBinPathRoot = "$env:systemdrive\Program Files\WindowsPowerShell\Modules\Microsoft.Azurestack.SecretRotation" Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.Utilities.dll" -ErrorAction Stop -Verbose:$false | Out-Null Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.Tracing.dll" -ErrorAction Stop -Verbose:$false | Out-Null Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.Contract.dll" -ErrorAction Stop -Verbose:$false | Out-Null Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.dll" -ErrorAction Stop -Verbose:$false | Out-Null $factory = New-Object Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.SSLBoundCertificateFactory $sslBoundCertificateWrapper = New-Object Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.SSLBoundCertificateWrapper if ([string]::IsNullOrEmpty($VMName)) { $SSLBoundCertClass = $factory.CreateSSLBoundCertificate($SSLHost, $Port) } else { $SSLBoundCertClass = $factory.CreateSSLBoundCertificate($SSLHost, $VMName, $Port) } Throw-IfNullOrEmpty -DataToCheck $SSLBoundCertClass -ErrorMessage "Failed to create the instance of SSLBoundCert class" $sslBoundCertificate = $sslBoundCertificateWrapper.GetCertBoundToEndpointAsync($SSLBoundCertClass).GetAwaiter().GetResult() $certThumbprintBoundToTheEndpoint = $sslBoundCertificate.GetCertHashString() Throw-IfNullOrEmpty -DataToCheck $certThumbprintBoundToTheEndpoint -ErrorMessage "Thumbprint for the certificate bound to the endpoint $SSLHost is null" Trace-Execution "Found the thumbprint for the certificate bound to the endpoint: $SSLHost it is: $certThumbprintBoundToTheEndpoint" #Compare both the thumbprints if($ExpectedCertThumbprint -ieq $certThumbprintBoundToTheEndpoint) { Trace-Execution "Thumbprint bound to the endpoint matched with the one expected from pfx file at rotation location. The thumbprint is: $certThumbprintBoundToTheEndpoint " } else { $RetryTimes = 4 if($Parameters -and $retryIteration -and $retryIteration -lt $RetryTimes) { # In case SF app failed over after we fetch the endpoints and before validation is done, retrying so that we do not incorrectly report failure # Retrying 3 times assumng that if all 3 times failover is the reason then anyway something is broken and we will give up $retryIteration++ Validate-InternalSSLCertBinding -Parameters $Parameters -retryIteration $retryIteration Trace-Execution "Retry attempt # $retryIteration " } else { throw "The certificate at rotation location is not bound to the endpoint. The one bound to the endpoint has the thumbprint: $certThumbprintBoundToTheEndpoint while the one at rotation location has thumbprint: $ExpectedCertThumbprint" } } } #This function handles the validation of SSL certificate bindings with the endpoints for: #1. External SSL Certificates with SF apps #2. External SSL Certificates with highly available endpoints #3. Internal SSL Certificates with static VIPs and have mapping info in VIPs section of parent Role.xml and certificates info in Role.xml - KeyVault apps #4. Internal SSL Certificates with static VIPs and have mapping info and certificate info both in VIPs section of Role.xml. These have NO separate renewal path - WAS, WASP #5. Internal SSL Certificates with static VIPs and have mapping info in Mapping section of VIPs of Parent Role.xml - Some Keyvault apps function Validate-SSLCertificateBinding { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Switch] $Internal, [Switch] $ValidateCurrentCertLocation ) $roleDefinition = $Parameters.Configuration.Role $RoleId = $Parameters.Configuration.Role.Id $ServiceType = $Parameters.Configuration.Role.ServiceType $certRoleDefinition = $Parameters.Roles["CertificateManagement"].PublicConfiguration $secureCertPwd = Get-CACertPassword -Parameters $Parameters -ErrorAction Stop -Verbose Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Failed to retrieve the password for the certificate." # Get external cert information $externalCertConfig = $certRoleDefinition.PublicInfo.ExternalCertConfigurations $externalCertsInfoList = $externalCertConfig.Certificates.Certificate # Get $certsForThisRole for internal SSL certs for given role if ($Internal) { if ($roleDefinition.ServiceType -eq "IIS") { $VIPForThisRole = $roleDefinition.PublicInfo.VIPs.VIP | Where-object {$_.NetworkId -eq "InternalVip"} $certsForThisRole = $VIPForThisRole.PortMapping.Mapping[0].PfxFilePath } else { $certsForThisRole = $roleDefinition.PublicInfo.Certificates.Certificate | Where-object {$_.Type -eq "SSL"} } } else { $certsForThisRole = $externalCertsInfoList.Consumers.Consumer | Where-Object { if ($_.ParentRoleId -eq $null) {$_.RoleId -eq $RoleId} else {$_.ParentRoleId -eq $RoleId} } } Throw-IfNullOrEmpty -DataToCheck $certsForThisRole -ErrorMessage "Failed to get the list of certificates for the role $RoleId" $count = $certsForThisRole.Count Trace-Execution "Got certs for this role: $RoleId Count: $count" $clusterName = Get-ManagementClusterName $Parameters Trace-Execution "Got clusterName $clusterName" [string[]] $roleVMs = $roleDefinition.Nodes.Node.Name Throw-IfNullOrEmpty -DataToCheck $roleVMs -ErrorMessage "Failed to get the VMs info for the role $RoleId" $vmCount = $roleVMs.Count Trace-Execution "Found $vmCount VMs for Role: $RoleId" $internalDomainFQDN = Get-InternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose Throw-IfNullOrEmpty -DataToCheck $internalDomainFQDN -ErrorMessage "Failed to get Internal Domain FQDN from the role for the role: $RoleId." $externalDomainFQDN = Get-ExternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose Throw-IfNullOrEmpty -DataToCheck $externalDomainFQDN -ErrorMessage "Failed to get External Domain FQDN from the role for the role: $RoleId." foreach ($consumer in $certsForThisRole) { if ($Internal) { if ($roleDefinition.ServiceType -eq "IIS") { $renewalFileLocation = $consumer } else { if ($ValidateCurrentCertLocation) { $renewalFileLocation = $consumer.CertFile } else { $renewalFileLocation = $consumer.CertFileRenewal } } } else { if ($ValidateCurrentCertLocation) { $renewalFileLocation = $consumer.Location } else { $renewalFileLocation = $consumer.RotationLocation } } $certLocation = Get-SharePath $Parameters $renewalFileLocation $clusterName if(-not $(Test-Path -Path $certLocation)) { throw "Could not find the certificate at this location: $certLocation for the role: $RoleId " } else { Trace-Execution "Got certLocation $certLocation" $pfxData = Get-PfxData -FilePath $certLocation -Password $secureCertPwd Throw-IfNullOrEmpty -DataToCheck $pfxData -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation" $thumbprintFromRotationPfx = $pfxData.EndEntityCertificates.Thumbprint Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx " Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId" #Get thumbprint for the cert bound to the endpoint if (!($roleDefinition.ServiceType -eq "IIS") -or !$internal) { $vipIdForTheEndpoint = $consumer.VIPId Throw-IfNullOrEmpty -DataToCheck $vipIdForTheEndpoint -ErrorMessage "Failed to get VIP ID for the endpoint." Trace-Execution "Retrieved the VIP Id for the endpoint: $vipIdForTheEndpoint." } if ($Internal) { if ($roleDefinition.ServiceType -eq "IIS") { $vipForEndpointFromRole = $VIPForThisRole } else { $parentRoleID = $consumer.ParentRoleId $parentRoleDefinition = $Parameters.Roles[$parentRoleID].PublicConfiguration $vipForEndpointFromRole = $parentRoleDefinition.PublicInfo.VIPs.VIP | Where-Object { $_.Id -eq $vipIdForTheEndpoint } } } else { $vipForEndpointFromRole = $roleDefinition.PublicInfo.VIPs.VIP | Where-Object { $_.Id -eq $vipIdForTheEndpoint } } if($consumer.MappingId) { $mappingId = $consumer.MappingId Trace-Execution "It has a MappingId: $mappingId" $mapping = $vipForEndpointFromRole.PortMapping.Mapping | Where-Object { $_.Name -eq $mappingId } $dnsEndpointFromRole = $mapping.DnsEndpoints.Endpoint $mappingsFromRole = $mapping } else { $dnsEndpointFromRole = $vipForEndpointFromRole.DnsEndpoints.Endpoint | Where-Object { $_.IsInternal -eq $null } # Some roles have different mapping info which have no certs associated. Filtering those out $mappingsFromRole = $vipForEndpointFromRole.PortMapping.Mapping | Where-Object { $_.SkipCertValidation -eq $null } } $endpointFromRole = $dnsEndpointFromRole.Path Throw-IfNullOrEmpty -DataToCheck $endpointFromRole -ErrorMessage "Failed to get endpoint for the role: $RoleId." Trace-Execution "Retrieved the endpoint: $endpointFromRole." Throw-IfNullOrEmpty -DataToCheck $mappingsFromRole -ErrorMessage "Failed to get certificate mapping info for the role: $RoleId and endpoint: $endpointFromRole." $endpoint = $endpointFromRole.Replace('*', 'a') if ($Internal) { $DomainFQDN = $internalDomainFQDN } else { $DomainFQDN = $externalDomainFQDN } $sslHost = $endpoint + "." + $DomainFQDN Trace-Execution "This is the SSL Host name: $sslHost for the endpoint: $endpoint." foreach ($mapping in $mappingsFromRole) { if ($serviceType -eq "SF") { # For dynamic VIPs we will go through load balancer and the port applicable is FrontEndPort $port = $mapping.FrontEndPort Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get port info for the endpoint: $endpointFromRole" Trace-Execution "Validating for SSLHost: $sslHost and port: $port for the endpoint: $endpoint." ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx } else { # To validate on each VM, the port applicable is BackEndPort $port = $mapping.BackEndPort Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get port info for the endpoint: $endpointFromRole" if ($Internal -and $roleDefinition.ServiceType -eq "IIS") { $renewalCertFileLocation = $mapping.PfxFilePath $certLocation = Get-SharePath $Parameters $renewalCertFileLocation $clusterName if(-not $(Test-Path -Path $certLocation)) { throw "Could not find the certificate at this location: $certLocation for the role: $RoleId." } else { Trace-Execution "Got certLocation $certLocation" $pfxDataForInternalIISCert = Get-PfxData -FilePath $certLocation -Password $secureCertPwd Throw-IfNullOrEmpty -DataToCheck $pfxDataForInternalIISCert -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation" $thumbprintFromRotationPfx = $pfxDataForInternalIISCert.EndEntityCertificates.Thumbprint Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx " Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId" } } if (($roleDefinition.ServiceType -eq "IIS") -or ($roleDefinition.ServiceType -eq "Standalone")) { # For highly available endpoints, validating binding on every node foreach ($vm in $roleVMs) { Trace-Execution "Validating for SSLHost: $sslHost, VM: $vm and port: $port for the endpoint: $endpoint." ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx -VMName $vm } } else { # For SF stateless apps we were validating binding on every node, but now changing it due to another change to limit SF app instances to three Trace-Execution "Validating for SSLHost: $sslHost and port: $port for the endpoint: $endpoint." ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx } } } } } } # Update Windows DISM features based on the configuration of Role.xml # Including installing features and removiing unused features function Update-WindowsDismFeature { [CmdletBinding()] Param ( [Parameter(Mandatory = $false)] [object] $WindowsFeature, [Parameter(Mandatory = $false)] [Switch] $Online, [Parameter(Mandatory = $false)] [string] $Path, [Parameter(Mandatory = $false)] [string] $WinSxSPath = "$env:windir\WinSxS" ) Trace-Execution "Update Windows DISM features" if ($null -ne $WindowsFeature) { if ($Online) { $dismFeatures = (Get-WindowsOptionalFeature -Online).FeatureName } else { $dismFeatures = (Get-WindowsOptionalFeature -Path $Path).FeatureName } if ($null -ne $WindowsFeature.Feature.Name) { $featuresToInstall = $dismFeatures | Where-Object { $_ -in $WindowsFeature.Feature.Name } if ($null -ne $featuresToInstall -and $featuresToInstall.Count -gt 0) { Trace-Execution "Install Windows DISM features: $featuresToInstall" if ($Online) { Enable-WindowsOptionalFeature -FeatureName $featuresToInstall -Online -All -Source $WinSxSPath -NoRestart } else { Enable-WindowsOptionalFeature -FeatureName $featuresToInstall -Path $path -All } } } if ($null -ne $WindowsFeature.RemoveFeature.Name) { $featuresToRemove = $dismFeatures | Where-Object { $_ -in $WindowsFeature.RemoveFeature.Name } if ($null -ne $featuresToRemove -and $featuresToRemove.Count -gt 0) { Trace-Execution "Remove unused Windows DISM features: $featuresToRemove" if ($Online) { Disable-WindowsOptionalFeature -FeatureName $featuresToRemove -Online -NoRestart } else { Disable-WindowsOptionalFeature -FeatureName $featuresToRemove -Path $Path } } } } } function Get-EndpointInfo { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $EndpointUrl ) $endpointUrl = $EndpointUrl.ToUpper() # e.g. HTTPS://V-XRP03.V.MASD.STBTEST.MICROSOFT.COM:14006 $pos = $EndpointUrl.ToUpper().IndexOf("HTTPS://") $endpoint = $endpointUrl.Substring($pos+8) # gives - V-XRP03.V.MASD.STBTEST.MICROSOFT.COM:14006 $endpointInfo = $endpoint.Split(":") return $endpointInfo } function Validate-EndpointAndPortWithThumbprint { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] $FullEndpointUrl, [Parameter(Mandatory = $true)] $ThumbprintFromRotationPfx, [Parameter(Mandatory = $false)] $retryIteration ) $endpoint = Get-EndpointInfo -EndpointUrl $FullEndpointUrl $port, $null = $endpoint[1] -split '/' Throw-IfNullOrEmpty -DataToCheck $endpoint -ErrorMessage "Failed to get the endpoint for $FullEndpointUrl" Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get the port for $FullEndpointUrl" Trace-Execution "Full endpoint url is: $endpoint and the port is: $port." $sslHost = $endpoint[0] Throw-IfNullOrEmpty -DataToCheck $sslHost -ErrorMessage "Failed to get the SSL host for $FullEndpointUrl" Trace-Execution "This is the SSL Host name: $sslHost for the endpoint: $endpoint." ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $ThumbprintFromRotationPfx -Parameters $Parameters -retryIteration $retryIteration } function Resolve-EndpointAndPort { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] $ConvertedEndpointInfo, [Parameter(Mandatory = $true)] $ThumbprintFromRotationPfx, [Parameter(Mandatory = $false)] $retryIteration ) $restEndpointStr = "OwinListener RestEndpoint" # This is for ACS Migration Service $serviceEndpointStr = "ServiceEndpoint" # For most of SF apps $serviceManagementEndpointStr = "ServiceManagementEndpoint" # For most of SF apps if ($ConvertedEndpointInfo.Count -gt 0) { foreach ($convertedEndpoint in $ConvertedEndpointInfo) { $fullRestEndpointUrl = $convertedEndpoint.$restEndpointStr if(![string]::IsNullOrEmpty($fullRestEndpointUrl)) { Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullRestEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration } $fullServiceEndpointUrl = $convertedEndpoint.$serviceEndpointStr if(![string]::IsNullOrEmpty($fullServiceEndpointUrl)) { Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullServiceEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration } $fullServiceManagementEndpointUrl = $convertedEndpoint.$serviceManagementEndpointStr if(![string]::IsNullOrEmpty($fullServiceManagementEndpointUrl)) { Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullServiceManagementEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration } } } else { throw "Failed to get the Endpoint information." } } function Validate-InternalSSLCertBinding { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $false)] $retryIteration, [Switch] $ValidateCurrentCertLocation ) $ErrorActionPreference = 'Stop' $roleDefinition = $Parameters.Configuration.Role $RoleId = $Parameters.Configuration.Role.Id $ServiceType = $Parameters.Configuration.Role.ServiceType $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo [string] $TimeString = Get-Date -Format "yyyyMMdd-HHmmss" $RemoteLOGFILE = "$env:SystemDrive\MASLogs\$($roleDefinition.Id)_$($MyInvocation.MyCommand.Name)_$TimeString.log" $secureCertPwd = Get-CACertPassword -Parameters $Parameters -ErrorAction Stop -Verbose Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Failed to retrieve the password for the certificate." $appName = $roleDefinition.PrivateInfo.ServiceFabricApplication.Name # Get the cert information $certsForThisRole = $roleDefinition.PublicInfo.Certificates.Certificate | Where-Object { $_.Type -eq "SSL" } Throw-IfNullOrEmpty -DataToCheck $certsForThisRole -ErrorMessage "Failed to get the list of certificates for the role $RoleId" $count = $certsForThisRole.Count if ($count -gt 1) { throw "Expected only one SSL certificate for role: $RoleId but got $count" } Trace-Execution "Got SSL certificate for this role: $RoleId" if ($ValidateCurrentCertLocation) { $certFileRenewal = $certsForThisRole.CertFile } else { $certFileRenewal = $certsForThisRole.CertFileRenewal } Trace-Execution "Certificate file for renewal for the role: $RoleId is at: $certFileRenewal" $clusterName = Get-ManagementClusterName $Parameters Trace-Execution "Got clusterName $clusterName" $certLocation = Get-SharePath $Parameters $certFileRenewal $clusterName if(-not $(Test-Path -Path $certLocation)) { throw "Could not find the certificate at this location: $certLocation for this role: $RoleId" } else { Trace-Execution "Got certLocation $certLocation and validated it" $pfxData = Get-PfxData -FilePath $certLocation -Password $secureCertPwd Throw-IfNullOrEmpty -DataToCheck $pfxData -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation" $thumbprintFromRotationPfx = $pfxData.EndEntityCertificates.Thumbprint Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx " Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId" [string[]] $roleVMs = $roleDefinition.Nodes.Node.Name Throw-IfNullOrEmpty -DataToCheck $roleVMs -ErrorMessage "Failed to get the VMs info for the role $RoleId" $vmCount = $roleVMs.Count Trace-Execution "Found $vmCount VMs for Role: $RoleId" #Getting endpoint and port info $domainFQDN = Get-InternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose Throw-IfNullOrEmpty -DataToCheck $DomainFQDN -ErrorMessage "Failed to get Internal Domain FQDN from the role for the role: $RoleId." $credential = Get-CredentialFromStore -Parameters $Parameters -SecurityInfoUsers $securityInfo.DomainUsers.User -AccountType $Parameters.Configuration.Role.PrivateInfo.Accounts.RunAsAccountID -DomainName $domainFqdn $firstRoleVM = "$(Get-FirstAvailableNode -Nodes $roleVMs).$domainFqdn" Trace-Execution "Getting endpoint and port information for the app: $appName" try { $remoteSession = New-PSSession -ComputerName $firstRoleVM -Credential $credential -Authentication Credssp $scriptBlock = { try { Start-Transcript -Append -Path $Using:RemoteLOGFILE $ErrorActionPreference = "stop" Connect-ServiceFabricCluster | Out-Null $sfApp = Get-ServiceFabricApplication -ApplicationName $using:appName if(!$sfApp) { throw "Failed to retrieve Service fabric application for the role: $Using:RoleId." } Write-Verbose "Got the service fabric application for: $using:appName for the role: $Using:RoleId." -Verbose $replicas = @() $sfServices = $sfApp | Get-ServiceFabricService foreach ($sfService in $sfServices) { if ($sfService.ServiceKind -eq 'Stateful') { $replicas += $sfService | Get-ServiceFabricPartition | Get-ServiceFabricReplica | Where Replicarole -eq Primary } else { $replicas += $sfService | Get-ServiceFabricPartition | Get-ServiceFabricReplica } } if(!$replicas) { throw "Failed to retrieve Service fabric replicas for the application: $sfApp in the role: $Using:RoleId." } Write-Verbose "Got the service fabric replicas for: $using:appName for the role: $Using:RoleId" -Verbose $allReplicasWithReplicaAddress = $replicas | Where-Object {![string]::IsNullOrEmpty($_.ReplicaAddress)} $secureReplicasWithReplicaAddress = $allReplicasWithReplicaAddress | Where-Object {$_.ReplicaAddress -match 'https:'} if(!$secureReplicasWithReplicaAddress) { throw "Failed to retrieve any replicas with replica address for the application: $sfApp in the role: $Using:RoleId." } Write-Verbose "Got the service fabric replicas with replica address for: $using:appName for the role: $Using:RoleId" -Verbose $endpointInfo = $secureReplicasWithReplicaAddress.ReplicaAddress if(!$endpointInfo) { throw "Failed to retrieve endpoint information from Service fabric replicas for the application: $sfApp in the role: $Using:RoleId." } Write-Verbose "Got the endpoint information for: $using:appName for the role: $Using:RoleId as $endpointInfo" -Verbose try { $convertedEndpointInfoCustom = $endpointInfo | ConvertFrom-Json Write-Verbose "Converted the endpoint info from json for endpoint: $endpointInfo" -Verbose } catch { Trace-Error "Failed to convert the endpoint information from json. Please check the json format for the endpoint: $($endpointInfo) throw $($_)" throw } # Explicitly converting PScustomObject to an array so that when it has one element it returns Count as 1. By default it will not convert it to an array with only one element $convertedEndpointArray = @([PScustomObject]$convertedEndpointInfoCustom) $convertedEndpointInfo = $convertedEndpointArray.Endpoints if(!$convertedEndpointInfo) { throw "Endpoints could not be retrieved for the app. $sfApp in the role: $Using:RoleId." } Write-Verbose "Got the endpoint information converted from json for: $using:appName for the role: $Using:RoleId" -Verbose return $convertedEndpointInfo } finally { Stop-Transcript -ErrorAction Ignore } } $dataFromRemote = Invoke-Command -Session $remoteSession -ScriptBlock $scriptBlock $dataFromRemoteArray = @([PScustomObject]$dataFromRemote) } finally { $remoteSession | Remove-Pssession -ErrorAction Ignore | Out-Null } Trace-Execution "Got the endpoint and port information for the app: $appName" if(!$retryIteration) { $retryIteration = 0 } Resolve-EndpointAndPort -Parameters $Parameters -convertedEndpointInfo $dataFromRemoteArray -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration } } <# .Synopsis Reliable atomic copy of file. .Description This function performs a reliable copy of a file by using temporary staging copy and hash validation mechanism. This function is designed to be re-entrant. #> function Copy-FileAtomic { [CmdletBinding()] param ( [Parameter(Mandatory = $true, HelpMessage="Literal path to the source file which is to be copied.")] [string] $SourceFilePath, [Parameter(Mandatory = $true, HelpMessage="Literal path to the destination file where the source file would be copied to.")] [string] $DestinationFilePath ) $destFile = Split-Path -path $DestinationFilePath -Leaf $destFolder = Split-Path -path $DestinationFilePath -Parent $newFileName = '{0}.new' -f $destFile $prevFileName = '{0}.prev' -f $destFile $newPath = Join-Path $destFolder $newFileName $prevPath = Join-Path $destFolder $prevFileName # Handle code re-entrancy if (Test-Path $prevPath) { if (Test-Path $DestinationFilePath) { Write-Verbose "Removing stale file: $prevPath" Remove-Item -Path $prevPath } else { Write-Verbose "Renaming stale file: $prevPath to $DestinationFilePath" Rename-Item -Path $prevPath -NewName $DestinationFilePath } } # Copy file to a temp file in destination folder Write-Verbose "Copying file $SourceFilePath to $newPath" Copy-Item -Path $SourceFilePath -Destination $newPath -Force -ErrorAction Stop # Validate copied staging file against original source file if ((Get-FileHash $SourceFilePath).hash -ne (Get-FileHash $newPath).hash) { throw "$newPath is not copied correctly." } # Handle atomicity and overwrite issues $overwrite = Test-Path $DestinationFilePath if ($overwrite) { Write-Verbose "Renaming existing destination file: $DestinationFilePath to $prevPath" Rename-Item -Path $DestinationFilePath -NewName $prevPath -ErrorAction Stop } Write-Verbose "Renaming new file: $newPath to $DestinationFilePath" Rename-Item -Path $newPath -NewName $DestinationFilePath -ErrorAction Stop if ($overwrite) { Write-Verbose "Removing stale file: $prevPath" Remove-Item -Path $prevPath } } function Get-ServiceFabricContainerSubnet { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = "Stop" # Attempt to query for the actual subnet value $privateInfo = $Parameters.Configuration.Role.PrivateInfo # Query for Container Network element $containerNetwork = $privateInfo.Configurations.ContainerNetwork # Query for Container Network if (-not $containerNetwork) { Trace-Error "Couldn't find 'ContainerNetwork' element in 'PrivateInfo/Configurations'" } # Query for subnet value if ([string]::IsNullOrEmpty($containerNetwork.Subnet)) { Trace-Error "Subnet of 'PrivateInfo/Configurations/ContainerNework' either is not defined, null or empty" } # Finally that is subnet value we've been looking for return $containerNetwork.Subnet } function Get-ServiceFabricTraceSharePath { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = "Stop" $privateInfo = $Parameters.Configuration.Role.PrivateInfo $clusterName = Get-ManagementClusterName $Parameters $sharePath = Get-SharePath $Parameters $privateInfo.Configurations.TraceRootShare.Path $clusterName # Verify share path is actually defined if ([string]::IsNullOrEmpty($sharePath)) { # NOTE: This error means role definition is missing <TraceRootShare Path="<...>" /> element Trace-Error "[$($Parameters.Configuration.Role.Id)]: Cannot get share for saving trace log of Service Fabric cluster" } return $sharePath } function Get-FullyQualifiedDomainName { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = "Stop" $publicInfo = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo return $publicInfo.DomainConfiguration.FQDN } function Get-ServiceFabricClusterRunAsAccount { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = "Stop" $fqdn = Get-FullyQualifiedDomainName -Parameters $Parameters $privateInfo = $Parameters.Configuration.Role.PrivateInfo $accountName = $privateInfo.Configurations.RunAs.AccountName # Cluster Credentials $clusterAccount = "$fqdn\$accountName$" return $clusterAccount } <# .Synopsis Returns the encryption certificate to be used by Service Fabric Central Secret Store (CSS) service. .Description Returns the encryption certificate thumbprint to be used by Service Fabric Central Secret Store (CSS) service by reading from the pfx file. Note that this may be different from the one currently in use by CSS. #> function Get-SfCssEncryptionCertThumbprintToUse { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node | ? IsManagementCluster -eq "true" | Select-Object -ExpandProperty Name $roleDefinition = $Parameters.Configuration.Role $certFilePathToUse = Get-SharePath $Parameters $roleDefinition.PrivateInfo.Configurations.SfCSS.CertPath $clusterName $updateCertFilePath = Get-SharePath $Parameters $roleDefinition.PrivateInfo.Configurations.SfCSS.RenewalCertPath $clusterName # If a pfx file exists in the Update directory, use that, otherwise use the one in the Current directory. if (Test-Path -Path $updateCertFilePath) { $certFilePathToUse = $updateCertFilePath } $certThumbprint = Get-SfCssEncryptionCertThumbprint -Parameters $Parameters -FilePath $certFilePathToUse return $certThumbprint } <# .Synopsis Returns the thumbprint for the specified pfx file. .Description Returns the thumbprint for the specified pfx file for use by SF CSS. #> function Get-SfCssEncryptionCertThumbprint { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [string] $FilePath ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $roleDefinition = $Parameters.Configuration.Role $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo $runasAccount = $securityInfo.DomainUsers.User | ? Role -EQ $roleDefinition.PrivateInfo.Accounts.RunAsAccountID $runasAccountCredential = $Parameters.GetCredential($runasAccount.Credential) # Explicitly storing node names as an array, because in OneNode environment, PowerShell would treat it as an object instead # since there is only one XRP node. [string[]] $nodeNames = $roleDefinition.Nodes.Node | ForEach-Object {$_.Name} $nodeName = $nodeNames[0] # SF CSS certificate generated in 2008 release was protected exclusively to the FileCopyAgent gMSA, so we need to execute # Get-PfxData against the certificate using that account. As we can't directly execute an Invoke-Command against # a gMSA account, we will use a scheduled task. # TODO: Certificates generated in 2104 release will be protected to Domain Admins + FCA gMSA, so we can # simplify this logic to directly call Get-PfxData post-2104. $domainName = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo.DomainConfiguration.DomainName $serviceAccount = "AzS-Xrp-Fca" $certProtectToUser = '{0}\{1}$' -f $domainName, $serviceAccount $getThumbprintScriptBlock = { $taskName = "GetSfCssEncryptionCertThumbprint-$(New-Guid)" $thumbprintFile = "C:\temp\ServiceFabricCssEncryption.txt" $SCHED_S_TASK_RUNNING = 267009 $SCHED_S_TASK_QUEUED = 267045 Unregister-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue -Confirm:$false | Out-Null Remove-Item -Path $thumbprintFile -ErrorAction SilentlyContinue | Out-Null $scriptBlockString = @" (Get-PfxData -FilePath $using:FilePath).EndEntityCertificates.Thumbprint | New-Item -Path "C:\temp\ServiceFabricCssEncryption.txt" -Force exit "@ $scriptBlock = [ScriptBlock]::Create($scriptBlockString) try { $TaskParameters = @{ TaskName = $taskName } $TaskParameters['Action'] = New-ScheduledTaskAction -Execute "Powershell.exe" -Argument "-NoExit -Command $scriptBlock" Trace-Execution "Executing task to fetch SF CSS encryption certificate thumbprint using user: $using:certProtectToUser" $TaskParameters['Principal'] = New-ScheduledTaskPrincipal -UserID $using:certProtectToUser -LogonType Password -RunLevel Highest Trace-Execution "Registering scheduled task for getting SF CSS encryption certificate thumbprint." $scheduledTask = Register-ScheduledTask @TaskParameters -ErrorAction Stop Trace-Execution "Starting scheduled task for getting SF CSS encryption certificate thumbprint. Task name: $taskName" $scheduledTask | Start-ScheduledTask -ErrorAction Stop do { Start-Sleep -Seconds 1 $taskResult = $ScheduledTask | Get-ScheduledTaskInfo } while (($taskResult.LastTaskResult -eq $SCHED_S_TASK_RUNNING) -or ($taskResult.LastTaskResult -eq $SCHED_S_TASK_QUEUED)) Trace-Execution "The scheduled task for getting SF CSS encryption certificate thumbprint completed. TaskResult: $($taskResult.LastTaskResult)" Get-Content -Path $thumbprintFile } catch { Trace-Error "Failed to fetch SF CSS encrytion certificate thumbprint. Reason: $_" } finally { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue | Out-Null Remove-Item -Path $thumbprintFile -ErrorAction SilentlyContinue | Out-Null } } # As we need to execute the script block to fetch the cert thumbprint using FCA's gMSA account, we need to # execute it from one of the XRP nodes. Trace-Execution "Executing script block to fetch SF CSS encryption certificate thumbprint from $nodeName." $session = New-PSSession -ComputerName $nodeName -Credential $runasAccountCredential -Authentication Credssp try { [string]$certThumbprint = Invoke-Command -Session $session -ScriptBlock $getThumbprintScriptBlock } finally { Remove-PSSession -Session $session -ErrorAction Ignore | Out-Null } if ([string]::IsNullOrEmpty($certThumbprint)) { throw "SF CSS certificate thumbprint is null or empty." } Trace-Execution "Determined the encryption certificate thumbprint for SF CSS to be: '$certThumbprint'" return $certThumbprint } <# .SYNOPSIS #### Reference from nuget in 1907 Returns a structured object containing information about each of the exceptions in an update summary #> function Read-SummaryXml { Param ( [parameter(Mandatory = $false)][string] $RoutingJsonPath = $RoutingJsonSharePath, [parameter(Mandatory = $false, ParameterSetName = 'Path')][string] $SummaryXMLPath, [parameter(Mandatory = $false, ParameterSetName = 'Xml')][xml] $SummaryXML, [parameter(Mandatory = $false)][switch] $Routing ) if ($SummaryXMLPath) { $SummaryXML = Get-Content "$SummaryXMLPath" -ErrorAction Stop } $foundException = $false $Exceptions = $SummaryXML.SelectNodes(".//Exception") $returnInformation = @() foreach ($exception in $Exceptions) { if ($exception.ParentNode.RolePath) { $RolePath = $exception.ParentNode.RolePath | select -first 1 $Interface = $exception.ParentNode.InterfaceType | select -first 1 $ExecutionContextRolePath = $null $ExecutionContextNode = "" $ActionType = $null $ActionRolePath = "" $curAncestor = $exception while ($curAncestor -and (!$ExecutionContextRolePath -or !$ActionType)) { if (!$ExecutionContextRolePath) { $ExecutionContextRolePath = $curAncestor.ExecutionContext.Roles.Role.RolePath | select -first 1 $ExecutionContextNode = $curAncestor.ExecutionContext.Roles.Role.Nodes.Node.Name | select -first 1 } if (!$ActionType) { $ActionType = $curAncestor.ActionType $ActionRolePath = $curAncestor.RolePath } $curAncestor = $curAncestor.ParentNode } if ($Routing) { $returnInformation += @(Get-RoutingInformation -RoutingJsonPath $RoutingJsonPath -Exception $exception -ExecutionContextRolePath $ExecutionContextRolePath -Interface $Interface -RolePath $RolePath) } else { $returnInformation += @{ 'Exception' = $exception.Raw 'ExecutionContextNode' = $ExecutionContextNode 'ExecutionContextRolePath' = $ExecutionContextRolePath 'RolePath' = $RolePath 'InterfaceType' = $Interface 'ActionType' = $ActionType 'ActionRolePath' = $ActionRolePath } } } } $returnInformation } function Get-FailureListFromMainActionPlan { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $runtimeParameters = $null $failureList = $null if($Parameters.RunInformation -ne $null -and $Parameters.RunInformation.ContainsKey('RuntimeParameter')) { $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] } if($runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('FailureSummaryXml')) { $failureSummaryXml = $runtimeParameters['FailureSummaryXml'] $failureList = @(Read-SummaryXml -SummaryXML $failureSummaryXml) } return $failureList } <# .SYNOPSIS A generic wrapper function to send telemetry events. #> function Send-TelemetryEvent { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string] $ComponentName, [Parameter(Mandatory=$true)] [string] $EventName, [Parameter(Mandatory=$true)] [int] $EventVersion, [Parameter(Mandatory=$true)] [hashtable] $EventData ) $ErrorActionPreference = "Stop" Trace-Execution "Telemetry event = Component:[$ComponentName], EventName:[$EventName], EventVersion:[$EventVersion], EventData:$($EventData | Out-String)" try { $telemetryPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Fabric.Health.Telemetry.TelemetryReporter" $telemetryDll = Join-Path $telemetryPath "lib\net46\Microsoft.AzureStack.Fabric.Health.Telemetry.TelemetryReporter.dll" Import-Module -Name $telemetryDll -Verbose:$false -DisableNameChecking } catch { Trace-Warning "Error loading telemetry library: $_" return } try { Trace-Execution "Sending telemetry event." Send-AzureStackTelemetryEvent -Component $ComponentName -Name $EventName -Version $EventVersion -TelemetryEventData $EventData -Verbose Trace-Execution "Finish sending telemetry event." } catch { Trace-Warning "Error sending telemetry event: $_" } } <# .SYNOPSIS A generic wrapper function to raise or clear secret rotation alert. #> function RaiseOrClear-SRAlerts { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) Trace-Execution "Inside RaiseOrClear-SRAlerts method." $azureStackNugetPath = Get-ASArtifactPath -NugetName "microsoft.azurestack.fault.sdk" $alertBinPath = Join-Path $azureStackNugetPath "lib\net46\" $dlls = (Get-ChildItem -Path $alertBinPath -Filter "*.dll").FullName foreach ($dll in $dlls) { Trace-Execution "Loading DLL ... $dll" $ret = [System.Reflection.Assembly]::LoadFile($dll) } $faultWriter = [Microsoft.AzureStack.Fault.Sdk.FaultWriterFactory]::Create() $runtimeParameters = $null if($Parameters.RunInformation -ne $null -and $Parameters.RunInformation.ContainsKey('RuntimeParameter')) { $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] } if($runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('SummaryXml')) { $resourceProviderNamespace = "Microsoft.Fabric.Admin" $summaryXml = [xml]$runtimeParameters['SummaryXml'] $actionStatus = $summaryXml.Action.Status if ($summaryXml.Action.Type -ieq "ExternalCertRotation") { $faultTypeId = "ExternalSecretRotation.Actionplan.Critical" } else { $faultTypeId = "InternalSecretRotation.Actionplan.Critical" } $resourceType = "infraRoleInstances" $resourceName = $env:ComputerName if ($actionStatus -eq "Success") { Trace-Execution "Secret rotation action plan completed successfully, hence close active alert if any." $faultWriter.CloseFault($resourceProviderNamespace,$faultTypeId,$resourceType,$resourceName) } else { Trace-Execution "Secret rotation action plan status is $actionStatus hence raising an alert." $faultWriter.OpenFault($resourceProviderNamespace,$faultTypeId,$resourceType,$resourceName) } } } <# .SYNOPSIS A generic wrapper function checks whether the current update is pnu or not by comparing minor build version. #> function Test-IsMonthlyUpdate { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) Trace-Execution "Inside Test-IsMonthlyUpdate method." $isPnu = $false if ((Test-IsUpdate -Parameters $Parameters) -and !(Test-IsHotfixUpdate -Parameters $Parameters)) { $isPnu = $true } Trace-Execution "Is it a pnu update :: $isPnu." return $isPnu } <# .SYNOPSIS A generic wrapper function checks whether the current update is a Hotfix pnu or not by checking the Update Name. #> function Test-IsHotfixUpdate { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) Trace-Execution "Inside Test-IsHotfixUpdate method." $isHotfixPNU = $false # In case of update, run time information will be populated if($Parameters.RunInformation -ne $null) { $runtimeParameters = $Parameters.RunInformation['RuntimeParameter'] if( $runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('UpdateVersion') ) { $updateMetadataXml = [xml]$runtimeParameters['metadata'] Trace-Execution "UpdateName : $($updateMetadataXml.UpdatePackageManifest.UpdateInfo.UpdateName)" if($updateMetadataXml.UpdatePackageManifest.UpdateInfo.UpdateName -match 'AzS Hotfix') { $isHotfixPNU = $true; } } } Trace-Execution "Is it a Hotfix Update :: $isHotfixPNU." return $isHotfixPNU } <# .SYNOPSIS A helper function to check if the current action plan is an Update action plan. #> function Test-IsUpdate { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $updateVersion = Get-InProgressUpdateVersion -Parameters $Parameters if ($updateVersion -ne $null) { return $true } else { return $false } } <# .SYNOPSIS Tests whether the execution context node is a member of the management cluster. #> function Test-ManagementClusterNode { [CmdletBinding()] [OutputType([System.Boolean])] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop # Default to true if there is no BareMetal execution context $result = $true # If there is a BareMetal execution context, compare management cluster name and RefClusterId of the node if ( ($nodeName = Get-ExecutionContextNodeName -Parameters $Parameters) -and ((Get-ExecutionContextRoleName -Parameters $Parameters) -eq 'BareMetal') ) { $managementClusterName = Get-ManagementClusterName $Parameters Trace-Execution "Management cluster name is '$managementClusterName'" $bareMetalRole = $Parameters.Roles["BareMetal"].PublicConfiguration $nodeClusterName = ($bareMetalRole.Nodes.Node | Where-Object {$_.Name -eq $nodeName}).RefClusterId Trace-Execution "Node '$nodeName' cluster is '$nodeClusterName'" if ($nodeClusterName -eq $managementClusterName) { Trace-Execution "Node '$nodeName' is a member of management cluster '$managementClusterName'" } else { Trace-Execution "Node '$nodeName' is not a member of management cluster '$managementClusterName'" $result = $false } } return $result } <# .SYNOPSIS A helper function to Skip nodes which are provided in runtime parameters. #> function Skip-NodesProvidedInRuntimeParameter { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory = $true)] [string[]] $Nodes ) Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of Nodes which are getting filtered $($Nodes -join ',')." $unreachableNodesStr = $Parameters.RunInformation['RuntimeParameter']['UnreachableNodes'] if([string]::IsNullOrEmpty($unreachableNodesStr)) { Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] No nodes are present in the list of unreachable nodes." return $Nodes } Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of unreachable nodes: $unreachableNodesStr." $unreachableNodes = $unreachableNodesStr -split ',' $newList = $Nodes | % { $_ -notin $unreachableNodes } Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of Nodes after Filtering $($newList -join ',')." return $newList } <# .SYNOPSIS A helper function to get first available node. #> function Get-FirstAvailableNode { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [string[]] $Nodes ) Trace-Execution "[Get-FirstAvailableNode] List of Nodes: $($Nodes -join ',')." foreach($node in $Nodes) { $returnValue = Invoke-Command {"reachable"} -ComputerName $node -ErrorAction Ignore if($returnValue -eq "reachable") { Trace-Execution "[Get-FirstAvailableNode] Node selected with WinRM connectivity: '$node'." return $node } else { Trace-Execution "[Get-FirstAvailableNode] Node is not reachable via WinRM: '$node'." } } Trace-Warning "[Get-FirstAvailableNode] No node is reachable via WinRM, fall back to Test-NetConnection (ping)." foreach($node in $Nodes) { $returnValue = Test-NetConnection -ComputerName $node -ErrorAction Ignore if($returnValue.PingSucceeded) { Trace-Execution "[Get-FirstAvailableNode] Node selected with Test-NetConnection connectivity: '$node'." return $node } else { Trace-Execution "[Get-FirstAvailableNode] Node is not reachable via Test-NetConnection: '$node'." } } # In few situations where this code is running on ECE agent then there may be permission issues, to work that around # returning first node if none of the nodes are reachable. # TODO: Fix the permission issue with ECE agent Trace-Warning "[Get-FirstAvailableNode] Fall back to return first node as No node is reachable from the list: $($Nodes -join ',')." return $Nodes[0] } <# .SYNOPSIS A helper function to check if the cloud admin operation status is update critical. #> function Test-CloudStatusUpdateCritical { $ErrorActionPreference = "Stop" Import-Module -Name ECEClient -Verbose:$false -DisableNameChecking $eceClient = Create-ECEClientWithServiceResolver $eceParamsXml = [XML]($eceClient.GetCloudParameters().getAwaiter().GetResult().CloudDefinitionAsXmlString) $statusNode = $eceParamsXml.SelectNodes("//Parameter[@Name='CloudAdminOperationStatus']") return ($statusNode.Value -eq "UpdateCritical") } function Get-AzureStackHostAllVirtualSwitchNames { Param( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) [System.String[]] $physicalMachines = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node | Select-Object -ExpandProperty "Name" [PSObject[]] $vmSwitch = Get-VMSwitch -ComputerName $physicalMachines[0] -SwitchType External if ((-not $vmSwitch) -or $vmSwitch.Count -eq 0) { Trace-Error "Cannot get Virtual Switch Name from [ $($physicalMachines[0]) ]" } [System.String[]] $retVal = $vmSwitch.Name return $retVal } function Get-AzureStackHostDefaultVirtualSwitchName { Param( [Parameter(Mandatory = $true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) [System.String[]] $allSwitches = Get-AzureStackHostAllVirtualSwitchNames -Parameters $Parameters return $allSwitches[0] } Export-ModuleMember -Function Add-AccountAcls Export-ModuleMember -Function Add-AccountMemberships Export-ModuleMember -Function Add-AllVIPs Export-ModuleMember -Function Add-DnsExternalPtrRecords Export-ModuleMember -Function Add-DnsRecord Export-ModuleMember -Function Add-DnsServerResourceRecordCNameRemoteOrLocal Export-ModuleMember -Function Add-DnsServerResourceRecordRemoteOrLocal Export-ModuleMember -Function Add-GroupMembership Export-ModuleMember -Function Add-GuestCluster Export-ModuleMember -Function Add-IDnsConfiguration Export-ModuleMember -Function Add-IPAddress Export-ModuleMember -Function Add-LoadBalancerToNetworkAdapter Export-ModuleMember -Function Add-LocalAdministrators Export-ModuleMember -Function Add-MachineMembership Export-ModuleMember -Function Add-NetworkAdapterToNetwork Export-ModuleMember -Function Add-ServiceAccountMembership Export-ModuleMember -Function Add-ServiceAccountSpn Export-ModuleMember -Function Add-UserMembership Export-ModuleMember -Function Assert-Service Export-ModuleMember -Function Clear-RemoteDNSServerCache Export-ModuleMember -Function ConnectPSSession Export-ModuleMember -Function ConvertFrom-IPAddress Export-ModuleMember -Function Convert-IPv4IntToString Export-ModuleMember -Function Convert-IPv4StringToInt Export-ModuleMember -Function ConvertTo-IPAddress Export-ModuleMember -Function ConvertTo-MacAddress Export-ModuleMember -Function ConvertTo-PrefixLength Export-ModuleMember -Function ConvertTo-SubnetMask Export-ModuleMember -Function Copy-FileAtomic Export-ModuleMember -Function Create-ImportModuleString Export-ModuleMember -Function Create-LocalJob Export-ModuleMember -Function Create-RemoteJob Export-ModuleMember -Function Disable-CredSSP Export-ModuleMember -Function Disable-RemoteClientCredSSP Export-ModuleMember -Function Dismount-Wim Export-ModuleMember -Function Enable-AutoLogon Export-ModuleMember -Function Enable-CredSSP Export-ModuleMember -Function Expand-DeploymentArtifacts Export-ModuleMember -Function Expand-NugetContent Export-ModuleMember -Function Expand-UpdateContent Export-ModuleMember -Function Find-LockedFiles Export-ModuleMember -Function Get-FirstAvailableNode Export-ModuleMember -Function Get-ActionPlanInstanceID Export-ModuleMember -Function Get-FullyQualifiedDomainName Export-ModuleMember -Function Get-AvailableADComputerName Export-ModuleMember -Function Get-AvailableNode Export-ModuleMember -Function Get-AvailableServer Export-ModuleMember -Function Get-BareMetalCredential Export-ModuleMember -Function Get-BroadcastAddress Export-ModuleMember -Function Get-CACertPassword Export-ModuleMember -Function Get-ClusterShare Export-ModuleMember -Function Get-ClusterShareNames Export-ModuleMember -Function Get-CredentialFromStore Export-ModuleMember -Function Get-CurrentOrchestrators Export-ModuleMember -Function Get-DnsServerResourceRecordRemoteOrLocal Export-ModuleMember -Function Get-DNSServers Export-ModuleMember -Function Get-DomainCredential Export-ModuleMember -Function Get-DomainIPMapping Export-ModuleMember -Function Get-EndpointAndPort Export-ModuleMember -Function Get-ExecutionContextClusterName Export-ModuleMember -Function Get-ExecutionContextNodeName Export-ModuleMember -Function Get-ExecutionContextRoleName Export-ModuleMember -Function Get-ExternalDnsMachineName Export-ModuleMember -Function Get-GatewayAddress Export-ModuleMember -Function Get-HostUpdateShare Export-ModuleMember -Function GetiDNSServersSettings Export-ModuleMember -Function Get-InProgressUpdateVersion Export-ModuleMember -Function Get-InProgressUpdatePackagePath Export-ModuleMember -Function Get-InProgressUpdateNugetStore Export-ModuleMember -Function Get-IsVirtualNetworkAlreadyConfigured Export-ModuleMember -Function Get-JeaSession Export-ModuleMember -Function Get-LocalCsvPathFromSharePath Export-ModuleMember -Function Get-MacAddress Export-ModuleMember -Function Get-MacAddressString Export-ModuleMember -Function Get-NCAccessControlList Export-ModuleMember -Function Get-NCCredential Export-ModuleMember -Function Get-NCGateway Export-ModuleMember -Function Get-NCGatewayPool Export-ModuleMember -Function Get-NCIPPool Export-ModuleMember -Function Get-NCLoadBalancer Export-ModuleMember -Function Get-NCLoadbalancerManager Export-ModuleMember -Function Get-NCLoadBalancerMux Export-ModuleMember -Function Get-NCLogicalNetwork Export-ModuleMember -Function Get-NCLogicalNetworkSubnet Export-ModuleMember -Function Get-NCMACPool Export-ModuleMember -Function Get-NCNetworkInterface Export-ModuleMember -Function Get-NCNetworkInterfaceInstanceId Export-ModuleMember -Function Get-NCNetworkInterfaceResourceId Export-ModuleMember -Function Get-NCPublicIPAddress Export-ModuleMember -Function Get-NCServer Export-ModuleMember -Function Get-NCSwitch Export-ModuleMember -Function Get-NCVirtualGateway Export-ModuleMember -Function Get-NCVirtualNetwork Export-ModuleMember -Function Get-NCVirtualServer Export-ModuleMember -Function Get-NCVirtualSubnet Export-ModuleMember -Function Get-NetworkAddress Export-ModuleMember -Function Get-NetworkDefinitionForCluster Export-ModuleMember -Function Get-NetworkDefinitions Export-ModuleMember -Function Get-NetworkMap Export-ModuleMember -Function Get-NetworkNameForCluster Export-ModuleMember -Function Get-NugetStorePath Export-ModuleMember -Function Get-NugetVersions Export-ModuleMember -Function Get-OfflineDjoinBlob Export-ModuleMember -Function Get-PortProfileId Export-ModuleMember -Function Get-PrimaryDomainController Export-ModuleMember -Function Get-RangeEndAddress Export-ModuleMember -Function Get-ServiceFabricTraceSharePath Export-ModuleMember -Function Get-ServiceFabricClusterRunAsAccount Export-ModuleMember -Function Get-ServiceFabricContainerSubnet Export-ModuleMember -Function Get-SfCssEncryptionCertThumbprintToUse Export-ModuleMember -Function Get-SfCssEncryptionCertThumbprint Export-ModuleMember -Function Get-ScopeRange Export-ModuleMember -Function Get-ServerResourceId Export-ModuleMember -Function Get-SharePath Export-ModuleMember -Function Get-StorageEndpointName Export-ModuleMember -Function Get-TemporaryDomainCredential Export-ModuleMember -Function Get-UserAccountCredential Export-ModuleMember -Function Get-TestLogsPath Export-ModuleMember -Function Get-VMWSManDiagnostics Export-ModuleMember -Function Initialize-ECESession Export-ModuleMember -Function Initialize-NugetScript Export-ModuleMember -Function Invoke-ECECommand Export-ModuleMember -Function Invoke-PSDirectOnVM Export-ModuleMember -Function Invoke-ScriptBlockInParallel Export-ModuleMember -Function Invoke-ScriptBlockWithRetries Export-ModuleMember -Function Invoke-WebRequestWithRetries Export-ModuleMember -Function IsIpPoolRangeValid Export-ModuleMember -Function IsIpWithinPoolRange Export-ModuleMember -Function IsOneNode Export-ModuleMember -Function IsOnVirtualMachine Export-ModuleMember -Function IsVirtualAzureStack Export-ModuleMember -Function JSONDelete Export-ModuleMember -Function JSONGet Export-ModuleMember -Function JSONPost Export-ModuleMember -Function Mount-Wim Export-ModuleMember -Function Mount-WindowsImageWithRetry Export-ModuleMember -Function New-ACL Export-ModuleMember -Function New-CimSessionVerify Export-ModuleMember -Function New-Credential Export-ModuleMember -Function New-ExecutionContextXmlForNode Export-ModuleMember -Function New-LoadBalancerVIP Export-ModuleMember -Function New-LocalUserWrapper Export-ModuleMember -Function New-NCAccessControlList Export-ModuleMember -Function New-NCAccessControlListRule Export-ModuleMember -Function New-NCBgpPeer Export-ModuleMember -Function New-NCBgpRouter Export-ModuleMember -Function New-NCBgpRoutingPolicy Export-ModuleMember -Function New-NCBgpRoutingPolicyMap Export-ModuleMember -Function New-NCCredential Export-ModuleMember -Function New-NCGateway Export-ModuleMember -Function New-NCGatewayPool Export-ModuleMember -Function New-NCGreTunnel Export-ModuleMember -Function New-NCIPPool Export-ModuleMember -Function New-NCIPSecTunnel Export-ModuleMember -Function New-NCL3Tunnel Export-ModuleMember -Function New-NCLoadBalancer Export-ModuleMember -Function New-NCLoadBalancerBackendAddressPool Export-ModuleMember -Function New-NCLoadBalancerFrontEndIPConfiguration Export-ModuleMember -Function New-NCLoadBalancerLoadBalancingRule Export-ModuleMember -Function New-NCLoadBalancerMux Export-ModuleMember -Function New-NCLoadBalancerMuxPeerRouterConfiguration Export-ModuleMember -Function New-NCLoadBalancerOutboundNatRule Export-ModuleMember -Function New-NCLoadBalancerProbe Export-ModuleMember -Function New-NCLoadBalancerProbeObject Export-ModuleMember -Function New-NCLogicalNetwork Export-ModuleMember -Function New-NCLogicalNetworkSubnet Export-ModuleMember -Function New-NCLogicalSubnet Export-ModuleMember -Function New-NCMACPool Export-ModuleMember -Function New-NCNetworkInterface Export-ModuleMember -Function New-NCPublicIPAddress Export-ModuleMember -Function New-NCServer Export-ModuleMember -Function New-NCServerConnection Export-ModuleMember -Function New-NCServerNetworkInterface Export-ModuleMember -Function New-NCSlbState Export-ModuleMember -Function New-NCSwitch Export-ModuleMember -Function New-NCSwitchPort Export-ModuleMember -Function New-NCVirtualGateway Export-ModuleMember -Function New-NCVirtualNetwork Export-ModuleMember -Function New-NCVirtualServer Export-ModuleMember -Function New-NCVirtualSubnet Export-ModuleMember -Function New-NCVpnClientAddressSpace Export-ModuleMember -Function New-RegistryPropertyNameValue Export-ModuleMember -Function New-ServiceAccount Export-ModuleMember -Function New-UserAccount Export-ModuleMember -Function New-UserGroup Export-ModuleMember -Function NormalizeIPv4Subnet Export-ModuleMember -Function PublishAndStartDscConfiguration Export-ModuleMember -Function PublishAndStartDscForJea Export-ModuleMember -Function RaiseOrClear-SRAlerts Export-ModuleMember -Function ReliableActionService Export-ModuleMember -Function ReliableGetService Export-ModuleMember -Function ReliableRestartService Export-ModuleMember -Function ReliableSetService Export-ModuleMember -Function ReliableStartService Export-ModuleMember -Function ReliableStopService Export-ModuleMember -Function Remove-ComputerAndDnsRecord Export-ModuleMember -Function Remove-DnsRecord Export-ModuleMember -Function Remove-DnsServerResourceRecordRemoteOrLocal Export-ModuleMember -Function Remove-LoadBalancerFromNetworkAdapter Export-ModuleMember -Function Remove-NCAccessControlList Export-ModuleMember -Function Remove-NCCredential Export-ModuleMember -Function Remove-NCGateway Export-ModuleMember -Function Remove-NCGatewayPool Export-ModuleMember -Function Remove-NCIPPool Export-ModuleMember -Function Remove-NCLoadBalancer Export-ModuleMember -Function Remove-NCLoadBalancerMux Export-ModuleMember -Function Remove-NCLogicalNetwork Export-ModuleMember -Function Remove-NCMACPool Export-ModuleMember -Function Remove-NCNetworkInterface Export-ModuleMember -Function Remove-NCPublicIPAddress Export-ModuleMember -Function Remove-NCServer Export-ModuleMember -Function Remove-NCSwitch Export-ModuleMember -Function Remove-NCVirtualGateway Export-ModuleMember -Function Remove-NCVirtualNetwork Export-ModuleMember -Function Remove-NCVirtualServer Export-ModuleMember -Function Remove-PortProfileId Export-ModuleMember -Function Remove-UserMembership Export-ModuleMember -Function Reset-RestartCallback Export-ModuleMember -Function Resolve-EndpointAndPort Export-ModuleMember -Function Restart-Machine Export-ModuleMember -Function Set-DNSForwarder Export-ModuleMember -Function Set-MacAndIPAddressSingleNode Export-ModuleMember -Function Set-MacAndIPAddress Export-ModuleMember -Function Set-NCConnection Export-ModuleMember -Function Set-NCLoadBalancerManager Export-ModuleMember -Function Set-PortProfileId Export-ModuleMember -Function Set-PortProfileIdHelper Export-ModuleMember -Function Set-RestartCallback Export-ModuleMember -Function Send-TelemetryEvent Export-ModuleMember -Function Skip-NodesProvidedInRuntimeParameter Export-ModuleMember -Function Start-CloudCluster Export-ModuleMember -Function Start-ParallelWork Export-ModuleMember -Function Start-ParallelWorkAndWait Export-ModuleMember -Function Start-Test Export-ModuleMember -Function Stop-CloudCluster Export-ModuleMember -Function Test-CloudStatusUpdateCritical Export-ModuleMember -Function Test-ClusterResourceExist Export-ModuleMember -Function Test-HostPhysicalDiskSize Export-ModuleMember -Function Test-IPConnection Export-ModuleMember -Function Test-NetworkMap Export-ModuleMember -Function Test-IsMonthlyUpdate Export-ModuleMember -Function Test-IsHotfixUpdate Export-ModuleMember -Function Test-IsUpdate Export-ModuleMember -Function Test-ManagementClusterNode Export-ModuleMember -Function Test-PSSession Export-ModuleMember -Function Test-PSSessionConnection Export-ModuleMember -Function Test-RoleNodeGroupAvailability Export-ModuleMember -Function Test-SFRingHealth Export-ModuleMember -Function Test-PrecheckClusterNodeHealth Export-ModuleMember -Function Test-PrecheckNCClusterNodeHealth Export-ModuleMember -Function Test-WSManConnection Export-ModuleMember -Function Test-WSmanForCredSSP Export-ModuleMember -Function Trace-Error Export-ModuleMember -Function Trace-Execution Export-ModuleMember -Function Trace-Warning Export-ModuleMember -Function Update-JEAEndpointsForUpdate Export-ModuleMember -Function Update-NCCredential Export-ModuleMember -Function Update-NCServer Export-ModuleMember -Function Update-NCVirtualServer Export-ModuleMember -Function Validate-EndpointAndPortWithThumbprint Export-ModuleMember -Function Validate-InternalSSLCertBinding Export-ModuleMember -Function Validate-SSLCertificateBinding Export-ModuleMember -Function Update-WindowsDismFeature Export-ModuleMember -Function Wait-ParallelWork Export-ModuleMember -Function Wait-Result Export-ModuleMember -Function Wait-VirtualNetwork Export-ModuleMember -Function Read-SummaryXml Export-ModuleMember -Function Get-FailureListFromMainActionPlan Export-ModuleMember -Function Get-AzureStackHostAllVirtualSwitchNames Export-ModuleMember -Function Get-AzureStackHostDefaultVirtualSwitchName # SIG # Begin signature block # MIIn0QYJKoZIhvcNAQcCoIInwjCCJ74CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCD2Hmcts5h2jPSk # EDHACfx0fTeWeI1XDtqn37nI7Zi2LqCCDYUwggYDMIID66ADAgECAhMzAAADTU6R # phoosHiPAAAAAANNMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI4WhcNMjQwMzE0MTg0MzI4WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDUKPcKGVa6cboGQU03ONbUKyl4WpH6Q2Xo9cP3RhXTOa6C6THltd2RfnjlUQG+ # Mwoy93iGmGKEMF/jyO2XdiwMP427j90C/PMY/d5vY31sx+udtbif7GCJ7jJ1vLzd # j28zV4r0FGG6yEv+tUNelTIsFmmSb0FUiJtU4r5sfCThvg8dI/F9Hh6xMZoVti+k # bVla+hlG8bf4s00VTw4uAZhjGTFCYFRytKJ3/mteg2qnwvHDOgV7QSdV5dWdd0+x # zcuG0qgd3oCCAjH8ZmjmowkHUe4dUmbcZfXsgWlOfc6DG7JS+DeJak1DvabamYqH # g1AUeZ0+skpkwrKwXTFwBRltAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUId2Img2Sp05U6XI04jli2KohL+8w # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMDUxNzAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # ACMET8WuzLrDwexuTUZe9v2xrW8WGUPRQVmyJ1b/BzKYBZ5aU4Qvh5LzZe9jOExD # YUlKb/Y73lqIIfUcEO/6W3b+7t1P9m9M1xPrZv5cfnSCguooPDq4rQe/iCdNDwHT # 6XYW6yetxTJMOo4tUDbSS0YiZr7Mab2wkjgNFa0jRFheS9daTS1oJ/z5bNlGinxq # 2v8azSP/GcH/t8eTrHQfcax3WbPELoGHIbryrSUaOCphsnCNUqUN5FbEMlat5MuY # 94rGMJnq1IEd6S8ngK6C8E9SWpGEO3NDa0NlAViorpGfI0NYIbdynyOB846aWAjN # fgThIcdzdWFvAl/6ktWXLETn8u/lYQyWGmul3yz+w06puIPD9p4KPiWBkCesKDHv # XLrT3BbLZ8dKqSOV8DtzLFAfc9qAsNiG8EoathluJBsbyFbpebadKlErFidAX8KE # usk8htHqiSkNxydamL/tKfx3V/vDAoQE59ysv4r3pE+zdyfMairvkFNNw7cPn1kH # Gcww9dFSY2QwAxhMzmoM0G+M+YvBnBu5wjfxNrMRilRbxM6Cj9hKFh0YTwba6M7z # ntHHpX3d+nabjFm/TnMRROOgIXJzYbzKKaO2g1kWeyG2QtvIR147zlrbQD4X10Ab # rRg9CpwW7xYxywezj+iNAc+QmFzR94dzJkEPUSCJPsTFMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAANNTpGmGiiweI8AAAAA # A00wDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIBY4 # q+Bo2vE9dWTtA27T9YeV2lbgCVFlbIrGNK2RyG4jMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAtCNRQvZuK/YSNPOJfbg7AMgTjIPrQjdAb2pm # 7bNB5wvXoz7yHnL9xN5loLoOGYKlymFtRHC3QDF6TD1ArFnkCADmg8Q1I6XrggSO # BjtdgeKM5bkefNZ+uBR4V7h7Wwc5px/mL9kFm7hqvjwgv9B2rKNBVHBsKC46usif # ZFe/Er3MY31hTx6zu9isu+GJXGcTyGtKpNCWadRp2oUO5CN4WPYEmyb1DTpcPEJf # 7pzJh/nJ7OeKjnfd5KsnL/yHDpwEdE8Vg/15I2301xfzBF+pgl+2Pi5jMWkHWxCZ # c8b9wmAaDvVyHt0qACoqedXa0+llQ1M6GCbhm1ST6hcONlArO6GCFywwghcoBgor # BgEEAYI3AwMBMYIXGDCCFxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFZBgsqhkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCDITCmQqA+GWwjVGdQijKYtn8E7Aw/SnMT0 # S5ieve8jwwIGZJNEvY5iGBMyMDIzMDcxMzE5MjUyOC43ODdaMASAAgH0oIHYpIHV # MIHSMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsT # HVRoYWxlcyBUU1MgRVNOOkZDNDEtNEJENC1EMjIwMSUwIwYDVQQDExxNaWNyb3Nv # ZnQgVGltZS1TdGFtcCBTZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAG59gAN # ZVRPvAMAAQAAAbkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAg # UENBIDIwMTAwHhcNMjIwOTIwMjAyMjE3WhcNMjMxMjE0MjAyMjE3WjCB0jELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9z # b2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMg # VFNTIEVTTjpGQzQxLTRCRDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt # U3RhbXAgU2VydmljZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAONJ # Pslh9RbHyQECbUIINxMF5uQkyN07VIShITXubLpWnANgBCLvCcJl7o/2HHORnsRc # mSINJ/qclAmLIrOjnYnrbocAnixiMEXC+a1sZ84qxYWtEVY7VYw0LCczY+86U/8s # hgxqsaezKpWriPOcpV1Sh8SsOxf30yO7jvld/IBA3T6lHM2pT/HRjWk/r9uyx0Q4 # atx0mkLVYS9y55/oTlKLE00h792S+maadAdy3VgTweiwoEOXD785wv3h+fwH/wTQ # tC9lhAxhMO4p+OP9888Wxkbl6BqRWXud54RTzqp2Vr+yen1Q1A6umyMB7Xq0snIY # G5B1Acc4UgJlPQ/ZiMkqgxQNFCWQvz0G9oLgSPD8Ky0AkX22PcDOboPuNT4RceWP # X0UVZUsX9IUgs7QF41HiQSwEeOOHGyrfQdmSslATrbmH/18M5QrsTM5JINjct9G4 # 2xqN8VF9Z8WOiGMjNbvlpcEmmysYl5QyhrEDoFnQTU7bFrD3JX0fIfu1sbLWeBqX # wbp4Z8yACTtphK2VbzOvi4vc0RCmRNzvYQQ2PjZ7NaTXE4Gu3vggAJ+rtzUTAfJo # tvOSqcMgNwLZa1Y+ET/lb0VyjrYwFuHtg0QWyQjP5350LTpv086pyVUh4A3w/Os5 # hTGFZgFe5bCyMnpY09M0yPdHaQ/56oYUsSIcyKyVAgMBAAGjggFJMIIBRTAdBgNV # HQ4EFgQUt7A4cdtYQ5oJjE1ZqrSonp41RFIwHwYDVR0jBBgwFoAUn6cVXQBeYl2D # 9OXSZacbUzUZ6XIwXwYDVR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3Nv # ZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy # MDIwMTAoMSkuY3JsMGwGCCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1l # LVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUB # Af8EDDAKBggrBgEFBQcDCDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQAD # ggIBAM3cZ7NFUHRMsLKzjl7rJPIkv7oJ+s9kkut0hZif9WSt60SzYGULp1zmdPqc # +w8eHTkhqX0GKCp2TTqSzBXBhwHOm8+p6hUxNlDewGMZUos952aTXblAT3OKBnfV # BLQyUavrSjuJGZAW30cNY3rjVDUlGD+VygQHySaDaviJQbK6/6fQvUUFoqIk3ldG # fjnAtnebsVlqh6WWamVc5AZdpWR1jSzN/oxKYqc1BG4SxxlPtcfrAdBz/cU4bxVX # qAAf02NZscvJNpRnOALf5kVo2HupJXCsk9TzP5PNW2sTS3TmwhIQmPxr0E0UqOoj # UrBJUOhbITAxcnSa/IMluL1HXRtLQZI+xs2eRtuPOUsKUW71/1YeqsYCLHLvu82c # eDVQQvP7GHEEkp2kEjiofbjYErBo2iCEaxxeX4Z9HvAgA4MsQkbn6e4EFQf13sP+ # Kn3XgMIvJbqLJeFcQja+SUeOXu5cfkxe0GzTNojdyIwzaHlhOflVRZNrxee3B+yZ # wd3JHDIvv71uSI/SIzzt9cU2GyHQVqxBSrRtKW6W8Vw7zpVvoVsIv3ljxg+7NiGS # lXX1s7zbBNDMUj9OnzOlHK/3mrOU8YEuRf6RwakW5UCeGamy5MiKu2YuyKiGBCv4 # OGhPstNe7ALkEOh8BX12t4ntuYu+gw9L6yCPY0jWYaQtzAP9MIIHcTCCBVmgAwIB # AgITMwAAABXF52ueAptJmQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UE # BhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAc # BgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0 # IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1 # WhcNMzAwOTMwMTgzMjI1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O # 1YLT/e6cBwfSqWxOdcjKNVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZn # hUYjDLWNE893MsAQGOhgfWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t # 1w/YJlN8OWECesSq/XJprx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxq # D89d9P6OU8/W7IVWTe/dvI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmP # frVUj9z6BVWYbWg7mka97aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSW # rAFKu75xqRdbZ2De+JKRHh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv # 231fgLrbqn427DZM9ituqBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zb # r17C89XYcz1DTsEzOUyOArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYcten # IPDC+hIK12NvDMk2ZItboKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQc # xWv2XFJRXRLbJbqvUAV6bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17a # j54WcmnGrnu3tz5q4i6tAgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQAB # MCMGCSsGAQQBgjcVAgQWBBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQU # n6cVXQBeYl2D9OXSZacbUzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEw # QTA/BggrBgEFBQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9E # b2NzL1JlcG9zaXRvcnkuaHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/ # MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJ # oEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01p # Y1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYB # BQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9v # Q2VyQXV0XzIwMTAtMDYtMjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3h # LB9nATEkW+Geckv8qW/qXBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x # 5MKP+2zRoZQYIu7pZmc6U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74p # y27YP0h1AdkY3m2CDPVtI1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1A # oL8ZthISEV09J+BAljis9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbC # HcNhcy4sa3tuPywJeBTpkbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB # 9s7GdP32THJvEKt1MMU0sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNt # yo4JvbMBV0lUZNlz138eW0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3 # rsjoiV5PndLQTHa1V1QJsWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcV # v7TOPqUxUYS8vwLBgqJ7Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A24 # 5oyZ1uEi6vAnQj0llOZ0dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lw # Y1NNje6CbaUFEMFxBmoQtB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB # 0jELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMk # TWljcm9zb2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1U # aGFsZXMgVFNTIEVTTjpGQzQxLTRCRDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsOAwIaAxUAx2IeGHhk58MQkzzS # WknGcLjfgTqggYMwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAN # BgkqhkiG9w0BAQUFAAIFAOhawyEwIhgPMjAyMzA3MTQwMjM4MjVaGA8yMDIzMDcx # NTAyMzgyNVowdzA9BgorBgEEAYRZCgQBMS8wLTAKAgUA6FrDIQIBADAKAgEAAgIg # TgIB/zAHAgEAAgIShjAKAgUA6FwUoQIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgor # BgEEAYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUA # A4GBAA0VHaQnZRwItOCX0CUpdfOMptYbl6XtIYD1ycnZainPjDgV0yFLK4YLzEqe # PRMY8MjUqxL+PKBlfrFiPBKnOWIua5ycAy6GEnT9SWcMaq4f7z7/lipMF5Bjw9Iy # DURfn8iAdVqDrq5fLVYGn/PcZncBjj0TrrYpApUsspWai471MYIEDTCCBAkCAQEw # gZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAG59gANZVRPvAMA # AQAAAbkwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0B # CRABBDAvBgkqhkiG9w0BCQQxIgQgpZN/wu93MRuwNYGBwYn2ovmam+O+UyFzUkFn # aB2dMRAwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9BCBk60bO8W85uTAfJVEO # 3vX2aLaQFcgcGpdwsOoi+foP9DCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwAhMzAAABufYADWVUT7wDAAEAAAG5MCIEIP/xny8MwjfoZng34WJL # cFvTdj5++ECqtNMNJqI/e10PMA0GCSqGSIb3DQEBCwUABIICALG2+PYCfVOpfCu3 # 0lxdtZmjpgtfEJSxhwSLDqfAkl3oufAMNhJW4lD/MlWS2RyDWQsTUK49NYum3YW3 # cl5Ojrgc3GLHE5D4ABOMdeiWBJ5+Aok3krKYXiTsBDibtQN9FkR9dbFJFQYTJdOd # N4iN3j8pRFV4eV4tbcAFp9Vd3/mzUXWDxEN9qgOFz3QMksx/jIn5fRUKtmPSXi72 # D0VWIhfePmLHLBr7j38G3mJk+UfnwamjlEo5Tl+JSPHGoxgGl/i+QI+2k2MyKdu3 # 5ODJATq7jMUoHWtrRC+k4u7cccguT0t39Q5i6cMOJ9+2r+xH/JtIjflo7NGjkprZ # 40SVA/KdNG5ueiyhE7BFET+W06VrZRPDN4Hhs3aJUgyJuotfxNGYOnzxYLNxubT2 # ZFAjgnZWV6HuW0C/iZ5TDxmLBTXM4odGuBi0714xduPrzSYQOGcrySWCq7IQ0Z27 # OQco9vQ4Uvpq2Az2yRiRqm1YmAeUaTw5oXEyDVkdXu2DZJtF2iiMwi7vsGAdYqiA # d+44pFhRpElBs2EnIzPDHxlmVt7WRGz5XWrGpT9IZVRLqesn8jGxjS/H32slKta/ # 1axA6utlh3tUz1EPHE6MfXFne1rV6dV/QVLypSTTRTp3rOFlks62fiNL60grQPOH # qpfqXEEzbDYfcGWCUlzbwIkV8Kxi # SIG # End signature block |