SDNExpress.ps1
# -------------------------------------------------------------- # Copyright © Microsoft Corporation. All Rights Reserved. # Microsoft Corporation (or based on where you live, one of its affiliates) licenses this sample code for your internal testing purposes only. # Microsoft provides the following sample code AS IS without warranty of any kind. The sample code arenot supported under any Microsoft standard support program or services. # Microsoft further disclaims all implied warranties including, without limitation, any implied warranties of merchantability or of fitness for a particular purpose. # The entire risk arising out of the use or performance of the sample code remains with you. # In no event shall Microsoft, its authors, or anyone else involved in the creation, production, or delivery of the code be liable for any damages whatsoever # (including, without limitation, damages for loss of business profits, business interruption, loss of business information, or other pecuniary loss) # arising out of the use of or inability to use the sample code, even if Microsoft has been advised of the possibility of such damages. # --------------------------------------------------------------- <# .SYNOPSIS Deploys and configures the Microsoft SDN infrastructure, including creation of the network controller, Software Load Balancer MUX and gateway VMs. Then the VMs and Hyper-V hosts are configured to be used by the Network Controller. When this script completes the SDN infrastructure is ready to be fully used for workload deployments. .EXAMPLE .\SDNExpress.ps1 -ConfigurationDataFile .\MyConfig.psd1 Reads in the configuration from a PSD1 file that contains a hash table of settings data. .EXAMPLE .\SDNExpress -ConfigurationData $MyConfigurationData Uses the hash table that is passed in as the configuration data. This parameter set is useful when programatically generating the configuration data. .EXAMPLE .\SDNExpress Displays a user interface for interactively defining the configuraiton data. At the end you have the option to save as a configuration file before deploying. .NOTES Prerequisites: * All Hyper-V hosts must have Hyper-V enabled and the Virtual Switch already created. * All Hyper-V hosts must be joined to Active Directory. * The physical network must be preconfigured for the necessary subnets and VLANs as defined in the configuration data. * The VHD specified in the configuration data must be reachable from the computer where this script is run. #> param( [Parameter(Mandatory=$true,ParameterSetName="ConfigurationFile")] [String] $ConfigurationDataFile=$null, [Parameter(Mandatory=$true,ParameterSetName="ConfigurationData")] [object] $ConfigurationData=$null, [Switch] $SkipValidation, [Switch] $SkipDeployment, [PSCredential] $DomainJoinCredential = $null, [PSCredential] $NCCredential = $null, [PSCredential] $LocalAdminCredential = $null ) function IsNotNullOrEmpty($value) { return -not [string]::IsNullOrEmpty($value) } function IsNotNullOrEmptyOrZero($value) { return -not ([string]::IsNullOrEmpty($value) -or ($value -eq 0)) } function ValidatePrivateVIPSubnetConfigNotEmpty { if ([string]::IsNullOrEmpty($ConfigData.PrivateVIPGateway)) { throw "PrivateVIPGateway is required to create a PrivateVIPSubnet" } if ([string]::IsNullOrEmpty($ConfigData.PrivateVIPPoolStart)) { throw "PrivateVIPPoolStart is required to create a PrivateVIPSubnet" } if ([string]::IsNullOrEmpty($ConfigData.PrivateVIPPoolEnd)) { throw "PrivateVIPPoolEnd is required to create a PrivateVIPSubnet" } } function ValidatePublicVIPSubnetConfigNotEmpty { if ([string]::IsNullOrEmpty($ConfigData.PublicVIPGateway)) { throw "PublicVIPGateway is required to create a PublicVIPSubnet" } if ([string]::IsNullOrEmpty($ConfigData.PublicVIPPoolStart)) { throw "PublicVIPPoolStart is required to create a PublicVIPSubnet" } if ([string]::IsNullOrEmpty($ConfigData.PublicVIPPoolEnd)) { throw "PublicVIPPoolEnd is required to create a PublicVIPSubnet" } } <# This function is used to resolve the credentials to use for the deployment. If the credentials are not provided on the command line, the user is prompted. #> function ResolveCredentials { param( [PSCredential] $Credential, [String] $Message ) if ($null -eq $Credential) { write-sdnexpresslog "No credentials found on command line or in config file. Prompting." $Credential = get-Credential -Message $Message # return secure string return $Credential.Password } if ($null -ne $Credential) { write-sdnexpresslog "Using credentials from the command line." # return secure string return $Credential.Password } write-sdnexpresslog "Using credentials from config file." # this is no longer deemed safe, password and credentials may not be provided through config files. throw "Providing credentials through the configuration file is not considered secure. Please provide credentials directly when invoking sdnexpress " } function GetNextMacAddress { param( [String] $MacAddress ) return ("{0:X12}" -f ([convert]::ToInt64($MacAddress.ToUpper().Replace(":", "").Replace("-", ""), 16) + 1)).Insert(2, "-").Insert(5, "-").Insert(8, "-").Insert(11, "-").Insert(14, "-") } function ValidateMacPools { $curInstallDirectory = $Global:installDirectory Import-Module Moc -ErrorAction SilentlyContinue if ($null -eq $(Get-Module -Name Moc)) { Write-SDNExpressLog "MOC Module not found. Skipping check for conflicting MOC and SDN resources." return } try { try { $mocNotInstalled = (Get-mocconfig -ErrorAction SilentlyContinue).installState -eq "NotInstalled" if ($mocnotInstalled) { Write-SDNExpressLog "No MOC installation found. Skipping check for conflicting MOC and SDN resources." return } } catch { Write-SDNExpressLog "No MOC installation found. Skipping check for conflicting MOC and SDN resources." return } $loc = get-moclocation -ErrorAction SilentlyContinue if (-not $loc) { Write-SDNExpressLog "No MOC location found. Skipping check for conflicting MOC and SDN resources." return } $sdnStart = $ConfigData.SDNMacPoolStart -Replace "-","" $sdnEnd = $ConfigData.SDNMacPoolEnd -Replace "-","" foreach($l in $loc) { $mocMacPool = Get-MocMacPool -location $l.Name if (-not $mocMacPool) { continue } try { $mocStart = $mocMacPool.Properties.range.startmacaddress -Replace ":","" $mocEnd = $mocMacPool.Properties.range.endmacaddress -Replace ":","" } catch { throw "Could not determine the MOC MAC pool start or end MAC address." } if ((IsMac1GreaterThanMac2 -mac1 $sdnStart -mac2 $mocStart) ` -and (IsMac1GreaterThanMac2 -mac1 $sdnStart -mac2 $mocEnd)) { continue } if ((IsMac1GreaterThanMac2 -mac1 $mocStart -mac2 $sdnEnd) ` -and (IsMac1GreaterThanMac2 -mac1 $mocEnd -mac2 $sdnEnd)) { continue } throw "The SDN MAC pool specified in the configuration file conflicts with the MOC MAC pools." } } finally { $Global:installDirectory = $curInstallDirectory } } function IsMac1GreaterThanMac2 { param( [string]$mac1, [string]$mac2 ) $mac1Bytes = [System.Net.NetworkInformation.PhysicalAddress]::Parse($mac1).GetAddressBytes() $mac2Bytes = [System.Net.NetworkInformation.PhysicalAddress]::Parse($mac2).GetAddressBytes() $i = 0 while ($i -lt $mac2Bytes.Count) { if ($mac1Bytes[$i] -gt $mac2Bytes[$i]) { return $true } if ($mac2Bytes[$i] -gt $mac1Bytes[$i]) { return $false } $i++ } return $false } # returns a simple lexicographical based comparison of two OS versions function IsOSVersionGreaterOrEqual { param( [string] $MinVersion, [string] $TestVersion ) return [string]::Compare($TestVersion, $MinVersion, [System.StringComparison]::Ordinal) -ge 0 } # Feature specific checks for supported versions and functionality function ValidateSdnConfiguration { param( [hashtable] $config, [switch] $SkipValidation ) $hostInfo = Invoke-Command -ComputerName $config.hypervhosts[0] -ScriptBlock { Get-ComputerInfo } if($SkipValidation) { Write-SDNExpressLog "Skipping validation checks." return } # Feature specific checks for supported versions and functionality # -- subjectName on < 22H2 if($config.UseCertBySubject -eq $true) { if (-not (IsOSVersionGreaterOrEqual -MinVersion "23H2" -TestVersion $hostInfo.OSDisplayVersion)) { throw "Subject name credential object is not supported on anything below 23H2. Please upgrade your target cluster" } } } try { # Script version, should be matched with the config files $ScriptVersion = "4.0" if ((get-wmiobject win32_operatingsystem).caption.Contains("Windows 10")) { get-windowscapability -name rsat.NetworkController.Tools* -online | Add-WindowsCapability -online } else { $feature = get-windowsfeature "RSAT-NetworkController" if ($null -eq $feature) { throw "SDN Express requires Windows Server 2016 or later." } if (!$feature.Installed) { add-windowsfeature "RSAT-NetworkController" } } import-module .\SDNExpress.psm1 -force write-SDNExpressLog "*** Begin SDN Express Deployment ***" write-SDNExpressLog "ParameterSet: $($psCmdlet.ParameterSetName)" write-SDNExpressLog " -ConfigurationDataFile: $ConfigurationDataFile" write-SDNExpressLog " -ConfigurationData: $ConfigurationData" write-SDNExpressLog " -SkipValidation: $SkipValidation" write-SDNExpressLog " -SkipDeployment: $SkipValidation" Write-SDNExpressLog "Host name $(hostname)" $sdnexpm = find-module sdnexpress if ($sdnexpm -ne $null) { Write-SDNExpressLog "SdnExpress version $($sdnexpm.version)" } if ($psCmdlet.ParameterSetName -eq "ConfigurationFile") { write-sdnexpresslog "Using configuration file passed in by parameter." $configdata = [hashtable] (Invoke-Expression (Get-Content $ConfigurationDataFile | out-string)) } elseif ($psCmdlet.ParameterSetName -eq "ConfigurationData") { write-sdnexpresslog "Using configuration data object passed in by parameter." $configdata = $configurationData } ValidateSdnConfiguration -config $configdata -SkipValidation $SkipValidation $IsL2ForwardingEnabled = $false if ($ConfigData.IsL2ForwardingEnabled) { $IsL2ForwardingEnabled = $true } write-sdnexpresslog "IsL2ForwardingEnabled : $IsL2ForwardingEnabled" # if FCNC is enabled, load the modules if ($configdata.UseFCNC) { if ((Get-CimInstance Win32_OperatingSystem).OperatingSystemSKU -eq "406" -and $configdata.UseFCNC -ne "1975") { write-sdnexpresslog "The native SDN is not supported on the Azure Stack HCI SKU" throw "The native SDN is not supported on the Azure Stack HCI SKU" } if(-not [string]::IsNullOrEmpty($Global:FCNC_MODULE_PATH_ROOT)) { ipmo (Join-Path $Global:FCNC_MODULE_PATH_ROOT -ChildPath NetworkControllerFc.psd1) -Force -Scope Global } else { import-Module NetworkControllerFc -ErrorAction SilentlyContinue if ($null -eq (Get-Module NetworkControllerFc)) { ipmo ..\NetworkControllerFc\NetworkControllerFc.psd1 -Force -Scope Global } } # rename and copy package if([string]::IsNullOrEmpty($configdata.FCNCPackage) -eq $false) { write-sdnexpresslog "looking for FCNC package $($configdata.FCNCPackage)" # check if the package exists if (Test-Path $configdata.FCNCPackage) { write-sdnexpresslog "FCNC package found" $configdata.FCNCBins = $configdata.FCNCPackage } else { write-sdnexpresslog "FCNC package not found" throw "FCNC package not found" } # copy the nuget to a temp file, rename to zip , decompress it and delete the temp file write-sdnexpresslog "copying FCNC package to $($configdata.FCNCBins)" Copy-Item $configdata.FCNCPackage "$($configdata.FCNCPackage).zip" -Verbose $configdata.FCNCBins = $configdata.FCNCPackage.Replace(".nupkg", ".zip") Copy-Item $configdata.FCNCPackage $configdata.FCNCBins -Force write-sdnexpresslog "unzipping FCNC package" Expand-Archive -Path $configdata.FCNCBins -DestinationPath $configdata.FCNCBins.Replace(".zip", "") -Force $configdata.FCNCBins = $configdata.FCNCBins.Replace(".zip", "") } } if ($Configdata.ScriptVersion -ne $scriptversion) { write-error "Configuration file version $($ConfigData.ScriptVersion) is not compatible with this version of SDN express. Please update your config file to match the version $scriptversion example." return } if ($DomainJoinCredential -eq $null) { throw "Domain Join credentials are required." } if($null -eq $NCCredential) { throw "NC credentials are required." } if($null -eq $LocalAdminCredential) { throw "Local Admin credentials are required." } # grab the secure passwords from the credentials $DomainJoinPassword = $DomainJoinCredential.Password $NCPassword = $NCCredential.Password $LocalAdminPassword = $LocalAdminCredential.Password $credential = $DomainJoinCredential if (![string]::IsNullOrEmpty($ConfigData.ManagementSubnet)) { $ManagementSubnetBits = $ConfigData.ManagementSubnet.Split("/")[1] } if ([string]::IsNullOrEmpty($ConfigData.PASubnet)) { if ($ConfigData.Muxes.Count -gt 0) { throw "Load Balancer Mux configuration requires a PA Subnet." } if ($ConfigData.Gateways.Count -gt 0) { throw "Gateway configuration requires a PA Subnet." } } # check for installed features Install-SdnWindowsFeatures -hyperVHosts $configdata.hypervhosts -credential $credential -isFCNC $configdata.UseFCNC $hasPrivateVIPSubnet = IsNotNullOrEmpty($ConfigData.PrivateVIPSubnet) $hasPrivateVIPGateway = IsNotNullOrEmpty($ConfigData.PrivateVIPGateway) $hasPrivateVIPPoolStart = IsNotNullOrEmpty($ConfigData.PrivateVIPPoolStart) $hasPrivateVIPPoolEnd = IsNotNullOrEmpty($ConfigData.PrivateVIPPoolEnd) if($hasPrivateVIPSubnet -and ($IsL2ForwardingEnabled -or $hasPrivateVIPGateway -or $hasPrivateVIPPoolStart -or $hasPrivateVIPPoolEnd)) { ValidatePrivateVIPSubnetConfigNotEmpty } $hasPublicVIPSubnet = IsNotNullOrEmpty($ConfigData.PublicVIPSubnet) $hasPublicVIPGateway = IsNotNullOrEmpty($ConfigData.PublicVIPGateway) $hasPublicVIPPoolStart = IsNotNullOrEmpty($ConfigData.PublicVIPPoolStart) $hasPublicVIPPoolEnd = IsNotNullOrEmpty($ConfigData.PublicVIPPoolEnd) if($hasPublicVIPSubnet -and ($IsL2ForwardingEnabled -or $hasPublicVIPGateway -or $hasPublicVIPPoolStart -or $hasPublicVIPPoolEnd)) { ValidatePublicVIPSubnetConfigNotEmpty } if($IsL2ForwardingEnabled -and (IsNotNullOrEmptyOrZero($ConfigData.PrivateVIPVLANID)) -and (IsNotNullOrEmptyOrZero($ConfigData.PublicVIPVLANID)) -and ($ConfigData.PrivateVIPVLANID -eq $ConfigData.PublicVIPVLANID)) { throw "Private/Public VIP subnet with non-zero VlanIds can not be same." } if (($ConfigData.Muxes.count -gt 0) -or ($ConfigData.Gateways.count -gt 0)) { $PASubnetBits = $ConfigData.PASubnet.Split("/")[1] } ValidateMacPools if ($ConfigData.DomainJoinUserName -ne $null) { $DomainJoinUserNameDomain = $ConfigData.DomainJoinUserName.Split("\")[0] $DomainJoinUserNameName = $ConfigData.DomainJoinUserName.Split("\")[1] } if ($ConfigData.LocalAdminDomainUser -ne $null) { $LocalAdminDomainUserDomain = $ConfigData.LocalAdminDomainUser.Split("\")[0] $LocalAdminDomainUserName = $ConfigData.LocalAdminDomainUser.Split("\")[1] } if ($null -eq $ConfigData.VMProcessorCount) {$ConfigData.VMProcessorCount = 8} if ($null -eq $ConfigData.VMMemory) {$ConfigData.VMMemory = 8GB} if ($null -eq $ConfigData.DisableIPv6DHCP) {$ConfigData.DisableIPv6DHCP = $false} if ([string]::IsNullOrEmpty($ConfigData.PoolName)) {$ConfigData.PoolName = "DefaultAll"} write-SDNExpressLog "STAGE 1: Create VMs" $createparams = @{ 'ComputerName'=''; 'VMLocation'=$ConfigData.VMLocation; 'VMName'=''; 'VHDSrcPath'=$ConfigData.VHDPath; 'VHDName'=$ConfigData.VHDFile; 'VMMemory'=$ConfigData.VMMemory; 'VMProcessorCount'=$ConfigData.VMProcessorCount; 'Nics'=@(); 'CredentialDomain'=$DomainJoinUserNameDomain; 'CredentialUserName'=$DomainJoinUserNameName; 'CredentialPassword'=$DomainJoinPassword; 'JoinDomain'=$ConfigData.JoinDomain; 'LocalAdminPassword'=$LocalAdminPassword; 'DomainAdminDomain'=$LocalAdminDomainUserDomain; 'DomainAdminUserName'=$LocalAdminDomainUserName; 'SwitchName'=$ConfigData.SwitchName; 'DisableIPv6DHCP'=$ConfigData.DisableIPv6DHCP } if (![String]::IsNullOrEmpty($ConfigData.ProductKey)) { $createparams.ProductKey = $ConfigData.ProductKey } if (![String]::IsNullOrEmpty($ConfigData.Locale)) { $createparams.Locale = $ConfigData.Locale } if (![String]::IsNullOrEmpty($ConfigData.TimeZone)) { $createparams.TimeZone = $ConfigData.TimeZone } write-SDNExpressLog "STAGE 1.0.1: Enable VFP" foreach ($h in $ConfigData.hypervhosts) { write-SDNExpressLog "Enabling VFP on $($h) $($ConfigData.SwitchName)" $s = New-SdnExpressPsSession -ComputerName $h $credential invoke-command -Session $s { param( [String] $VirtualSwitchName ) Enable-VmSwitchExtension -VMSwitchName $VirtualSwitchName -Name "Microsoft Azure VFP Switch Extension" } -ArgumentList $ConfigData.SwitchName $s = New-SdnExpressPsSession -ComputerName $h $credential invoke-command -Session $s { Set-Service -Name NCHostAgent -StartupType Automatic; Start-Service -Name NCHostAgent } } $HostNameIter = 0 $useCertBySubject = $false if ($ConfigData.UseCertBySubject) { $useCertBySubject = $true } if (-not $ConfigData.UseFCNC) { foreach ($NC in $ConfigData.NCs) { if ([string]::IsNullOrEmpty($nc.macaddress)) { $nc.macaddress = $ConfigData.SDNMacPoolStart $configdata.SDNMacPoolStart = GetNextMacAddress($ConfigData.SDNMacPoolStart) } if ([string]::IsNullOrEmpty($nc.HostName)) { $nc.HostName = $ConfigData.HyperVHosts[$HostNameIter] $HostNameIter = ($HostNameIter + 1) % $ConfigData.HyperVHosts.Count } } } foreach ($Mux in $ConfigData.Muxes) { if ([string]::IsNullOrEmpty($Mux.HostName)) { $Mux.HostName = $ConfigData.HyperVHosts[$HostNameIter] $HostNameIter = ($HostNameIter + 1) % $ConfigData.HyperVHosts.Count } # In Bgpless(L2Forwarding) mode PAIPaddress will be created later at SLB configuration stage, so initializing Mux.PAIPAddress only if L2Forwarding is disabled if (-not $IsL2ForwardingEnabled -and [string]::IsNullOrEmpty($Mux.PAIPAddress)) { $Mux.PAIPAddress = $ConfigData.PAPoolStart $ConfigData.PAPoolStart = Get-IPAddressInSubnet -Subnet $ConfigData.PAPoolStart -Offset 1 } } #Allocate GW management MACs from outside of SDN pool foreach ($gateway in $ConfigData.Gateways) { if ([string]::IsNullOrEmpty($Gateway.macaddress)) { $gateway.macaddress = $ConfigData.SDNMacPoolStart $configdata.SDNMacPoolStart = GetNextMacAddress($ConfigData.SDNMacPoolStart) } if ([string]::IsNullOrEmpty($Gateway.HostName)) { $Gateway.HostName = $ConfigData.HyperVHosts[$HostNameIter] $HostNameIter = ($HostNameIter + 1) % $ConfigData.HyperVHosts.Count } } #Allocate GW FE & BE macs, FE IP from within SDN mac and PA pools $nextmac = $configdata.SDNMacPoolStart $PAOffset = 0 foreach ($gateway in $ConfigData.Gateways) { if ([string]::IsNullOrEmpty($Gateway.FrontEndMac)) { $gateway.FrontEndMac = $nextmac $nextmac = GetNextMacAddress($nextmac) } if ([string]::IsNullOrEmpty($Gateway.BackEndMac)) { $gateway.BackEndMac = $nextmac $nextmac = GetNextMacAddress($nextmac) } if ([string]::IsNullOrEmpty($Gateway.FrontEndIP)) { $Gateway.FrontEndIP = Get-IPAddressInSubnet -Subnet $ConfigData.PAPoolStart -Offset $PAOffset $PAOffset += 1 } } if (-not $ConfigData.UseFCNC) { write-SDNExpressLog "STAGE 1.1: Create NC VMs" foreach ($NC in $ConfigData.NCs) { $createparams.ComputerName=$NC.HostName; $createparams.VMName=$NC.ComputerName; if ([string]::IsNullOrEmpty($NC.ManagementIP)) { $createparams.Nics=@( @{Name="Management"; MacAddress=$NC.MacAddress; VLANID=$ConfigData.ManagementVLANID; SwitchName=$NC.ManagementSwitch} ) } else { $createparams.Nics=@( @{Name="Management"; MacAddress=$NC.MacAddress; IPAddress="$($NC.ManagementIP)/$ManagementSubnetBits"; Gateway=$ConfigData.ManagementGateway; DNS=$ConfigData.ManagementDNS; VLANID=$ConfigData.ManagementVLANID; SwitchName=$NC.ManagementSwitch} ) } $createparams.Roles=@("NetworkController","NetworkControllerTools") New-SDNExpressVM @createparams } } write-SDNExpressLog "STAGE 1.2: Create Mux VMs" foreach ($Mux in $ConfigData.Muxes) { $createparams.ComputerName=$mux.HostName; $createparams.VMName=$mux.ComputerName; if ([string]::IsNullOrEmpty($Mux.ManagementIP)) { $createparams.Nics=@( @{Name="Management"; MacAddress=$Mux.MacAddress; VLANID=$ConfigData.ManagementVLANID; SwitchName=$Mux.ManagementSwitch} ) } else { $createparams.Nics=@( @{Name="Management"; MacAddress=$Mux.MacAddress; IPAddress="$($Mux.ManagementIP)/$ManagementSubnetBits"; Gateway=$ConfigData.ManagementGateway; DNS=$ConfigData.ManagementDNS; VLANID=$ConfigData.ManagementVLANID; SwitchName=$Mux.ManagementSwitch} ) } # In BgpLess Mux (L2Forwarding) mode HNVPA adapter will be created later during SLB configuration stage, so only HNVPA adapter if L2Forwarding is disabled if (-not $IsL2ForwardingEnabled) { $createparams.Nics += @{Name="HNVPA"; MacAddress=$Mux.PAMacAddress; IPAddress="$($Mux.PAIPAddress)/$PASubnetBits"; VLANID=$ConfigData.PAVLANID; IsMuxPA=$true} } $createparams.Roles=@("SoftwareLoadBalancer") New-SDNExpressVM @createparams } $ncVmUpgradeNeeded = 0 if ($ConfigData.NCs.count -gt 0 -or $ConfigData.UseFCNC) { write-SDNExpressLog "STAGE 2: Network Controller Configuration" $NCNodes = @() if ($ConfigData.UseFCNC) { if ([string]::IsNullOrEmpty($ConfigData.FCNCBins)) { $ConfigData.FCNCBins = "C:\Windows\NetworkController" } $NCNodes = $ConfigData.HyperVHosts $params = @{ 'Credential'=$Credential 'RestName'=$ConfigData.RestName 'RestIpAddress'=$ConfigData.RestIpAddress 'ComputerNames'=$NCNodes 'FCNCBins' = $ConfigData.FCNCBins 'FCNCDBs' = $ConfigData.FCNCDBs 'ClusterNetworkName' = $ConfigData.ClusterNetworkName 'UseCertBySubject' = $useCertBySubject 'CertificatePassword' = $NCPassword } New-FCNCNetworkController @params } else { foreach ($NC in $ConfigData.NCs) { $NCNodes += $NC.ComputerName } WaitforComputerToBeReady -ComputerName $NCNodes -Credential $Credential $params = @{ 'Credential'=$Credential 'RestName'=$ConfigData.RestName 'RestIpAddress'=$ConfigData.RestIpAddress 'ComputerNames'=$NCNodes 'UseCertBySubject' = $useCertBySubject 'CertificatePassword' = $NCPassword } if (![string]::IsNullOrEmpty($ConfigData.ManagementSecurityGroup)) { $params.ManagementSecurityGroupName = $ConfigData.ManagementSecurityGroup $params.ClientSecurityGroupName = $ConfigData.ClientSecurityGroup } $ncVmUpgradeNeeded = New-SDNExpressNetworkController @params Import-SdnExpressCARootIntoNCVMs -NCVMs $NCNodes -Hosts $ConfigData.hypervhosts -Credential $Credential } write-SDNExpressLog "STAGE 2.1: Getting REST cert thumbprint in order to find it in local root store." # Check through nodes until we find a node that was originally set up with $NCHostCertThumb = $null $nodeIdx = 0 while ($null -eq $NCHostCertThumb -and $nodeIdx -lt $NCNodes.length) { $s = New-SdnExpressPsSession -ComputerName $NCNodes[$nodeIdx] $credential $NCHostCertThumb = invoke-command -Session $s { param( $RESTName, [String] $funcDefGetSdnCert ) . ([ScriptBlock]::Create($funcDefGetSdnCert)) $Cert = GetSdnCert -subjectName $RestName.ToUpper() return $cert.Thumbprint } -ArgumentList $ConfigData.RestName, $Global:fdGetSdnCert $nodeIdx++ } $NCHostCert = get-childitem "cert:\localmachine\root\$NCHostCertThumb" $params = @{ 'RestName' = $ConfigData.RestName; 'MacAddressPoolStart' = $ConfigData.SDNMacPoolStart; 'MacAddressPoolEnd' = $ConfigData.SDNMacPoolEnd; 'NCHostCert' = $NCHostCert 'NCUsername' = $ConfigData.NCUsername; 'NCPassword' = $NCPassword 'UseCertBySubject' = $useCertBySubject } New-SDNExpressVirtualNetworkManagerConfiguration @Params -Credential $Credential if (![string]::IsNullOrEmpty($ConfigData.PASubnet)) { $params = @{ 'RestName' = $ConfigData.RestName; 'AddressPrefix' = $ConfigData.PASubnet; 'VLANID' = $ConfigData.PAVLANID; 'DefaultGateways' = $ConfigData.PAGateway; 'IPPoolStart' = $ConfigData.PAPoolStart; 'IPPoolEnd' = $ConfigData.PAPoolEnd } Add-SDNExpressVirtualNetworkPASubnet @params -Credential $Credential } else { write-SDNExpressLog "PA subnets not specified in configuration, skipping Virtual Network PA configuration." } } else { $NCHostCert = GetSdnCert -subjectName $configdata.RestName -store "cert:\localmachine\root" if ($null -eq $NCHostCert) { $ErrorText = "Network Controller cert with CN=$($configdata.RestName) not found on $(hostname) in cert:\localmachine\root" write-SDNExpressLog $ErrorText throw $ErrorText } } $useFcNc = $false if ($ConfigData.UseFCNC) { $useFcNc = $true } if ($ConfigData.Muxes.Count -gt 0) { write-SDNExpressLog "STAGE 3: SLB Configuration" if (![string]::IsNullOrEmpty($ConfigData.PrivateVIPSubnet)) { $params = @{ 'RestName' = $ConfigData.RestName; 'PrivateVIPPrefix' = $ConfigData.PrivateVIPSubnet; 'PublicVIPPrefix' = $ConfigData.PublicVIPSubnet; 'IsL2ForwardingEnabled' = $IsL2ForwardingEnabled; } if ($hasPrivateVIPGateway) { $params.Add('PrivateVIPGateway', $ConfigData.PrivateVIPGateway) } if ($hasPrivateVIPPoolStart) { $params.Add('PrivateVIPPoolStart', $ConfigData.PrivateVIPPoolStart) } if ($hasPrivateVIPPoolEnd) { $params.Add('PrivateVIPPoolEnd', $ConfigData.PrivateVIPPoolEnd) } if (IsNotNullOrEmpty($ConfigData.PrivateVIPVLANID)) { $params.Add('PrivateVIPVLANID', $ConfigData.PrivateVIPVLANID) } if ($hasPublicVIPGateway) { $params.Add('PublicVIPGateway', $ConfigData.PublicVIPGateway) } if ($hasPublicVIPPoolStart) { $params.Add('PublicVIPPoolStart', $ConfigData.PublicVIPPoolStart) } if ($hasPublicVIPPoolEnd) { $params.Add('PublicVIPPoolEnd', $ConfigData.PublicVIPPoolEnd) } if (IsNotNullOrEmpty($ConfigData.PublicVIPVLANID)) { $params.Add('PublicVIPVLANID', $ConfigData.PublicVIPVLANID) } New-SDNExpressLoadBalancerManagerConfiguration @Params -Credential $Credential } else { write-SDNExpressLog "VIP subnets not specified in configuration, skipping load balancer manager configuration." } WaitforComputerToBeReady -ComputerName $ConfigData.Muxes.ComputerName -Credential $Credential foreach ($Mux in $ConfigData.muxes) { if($IsL2ForwardingEnabled) { $params = @{ 'RestName'=$ConfigData.RestName 'ComputerName'=$Mux.computername 'JoinDomain'=$ConfigData.JoinDomain 'InternalNicLogicalNetworkName' = "HNVPA" 'InternalNicAddressPrefix' = $ConfigData.PASubnet } $Result = Initialize-SDNExpressMuxNic @params -Credential $Credential $Mux.ExtMacAddress = $Result.ExternalMac $Mux.PAIPAddress = $Result.InternalIP $Mux.PAMacAddress = $Result.InternalMac $params = @{ 'ComputerName' = $mux.HostName; 'VMName' = $mux.ComputerName; 'SwitchName' = $ConfigData.SwitchName; 'Nics' = @( @{Name="HNVPA"; MacAddress=$Mux.PAMacAddress; InstanceId="{$([Guid]::Empty)}"; VlanId=$ConfigData.PAVLANID; IsMuxPA=$true}, @{Name="External"; MacAddress=$Mux.ExtMacAddress; InstanceId="{$($Result.ExternalNicInstanceId)}"; IsMuxExt=$true} ); } Add-SDNExpressMuxNic @params -Credential $Credential $params = @{ 'ComputerName' = $mux.ComputerName; 'MacAddress' = $Mux.PAMacAddress; 'IPAddress' = $Mux.PAIPAddress; 'AddressPrefix' = $ConfigData.PASubnet.Split("/") | Select-Object -Last 1; } # Configuring the HNVPA IPAddress on mux HNVPA adapter Add-IPAddress @params -Credential $Credential } $params = @{ 'ComputerName' = $Mux.ComputerName 'PAMacAddress' = $Mux.PAMacAddress 'ExtMacAddress' = $Mux.ExtMacAddress 'PAGateway' = $ConfigData.PAGateway 'LocalPeerIP' = $Mux.PAIPAddress 'MuxASN' = $ConfigData.SDNASN 'Routers' = $ConfigData.Routers 'RestName' = $ConfigData.RestName 'NCHostCert' = $NCHostCert 'Credential' = $Credential 'IsFC' = $useFcNc 'IsL2ForwardingEnabled' = $IsL2ForwardingEnabled } Add-SDNExpressMux @params } } write-SDNExpressLog "STAGE 4: Host Configuration" $params = @{} if (![string]::IsNullOREmpty($ConfigData.PASubnet)) { $params.HostPASubnetPrefix = $ConfigData.PASubnet; } foreach ($h in $ConfigData.hypervhosts) { if($null -ne $ConfigData.Port -and $ConfigData.Port -ne 0) { write-SDNExpressLog "Using port $($ConfigData.Port) for host $h" $params.Port = $ConfigData.Port } $ncNodes = $ConfigData.NCs.ComputerName Add-SDNExpressHost @params -ComputerName $h ` -RestName $ConfigData.RestName ` -NCHostCert $NCHostCert ` -Credential $Credential ` -VirtualSwitchName $ConfigData.SwitchName ` -IsFC $useFcNc ` -AddToFcCluster $false ` -CertificatePassword $NCPassword ` -NCNodes $ncNodes } if ($ConfigData.Gateways.Count -gt 0) { write-SDNExpressLog "STAGE 5.1: Create Gateway VMs" foreach ($Gateway in $ConfigData.Gateways) { $params = @{ 'RestName'=$ConfigData.RestName 'ComputerName'=$gateway.computername 'HostName'=$gateway.Hostname 'JoinDomain'=$ConfigData.JoinDomain 'FrontEndLogicalNetworkName'='HNVPA' 'FrontEndAddressPrefix'=$ConfigData.PASubnet } $Result = Initialize-SDNExpressGateway @params -Credential $Credential $Gateway.FrontEndMac = $Result.FrontEndMac $Gateway.FrontEndIP = $Result.FrontEndIP $Gateway.BackEndMac = $Result.BackEndMac $createparams.ComputerName=$Gateway.HostName; $createparams.VMName=$Gateway.ComputerName; if ([string]::IsNullOrEmpty($Gateway.ManagementIP)) { $createparams.Nics=@( @{Name="Management"; MacAddress=$Gateway.MacAddress; VLANID=$ConfigData.ManagementVLANID; SwitchName=$Mux.ManagementSwitch} @{Name="FrontEnd"; MacAddress=$Gateway.FrontEndMac; IPAddress="$($Gateway.FrontEndIp)/$PASubnetBits"; VLANID=$ConfigData.PAVLANID}, @{Name="BackEnd"; MacAddress=$Gateway.BackEndMac; VLANID=$ConfigData.PAVLANID} ); } else { $createparams.Nics=@( @{Name="Management"; MacAddress=$Gateway.MacAddress; IPAddress="$($Gateway.ManagementIP)/$ManagementSubnetBits"; Gateway=$ConfigData.ManagementGateway; DNS=$ConfigData.ManagementDNS; VLANID=$ConfigData.ManagementVLANID; SwitchName=$Mux.ManagementSwitch} @{Name="FrontEnd"; MacAddress=$Gateway.FrontEndMac; IPAddress="$($Gateway.FrontEndIp)/$PASubnetBits"; VLANID=$ConfigData.PAVLANID}, @{Name="BackEnd"; MacAddress=$Gateway.BackEndMac; VLANID=$ConfigData.PAVLANID} ); } $createparams.Roles=@("RemoteAccess", "RemoteAccessServer", "RemoteAccessMgmtTools", "RemoteAccessPowerShell", "RasRoutingProtocols", "Web-Application-Proxy") New-SDNExpressVM @createparams } write-SDNExpressLog "STAGE 5.3: Configure Gateways" if ([String]::IsNullOrEmpty($ConfigData.RedundantCount)) { $ConfigData.RedundantCount = 1 } if ([string]::IsNullOrEmpty($configdata.GatewayPoolType) -or ($configdata.GatewayPoolType -eq "All")) { write-SDNExpressLog "Gateway pool type is All." New-SDNExpressGatewayPool -IsTypeAll -PoolName $ConfigData.PoolName -Capacity $ConfigData.Capacity -GreSubnetAddressPrefix $ConfigData.GreSubnet -RestName $ConfigData.RestName -Credential $Credential -RedundantCount $ConfigData.RedundantCount } elseif ($configdata.GatewayPoolType -eq "GRE") { write-SDNExpressLog "Gateway pool type is GRE." New-SDNExpressGatewayPool -IsTypeGRE -PoolName $ConfigData.PoolName -Capacity $ConfigData.Capacity -GreSubnetAddressPrefix $ConfigData.GreSubnet -RestName $ConfigData.RestName -Credential $Credential -RedundantCount $ConfigData.RedundantCount } elseif ($configdata.GatewayPoolType -eq "Forwarding") { write-SDNExpressLog "Gateway pool type is Forwarding." New-SDNExpressGatewayPool -IsTypeForwarding -PoolName $ConfigData.PoolName -Capacity $ConfigData.Capacity -RestName $ConfigData.RestName -Credential $Credential -RedundantCount $ConfigData.RedundantCount } elseif ($configdata.GatewayPoolType -eq "IPSec") { write-SDNExpressLog "Gateway pool type is IPSec." New-SDNExpressGatewayPool -IsTypeIPSec -PoolName $ConfigData.PoolName -Capacity $ConfigData.Capacity -RestName $ConfigData.RestName -Credential $Credential -RedundantCount $ConfigData.RedundantCount } else { write-SDNExpressLog "Gateway pool type is Invalid." throw "Invalid GatewayPoolType specified in config file." } WaitforComputerToBeReady -ComputerName $ConfigData.Gateways.ComputerName -Credential $Credential foreach ($G in $ConfigData.Gateways) { $params = @{ 'RestName'=$ConfigData.RestName 'ComputerName'=$g.computername 'HostName'=$g.Hostname 'NCHostCert'= $NCHostCert 'PoolName'=$ConfigData.PoolName 'FrontEndIp'=$G.FrontEndIP 'FrontEndLogicalNetworkName'='HNVPA' 'FrontEndAddressPrefix'=$ConfigData.PASubnet 'FrontEndMac'=$G.FrontEndMac 'BackEndMac'=$G.BackEndMac 'Routers'=$ConfigData.Routers 'PAGateway'=$ConfigData.PAGateway 'ManagementRoutes'=$ConfigData.ManagementRoutes 'LocalASN'=$ConfigData.SDNASN } if ($ConfigData.UseGatewayFastPath -eq $true) { New-SDNExpressGateway @params -Credential $Credential -UseFastPath -IsFC $useFcNc } else { New-SDNExpressGateway @params -Credential $Credential -IsFC $useFcNc } } } [bool] $testMux = ($configdata.Muxes.Count -gt 0 ) [bool] $testGateway = ($configData.Gateways.Count -gt 0) Test-SdnExpressHealth -restname $ConfigData.RestName ` -Credential $Credential ` -testmux $testMux ` -testgateway $testgateway } catch { try { write-SDNExpressLog "An error occurred during the deployment. $_" } catch { Write-Host "Unhandled error: $_" } $pscmdlet.throwterminatingerror($PSItem) } if ($ncVmUpgradeNeeded -eq 1) { $errMsg = "This older release of HCI runnning in the Network Controller VMs contains a known issue that breaks communication to hosts. Please upgrade the VMs to any patch released after February 2024" Write-Host $errMsg -ForegroundColor Red -BackgroundColor White Write-SDNExpressLog $errMsg } write-SDNExpressLog "SDN Express deployment complete." # SIG # Begin signature block # MIIoKgYJKoZIhvcNAQcCoIIoGzCCKBcCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAlm2tvtRS9elNK # gIzZZlY9X0kZ5JHt96IRdNg5+CSLiqCCDXYwggX0MIID3KADAgECAhMzAAAEBGx0 # Bv9XKydyAAAAAAQEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTE0WhcNMjUwOTExMjAxMTE0WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC0KDfaY50MDqsEGdlIzDHBd6CqIMRQWW9Af1LHDDTuFjfDsvna0nEuDSYJmNyz # NB10jpbg0lhvkT1AzfX2TLITSXwS8D+mBzGCWMM/wTpciWBV/pbjSazbzoKvRrNo # DV/u9omOM2Eawyo5JJJdNkM2d8qzkQ0bRuRd4HarmGunSouyb9NY7egWN5E5lUc3 # a2AROzAdHdYpObpCOdeAY2P5XqtJkk79aROpzw16wCjdSn8qMzCBzR7rvH2WVkvF # HLIxZQET1yhPb6lRmpgBQNnzidHV2Ocxjc8wNiIDzgbDkmlx54QPfw7RwQi8p1fy # 4byhBrTjv568x8NGv3gwb0RbAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU8huhNbETDU+ZWllL4DNMPCijEU4w # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMjkyMzAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAIjmD9IpQVvfB1QehvpC # Ge7QeTQkKQ7j3bmDMjwSqFL4ri6ae9IFTdpywn5smmtSIyKYDn3/nHtaEn0X1NBj # L5oP0BjAy1sqxD+uy35B+V8wv5GrxhMDJP8l2QjLtH/UglSTIhLqyt8bUAqVfyfp # h4COMRvwwjTvChtCnUXXACuCXYHWalOoc0OU2oGN+mPJIJJxaNQc1sjBsMbGIWv3 # cmgSHkCEmrMv7yaidpePt6V+yPMik+eXw3IfZ5eNOiNgL1rZzgSJfTnvUqiaEQ0X # dG1HbkDv9fv6CTq6m4Ty3IzLiwGSXYxRIXTxT4TYs5VxHy2uFjFXWVSL0J2ARTYL # E4Oyl1wXDF1PX4bxg1yDMfKPHcE1Ijic5lx1KdK1SkaEJdto4hd++05J9Bf9TAmi # u6EK6C9Oe5vRadroJCK26uCUI4zIjL/qG7mswW+qT0CW0gnR9JHkXCWNbo8ccMk1 # sJatmRoSAifbgzaYbUz8+lv+IXy5GFuAmLnNbGjacB3IMGpa+lbFgih57/fIhamq # 5VhxgaEmn/UjWyr+cPiAFWuTVIpfsOjbEAww75wURNM1Imp9NJKye1O24EspEHmb # DmqCUcq7NqkOKIG4PVm3hDDED/WQpzJDkvu4FrIbvyTGVU01vKsg4UfcdiZ0fQ+/ # V0hf8yrtq9CkB8iIuk5bBxuPMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGgowghoGAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAQEbHQG/1crJ3IAAAAABAQwDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIBe5qrNSUh4jGlr28VT9WEj8 # yrb5NdHWcd7ialZjLDHzMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAFP0RwlYpuDxilo6YvVzRuaPJlvJH45WnnuWJi0XXU9d0MlbKZjPVxTsu # I4HwudQ7C9cUQ9iAaCnvwAq78/p82Z37YPzxosGhDxy9UF1XwgsdhQ4B5r1dvXx/ # M1orRpigK414T3RskJvi3mey/BmLEncV9IY1G8wnqUYN41w1OBOTP+d7cjUt6UvI # bw65jhFEIcJMQSyFZbzeTn3Hc+CT2SEbOJ/xdO4vbb/X+dli4uCBiDd5tgiepEeS # PNGr+3N2Ltp8D1Dt6PlmjaCm5ewKdZk1PBkfTw38EGVsL+tCQY2kEbcurSS6WnFs # 6I221ss8GRBwqxlGBgDkX33lup2mMaGCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCC # F3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq # hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCAIpfv7i+NhHESJjfilTSQAZQKtmGaH7Tw/gb5RWxwxYgIGZ1sAXxO6 # GBMyMDI1MDEwMjE5MzgxOS4yMzNaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l # cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046RjAwMi0w # NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg # ghHqMIIHIDCCBQigAwIBAgITMwAAAfI+MtdkrHCRlAABAAAB8jANBgkqhkiG9w0B # AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzEyMDYxODQ1 # NThaFw0yNTAzMDUxODQ1NThaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z # MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046RjAwMi0wNUUwLUQ5NDcxJTAjBgNV # BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQC85fPLFwppYgxwYxkSEeYvQBtnYJTtKKj2FKxzHx0f # gV6XgIIrmCWmpKl9IOzvOfJ/k6iP0RnoRo5F89Ad29edzGdlWbCj1Qyx5HUHNY8y # u9ElJOmdgeuNvTK4RW4wu9iB5/z2SeCuYqyX/v8z6Ppv29h1ttNWsSc/KPOeuhzS # AXqkA265BSFT5kykxvzB0LxoxS6oWoXWK6wx172NRJRYcINfXDhURvUfD70jioE9 # 2rW/OgjcOKxZkfQxLlwaFSrSnGs7XhMrp9TsUgmwsycTEOBdGVmf1HCD7WOaz5EE # cQyIS2BpRYYwsPMbB63uHiJ158qNh1SJXuoL5wGDu/bZUzN+BzcLj96ixC7wJGQM # BixWH9d++V8bl10RYdXDZlljRAvS6iFwNzrahu4DrYb7b8M7vvwhEL0xCOvb7WFM # sstscXfkdE5g+NSacphgFfcoftQ5qPD2PNVmrG38DmHDoYhgj9uqPLP7vnoXf7j6 # +LW8Von158D0Wrmk7CumucQTiHRyepEaVDnnA2GkiJoeh/r3fShL6CHgPoTB7oYU # /d6JOncRioDYqqRfV2wlpKVO8b+VYHL8hn11JRFx6p69mL8BRtSZ6dG/GFEVE+fV # mgxYfICUrpghyQlETJPITEBS15IsaUuW0GvXlLSofGf2t5DAoDkuKCbC+3VdPmlY # VQIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFJVbhwAm6tAxBM5cH8Bg0+Y64oZ5MB8G # A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG # Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy # MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w # XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy # dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG # A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD # AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQA9S6eO4HsfB00XpOgPabcN3QZeyipgilcQ # SDZ8g6VCv9FVHzdSq9XpAsljZSKNWSClhJEz5Oo3Um/taPnobF+8CkAdkcLQhLdk # Shfr91kzy9vDPrOmlCA2FQ9jVhFaat2QM33z1p+GCP5tuvirFaUWzUWVDFOpo/O5 # zDpzoPYtTr0cFg3uXaRLT54UQ3Y4uPYXqn6wunZtUQRMiJMzxpUlvdfWGUtCvnW3 # eDBikDkix1XE98VcYIz2+5fdcvrHVeUarGXy4LRtwzmwpsCtUh7tR6whCrVYkb6F # udBdWM7TVvji7pGgfjesgnASaD/ChLux66PGwaIaF+xLzk0bNxsAj0uhd6QdWr6T # T39m/SNZ1/UXU7kzEod0vAY3mIn8X5A4I+9/e1nBNpURJ6YiDKQd5YVgxsuZCWv4 # Qwb0mXhHIe9CubfSqZjvDawf2I229N3LstDJUSr1vGFB8iQ5W8ZLM5PwT8vtsKEB # wHEYmwsuWmsxkimIF5BQbSzg9wz1O6jdWTxGG0OUt1cXWOMJUJzyEH4WSKZHOx53 # qcAvD9h0U6jEF2fuBjtJ/QDrWbb4urvAfrvqNn9lH7gVPplqNPDIvQ8DkZ3lvbQs # Yqlz617e76ga7SY0w71+QP165CPdzUY36et2Sm4pvspEK8hllq3IYcyX0v897+X9 # YeecM1Pb1jCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI # hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy # MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC # AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg # M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF # dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6 # GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp # Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu # yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E # XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0 # lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q # GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ # +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA # PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw # EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG # NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV # MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj # cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK # BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC # AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX # zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v # cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI # KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG # 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x # M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC # VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449 # xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM # nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS # PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d # Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn # GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs # QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL # jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL # 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNN # MIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn # MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOkYwMDItMDVFMC1EOTQ3MSUwIwYDVQQD # ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQBr # i943cFLH2TfQEfB05SLICg74CKCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6yEtNjAiGA8yMDI1MDEwMjE1MjAy # MloYDzIwMjUwMTAzMTUyMDIyWjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDrIS02 # AgEAMAcCAQACAgzRMAcCAQACAhNMMAoCBQDrIn62AgEAMDYGCisGAQQBhFkKBAIx # KDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZI # hvcNAQELBQADggEBAKGtIc76V72bjo5UNVewIqOiodsAM8lYZA0Dh49qaCdyfnLY # vg3pgUfQ08nb0BaJJxLv6dFYz7+74m6plry9G9jczuxERfh7/a9ZbUSSTbdn1ooa # VBWb+e4B3JyqB9bF8RO914iyw8rHy5JaAVl9ymKAIs7kEhwH2AoC1Jn+LT3R+nz9 # z3Ddp8XZuhGaPg094AbOzLeKLRyskRZFHdppLNSVF/F2c47yCQANSggzdE7XPiiO # AvwKYSYoCwZIWg8Z+0YHkpGrcA2dyGgae65Yv+79j0dixdl5GktNRajx8diJUr0E # bbque0w/6TF0kyZe3keaxHKsGbJP4WwNEZJF20QxggQNMIIECQIBATCBkzB8MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNy # b3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAfI+MtdkrHCRlAABAAAB8jAN # BglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8G # CSqGSIb3DQEJBDEiBCC9+ltkQSnaPJ05D14NIwF1K7/ue2wsdG+XJ7xK1qH0lTCB # +gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIPjaPh0uMVJc04+Y4Ru5BUUbHE4s # uZ6nRHSUu0XXSkNEMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh # c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD # b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw # MTACEzMAAAHyPjLXZKxwkZQAAQAAAfIwIgQgfH410yoCsZUqTjCnWntpf5hngZzq # wlgYC99Zk8n7KQMwDQYJKoZIhvcNAQELBQAEggIAGrCKr0UUV+T4g3i9uNdrb/9Q # vSVlotJ+b5qUhOEGrkB/E0yR5zLnjpmTB1HEKl/L9SVAQTj/xWVWulHfASZV7/3H # FBiAFDKaxxTX5p2GLXC6lLDfrXL6+yaXilU7jekljtWvwJyuJ1E4wtVeBP1SIdMn # NxfVmrMSZp33FISMTWpXiKqIMlOHWKBNToD5HDW6SNJNAB9s9LrDEN96E+fYdIzW # XbjsXPV6AKMCG/GZMVwuM1zoG0yjZ5sD85/ndTAoNeESvycxoICRA9mr3zW6xXVK # oprzHjvuTx2xj4vOynexB/NqF0Kykk79AFRbNYxsJmY+wpWaeCanuo12C/m/9hC7 # KaijlqXUmslEq0WHspqrYy83H70sEaHFz9aMt+He4WKQhi5CjHlQeSpv00wAxrRS # nsBlpSyMWZrQhR9u9FLDNocY1hKLIcTAqRc7aeW+fUANBlD7hGi6v1B8lpr0SzFP # l/YwQuINlLYHiEJ3atzPSJ9wPyPkwWcke1DBMYkjjPUaFPvemHHiNEpbEJ2cwBYs # SKPTYOMFHon+6sTCyL28AI8sKytkVL4PW1glbTmbjkag/FrV8B49+PRfwc4xuEQc # m1qBUSZmlJJVVE1GVYf6CaPQGsR6pU1++r121lWMbljY8OYC8r2QBLM69gP8FhVg # XzPQXtBQDi+YWPi8W88= # SIG # End signature block |