TrafficUtil.psm1
|
Import-Module $PSScriptRoot\Logger.psm1 Import-Module $PSScriptRoot\ConfigManager.psm1 $script:MaxTrafficRetryCount = 5 class TrafficEndpoint { [string] $VmName = "VmName" [string] $HostName = "HostName" [pscredential] $VmCredential = $null [pscredential] $hostCredential = $null [string] $IpAddress = "0.0.0.0" [int] $Port = 5001 [string] $PublicIP = $null [string] ToString() { return "vm:$($this.VmName) ip:$($this.IpAddress):$($this.Port) pip:$($this.PublicIP)" } } class TrafficPattern { [int] $Connections = 10 [int] $Duration = 1 [int] $Iterations = 1 # 10 GB [int64] $TransferSizeInBytes = 10737418240 [string] $Protocol = "TCP" [uint64] $BitsPerSecond = 0 [uint64] $FrameRate = 0 [uint64] $BufferDepth = 0 [uint64] $StreamLength = 0 [string] $PushPattern = "pushpull" [string] ToString() { if("UDP" -eq $this.Protocol) { return "udp cxn:$($this.Connections) iters:$($this.Iterations) xfer:$($this.TransferSizeInBytes) bps:$($this.BitsPerSecond) fr:$($this.FrameRate) bd:$($this.BufferDepth)" } if("TCP" -eq $this.Protocol) { return "tcp cxn:$($this.Connections) iters:$($this.Iterations)bps:$($this.BitsPerSecond) bd:$($this.BufferDepth) ptrn:$($this.PushPattern)" } return "" } } class WorkloadSummary { [int] $TotalConnections = 0 [int] $FailedConnections = 0 [int] $SuccessfulConnections = 0 [int] $TotalBytesTransferred = 0 } Set-Variable VNET_DIP -Value 0 -Option Constant Set-Variable PRIV_VIP -Value 1 -Option Constant Set-Variable PUB_VIP -Value 2 -Option Constant Set-Variable ILB_VIP -Value 3 -Option Constant Set-Variable CLIENT_IP -Value 4 -Option Constant Set-Variable DIPS -Value "DIPS" -Option Constant Set-Variable VIP_IP -Value "VIP_IP" -Option Constant Set-Variable VIP_PORT -Value "VIP_PORT" -Option Constant Set-Variable VIP_PROTOCOL -Value "VIP_PROTOCOL" -Option Constant Set-Variable VIP_TYPE -Value "VIP_TYPE" -Option Constant Set-Variable DIP_IP -Value "DIP_IP" -Option Constant Set-Variable DIP_PORT -Value "DIP_PORT" -Option Constant Set-Variable DIP_VMNAME -Value "DIP_VMNAME" -Option Constant Set-Variable FRONTEND_IP_CONFIG -Value "FRONTEND_IP_CONFIG" -Option Constant Set-Variable BACKEND_IP_CONFIG -Value "BACKEND_IP_CONFIG" -Option Constant Set-Variable INBOUND_NAT_TRAFFIC_RULES -Value "INBOUND_NAT_TRAFFIC_RULES" -Option Constant Set-Variable LOADBALANCER_TRAFFIC_RULES -Value "LOADBALANCER_TRAFFIC_RULES" -Option Constant Set-Variable OUTBOUND_NAT_TRAFFIC_RULES -Value "OUTBOUND_NAT_TRAFFIC_RULES" -Option Constant Set-Variable DIP_IPCONFIG -Value "DIP_IPCONFIG" -Option Constant Set-Variable CLIENT_VMNAME -Value "InternetClient" -Option Constant Set-Variable DIP_VM_HOSTNAME -Value "DIP_VM_HOSTNAME" -Option Constant Set-Variable TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -Value 5 -Option Constant Set-Variable DEFAULT_CTSTRAFFICPORT -Value 5001 -Option Constant $script:CtsTrafficVMFolderPath = "$env:SystemDrive\tools" $script:CtsTrafficVMPath = "$env:SystemDrive\tools\CtsTraffic.exe" <# This function will create a new traffic pattern object The traffic pattern object will be used to run traffic between two VMs #> function New-TrafficPattern { param( [int] $connections, [int] $duration, [int] $iterations, [int64] $TransferSizeInBytes, [string] $protocol = "TCP", [int64] $bitsPerSecond = 0, [int64] $frameRate = 0, [int64] $bufferDepth = 0, [uint64] $streamLength = 0, [ValidateSet("push", "pullpush","pull","duplex", "")] [string] $pushPattern = "pushpull" ) $pattern = New-Object TrafficPattern $pattern.Connections = $connections $pattern.Duration = $duration $pattern.Iterations = $iterations $pattern.TransferSizeInBytes = $TransferSizeInBytes $pattern.Protocol = $protocol $pattern.BitsPerSecond = $bitsPerSecond $pattern.FrameRate = $frameRate $pattern.BufferDepth = $bufferDepth $pattern.PushPattern = $pushPattern $pattern.StreamLength = $streamLength return $pattern } <# Creates a new traffic endpoint object. Endpoints can run/receieve traffic between two VMs endpointType : 0 - Vnet DIP 1 - Private IP Address 2 - PublicIP 3 - ILB 4 - Client IP #> function New-TrafficEndpoint { param( [string] $vmName, [string] $hostName, [pscredential] $vmCredential, [pscredential] $hostCredential, [string] $ipAddress, [int] $port, [bool] $resolveHostName = $false, [string] $publicIp = $null, [ValidateSet(0,1,2,3,4)] [parameter(Mandatory=$true)] [int] $endpointType ) if($resolveHostName -eq $true) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCredential -vmName $vmName -Force $true } $endpoint = New-Object TrafficEndpoint $endpoint.VmName = $vmName $endpoint.HostName = $hostName $endpoint.VmCredential = $vmCredential $endpoint.HostCredential = $hostCredential $endpoint.IpAddress = $ipAddress $endpoint.Port = $port $endpoint.PublicIP = $publicIp return $endpoint } <# Checks if the VM has ctstraffic installed and physically ready to run traffic workloads - Checks if the VM is powered on - Checks if the VM has ctstraffic installed - Checks if the VM has a valid IP address - Checks if the VM has a ctstraffic is enabled on firewall #> function Initialize-VmForCtstraffic { param( [object[]] $trafficEndpoints, [pscredential] $hostCred, [pscredential] $vmCred ) $ctsTraffic = Get-DefaultCtsTrafficFile if (-not (Test-Path $ctsTraffic)) { throw "Initialize-VmForCtstraffic: CtsTraffic not found at $ctsTraffic" } foreach($endpoint in $trafficEndpoints) { # log all properties of endpoint Write-SDNExpressLog -Message "Copy-VMFile $($endpoint.VmName) ` Host:$($endpoint.HostName) ` ToolPath:$script:CtsTrafficVMPath ` SourcePath:$ctsTraffic ` DestPath:$script:CtsTrafficVMPath ` " # stop any previous running instances of ctstraffic Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" ` -hostCred $hostCred ` -vmCred $vmCred $ctsEnableFirewall = "`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Inbound)"" -Direction Inbound -Program $script:CtsTrafficVMPath -Action Allow" $ctsEnableFirewall += ";`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Outbound)"" -Direction Outbound -Program $script:CtsTrafficVMPath -Action Allow" $hostCred = Get-TurnKeySdnCred $vmCred = Get-TurnKeySdnWorkloadVmCred $results = Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd $ctsEnableFirewall ` -hostCred $hostCred ` -vmCred $vmCred # ctstraffic.exe is detected as malware by windows defender, so we need to disable it # it also does full FS scan which slows deployment considerably $disableDefender = "Set-MpPreference -DisableRealtimeMonitoring `$true;" $disableDefender += "Set-MpPreference -DisableBehaviorMonitoring `$true;" $disableDefender += "Set-MpPreference -DisableBlockAtFirstSeen `$true;" $disableDefender += "Add-MpPreference -ExclusionPath $script:CtsTrafficVMFolderPath;" Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd $disableDefender ` -hostCred $hostCred ` -vmCred $vmCred # now copy ctstraffic to the vm Copy-VMFile -VMName $endpoint.VmName ` ` -SourcePath $ctsTraffic ` -DestinationPath $script:CtsTrafficVMPath ` -FileSource Host ` -ComputerName $endpoint.HostName ` -CreateFullPath ` -Force } } <# Runs traffic between given endpoints - Completes a full mesh (NXN endpoints with the traffic pattern provided), with one at a time. - Assumes that appropriate firewall rules are enabled on the VMs (From SDN) #> function Start-IntraVmTraffic { param( [object[]] $trafficEndpoints, [TrafficPattern[]] $trafficPatterns, [pscredential] $hostCred, [pscredential] $vmCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet # todo : (optimization) this can be sped up if we keep the destination running and go over the targets, this # way the init time can be saved (needs to handle different outFileNames for each target) foreach($trafficPattern in $trafficPatterns) { foreach($srcEndpoint in $trafficEndpoints) { foreach($dstEndpoint in $trafficEndpoints) { if($srcEndpoint -eq $dstEndpoint) { continue } if($srcEndpoint.VmName -like "*ilb_client*" -or $dstEndpoint.VmName -like "*ilb_client*") { Write-TraceLog "Start-IntraVmTraffic: ILB & clients are skipped from EW traffic tests." continue } $retryCount = 0 while($true) { try { $guid = New-Guid Write-TraceLog "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) starting.." -Warning LogTrafficInfo -listenEndpoint $dstEndpoint -sendEndpoint $srcEndpoint -trafficPattern $trafficPattern $outFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".csv" $errorFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".err" Start-CtstrafficListener -listerningPort $dstEndpoint.Port ` -vmName $dstEndpoint.VmName ` -hostName $dstEndpoint.HostName ` -vmCred $vmCred ` -hostCred $hostCred ` -trafficPattern $trafficPattern ` -outFileName $outFileName ` -uri $uri ` -errorFileName $errorFileName ` -isILB $false Start-CtstrafficSender -targetIpAddress $dstEndpoint.IpAddress ` -targetPort $dstEndpoint.Port ` -hostCred $hostCred ` -vmCred $vmCred ` -vmName $srcEndpoint.VmName ` -hostName $srcEndpoint.HostName ` -trafficPattern $trafficPattern ` -outFileName $outFileName ` -uri $uri ` -wait $true $srcWorkloadResult = Wait-ForWorloadCompletion -vmName $srcEndpoint.VmName ` -hostName $srcEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName Test-WorkloadResult -workloadResults $srcWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $srcEndpoint.VmName $dstWorkloadResult = Wait-ForWorloadCompletion -vmName $dstEndpoint.VmName ` -hostName $dstEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $dstWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dstEndpoint.VmName Write-Host "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) PASSED!" break } catch { $retryCount += 1 if ($retryCount -eq $script:MaxTrafficRetryCount) { throw "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) FAILED! Error $($_.Exception.Message) after $retryCount retries" } Write-Host "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) FAILED! Error $($_.Exception.Message), retrying... (retry: $retryCount)" } } } } } } <# Runs SLB Traffic on a given set of endpoints. The traffic pattern is defined by the following parameters: - RulesToTarget : The type of rules to target on the load balancer. Valid values are: - LoadbalancerRules : All load balancer rules - Inbound : All inbound NAT rules - Outbound : All outbound NAT rules The caller must create the metadata json which describes the topology of DIPs and VIPs before calling this method. Use the helper Get-EndpointsFromLoadBalancer to extract the metadata from a given loadbalancer and pass it to this method to initiate traffic. NOTE: the Get-EndpointsFromLoadBalancer does not detect if probes are down for the given endpoints. Please disable probes before calling this method. For inbound/lb rules the server instances are #> function Start-SLBTraffic { param( [ValidateSet("LoadbalancerRules", "Inbound", "Outbound")] [string] $TargetType, [object] $vips, [TrafficPattern[]] $trafficPatterns, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) $retryCount = 0 while($true) { try { if($TargetType -eq "LoadbalancerRules") { $vipInfo = $vips[$LOADBALANCER_TRAFFIC_RULES] if($null -eq $vipInfo -or $vipInfo.Count -eq 0) { throw "No loadbalancer rules found" } foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(lb) $vipEndpoint TRAFFICPATTERN: $trafficPattern" if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") { Write-TraceLog "Start-SLBTraffic:(lb) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)" continue; } Start-SLBLBRuleTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } elseif($TargetType -eq "Inbound") { $vipInfo = $vips[$INBOUND_NAT_TRAFFIC_RULES] foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(inbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern" if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") { Write-TraceLog "Start-SLBTraffic:(inbound) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)" continue; } Start-SLBInboundTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } elseif($TargetType -eq "Outbound") { $vipInfo = $vips[$OUTBOUND_NAT_TRAFFIC_RULES] foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(outbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern" if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") { Write-TraceLog "Start-SLBTraffic:(outbound) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)" continue; } Start-SLBOutboundTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } else { throw "Invalid TargetType parameter" } break } catch { $retryCount += 1 if ($retryCount -eq $script:MaxTrafficRetryCount) { throw "Start-SLBTraffic: Error $($_.Exception.Message) after $retryCount retries" } Write-TraceLog "Start-SLBTraffic: Error $($_.Exception.Message) retrying... (retry: $retryCount)" } } } <# Gets a suitable endpoint to run the client traffic on, if its ILB, it will pick up a VNET location (for ILB: The ILB client NIC is used) for everything else, it will use the PublicIP on the ILB CLient todo: make this configurable to run the workload in different locations #> function Get-SLBClientEndpoint { param( [object[]] $dips, [int] $vipType, [string] $uri, [pscredential] $hostCred, [pscredential] $vmCred, [string] $port ) if($dips -eq $null -or $dips.Count -eq 0) { throw "Update-DipInfoIfILBEnabledDips: Dips is null or empty" } $ipConfigTokens = $dips[0][$DIP_IPCONFIG].split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $subnetNameResourceRef = $ipConfig.properties.Subnet.resourceRef $subnetNameResRefTokens = $subnetNameResourceRef.split("/") $subnetName = $subnetNameResRefTokens[4] $vmName = "$($subnetName)_ilb_client" $nic = Get-NetworkControllerNetworkInterface -ResourceId $vmName -ConnectionUri $uri -PassInnerException if($nic -eq $null) { throw "Could not find NIC $vmName" } # for ILB cases, use the ILB VNET IP Address for traffic if($vipInfo[$VIP_TYPE] -eq 3) { New-TrafficEndpoint -vmName $vmName ` -hostName $null ` -vmCredential $vmCred ` -hostCredential $hostCred ` -ipAddress $nic.properties.IpConfigurations[0].properties.privateIpAddress ` -port 5001 ` -endpointType $CLIENT_IP ` -resolveHostName $true ` -publicIp $null } else { # for other LB scenarios use the PublicIP address for traffic # get public ip from the ilb client nic $publicIpResRef = $nic.properties.IpConfigurations[0].properties.PublicIPAddress.resourceRef $publicIpResRefTokens = $publicIpResRef.split("/") $publicIpResource = Get-NetworkControllerPublicIPAddress -ResourceId $publicIpResRefTokens[2] -ConnectionUri $uri -PassInnerException $publicIpAddress = $publicIpResource.properties.IpAddress New-TrafficEndpoint -vmName $vmName ` -hostName $null ` -vmCredential $vmCred ` -hostCredential $hostCred ` -ipAddress $null ` -port 5001 ` -endpointType $CLIENT_IP ` -resolveHostName $true ` -publicIp $publicIpAddress } } function Start-SLBInboundTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) # todo : this can be sped up if we pre resolve the host names for all the DIPs $dipHostName = Resolve-HostName -hostName $vipEndpoint[$DIPS][0][$DIP_VMNAME] ` -hostCred $hostCred ` -vmName $vipEndpoint[$DIPS][0][$DIP_VMNAME] ` -Force $true # select the best client based on the VIP type $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBInboundTraffic: Starting INBOUND Traffic ($($guid))" -Warning Write-TraceLog "Start-SLBInboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBInboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBInboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBInboundTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress) " Write-TraceLog "Start-SLBInboundTraffic: `t TRAFFIC : $($trafficPattern.ToString())" Write-TraceLog "----------------------------------------------" # set up dip to recv Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] ` -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] ` -hostName $dipHostName ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern Start-CtstrafficSender -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -targetIpAddress $vipEndpoint[$VIP_IP] ` -targetPort $vipEndpoint[$VIP_PORT] ` -hostCred $hostCred ` -vmCred $vmCred ` -ctsTrafficPathLocal $script:CtsTrafficVMPath ` -outFileName $outFileName ` -trafficPattern $trafficPattern # once the sender is complete, the receiver can be forced killed $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] ` -hostName $dipHostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $vipEndpoint["DIPS"][0][$DIP_VMNAME] $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBInboundTraffic: END --- " } function Test-WorkloadResult { param( [object] $workloadResults, [int] $maxFailedPercentage = 10, [string] $EndpointName ) if($workloadResults -eq $null) { throw "Workload result is null" } if($workloadResults.count -eq 0) { throw "Workload result is empty" } [int] $total = 0 [int] $failed = 0 [int] $successful = 0 foreach($result in $workloadResults) { if($result.Result -eq "Failed") { $failed += 1 } else { $successful += 1 } $total += 1 } if($total -eq 0) { throw "Workload result is empty" } $failedPercentage = $failed / $total * 100 if($failedPercentage -gt $maxFailedPercentage) { throw "Workload failed for more than 10% of the connections" } Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" -Warning Write-TraceLog "Test-WorkloadResult: Connections Total:$total Failed:$failed Successful:$successful Failed %:$failedPercentage" -Warning Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" -Warning return $true } function Start-SLBLBRuleTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) # select the best client based on the VIP type $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBLBRuleTraffic: Starting LB Rule Traffic ($($guid))" -Warning Write-TraceLog "Start-SLBLBRuleTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBLBRuleTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBLBRuleTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBLBRuleTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress):$($clientEndpoint.port) publicIp:$($clientEndpoint.PublicIP)" Write-TraceLog "----------------------------------------------" foreach($dipInfo in $vipEndpoint[$DIPS]) { Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] ` -vmName $dipInfo[$DIP_VMNAME] ` -hostName $null ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern } Start-CtstrafficSender -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -targetIpAddress $vipEndpoint[$VIP_IP] ` -targetPort $vipEndpoint[$VIP_PORT] ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName ` -trafficPattern $trafficPattern foreach($dipInfo in $vipEndpoint[$DIPS]) { # once the sender is complete, the receiver can be forced killed $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME] ` -hostName $null ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME] } $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBLBRuleTraffic: LB Rule Traffic ($($guid)) PASSED" Write-TraceLog "----------------------------------------------" } function Start-SLBOutboundTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) Write-TraceLog "Start-SLBOutboundTraffic: $dipInfo $vipInfo" # for Get-SLBClientEndpoint gives us the ILB client VM which has a PublicIP on it, so it should work from there $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $vipPort = 5001 $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBOutboundTraffic: Starting OUTBOUND LB Rule Traffic ($($guid))" -Warning Write-TraceLog "Start-SLBOutboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBOutboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBOutboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBOutboundTraffic: `t CLIENT : $($clientEndpoint.vmName) IP $($clientEndpoint.IpAddress) PORT;$($clientEndpoint.port) PUBLICIP:$($clientEndpoint.PublicIP)" Write-TraceLog "----------------------------------------------" # set up client to recv Start-CtstrafficListener -listerningPort $vipPort ` -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern # start senders foreach($dipInfo in $vipEndpoint[$DIPS]) { Start-CtstrafficSender -targetIpAddress $clientEndpoint.PublicIP ` -targetPort $vipPort ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -vmName $dipInfo[$DIP_VMNAME] ` -trafficPattern $trafficPattern } # wait for senders $workloadResults = @() foreach($dipInfo in $vipEndpoint[$DIPS]) { $workloadResult = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME] ` -hostName "" ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true # test workload results per DIP Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME] $workloadResults += $workloadResult } # also validate the total workload results Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName "SLB-OUTBOUND-DIPS_SUMMARY" # validate workload results on the oubound receiver $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBOutboundTraffic: Passed" } function Start-Traffic { param( [ValidateSet("Listen", "Target")] [string] $Role, [object] $trafficPattern, [object] $currEndpoint, [object] $targetEndpoint, [pscredential] $hostCred, [pscredential] $vmCred ) # note: # currEndpoint contains context on where the instance of ctstraffic is going to be run (either listen or target) # targetEndpoint contains context on where the traffic is going to be sent to, its required for sender to know where to send the traffic to $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;" if($Role -eq "Listen") { $cmd += "Start-Process " + $script:CtsTrafficVMPath + " -ArgumentList ""-listen:* -port:$($targetEndpoint.Port)"""; } else { $cmd += "Start-Process -FilePath " + $script:CtsTrafficVMPath + " -Wait -ArgumentList ""-target:$($targetEndpoint.IpAddress) -port:$($targetEndpoint.Port) -protocol:tcp -connections:$($trafficPattern.Connections) -iterations:$($trafficPattern.Iterations)"""; } Write-TraceLog -Message "Start-Traffic ($Role): $cmd" Invoke-PowershellCommandOnVmInternal -vmName $currEndpoint.VmName ` -hostName $currEndpoint.HostName ` -cmd $cmd ` -hostCredentials $hostCred ` -vmCredentials $vmCred } function Resolve-HostName { param( [string] $hostName, [pscredential] $hostCred, [string] $vmName, [bool] $Force = $false ) if(-not [string]::IsNullOrEmpty($hostName) -and $Force -eq $false) { Write-TraceLog "Resolve-HostName: HostName is not null or empty, returning $hostName" return $hostName } if([string]::IsNullOrEmpty($vmName)) { Write-TraceLog "Resolve-HostName: VMName is null, returning $hostName" return $hostName; } $nodes = (get-clusternode).Name foreach($node in $nodes) { $vm = Get-VM -VMName $vmName -ComputerName $node -ErrorAction SilentlyContinue if($vm -ne $null) { Write-TraceLog "Resolve-HostName: Resolved VM $vmName on node $node" return $node } } throw "Could not resolve host name for vm $vmName" } function Wait-ForWorloadCompletion { param( [string] $vmName, [string] $hostName, [pscredential] $hostCred, [pscredential] $vmCred, [string] $outFileName, [int] $timeoutInSeconds = 3600, [string] $ctstrafficPathLocal = $script:CtsTrafficVMPath, [bool] $force = $false ) # Write-FunctionEntryWithParams -FunctionName "Wait-ForWorloadCompletion" -BoundParameters $PSBoundParameters -UnboundArguments $args # todo : avoid resolution of VM to speed things up a bit $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $false Write-TraceLog "Wait-ForWorloadCompletion: host:$hostName vm:$vmName.." $ctsTrafficFolderPath = $ctstrafficPathLocal.Substring(0, $ctstrafficPathLocal.LastIndexOf("\")) $csvFilePath = Join-Path $ctsTrafficFolderPath $outFileName if([string]::IsNullOrEmpty($vmName) -and [string]::IsNullOrEmpty($hostName)) { # wait for ctstraffic.exe to complete or kill and exit $done = $false [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $stopwatch.Start() while(-not $done -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) { Write-Host "Waiting for ctstraffic to complete" $p = Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue if($null -eq $p) { $done = $true break } } if(-not $done) { Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue } if(-not $force -and -not $done) { throw "Workload did not complete in time" } $WorkloadResult = Get-Content -Path $csvFilePath | ConvertFrom-Csv Write-TraceLog "Wait-ForWorloadCompletion: host:$hostName vm:$vmName..done" return $WorkloadResult } # the following is only if we are running traffic from the host (which is running turnkey) $done = $false [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $stopwatch.Start() while(-not $done -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) { Write-Host "Waiting for ctstraffic to complete" $cmd = "Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue" $p = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred if($null -eq $p) { $done = $true break } } if ($force -or -not $done) { Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" ` -hostCred $hostCred ` -vmCred $vmCred } if (-not $done -and -not $force) { throw "Workload did not complete in time" } $cmd = "Get-Content -Path $csvFilePath | ConvertFrom-Csv" $WorkloadResult = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred return $WorkloadResult } function Unblock-AllPortsOnVm { param( [string] $nicResourceId, [string] $restName ) $uri = "https://$restName"; $vnic = Get-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -PassInnerException if($vnic.properties.IpConfigurations -eq $null -or $vnic.properties.IpConfigurations.Count -eq 0) { throw "No IP configuration found on the NIC" } if($vnic.properties.IpConfigurations.Count -gt 1) { throw "Multiple IP configurations found on the NIC" } # create the NICs ACL $nicAclResourceId = $nicResourceId + "_ACL" $newNicAcl = New-Object Microsoft.Windows.NetworkController.AccessControlList $newNicAcl.properties = New-Object Microsoft.Windows.NetworkController.AccessControlListProperties $newNicAcl.properties.AclRules = @() $newNicAclRuleIdBase = "ACL_RULE_"; $newNicAcl.properties.AclRules += GetAclRule -resourceId $($newNicAclRuleIdBase + 1) ` -protocol "All" ` -action "Allow" ` -sourcePrefix "*" ` -destPrefix "*" ` -type "Inbound" ` -priority 1000 $newNicAcl.properties.AclRules += GetAclRule -resourceId $($newNicAclRuleIdBase + 2) ` -protocol "All" ` -action "Allow" ` -sourcePrefix "*" ` -destPrefix "*" ` -type "Outbound" ` -priority 1000 $newAcl = New-NetworkControllerAccessControlList -ResourceId $nicAclResourceId -ConnectionUri $uri -Properties $newNicAcl.properties -PassInnerException -Force Write-TraceLog "Created/updated allow all acl $nicAclResourceId, resourceRef : $($newAcl.resourceRef)" #bind this ACL to the NIC $vnic.properties.IpConfigurations[0].Properties.AccessControlList = $newAcl $nic = New-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -Properties $vnic.properties -PassInnerException -Force Write-TraceLog "Applied unblock-all ports ACL for NIC $nicResourceId" } function GetAclRule { param( [parameter(Mandatory=$true)] [String]$resourceId, [parameter(Mandatory=$true)] [String]$protocol, [parameter(Mandatory=$true)] [String]$action, [parameter(Mandatory=$true)] [String]$sourcePrefix, [parameter(Mandatory=$true)] [String]$destPrefix, [parameter(Mandatory=$true)] [String]$type, [parameter(Mandatory=$true)] [int]$priority ) $ruleproperties = new-object Microsoft.Windows.NetworkController.AclRuleProperties $ruleproperties.Protocol = $protocol $ruleproperties.SourcePortRange = "*" $ruleproperties.DestinationPortRange = "*" $ruleproperties.Action = $action $ruleproperties.SourceAddressPrefix = $sourcePrefix $ruleproperties.DestinationAddressPrefix = $destPrefix $ruleproperties.Priority = $priority $ruleproperties.Type = $type $ruleproperties.Logging = "Enabled" $aclrule = new-object Microsoft.Windows.NetworkController.AclRule $aclrule.Properties = $ruleproperties $aclrule.ResourceId = $resourceId return $aclrule } function Get-EndpointsFromLoadBalancer { param( [string] $lbResourceId, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) $lb = Get-NetworkControllerLoadBalancer -ResourceId $lbResourceId -ConnectionUri $uri -PassInnerException if($lb.properties.frontendIPConfigurations -eq $null -or $lb.properties.frontendIPConfigurations.Count -eq 0) { return $lbEndpoints } if( $lb.properties.frontendIPConfigurations.Count -gt 1) { throw "Multiple frontend IP configurations found on the load balancer" } # all Traffic rules $inboundNATTrafficRules = @() $loadbalancerTrafficRules = @() $outboundNATTrafficRules = @() # parse and create workload patterns for all INBOUND NAT RULES foreach($inboundNatRule in $lb.properties.inboundNatRules) { $inboundNatTraffic = @{} if( $inboundNatRule.properties.backendIpConfiguration -eq $null -or $inboundNatRule.properties.backendIpConfiguration.Count -eq 0) { continue } $inboundNatTraffic["frontendIpConfig"] = $inboundNatRule.properties.frontendIpConfigurations[0].resourceRef $inboundNatTraffic["VIP_PORT"] = $inboundNatRule.properties.frontendPort $inboundNatTraffic["VIP_PROTOCOL"] = $inboundNatRule.properties.protocol $inboundNatTraffic["backendIpConfig"] = $inboundNatRule.properties.backendIpConfiguration.resourceRef $inboundNatTraffic["DIP_PORT"] = $inboundNatRule.properties.backendPort $inboundNATTrafficRules += $inboundNatTraffic $inboundNatTraffic["DIPS"] = @() } # resolve all DIPs to IPs foreach($inboundNatTraffic in $inboundNATTrafficRules) { $dipInfo = @{} $ipConfigTokens = $inboundNatTraffic["backendIpConfig"].split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $dipInfo["DIP_PORT"] = $inboundNatTraffic["DIP_PORT"] $inboundNatTraffic["DIPS"] += $dipInfo } # resolve all VIPs foreach($inboundNatTraffic in $inboundNATTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $inboundNatTraffic["frontendIpConfig"] -uri $uri $inboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"] $inboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # parse and create workload patterns for all LB RULES foreach($lbRule in $lb.properties.loadBalancingRules) { $lbTrafficRule = @{} $lbTrafficRule["frontendIpConfig"] = $lbRule.properties.frontendIpConfigurations[0].resourceRef $lbTrafficRule["VIP_PORT"] = $lbRule.properties.frontendPort $lbTrafficRule["VIP_PROTOCOL"] = $lbRule.properties.protocol $lbTrafficRule["DIPS"] = @() #resolve DIPs $bePoolTokens = $lbRule.properties.backendAddressPool.resourceRef.split("/") $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) { $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo = @{} $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $lbTrafficRule["DIPS"] += $dipInfo } $lbTrafficRule["DIP_PORT"] = $lbRule.properties.backendPort $loadbalancerTrafficRules += $lbTrafficRule } # resolve all VIPs foreach($lbTrafficRule in $loadbalancerTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $lbTrafficRule["frontendIpConfig"] -uri $uri $lbTrafficRule["VIP_IP"] = $vipInfo["VIP_IP"] $lbTrafficRule["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # parse and create workload patterns for all OUTBOUND NAT RULES foreach($outboundNatRule in $lb.properties.outboundNatRules) { $outboundNatTraffic = @{} $outboundNatTraffic["frontendIpConfig"] = $outboundNatRule.properties.frontendIpConfigurations[0].resourceRef $outboundNatTraffic["VIP_PORT"] = 0 $outboundNatTraffic["VIP_PROTOCOL"] = "TCP" $outboundNatTraffic["DIPS"] = @() # resolve be pool to ipconfigurations (DIPS) $bePoolTokens = $outboundNatRule.properties.backendAddressPool.resourceRef.split("/") $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) { $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo = @{} $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $outboundNatTraffic["DIPS"] += $dipInfo } $outboundNatTraffic["DIP_PORT"] = $outboundNatRule.properties.backendPort $outboundNATTrafficRules += $outboundNatTraffic } # resolve all VIPs foreach($outboundNatTraffic in $outboundNATTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $outboundNatTraffic["frontendIpConfig"] -uri $uri $outboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"] $outboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # Add all traffic rules to the endpoint $out = @{} $out["INBOUND_NAT_TRAFFIC_RULES"] = $inboundNATTrafficRules $out["LOADBALANCER_TRAFFIC_RULES"] = $loadbalancerTrafficRules $out["OUTBOUND_NAT_TRAFFIC_RULES"] = $outboundNATTrafficRules return $out } function GetVIPInfoFromFroneEndIpConfig { param( [ValidateNotNullOrEmpty()] [parameter(Mandatory=$true)] [string] $feConfigResourceRef, [ValidateNotNullOrEmpty()] [parameter(Mandatory=$false)] [string] $uri ) $feConfigTokens = $feConfigResourceRef.split("/") $feConfig = Get-NetworkControllerLoadBalancerFrontendIpConfiguration -LoadBalancerId $feConfigTokens[2] -ResourceId $feConfigTokens[4] -ConnectionUri $uri -PassInnerException $vipInfo = @{} if($null -ne $feConfig.properties.PublicIPAddress) { $publicIpResRef = $feConfig.properties.PublicIPAddress.resourceRef; $publicIpResRef = $publicIpResRef.Substring($publicIpResRef.LastIndexOf("/") + 1) $publicIPResource = Get-NetworkControllerPublicIpAddress -ResourceId $publicIpResRef -ConnectionUri $uri -PassInnerException $vipInfo["VIP_IP"] = $publicIPResource.properties.IpAddress $vipInfo["VIP_TYPE"] = 2; # public IP } else { $vipInfo["VIP_IP"] = $feConfig.properties.PrivateIPAddress # check if the subnet is vnet or lnet if($feConfig.Properties.Subnet.resourceRef.StartsWith("/virtualNetworks/")) { $vipInfo["VIP_TYPE"] = 3 ; # ILB } else { $vipInfo["VIP_TYPE"] = 1 ; # private IP } } return $vipInfo } function Stop-CtstrafficListener { paramn( [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet # hostName will be empty for hostName as the client is the local machine if([string]::IsNullOrEmpty($hostName)) { Stop-Process -Name Ctstraffic -Verbose -Force Write-traceLog "Stopped ctstraffic listener on $hostName " return } $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName # run on remote machine $cmd = "Start-Process -FilePath $ctsTrafficPath -ArgumentList ""-listen:* -port:$($listerningPort)""" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Start-CtstrafficListener { param( [int] $listerningPort, [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred, [string] $uri, [string] $outFileName, [TrafficPattern] $trafficPattern, [string] $errorFileName = "error.log" ) [string] $ctsTrafficPath = $script:CtsTrafficVMPath # Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") { $ctsTrafficPath = $script:CtsTrafficVMPath } $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\")); if($listerningPort -eq 0) { # for HA ports, the port value may be set to 0 , override it in that case $listerningPort = $DEFAULT_CTSTRAFFICPORT write-TraceLog "Start-CtstrafficListener: Defaulting to port $listerningPort" } write-TraceLog "Start-CtstrafficListener: Starting ctstraffic ($ctsTrafficPath) listener on $hostName $vmName, ctsfolder : $ctsFolder port:$listerningPort" # use local machine if host and vmname are null if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) { if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd = "-listen:* -port:$($listerningPort) -connectionFileName:$ctsFolder\$outFileName -transfer:$($trafficPattern.transferSizeInBytes)" } else { $cmd = "-listen:* -port:$($listerningPort) -transfer:$($trafficPattern.transferSizeInBytes)" } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose Write-traceLog "Started ctstraffic listener on $hostName with command $cmd" return } if([string]::isNullOrEmpty($hostName)) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true } # build cts traffic parameter list $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;" $cmd += "del $ctsFolder\*.csv -Force;" $cmd += "`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();" $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -PassThru -ArgumentList """ $cmd += " -listen:* -port:$($listerningPort)" if($trafficPattern.Protocol -eq "UDP") { $cmd += " -protocol:udp" $cmd += " -BitsPerSecond:$($trafficPattern.BitsPerSecond)" $cmd += " -FrameRate:$($trafficPattern.FrameRate)" $cmd += " -StreamLength:$($trafficPattern.StreamLength)" } else { $cmd += " -protocol:tcp" if($trafficPattern.transferSizeInBytes -gt 0) { $cmd += " -transfer:$($trafficPattern.transferSizeInBytes)" } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } } if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$outFileName" } if(-not [string]::IsNullOrEmpty($errorFileName)) { $cmd += " -errorFileName:$ctsFolder\$errorFileName" } $cmd += "`";" + [System.Environment]::NewLine # end the -argument list # detect failures in the command and throw exceptions # for listerner, wait for 2 seconds to see if the process has exited (bad params or something) $cmd += [System.Environment]::NewLine + "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalSeconds -lt 2) {Start-Sleep -Seconds 1};" # if process is alive, means it seems stable, proceed $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$false ) { return };" # if process is dead, even if it exited with 0 error code and capture / throw error # if ctstraffic does not generate the error file for some reason then throw a generic error or process return code $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$true ) {`$c = get-content $ctsFolder\$errorFileName -ErrorAction SilentlyContinue; if(-not [string]::IsNullOrEmpty(`$c)) { throw `$c } else { throw ""No error log found. CtsTraffic Exit With code `$p.exitCode""} };" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Start-CtstrafficSender { param( [string] $targetIpAddress, [int] $targetPort, [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred, [string] $uri, [string] $ctsTrafficPath = $script:CtsTrafficVMPath, [string] $outFileName, [bool] $Wait = $true, [TrafficPattern] $trafficPattern ) #Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") { $ctsTrafficPath = $script:CtsTrafficVMPath } $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\")); $errorFileName = "$ctsFolder\error.log" write-TraceLog "Start-CtstrafficSender Starting ctstraffic ($ctsTrafficPath) sender on $hostName $vmName, ctsfolder : $ctsFolder" if($targetPort -eq 0) { # for HA ports, the port value may be set to 0 , override it in that case $targetPort = $DEFAULT_CTSTRAFFICPORT write-TraceLog "Start-CtstrafficSender: Defaulting to port $targetPort" } # use local machine if host and vmname are null if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) { # note: this is only to test it locally, this will not be used finally $cmd = " -target:$($targetIpAddress) -port:$($targetPort) -protocol:$($trafficPattern.protocol) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) -transfer:$($trafficPattern.transferSizeInBytes)" if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$outFileName " } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose -Wait Write-traceLog "Started ctstraffic listener on local machine with command $cmd" return } if([string]::isNullOrEmpty($hostName)) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true } # run on remote machine $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;" $cmd += "del $ctsFolder\*.csv -Force;`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();" $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -ArgumentList """ if($trafficPattern.Protocol -eq "UDP") { $cmd += " -protocol:udp" $cmd += " -BitsPerSecond:$($trafficPattern.BitsPerSecond)" $cmd += " -FrameRate:$($trafficPattern.FrameRate)" $cmd += " -StreamLength:$($trafficPattern.StreamLength)" } else { $cmd += " -protocol:tcp" if($trafficPattern.transferSizeInBytes -gt 0) { $cmd += " -transfer:$($trafficPattern.transferSizeInBytes)" } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } } $cmd += " -target:$($targetIpAddress) -port:$($targetPort) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) " if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$($outFileName) -errorfilename:$errorFileName" } $cmd += """" $cmd += " -PassThru;" $cmd += "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalMinutes -lt 90) {Start-Sleep -Seconds 15};" $cmd += "if(`$p.HasExited -eq `$false ) { `$p.Kill(); throw ""Ctstraffic did not complete on time"" };" $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -ne 0) {`$c = get-content $errorFileName; throw `$c };" $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -eq 0) { return };" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Get-TrafficPattens { # [CmdletBinding()] # param ( # [Parameter()] # [ValidateSet("TCP", "UDP")] # $Protocol, # [Parameter()] # [ValidateSet("ShortBursts", "LongRunning") # [string] $ProfileType # ) # todo : implement different pattern types based on profile # for HLK , we just hardcode some default patterns for now $trafficPatterns = @() $OneKBInBytes = 1024 $OneMBInBytes = 1048576 $OneGBInBytes = 1073741824 # few long runnning connections $trafficPatterns += New-TrafficPattern -connections 5 ` -duration 60 ` -iterations 2 ` -TransferSizeInBytes ($OneGBInBytes * 2) ` -protocol "TCP" # # medium set of connections # $trafficPatterns += New-TrafficPattern -connections 20 ` # -duration 60 ` # -iterations 20 ` # -TransferSizeInBytes ($OneMBInBytes * 2) ` # -protocol "TCP" # udp connections # 10mbps stream over 1000 frames $mbps10 = 25000000 $fr = 60 $trafficPatterns += New-TrafficPattern -connections 5 ` -duration 60 ` -iterations 2 ` -protocol "UDP" ` -bitsPerSecond $mbps10 ` -FrameRate $fr ` -BufferDepth 1 ` -pushPattern "" ` -streamLength 100 return $trafficPatterns } <# Triggers a flurry of live mirgation operations of the given VMs. The VMs are represented by DIP endpoints, which have a VM name and IP address. #> function Start-SDNVmMigrationValidation { param( [int] $percentageOfVmsToMove = 50, [object[]] $trafficEndpoints ) # calculate how many migrations need to be performed [int] $numOfVmsToMove = $trafficEndpoints.Count * $percentageOfVmsToMove / 100 # track vms already attempted to move $vmsMoved = @() # track vms that failed to move $vmsFailedToMove = @() # track vms that moved successfully $vmsMovedSuccessfully = @() $done = $false do { [int] $breaker = 0 $vmToMove = $null # find a target vm to move $vmToMove = Get-Random -InputObject $trafficEndpoints if($null -eq $vmToMove) { throw "Could not find a VM to move" } # move the vm try { # resolve host for the VM $hostName = Resolve-HostName -vmName $vmToMove.VmName -hostCred $vmToMove.HostCredential -vmCred $vmToMove.VmCredential $allHostnames = Get-ClusterNode | ?{$_.State -eq "UP" -and $_.Name -ne $hostName} $target = Get-Random -InputObject $allHostnames # for now using only live migration # $i = Get-Random -Maximum 4 # if($i -eq 0) { # $migrationType = "Quick" # } # if($i -eq 1) { # $migrationType = "Live" # } # if($i -eq 2) { # $migrationType = "Shutdown" # } # if($i -eq 3) { # $migrationType = "TurnOff" # } Write-TraceLog "Start-SDNVmMigrationValidation: Moving VM $($vmToMove.VmName) from host $hostName to host $($target.Name)" Move-ClusterVirtualMachineRole -Name $vmToMove.VMName -Node $target.Name -MigrationType Live $vmsMovedSuccessfully += $vmToMov if($vmsMovedSuccessfully.Count -ge $numOfVmsToMove) { $done = $true } $trafficEndpoints = $trafficEndpoints | Where-Object {$_.VmName -ne $vmToMove.VmName} } catch { Write-TraceLog "Start-SDNVmMigrationValidation: Failed to move VM $($vmToMove.VmName) from host $hostName to host $($target.Name) Error : $_" $vmsFailedToMove += $vmToMove # give up on 2nd failure if($vmsFailedToMove.Count -ge 2) { throw "Multiple live migrations failure, please check your configuration and run the test again" } continue } }until($done) Write-TraceLog "Start-SDNVmMigrationValidation: Successfully moved $numOfVmsToMove VMs" } function Test-TenantVirtualMachine { param( [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet $cmd = "`$process = Get-Process -Name winlogon; if(`$null -ne `$process) { return `$true } else { return `$false }" $hostCred = Get-TurnKeySdnCred $vmCred = Get-TurnKeySdnWorkloadVmCred $retryCount = 0 $maxRetryCount = 10 do { try { $result = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } catch { Write-TraceLog "Test-TenantVirtualMachine: Failed to connect to VM $vmName, retrying... (retry: $retryCount)" Write-TraceLog "Test-TenantVirtualMachine: Error : $_" $result = $false } if($result -eq $true) { return $true } $retryCount += 1 Start-Sleep -Seconds 15 } while($retryCount -lt $maxRetryCount) if($retryCount -ge $maxRetryCount) { throw "Failed to connect to VM $vmName" } Write-TraceLog "Test-TenantVirtualMachine: VM $vmName is available" } function LogTrafficInfo { param( [TrafficEndpoint] $listenEndpoint, [TrafficEndpoint] $sendEndpoint, [TrafficPattern] $trafficPattern ) Write-TraceLog "`t listenEndpoint: $($listenEndpoint.ToString())" Write-TraceLog "`t sendEndpoint: $($sendEndpoint.ToString())" Write-TraceLog "`t traffic: $($trafficPattern.ToString())" } # SIG # Begin signature block # MIIoKgYJKoZIhvcNAQcCoIIoGzCCKBcCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBNDlZCaVfQwho7 # pQgPO6qXHc7+U7714qvPX6vxaHdRjqCCDXYwggX0MIID3KADAgECAhMzAAADrzBA # DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA # hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG # 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN # xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL # go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB # tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd # mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ # 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY # 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp # XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn # TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT # e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG # OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O # PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk # ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx # HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt # CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGgowghoGAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIEZzGWYjc92rhz2JUcKZY8+g # 5Ns/+3ssP+D+90JGYwucMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAVsGkQdOjJrclTxo+pv6hnG6x6ugKapSFSODL+BVvXMCPARgynleNLgn2 # okvWfa1M9NkrW6iHM/kfAZfq77wPuS/95ImvKMLu5nj5ICzyfFQ8eebLick3bYg+ # LxL9En8vy1T5VgsdQRE5Fk77zvMUbFm9Ljoe0HHLYE+dy1Vt4l4qgKgKQKZN5O8X # vOn+ltAxQI1zPLSZMfjVz98s7o102E8m7TBlvv6swbp9nL9L6G/LcPNVI9ne+LkN # 6FqAdysWvZm0R0/4Fpbgyl2AJEiL7Z/ICqiaRLvLO/unw4+fY5UDDwSCs7I/u+XD # H9ldcHa2kdMEsIBLIDKq8qJPcGLdaKGCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCC # F3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq # hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCBlCBdy1x0PJ6PRZ4+dze3q1/2BosWlSzGBx2n0/mHBSwIGZmrkdUtF # GBMyMDI0MDcwOTE3MTYxNC4yMTRaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l # cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTIwMC0w # NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg # ghHqMIIHIDCCBQigAwIBAgITMwAAAecujy+TC08b6QABAAAB5zANBgkqhkiG9w0B # AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzEyMDYxODQ1 # MTlaFw0yNTAzMDUxODQ1MTlaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z # MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTIwMC0wNUUwLUQ5NDcxJTAjBgNV # BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQDCV58v4IuQ659XPM1DtaWMv9/HRUC5kdiEF89YBP6/ # Rn7kjqMkZ5ESemf5Eli4CLtQVSefRpF1j7S5LLKisMWOGRaLcaVbGTfcmI1vMRJ1 # tzMwCNIoCq/vy8WH8QdV1B/Ab5sK+Q9yIvzGw47TfXPE8RlrauwK/e+nWnwMt060 # akEZiJJz1Vh1LhSYKaiP9Z23EZmGETCWigkKbcuAnhvh3yrMa89uBfaeHQZEHGQq # dskM48EBcWSWdpiSSBiAxyhHUkbknl9PPztB/SUxzRZjUzWHg9bf1mqZ0cIiAWC0 # EjK7ONhlQfKSRHVLKLNPpl3/+UL4Xjc0Yvdqc88gOLUr/84T9/xK5r82ulvRp2A8 # /ar9cG4W7650uKaAxRAmgL4hKgIX5/0aIAsbyqJOa6OIGSF9a+DfXl1LpQPNKR79 # 2scF7tjD5WqwIuifS9YUiHMvRLjjKk0SSCV/mpXC0BoPkk5asfxrrJbCsJePHSOE # blpJzRmzaP6OMXwRcrb7TXFQOsTkKuqkWvvYIPvVzC68UM+MskLPld1eqdOOMK7S # bbf2tGSZf3+iOwWQMcWXB9gw5gK3AIYK08WkJJuyzPqfitgubdRCmYr9CVsNOuW+ # wHDYGhciJDF2LkrjkFUjUcXSIJd9f2ssYitZ9CurGV74BQcfrxjvk1L8jvtN7mul # IwIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFM/+4JiAnzY4dpEf/Zlrh1K73o9YMB8G # A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG # Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy # MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w # XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy # dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG # A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD # AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQB0ofDbk+llWi1cC6nsfie5Jtp09o6b6ARC # pvtDPq2KFP+hi+UNNP7LGciKuckqXCmBTFIhfBeGSxvk6ycokdQr3815pEOaYWTn # HvQ0+8hKy86r1F4rfBu4oHB5cTy08T4ohrG/OYG/B/gNnz0Ol6v7u/qEjz48zXZ6 # ZlxKGyZwKmKZWaBd2DYEwzKpdLkBxs6A6enWZR0jY+q5FdbV45ghGTKgSr5ECAOn # LD4njJwfjIq0mRZWwDZQoXtJSaVHSu2lHQL3YHEFikunbUTJfNfBDLL7Gv+sTmRi # DZky5OAxoLG2gaTfuiFbfpmSfPcgl5COUzfMQnzpKfX6+FkI0QQNvuPpWsDU8sR+ # uni2VmDo7rmqJrom4ihgVNdLaMfNUqvBL5ZiSK1zmaELBJ9a+YOjE5pmSarW5sGb # n7iVkF2W9JQIOH6tGWLFJS5Hs36zahkoHh8iD963LeGjZqkFusKaUW72yMj/yxTe # GEDOoIr35kwXxr1Uu+zkur2y+FuNY0oZjppzp95AW1lehP0xaO+oBV1XfvaCur/B # 5PVAp2xzrosMEUcAwpJpio+VYfIufGj7meXcGQYWA8Umr8K6Auo+Jlj8IeFS6lSv # KhqQpmdBzAMGqPOQKt1Ow3ZXxehK7vAiim3ZiALlM0K546k0sZrxdZPgpmz7O8w9 # gHLuyZAQezCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI # hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy # MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC # AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg # M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF # dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6 # GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp # Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu # yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E # XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0 # lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q # GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ # +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA # PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw # EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG # NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV # MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj # cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK # BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC # AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX # zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v # cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI # KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG # 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x # M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC # VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449 # xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM # nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS # PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d # Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn # GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs # QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL # jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL # 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNN # MIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn # MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjkyMDAtMDVFMC1EOTQ3MSUwIwYDVQQD # ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQCz # cgTnGasSwe/dru+cPe1NF/vwQ6CBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6jen4DAiGA8yMDI0MDcwOTEyMTMy # MFoYDzIwMjQwNzEwMTIxMzIwWjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDqN6fg # AgEAMAcCAQACAgKDMAcCAQACAhjxMAoCBQDqOPlgAgEAMDYGCisGAQQBhFkKBAIx # KDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZI # hvcNAQELBQADggEBAIPnw83ehDF1/uDkm+sCARcNNG1DKxvPU4tU18KBykIlJHrW # Gqi7HCRX7yDRn6mcaEcNmdRxLclyk/NeNbeg+PcRQMAHsISLe97KyX0rk7gxdal0 # Lnjqj/i53RkZReDNB0FG1RzcwPdZ2p2IPT6a4lh2i/sy7dhthrrXe9aku+H9unjA # DYkZcjRclibY4JyFWlasmuC9tNgKWXBgt7U2e//D27maDwFBcnECBJrHF5pmSwCL # MrujpLxVizXncsVPtSJFZIRkVTSVewy4mGadje3BUDJSLBPq6qLjWTM40S+B2y9L # NfdgIWjqsDKzi4zCDw/LeIn/iLxdWF6RzV3ZwJoxggQNMIIECQIBATCBkzB8MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNy # b3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAecujy+TC08b6QABAAAB5zAN # BglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8G # CSqGSIb3DQEJBDEiBCCdRN2RUdNZ7WvOeWTmrdAvbNKD1cqLzXL2tP4WmNCVwTCB # +gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIOU2XQ12aob9DeDFXM9UFHeEX74F # v0ABvQMG7qC51nOtMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh # c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD # b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw # MTACEzMAAAHnLo8vkwtPG+kAAQAAAecwIgQgF8BCWne0XzZOAF8zeJUIdMK1TQKx # UE6N313cG4grKrAwDQYJKoZIhvcNAQELBQAEggIALYI2OngTJj+ZMPIRB7Gb5ksm # 5L9m86okkC+Smpum6Dps8nV0EdjCUmZpVcYqhC6uVXFveK1LAF8ZhJ+rq72Z8+4f # ZTCxM94HcaVEbpeMmNqevA8kch+e2yHi6kY1HeTnZ4jh7TTalew0VuWHv2r0tRX/ # r2tbTitAldZZtQFLS++tpJ1e2IIDQx+0iGo3Tj7xOMss4WzbQYmroAt1EzKmVFYC # x7Z/6PuFQ4phsYx/owAFQuG/su8UxEJSynoCoJDhs+UNo+H7YzxrP6rgETAu/KjV # aaW0lYWN6eFZwQCM537n+fDbxjL3lO/SeKkD8PJn42tEucdf6CXwKJ6ue/fGF9L6 # ITMZpIGpHRgYTpLq0272Zi5xzvkN556o44i2u43dIF2h3pcs8/lqDnjQEJwhwjE/ # pcjtYR0iG458LaPCWlxEyPnAOzHn1JlTzigEAEKffjMEU0oLDrhk4u8J5rsbEO7d # MMpSI6rt374azTvHp2IanftuIl6xbunqeakBm3Tas7gGt9urapGAY2V70P5ek6t4 # dGmi+is5HJ6lugj7aUKL8WbSmtwzlwsQoWeEMM4NSPrtWR90ifsMMQ9fhVSl2OuI # QqUDfEljxynSwsBP4U1ckgHwQLvwDJgg+3wO2PXFcKr3AGx31V9asw5KGZ6P2ndn # C4YhIYQZYDR3rxsz1BE= # SIG # End signature block |