TrafficUtil.psm1
|
Import-Module $PSScriptRoot\Logger.psm1 Import-Module $PSScriptRoot\ConfigManager.psm1 class TrafficEndpoint { [string] $VmName = "VmName" [string] $HostName = "HostName" [pscredential] $VmCredential = $null [pscredential] $hostCredential = $null [string] $IpAddress = "0.0.0.0" [int] $Port = 5001 [string] $PublicIP = $null } class TrafficPattern { [int] $Connections = 10 [int] $Duration = 1 [int] $Iterations = 1 # 10 GB [int64] $TransferSizeInBytes = 10737418240 [string] $Protocol = "TCP" [uint64] $BitsPerSecond = 0 [uint64] $FrameRate = 0 [uint64] $BufferDepth = 0 [string] $PushPattern = "pushpull" } class WorkloadSummary { [int] $TotalConnections = 0 [int] $FailedConnections = 0 [int] $SuccessfulConnections = 0 [int] $TotalBytesTransferred = 0 } Set-Variable VNET_DIP -Value 0 -Option Constant Set-Variable PRIV_VIP -Value 1 -Option Constant Set-Variable PUB_VIP -Value 2 -Option Constant Set-Variable ILB_VIP -Value 3 -Option Constant Set-Variable CLIENT_IP -Value 4 -Option Constant Set-Variable DIPS -Value "DIPS" -Option Constant Set-Variable VIP_IP -Value "VIP_IP" -Option Constant Set-Variable VIP_PORT -Value "VIP_PORT" -Option Constant Set-Variable VIP_PROTOCOL -Value "VIP_PROTOCOL" -Option Constant Set-Variable VIP_TYPE -Value "VIP_TYPE" -Option Constant Set-Variable DIP_IP -Value "DIP_IP" -Option Constant Set-Variable DIP_PORT -Value "DIP_PORT" -Option Constant Set-Variable DIP_VMNAME -Value "DIP_VMNAME" -Option Constant Set-Variable FRONTEND_IP_CONFIG -Value "FRONTEND_IP_CONFIG" -Option Constant Set-Variable BACKEND_IP_CONFIG -Value "BACKEND_IP_CONFIG" -Option Constant Set-Variable INBOUND_NAT_TRAFFIC_RULES -Value "INBOUND_NAT_TRAFFIC_RULES" -Option Constant Set-Variable LOADBALANCER_TRAFFIC_RULES -Value "LOADBALANCER_TRAFFIC_RULES" -Option Constant Set-Variable OUTBOUND_NAT_TRAFFIC_RULES -Value "OUTBOUND_NAT_TRAFFIC_RULES" -Option Constant Set-Variable DIP_IPCONFIG -Value "DIP_IPCONFIG" -Option Constant Set-Variable CLIENT_VMNAME -Value "InternetClient" -Option Constant Set-Variable DIP_VM_HOSTNAME -Value "DIP_VM_HOSTNAME" -Option Constant Set-Variable TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -Value 5 -Option Constant $script:ctsTrafficFolderPath = "C:\tools\" $script:ctsTrafficPath = "C:\tools\ctsTraffic.exe" <# This function will create a new traffic pattern object The traffic pattern object will be used to run traffic between two VMs #> function New-TrafficPattern { param( [int] $connections, [int] $duration, [int] $iterations, [int64] $TransferSizeInBytes, [string] $protocol = "TCP", [int64] $bitsPerSecond = 0, [int64] $frameRate = 0, [int64] $bufferDepth = 0, [ValidateSet("push", "pullpush","pull","duplex", "")] [string] $pushPattern = "pushpull" ) $pattern = New-Object TrafficPattern $pattern.Connections = $connections $pattern.Duration = $duration $pattern.Iterations = $iterations $pattern.TransferSizeInBytes = $TransferSizeInBytes $pattern.Protocol = $protocol $pattern.BitsPerSecond = $bitsPerSecond $pattern.FrameRate = $frameRate $pattern.BufferDepth = $bufferDepth $pattern.PushPattern = $pushPattern return $pattern } <# Creates a new traffic endpoint object. Endpoints can run/receieve traffic between two VMs endpointType : 0 - Vnet DIP 1 - Private IP Address 2 - PublicIP 3 - ILB 4 - Client IP #> function New-TrafficEndpoint { param( [string] $vmName, [string] $hostName, [pscredential] $vmCredential, [pscredential] $hostCredential, [string] $ipAddress, [int] $port, [bool] $resolveHostName = $false, [string] $publicIp = $null, [ValidateSet(0,1,2,3,4)] [parameter(Mandatory=$true)] [int] $endpointType ) if($resolveHostName -eq $true) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCredential -vmName $vmName -Force $true } $endpoint = New-Object TrafficEndpoint $endpoint.VmName = $vmName $endpoint.HostName = $hostName $endpoint.VmCredential = $vmCredential $endpoint.HostCredential = $hostCredential $endpoint.IpAddress = $ipAddress $endpoint.Port = $port $endpoint.PublicIP = $publicIp return $endpoint } <# Checks if the VM has ctstraffic installed and physically ready to run traffic workloads - Checks if the VM is powered on - Checks if the VM has ctstraffic installed - Checks if the VM has a valid IP address - Checks if the VM has a ctstraffic is enabled on firewall #> function Init-VmForCtstraffic { param( [object[]] $trafficEndpoints, [pscredential] $hostCred, [pscredential] $vmCred ) [bool] $isCluster = (Test-IsCluster) #test if ctstraffic is available or not, if not, download it if( $isCluster) { $csvPath = get-csv $ctsTrafficClusterPath = Join-Path -Path $csvPath -ChildPath "\WorkloadTools\CtsTraffic.exe" } else { $ctsTrafficClusterPath = $script:ctsTrafficPath } if(-not (Test-Path $script:ctsTrafficPath) -or -not (Test-Path $ctsTrafficClusterPath)) { Write-TraceLog "Init-VmForCtstraffic: ctstraffic not found, downloading it" #download ctstraiffic from github Start-BitsTransfer https://github.com/microsoft/ctsTraffic/raw/master/Releases/2.0.3.2/x64/ctsTraffic.exe $script:ctsTrafficPath if($ctsTrafficClusterPath -ne $script:ctsTrafficPath) { mkdir "$csvPath\WorkloadTools\" -ErrorAction SilentlyContinue copy "$script:ctsTrafficPath" "$csvPath\WorkloadTools\" -Verbose -ErrorAction SilentlyContinue } Write-SDNExpressLog "Copied ctstraffic.exe from $script:ctsTrafficPath to $path" } foreach($endpoint in $trafficEndpoints) { # log all properties of endpoint Write-SDNExpressLog -Message "Copy-VMFile $($endpoint.VmName) ` Host:$($endpoint.HostName) ` ToolPath:$script:ctsTrafficPath ` SourcePath:$script:ctsTrafficPath ` DestPath:$script:ctsTrafficPath ` " # stop any previous running instances of ctstraffic Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" ` -hostCred $hostCred ` -vmCred $vmCred [string] $ctsTrafficLocalMachineSourcePath = $script:ctsTrafficPath if($isCluster) { $ctsTrafficLocalMachineSourcePath = $ctsTrafficClusterPath } $ctsEnableFirewall = "`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Inbound)"" -Direction Inbound -Program $script:ctsTrafficPath -Action Allow" $ctsEnableFirewall += ";`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Outbound)"" -Direction Outbound -Program $script:ctstrafficPath -Action Allow" $hostCred = Get-TurnKeySdnCred $vmCred = Get-TurnKeySdnWorkloadVmCred $results = Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd $ctsEnableFirewall ` -hostCred $hostCred ` -vmCred $vmCred # ctstraffic.exe is detected as malware by windows defender, so we need to disable it # it also does full FS scan which slows deployment considerably $disableDefender = "Set-MpPreference -DisableRealtimeMonitoring `$true;" $disableDefender += "Set-MpPreference -DisableBehaviorMonitoring `$true;" $disableDefender += "Set-MpPreference -DisableBlockAtFirstSeen `$true;" $disableDefender += "Add-MpPreference -ExclusionPath $script:ctsTrafficFolderPath;" Invoke-PowershellCommandOnVm -vmName $endpoint.VmName ` -hostName $endpoint.HostName ` -cmd $disableDefender ` -hostCred $hostCred ` -vmCred $vmCred # now copy ctstraffic to the vm Copy-VMFile -VMName $endpoint.VmName ` ` -SourcePath $ctsTrafficLocalMachineSourcePath ` -DestinationPath $script:ctsTrafficPath ` -FileSource Host ` -ComputerName $endpoint.HostName ` -CreateFullPath ` -Force } } <# Runs traffic between given endpoints - Completes a full mesh (NXN endpoints with the traffic pattern provided), with one at a time. - Assumes that appropriate firewall rules are enabled on the VMs (From SDN) #> function Start-IntraVmTraffic { param( [object[]] $trafficEndpoints, [TrafficPattern[]] $trafficPatterns, [pscredential] $hostCred, [pscredential] $vmCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet # todo : (optimization) this can be sped up if we keep the destination running and go over the targets, this # way the init time can be saved (needs to handle different outFileNames for each target) foreach($trafficPattern in $trafficPatterns) { foreach($srcEndpoint in $trafficEndpoints) { foreach($dstEndpoint in $trafficEndpoints) { if($srcEndpoint -eq $dstEndpoint) { continue } if($srcEndpoint.VmName -like "*ilb_client*" -or $dstEndpoint.VmName -like "*ilb_client*") { Write-TraceLog "Start-IntraVmTraffic: ILB & clients are skipped from EW traffic tests." continue } $guid = New-Guid Write-Host "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) starting.." $outFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".csv" $errorFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".err" Start-CtstrafficListener -listerningPort $dstEndpoint.Port ` -vmName $dstEndpoint.VmName ` -hostName $dstEndpoint.HostName ` -vmCred $vmCred ` -hostCred $hostCred ` -trafficPattern $trafficPattern ` -outFileName $outFileName ` -uri $uri ` -errorFileName $errorFileName ` -isILB $false Start-CtstrafficSender -targetIpAddress $dstEndpoint.IpAddress ` -targetPort $dstEndpoint.Port ` -hostCred $hostCred ` -vmCred $vmCred ` -vmName $srcEndpoint.VmName ` -hostName $srcEndpoint.HostName ` -trafficPattern $trafficPattern ` -outFileName $outFileName ` -uri $uri ` -wait $true $srcWorkloadResult = Wait-ForWorloadCompletion -vmName $srcEndpoint.VmName ` -hostName $srcEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName Test-WorkloadResult -workloadResults $srcWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $srcEndpoint.VmName $dstWorkloadResult = Wait-ForWorloadCompletion -vmName $dstEndpoint.VmName ` -hostName $dstEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $dstWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dstEndpoint.VmName Write-Host "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) PASSED!" } } } } <# Runs SLB Traffic on a given set of endpoints. The traffic pattern is defined by the following parameters: - RulesToTarget : The type of rules to target on the load balancer. Valid values are: - LoadbalancerRules : All load balancer rules - Inbound : All inbound NAT rules - Outbound : All outbound NAT rules The caller must create the metadata json which describes the topology of DIPs and VIPs before calling this method. Use the helper Get-EndpointsFromLoadBalancer to extract the metadata from a given loadbalancer and pass it to this method to initiate traffic. NOTE: the Get-EndpointsFromLoadBalancer does not detect if probes are down for the given endpoints. Please disable probes before calling this method. For inbound/lb rules the server instances are #> function Start-SLBTraffic { param( [ValidateSet("LoadbalancerRules", "Inbound", "Outbound")] [string] $TargetType, [object] $vips, [TrafficPattern[]] $trafficPatterns, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) if($TargetType -eq "LoadbalancerRules") { $vipInfo = $vips[$LOADBALANCER_TRAFFIC_RULES] if($null -eq $vipInfo -or $vipInfo.Count -eq 0) { throw "No loadbalancer rules found" } foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(lb) $vipEndpoint TRAFFICPATTERN: $trafficPattern" Start-SLBLBRuleTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } elseif($TargetType -eq "Inbound") { $vipInfo = $vips[$INBOUND_NAT_TRAFFIC_RULES] foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(inbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern" Start-SLBInboundTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } elseif($TargetType -eq "Outbound") { $vipInfo = $vips[$OUTBOUND_NAT_TRAFFIC_RULES] foreach($vipEndpoint in $vipInfo) { foreach($trafficPattern in $trafficPatterns) { Write-TraceLog "Start-SLBTraffic:(outbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern" Start-SLBOutboundTraffic -vipEndpoint $vipEndpoint ` -trafficPattern $trafficPattern ` -hostCred $hostCred ` -vmCred $vmCred ` -uri $uri } } } else { throw "Invalid TargetType parameter" } } <# Gets a suitable endpoint to run the client traffic on, if its ILB, it will pick up a VNET location (for ILB: The ILB client NIC is used) for everything else, it will use the PublicIP on the ILB CLient todo: make this configurable to run the workload in different locations #> function Get-SLBClientEndpoint { param( [object[]] $dips, [int] $vipType, [string] $uri, [pscredential] $hostCred, [pscredential] $vmCred, [string] $port ) if($dips -eq $null -or $dips.Count -eq 0) { throw "Update-DipInfoIfILBEnabledDips: Dips is null or empty" } $ipConfigTokens = $dips[0][$DIP_IPCONFIG].split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $subnetNameResourceRef = $ipConfig.properties.Subnet.resourceRef $subnetNameResRefTokens = $subnetNameResourceRef.split("/") $subnetName = $subnetNameResRefTokens[4] $vmName = "$($subnetName)_ilb_client" $nic = Get-NetworkControllerNetworkInterface -ResourceId $vmName -ConnectionUri $uri -PassInnerException if($nic -eq $null) { throw "Could not find NIC $vmName" } # for ILB cases, use the ILB VNET IP Address for traffic if($vipInfo[$VIP_TYPE] -eq 3) { New-TrafficEndpoint -vmName $vmName ` -hostName $null ` -vmCredential $vmCred ` -hostCredential $hostCred ` -ipAddress $nic.properties.IpConfigurations[0].properties.privateIpAddress ` -port 5001 ` -endpointType $CLIENT_IP ` -resolveHostName $true ` -publicIp $null } else { # for other LB scenarios use the PublicIP address for traffic # get public ip from the ilb client nic $publicIpResRef = $nic.properties.IpConfigurations[0].properties.PublicIPAddress.resourceRef $publicIpResRefTokens = $publicIpResRef.split("/") $publicIpResource = Get-NetworkControllerPublicIPAddress -ResourceId $publicIpResRefTokens[2] -ConnectionUri $uri -PassInnerException $publicIpAddress = $publicIpResource.properties.IpAddress New-TrafficEndpoint -vmName $vmName ` -hostName $null ` -vmCredential $vmCred ` -hostCredential $hostCred ` -ipAddress $null ` -port 5001 ` -endpointType $CLIENT_IP ` -resolveHostName $true ` -publicIp $publicIpAddress } } function Start-SLBInboundTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) # todo : this can be sped up if we pre resolve the host names for all the DIPs $dipHostName = Resolve-HostName -hostName $vipEndpoint[$DIPS][0][$DIP_VMNAME] ` -hostCred $hostCred ` -vmName $vipEndpoint[$DIPS][0][$DIP_VMNAME] ` -Force $true # select the best client based on the VIP type $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBInboundTraffic: Starting INBOUND Traffic ($($guid))" Write-TraceLog "Start-SLBInboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBInboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBInboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBInboundTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress) " Write-TraceLog "----------------------------------------------" # set up dip to recv Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] ` -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] ` -hostName $dipHostName ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern Start-CtstrafficSender -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -targetIpAddress $vipEndpoint[$VIP_IP] ` -targetPort $vipEndpoint[$VIP_PORT] ` -hostCred $hostCred ` -vmCred $vmCred ` -ctsTrafficPathLocal $script:ctsTrafficPath ` -outFileName $outFileName ` -trafficPattern $trafficPattern # once the sender is complete, the receiver can be forced killed $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] ` -hostName $dipHostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $vipEndpoint["DIPS"][0][$DIP_VMNAME] $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBInboundTraffic: END --- " } function Test-WorkloadResult { param( [object] $workloadResults, [int] $maxFailedPercentage = 10, [string] $EndpointName ) if($workloadResults -eq $null) { throw "Workload result is null" } if($workloadResults.count -eq 0) { throw "Workload result is empty" } [int] $total = 0 [int] $failed = 0 [int] $successful = 0 foreach($result in $workloadResults) { if($result.Result -eq "Failed") { $failed += 1 } else { $successful += 1 } $total += 1 } if($total -eq 0) { throw "Workload result is empty" } $failedPercentage = $failed / $total * 100 if($failedPercentage -gt $maxFailedPercentage) { throw "Workload failed for more than 10% of the connections" } Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" Write-TraceLog "Test-WorkloadResult: Connections Total:$total Failed:$failed Successful:$successful Failed %:$failedPercentage" Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" return $true } function Start-SLBLBRuleTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) # select the best client based on the VIP type $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBLBRuleTraffic: Starting LB Rule Traffic ($($guid))" Write-TraceLog "Start-SLBLBRuleTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBLBRuleTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBLBRuleTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBLBRuleTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress):$($clientEndpoint.port) publicIp:$($clientEndpoint.PublicIP)" Write-TraceLog "----------------------------------------------" foreach($dipInfo in $vipEndpoint[$DIPS]) { Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] ` -vmName $dipInfo[$DIP_VMNAME] ` -hostName $null ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern } Start-CtstrafficSender -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -targetIpAddress $vipEndpoint[$VIP_IP] ` -targetPort $vipEndpoint[$VIP_PORT] ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName ` -trafficPattern $trafficPattern foreach($dipInfo in $vipEndpoint[$DIPS]) { # once the sender is complete, the receiver can be forced killed $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME] ` -hostName $null ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME] } $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBLBRuleTraffic: LB Rule Traffic ($($guid)) PASSED" Write-TraceLog "----------------------------------------------" } function Start-SLBOutboundTraffic { param( [object] $vipEndpoint, [TrafficPattern] $trafficPattern, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) Write-TraceLog "Start-SLBOutboundTraffic: $dipInfo $vipInfo" # for Get-SLBClientEndpoint gives us the ILB client VM which has a PublicIP on it, so it should work from there $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] ` -vipType $vipEndpoint[$VIP_TYPE] ` -uri $uri ` -hostCred $hostCred ` -vmCred $vmCred ` -port $vipEndpoint[$VIP_PORT] $vipPort = 5001 $guid = New-Guid $outFileName = ($guid.ToString()).Substring(0,8) + ".csv" Write-TraceLog "----------------------------------------------" Write-TraceLog "Start-SLBOutboundTraffic: Starting OUTBOUND LB Rule Traffic ($($guid))" Write-TraceLog "Start-SLBOutboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) " Write-TraceLog "Start-SLBOutboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])" foreach($dipInfo in $vipEndpoint[$DIPS]) { Write-TraceLog "Start-SLBOutboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" } Write-TraceLog "Start-SLBOutboundTraffic: `t CLIENT : $($clientEndpoint.vmName) IP $($clientEndpoint.IpAddress) PORT;$($clientEndpoint.port) PUBLICIP:$($clientEndpoint.PublicIP)" Write-TraceLog "----------------------------------------------" # set up client to recv Start-CtstrafficListener -listerningPort $vipPort ` -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -vmCred $vmCred ` -hostCred $hostCred ` -outFileName $outFileName ` -trafficPattern $trafficPattern # start senders foreach($dipInfo in $vipEndpoint[$DIPS]) { Start-CtstrafficSender -targetIpAddress $clientEndpoint.PublicIP ` -targetPort $vipPort ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -vmName $dipInfo[$DIP_VMNAME] ` -trafficPattern $trafficPattern } # wait for senders $workloadResults = @() foreach($dipInfo in $vipEndpoint[$DIPS]) { $workloadResult = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME] ` -hostName "" ` -hostCred $hostCred ` -vmCred $vmCred ` -outFileName $outFileName ` -force $true # test workload results per DIP Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME] $workloadResults += $workloadResult } # also validate the total workload results Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName "SLB-OUTBOUND-DIPS_SUMMARY" # validate workload results on the oubound receiver $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName ` -hostName $clientEndpoint.HostName ` -hostCred $clientEndpoint.HostCredential ` -vmCred $clientEndpoint.VmCredential ` -outFileName $outFileName ` -force $true Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName Write-TraceLog "Start-SLBOutboundTraffic: Passed" } function Start-Traffic { param( [ValidateSet("Listen", "Target")] [string] $Role, [object] $trafficPattern, [object] $currEndpoint, [object] $targetEndpoint, [pscredential] $hostCred, [pscredential] $vmCred ) # note: # currEndpoint contains context on where the instance of ctstraffic is going to be run (either listen or target) # targetEndpoint contains context on where the traffic is going to be sent to, its required for sender to know where to send the traffic to $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue;" if($Role -eq "Listen") { $cmd += "Start-Process " + $script:ctsTrafficPath + " -ArgumentList ""-listen:* -port:$($targetEndpoint.Port)"""; } else { $cmd += "Start-Process -FilePath " + $script:ctsTrafficPath + " -Wait -ArgumentList ""-target:$($targetEndpoint.IpAddress) -port:$($targetEndpoint.Port) -protocol:tcp -connections:$($trafficPattern.Connections) -iterations:$($trafficPattern.Iterations)"""; } Write-TraceLog -Message "Start-Traffic ($Role): $cmd" Invoke-PowershellCommandOnVmInternal -vmName $currEndpoint.VmName ` -hostName $currEndpoint.HostName ` -cmd $cmd ` -hostCredentials $hostCred ` -vmCredentials $vmCred } function Resolve-HostName { param( [string] $hostName, [pscredential] $hostCred, [string] $vmName, [bool] $Force = $false ) if(-not [string]::IsNullOrEmpty($hostName) -and $Force -eq $false) { Write-TraceLog "Resolve-HostName: HostName is not null or empty, returning $hostName" return $hostName } if([string]::IsNullOrEmpty($vmName)) { Write-TraceLog "Resolve-HostName: VMName is null, returning $hostName" return $hostName; } $nodes = (get-clusternode).Name foreach($node in $nodes) { $vm = Get-VM -VMName $vmName -ComputerName $node -ErrorAction SilentlyContinue if($vm -ne $null) { Write-TraceLog "Resolve-HostName: Resolved VM $vmName on node $node" return $node } } throw "Could not resolve host name for vm $vmName" } function Wait-ForWorloadCompletion { param( [string] $vmName, [string] $hostName, [pscredential] $hostCred, [pscredential] $vmCred, [string] $outFileName, [int] $timeoutInSeconds = 3600, [string] $ctstrafficPathLocal = $script:ctsTrafficPath, [bool] $force = $false ) Write-FunctionEntryWithParams -FunctionName "Wait-ForWorloadCompletion" -BoundParameters $PSBoundParameters -UnboundArguments $args # todo : avoid resolution of VM to speed things up a bit $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $false Write-TraceLog "Wait-ForWorloadCompletion: Resolved hostName $hostName" $ctsTrafficFolderPath = $ctstrafficPathLocal.Substring(0, $ctstrafficPathLocal.LastIndexOf("\")) $csvFilePath = Join-Path $ctsTrafficFolderPath $outFileName if([string]::IsNullOrEmpty($vmName) -and [string]::IsNullOrEmpty($hostName)) { # wait for ctstraffic.exe to complete or kill and exit $done = $false [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $stopwatch.Start() while(-not $done -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) { Write-Host "Waiting for ctstraffic to complete" $p = Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue if($null -eq $p) { $done = $true break } } if(-not $done) { Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue } if(-not $force -and -not $done) { throw "Workload did not complete in time" } $WorkloadResult = Get-Content -Path $csvFilePath | ConvertFrom-Csv return $WorkloadResult } # the following is only if we are running traffic from the host (which is running turnkey) $done = $false [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $stopwatch.Start() while(-not $done -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) { Write-Host "Waiting for ctstraffic to complete" $cmd = "Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue" $p = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred if($null -eq $p) { $done = $true break } } if ($force -or -not $done) { Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" ` -hostCred $hostCred ` -vmCred $vmCred } if (-not $done -and -not $force) { throw "Workload did not complete in time" } $cmd = "Get-Content -Path $csvFilePath | ConvertFrom-Csv" $WorkloadResult = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred return $WorkloadResult } function Unblock-AllPortsOnVm { param( [string] $nicResourceId, [string] $restName ) $uri = "https://$restName"; $vnic = Get-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -PassInnerException if($vnic.properties.IpConfigurations -eq $null -or $vnic.properties.IpConfigurations.Count -eq 0) { throw "No IP configuration found on the NIC" } if($vnic.properties.IpConfigurations.Count -gt 1) { throw "Multiple IP configurations found on the NIC" } # create the NICs ACL $nicAclResourceId = $nicResourceId + "_ACL" $newNicAcl = New-Object Microsoft.Windows.NetworkController.AccessControlList $newNicAcl.properties = New-Object Microsoft.Windows.NetworkController.AccessControlListProperties $newNicAcl.properties.AclRules = @() $newNicAclRuleIdBase = "ACL_RULE_"; $newNicAcl.properties.AclRules += GetAclRule -resourceId $($newNicAclRuleIdBase + 1) ` -protocol "All" ` -action "Allow" ` -sourcePrefix "*" ` -destPrefix "*" ` -type "Inbound" ` -priority 1000 $newNicAcl.properties.AclRules += GetAclRule -resourceId $($newNicAclRuleIdBase + 2) ` -protocol "All" ` -action "Allow" ` -sourcePrefix "*" ` -destPrefix "*" ` -type "Outbound" ` -priority 1000 $newAcl = New-NetworkControllerAccessControlList -ResourceId $nicAclResourceId -ConnectionUri $uri -Properties $newNicAcl.properties -PassInnerException -Force Write-TraceLog "Created/updated allow all acl $nicAclResourceId, resourceRef : $($newAcl.resourceRef)" #bind this ACL to the NIC $vnic.properties.IpConfigurations[0].Properties.AccessControlList = $newAcl $nic = New-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -Properties $vnic.properties -PassInnerException -Force Write-TraceLog "Applied unblock-all ports ACL for NIC $nicResourceId" } function GetAclRule { param( [parameter(Mandatory=$true)] [String]$resourceId, [parameter(Mandatory=$true)] [String]$protocol, [parameter(Mandatory=$true)] [String]$action, [parameter(Mandatory=$true)] [String]$sourcePrefix, [parameter(Mandatory=$true)] [String]$destPrefix, [parameter(Mandatory=$true)] [String]$type, [parameter(Mandatory=$true)] [int]$priority ) $ruleproperties = new-object Microsoft.Windows.NetworkController.AclRuleProperties $ruleproperties.Protocol = $protocol $ruleproperties.SourcePortRange = "*" $ruleproperties.DestinationPortRange = "*" $ruleproperties.Action = $action $ruleproperties.SourceAddressPrefix = $sourcePrefix $ruleproperties.DestinationAddressPrefix = $destPrefix $ruleproperties.Priority = $priority $ruleproperties.Type = $type $ruleproperties.Logging = "Enabled" $aclrule = new-object Microsoft.Windows.NetworkController.AclRule $aclrule.Properties = $ruleproperties $aclrule.ResourceId = $resourceId return $aclrule } function Get-EndpointsFromLoadBalancer { param( [string] $lbResourceId, [pscredential] $hostCred, [pscredential] $vmCred, [string] $uri ) $lb = Get-NetworkControllerLoadBalancer -ResourceId $lbResourceId -ConnectionUri $uri -PassInnerException if($lb.properties.frontendIPConfigurations -eq $null -or $lb.properties.frontendIPConfigurations.Count -eq 0) { return $lbEndpoints } if( $lb.properties.frontendIPConfigurations.Count -gt 1) { throw "Multiple frontend IP configurations found on the load balancer" } # all Traffic rules $inboundNATTrafficRules = @() $loadbalancerTrafficRules = @() $outboundNATTrafficRules = @() # parse and create workload patterns for all INBOUND NAT RULES foreach($inboundNatRule in $lb.properties.inboundNatRules) { $inboundNatTraffic = @{} if( $inboundNatRule.properties.backendIpConfiguration -eq $null -or $inboundNatRule.properties.backendIpConfiguration.Count -eq 0) { continue } $inboundNatTraffic["frontendIpConfig"] = $inboundNatRule.properties.frontendIpConfigurations[0].resourceRef $inboundNatTraffic["VIP_PORT"] = $inboundNatRule.properties.frontendPort $inboundNatTraffic["VIP_PROTOCOL"] = $inboundNatRule.properties.protocol $inboundNatTraffic["backendIpConfig"] = $inboundNatRule.properties.backendIpConfiguration.resourceRef $inboundNatTraffic["DIP_PORT"] = $inboundNatRule.properties.backendPort $inboundNATTrafficRules += $inboundNatTraffic $inboundNatTraffic["DIPS"] = @() } # resolve all DIPs to IPs foreach($inboundNatTraffic in $inboundNATTrafficRules) { $dipInfo = @{} $ipConfigTokens = $inboundNatTraffic["backendIpConfig"].split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $dipInfo["DIP_PORT"] = $inboundNatTraffic["DIP_PORT"] $inboundNatTraffic["DIPS"] += $dipInfo } # resolve all VIPs foreach($inboundNatTraffic in $inboundNATTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $inboundNatTraffic["frontendIpConfig"] -uri $uri $inboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"] $inboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # parse and create workload patterns for all LB RULES foreach($lbRule in $lb.properties.loadBalancingRules) { $lbTrafficRule = @{} $lbTrafficRule["frontendIpConfig"] = $lbRule.properties.frontendIpConfigurations[0].resourceRef $lbTrafficRule["VIP_PORT"] = $lbRule.properties.frontendPort $lbTrafficRule["VIP_PROTOCOL"] = $lbRule.properties.protocol $lbTrafficRule["DIPS"] = @() #resolve DIPs $bePoolTokens = $lbRule.properties.backendAddressPool.resourceRef.split("/") $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) { $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo = @{} $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $lbTrafficRule["DIPS"] += $dipInfo } $lbTrafficRule["DIP_PORT"] = $lbRule.properties.backendPort $loadbalancerTrafficRules += $lbTrafficRule } # resolve all VIPs foreach($lbTrafficRule in $loadbalancerTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $lbTrafficRule["frontendIpConfig"] -uri $uri $lbTrafficRule["VIP_IP"] = $vipInfo["VIP_IP"] $lbTrafficRule["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # parse and create workload patterns for all OUTBOUND NAT RULES foreach($outboundNatRule in $lb.properties.outboundNatRules) { $outboundNatTraffic = @{} $outboundNatTraffic["frontendIpConfig"] = $outboundNatRule.properties.frontendIpConfigurations[0].resourceRef $outboundNatTraffic["VIP_PORT"] = 0 $outboundNatTraffic["VIP_PROTOCOL"] = "TCP" $outboundNatTraffic["DIPS"] = @() # resolve be pool to ipconfigurations (DIPS) $bePoolTokens = $outboundNatRule.properties.backendAddressPool.resourceRef.split("/") $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) { $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/") $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException $dipInfo = @{} $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2] $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef $outboundNatTraffic["DIPS"] += $dipInfo } $outboundNatTraffic["DIP_PORT"] = $outboundNatRule.properties.backendPort $outboundNATTrafficRules += $outboundNatTraffic } # resolve all VIPs foreach($outboundNatTraffic in $outboundNATTrafficRules) { $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $outboundNatTraffic["frontendIpConfig"] -uri $uri $outboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"] $outboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"] } # Add all traffic rules to the endpoint $out = @{} $out["INBOUND_NAT_TRAFFIC_RULES"] = $inboundNATTrafficRules $out["LOADBALANCER_TRAFFIC_RULES"] = $loadbalancerTrafficRules $out["OUTBOUND_NAT_TRAFFIC_RULES"] = $outboundNATTrafficRules return $out } function GetVIPInfoFromFroneEndIpConfig { param( [ValidateNotNullOrEmpty()] [parameter(Mandatory=$true)] [string] $feConfigResourceRef, [ValidateNotNullOrEmpty()] [parameter(Mandatory=$false)] [string] $uri ) $feConfigTokens = $feConfigResourceRef.split("/") $feConfig = Get-NetworkControllerLoadBalancerFrontendIpConfiguration -LoadBalancerId $feConfigTokens[2] -ResourceId $feConfigTokens[4] -ConnectionUri $uri -PassInnerException $vipInfo = @{} if($null -ne $feConfig.properties.PublicIPAddress) { $publicIpResRef = $feConfig.properties.PublicIPAddress.resourceRef; $publicIpResRef = $publicIpResRef.Substring($publicIpResRef.LastIndexOf("/") + 1) $publicIPResource = Get-NetworkControllerPublicIpAddress -ResourceId $publicIpResRef -ConnectionUri $uri -PassInnerException $vipInfo["VIP_IP"] = $publicIPResource.properties.IpAddress $vipInfo["VIP_TYPE"] = 2; # public IP } else { $vipInfo["VIP_IP"] = $feConfig.properties.PrivateIPAddress # check if the subnet is vnet or lnet if($feConfig.Properties.Subnet.resourceRef.StartsWith("/virtualNetworks/")) { $vipInfo["VIP_TYPE"] = 3 ; # ILB } else { $vipInfo["VIP_TYPE"] = 1 ; # private IP } } return $vipInfo } function Stop-CtstrafficListener { paramn( [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet # hostName will be empty for hostName as the client is the local machine if([string]::IsNullOrEmpty($hostName)) { Stop-Process -Name Ctstraffic -Verbose -Force Write-traceLog "Stopped ctstraffic listener on $hostName " return } $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName # run on remote machine $cmd = "Start-Process -FilePath $ctsTrafficPath -ArgumentList ""-listen:* -port:$($listerningPort)""" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Start-CtstrafficListener { param( [string] $listerningPort, [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred, [string] $uri, [string] $outFileName, [TrafficPattern] $trafficPattern, [string] $errorFileName = "error.log" ) [string] $ctsTrafficPath = $script:ctsTrafficPath Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") { $ctsTrafficPath = $script:ctsTrafficPath } $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\")); write-TraceLog "Start-CtstrafficListener: Starting ctstraffic ($ctsTrafficPath) listener on $hostName $vmName, ctsfolder : $ctsFolder" # use local machine if host and vmname are null if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) { if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd = "-listen:* -port:$($listerningPort) -connectionFileName:$ctsFolder\$outFileName -transfer:$($trafficPattern.transferSizeInBytes)" } else { $cmd = "-listen:* -port:$($listerningPort) -transfer:$($trafficPattern.transferSizeInBytes)" } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose Write-traceLog "Started ctstraffic listener on $hostName with command $cmd" return } if([string]::isNullOrEmpty($hostName)) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true } # build cts traffic parameter list $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue;" $cmd += "del $ctsFolder\*.csv -Force;" $cmd += "`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();" $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -PassThru -ArgumentList """ $cmd += " -listen:* -port:$($listerningPort)" if($trafficPattern.transferSizeInBytes -gt 0) { $cmd += " -transfer:$($trafficPattern.transferSizeInBytes)" } if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$outFileName" } if(-not [string]::IsNullOrEmpty($errorFileName)) { $cmd += " -errorFileName:$ctsFolder\$errorFileName" } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } $cmd += "`";" + [System.Environment]::NewLine # end the -argument list # detect failures in the command and throw exceptions # for listerner, wait for 2 seconds to see if the process has exited (bad params or something) $cmd += [System.Environment]::NewLine + "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalSeconds -lt 2) {Start-Sleep -Seconds 1};" # if process is alive, means it seems stable, proceed $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$false ) { return };" # if process is dead, even if it exited with 0 error code and capture / throw error # if ctstraffic does not generate the error file for some reason then throw a generic error or process return code $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$true ) {`$c = get-content $ctsFolder\$errorFileName -ErrorAction SilentlyContinue; if(-not [string]::IsNullOrEmpty(`$c)) { throw `$c } else { throw ""No error log found. CtsTraffic Exit With code `$p.exitCode""} };" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Start-CtstrafficSender { param( [string] $targetIpAddress, [string] $targetPort, [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred, [string] $uri, [string] $ctsTrafficPath = $script:ctsTrafficPath, [string] $outFileName, [bool] $Wait = $true, [TrafficPattern] $trafficPattern ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") { $ctsTrafficPath = $script:ctsTrafficPath } $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\")); $errorFileName = "$ctsFolder\error.log" write-TraceLog "Starting ctstraffic ($ctsTrafficPath) sender on $hostName $vmName, ctsfolder : $ctsFolder" # use local machine if host and vmname are null if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) { # note: this is only to test it locally, this will not be used finally $cmd = " -target:$($targetIpAddress) -port:$($targetPort) -protocol:$($trafficPattern.protocol) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) -transfer:$($trafficPattern.transferSizeInBytes)" if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$outFileName " } if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose -Wait Write-traceLog "Started ctstraffic listener on local machine with command $cmd" return } if([string]::isNullOrEmpty($hostName)) { $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true } # run on remote machine $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue;" $cmd += "del $ctsFolder\*.csv -Force;`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();" $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -ArgumentList " $cmd += """ -target:$($targetIpAddress) -protocol:$($trafficPattern.protocol) -port:$($targetPort) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) -transfer:$($trafficPattern.TransferSizeInBytes)" if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) { $cmd += " -pattern:$($trafficPattern.PushPattern)" } if(-not [string]::IsNullOrEmpty($outFileName)) { $cmd += " -connectionFileName:$ctsFolder\$($outFileName) -errorfilename:$errorFileName""" } $cmd += " -PassThru;" $cmd += "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalMinutes -lt 15) {Start-Sleep -Seconds 1};" $cmd += "if(`$p.HasExited -eq `$false ) { `$p.Kill(); throw ""Ctstraffic did not complete on time"" };" $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -ne 0) {`$c = get-content $errorFileName; throw `$c };" $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -eq 0) { return };" invoke-powershellcommandonvm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } function Get-TrafficPattens { # [CmdletBinding()] # param ( # [Parameter()] # [ValidateSet("TCP", "UDP")] # $Protocol, # [Parameter()] # [ValidateSet("ShortBursts", "LongRunning") # [string] $ProfileType # ) # todo : implement different pattern types based on profile # for HLK , we just hardcode some default patterns for now $trafficPatterns = @() $OneKBInBytes = 1024 $OnMBInBytes = 1048576 $OneGBInBytes = 1073741824 $OneGBInBytes = 1073741824 # todo : traffic needs to be updated (this is reduced due to data path issue) # few long runnning connections $trafficPatterns += New-TrafficPattern -connections 10 ` -duration 60 ` -iterations 2 ` -TransferSizeInBytes ($OnMBInBytes * 1) ` -protocol "TCP" # # medium set of connections # $trafficPatterns += New-TrafficPattern -connections 200 ` # -duration 60 ` # -iterations 100 ` # -TransferSizeInBytes ($OneMBInBytes * 2) ` # -protocol "TCP" # # # burst of very small connections # $trafficPatterns += New-TrafficPattern -connections 200 ` # -duration 60 ` # -iterations 300 ` # -TransferSizeInBytes $OneKBInBytes ` # -protocol "TCP" # # udp connections # $bps = ($OnMBInBytes*8) # $trafficPatterns += New-TrafficPattern -connections 20 ` # -duration 60 ` # -iterations 3 ` # -protocol "UDP" ` # -bitsPerSecond $bps ` # -FrameRate 100 ` # -BufferDepth 10 ` # -pushPattern "" return $trafficPatterns } <# Triggers a flurry of live mirgation operations of the given VMs. The VMs are represented by DIP endpoints, which have a VM name and IP address. #> function Start-SDNVmMigrationValidation { param( [int] $percentageOfVmsToMove = 50, [object[]] $trafficEndpoints ) # calculate how many migrations need to be performed [int] $numOfVmsToMove = $trafficEndpoints.Count * $percentageOfVmsToMove / 100 # track vms already attempted to move $vmsMoved = @() # track vms that failed to move $vmsFailedToMove = @() # track vms that moved successfully $vmsMovedSuccessfully = @() $done = $false do { [int] $breaker = 0 $vmToMove = $null # find a target vm to move do { $vmToMove = Get-Random -InputObject $trafficEndpoints $breaker += 1 if($breaker -gt ( $trafficEndpoints.Count * 3)) { throw "Could not find a VM to move, ran out of attempts" } } while($vmsMoved.Contains($vmToMove) -or $vmsFailedToMove.Contains($vmToMove)) if($null -eq $vmToMove) { throw "Could not find a VM to move" } # move the vm try { # resolve host for the VM $hostName = Resolve-HostName -vmName $vmToMove.VmName -hostCred $vmToMove.HostCredential -vmCred $vmToMove.VmCredential $allHostnames = Get-ClusterNode | ?{$_.State -eq "UP" -and $_.Name -ne $hostName} $target = Get-Random -InputObject $allHostnames # for now using only live migration # $i = Get-Random -Maximum 4 # if($i -eq 0) { # $migrationType = "Quick" # } # if($i -eq 1) { # $migrationType = "Live" # } # if($i -eq 2) { # $migrationType = "Shutdown" # } # if($i -eq 3) { # $migrationType = "TurnOff" # } Write-TraceLog "Start-SDNVmMigrationValidation: Moving VM $($vmToMove.VmName) from host $hostName to host $($target.Name)" Move-ClusterVirtualMachineRole -Name $vmToMove.VMName -Node $target.Name -MigrationType Live $vmsMovedSuccessfully += $vmToMov if($vmsMovedSuccessfully.Count -ge $numOfVmsToMove) { $done = $true } } catch { Write-TraceLog "Start-SDNVmMigrationValidation: Failed to move VM $($vmToMove.VmName) from host $hostName to host $($target.Name) Error : $_" $vmsFailedToMove += $vmToMove # No particular reason for 3, just that a live migration should not fail at this point if($vmsFailedToMove.Count -gt 3) { throw "Multiple live migrations failure, please check your configuration and run the test again" } continue } }until($done) Write-TraceLog "Start-SDNVmMigrationValidation: Successfully moved $numOfVmsToMove VMs" } function Initialize-TrafficUtil { param([string] $ctsTrafficPath = $script:ctsTrafficPath) if(Test-Path $script:ctsTrafficPath) { Write-Host "ctsTraffic already available at $ctsTrafficPath, skipping download" return } Start-BitsTransfer https://github.com/microsoft/ctsTraffic/raw/master/Releases/2.0.3.2/x64/ctsTraffic.exe $ctsTrafficPath -Verbose } function Test-TenantVirtualMachine { param( [string] $vmName, [string] $hostName, [pscredential] $vmCred, [pscredential] $hostCred ) Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet $cmd = "`$process = Get-Process -Name winlogon; if(`$null -ne `$process) { return `$true } else { return `$false }" $hostCred = Get-TurnKeySdnCred $vmCred = Get-TurnKeySdnWorkloadVmCred $retryCount = 0 $maxRetryCount = 10 do { try { $result = Invoke-PowershellCommandOnVm -vmName $vmName ` -hostName $hostName ` -cmd $cmd ` -hostCred $hostCred ` -vmCred $vmCred } catch { Write-TraceLog "Test-TenantVirtualMachine: Failed to connect to VM $vmName, retrying... (retry: $retryCount)" Write-TraceLog "Test-TenantVirtualMachine: Error : $_" $result = $false } if($result -eq $true) { return $true } $retryCount += 1 Start-Sleep -Seconds 15 } while($retryCount -lt $maxRetryCount) if($retryCount -ge $maxRetryCount) { throw "Failed to connect to VM $vmName" } Write-TraceLog "Test-TenantVirtualMachine: VM $vmName is available" } |