TurnKeySdn

0.0.29

TrafficUtil.psm1

Import-Module $PSScriptRoot\Logger.psm1
Import-Module $PSScriptRoot\ConfigManager.psm1

$script:MaxTrafficRetryCount = 5
class TrafficEndpoint {
    [string] $VmName = "VmName"
    [string] $HostName = "HostName"
    [pscredential] $VmCredential = $null
    [pscredential] $hostCredential = $null
    [string] $IpAddress = "0.0.0.0"
    [int] $Port = 5001
    [string] $PublicIP = $null

    [string] ToString() {
        return "vm:$($this.VmName) ip:$($this.IpAddress):$($this.Port) pip:$($this.PublicIP)"
    }
}

class TrafficPattern {
    [int] $Connections = 10
    [int] $Duration = 1
    [int] $Iterations = 1
    # 10 GB
    [int64] $TransferSizeInBytes = 10737418240
    [string] $Protocol = "TCP"
    [uint64] $BitsPerSecond = 0
    [uint64] $FrameRate = 0
    [uint64] $BufferDepth = 0
    [uint64] $StreamLength = 0
    [string] $PushPattern = "pushpull"

    [string] ToString() {
        if("UDP" -eq $this.Protocol) {
            return "udp cxn:$($this.Connections) iters:$($this.Iterations) xfer:$($this.TransferSizeInBytes) bps:$($this.BitsPerSecond) fr:$($this.FrameRate) bd:$($this.BufferDepth)"
        }
        if("TCP" -eq $this.Protocol) {
            return "tcp cxn:$($this.Connections) iters:$($this.Iterations)bps:$($this.BitsPerSecond) bd:$($this.BufferDepth) ptrn:$($this.PushPattern)"
        }
        return ""
    }
}

class WorkloadSummary {
    [int] $TotalConnections = 0
    [int] $FailedConnections = 0
    [int] $SuccessfulConnections = 0
    [int] $TotalBytesTransferred = 0
}

Set-Variable VNET_DIP -Value 0 -Option Constant
Set-Variable PRIV_VIP -Value 1 -Option Constant
Set-Variable PUB_VIP -Value 2 -Option Constant
Set-Variable ILB_VIP -Value 3 -Option Constant
Set-Variable CLIENT_IP -Value 4 -Option Constant
Set-Variable DIPS -Value "DIPS" -Option Constant
Set-Variable VIP_IP -Value "VIP_IP" -Option Constant
Set-Variable VIP_PORT -Value "VIP_PORT" -Option Constant
Set-Variable VIP_PROTOCOL -Value "VIP_PROTOCOL" -Option Constant
Set-Variable VIP_TYPE -Value "VIP_TYPE" -Option Constant
Set-Variable DIP_IP -Value "DIP_IP" -Option Constant
Set-Variable DIP_PORT -Value "DIP_PORT" -Option Constant
Set-Variable DIP_VMNAME -Value "DIP_VMNAME" -Option Constant
Set-Variable FRONTEND_IP_CONFIG -Value "FRONTEND_IP_CONFIG" -Option Constant
Set-Variable BACKEND_IP_CONFIG -Value "BACKEND_IP_CONFIG" -Option Constant
Set-Variable INBOUND_NAT_TRAFFIC_RULES -Value "INBOUND_NAT_TRAFFIC_RULES" -Option Constant
Set-Variable LOADBALANCER_TRAFFIC_RULES -Value "LOADBALANCER_TRAFFIC_RULES" -Option Constant
Set-Variable OUTBOUND_NAT_TRAFFIC_RULES -Value "OUTBOUND_NAT_TRAFFIC_RULES" -Option Constant
Set-Variable DIP_IPCONFIG -Value "DIP_IPCONFIG" -Option Constant
Set-Variable CLIENT_VMNAME -Value "InternetClient" -Option Constant
Set-Variable DIP_VM_HOSTNAME -Value "DIP_VM_HOSTNAME" -Option Constant
Set-Variable TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -Value 5 -Option Constant
Set-Variable DEFAULT_CTSTRAFFICPORT -Value 5001 -Option Constant

$script:CtsTrafficVMFolderPath = "$env:SystemDrive\tools"
$script:CtsTrafficVMPath = "$env:SystemDrive\tools\CtsTraffic.exe" 


<#
    This function will create a new traffic pattern object
    The traffic pattern object will be used to run traffic between two VMs
#>
function New-TrafficPattern {
    param(
        [int] $connections,
        [int] $duration,
        [int] $iterations,
        [int64] $TransferSizeInBytes,
        [string] $protocol = "TCP",
        [int64] $bitsPerSecond = 0,
        [int64] $frameRate = 0,
        [int64] $bufferDepth = 0,
        [uint64] $streamLength = 0,

        [ValidateSet("push", "pullpush","pull","duplex", "")]
        [string] $pushPattern = "pushpull"

    )

    $pattern = New-Object TrafficPattern
    $pattern.Connections = $connections
    $pattern.Duration = $duration
    $pattern.Iterations = $iterations
    $pattern.TransferSizeInBytes = $TransferSizeInBytes
    $pattern.Protocol = $protocol
    $pattern.BitsPerSecond = $bitsPerSecond
    $pattern.FrameRate = $frameRate
    $pattern.BufferDepth = $bufferDepth
    $pattern.PushPattern = $pushPattern
    $pattern.StreamLength = $streamLength
    
    return $pattern
}



<#
    Creates a new traffic endpoint object. Endpoints can run/receieve traffic between two VMs
 
    endpointType : 0 - Vnet DIP
                   1 - Private IP Address
                   2 - PublicIP
                   3 - ILB
                   4 - Client IP
#>
function New-TrafficEndpoint {

    param(
        [string] $vmName,
        [string] $hostName,
        [pscredential] $vmCredential,
        [pscredential] $hostCredential,
        [string] $ipAddress,
        [int] $port,
        [bool] $resolveHostName = $false,
        [string] $publicIp = $null,

        [ValidateSet(0,1,2,3,4)]
        [parameter(Mandatory=$true)]
        [int] $endpointType
    )

    if($resolveHostName -eq $true) {
        $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCredential -vmName $vmName -Force $true
    }

    $endpoint = New-Object TrafficEndpoint
    $endpoint.VmName = $vmName
    $endpoint.HostName = $hostName
    $endpoint.VmCredential = $vmCredential
    $endpoint.HostCredential = $hostCredential
    $endpoint.IpAddress = $ipAddress
    $endpoint.Port = $port
    $endpoint.PublicIP = $publicIp

    return $endpoint
}

<#
    Checks if the VM has ctstraffic installed and physically ready to run traffic workloads
    - Checks if the VM is powered on
    - Checks if the VM has ctstraffic installed
    - Checks if the VM has a valid IP address
    - Checks if the VM has a ctstraffic is enabled on firewall
#>
function Initialize-VmForCtstraffic {

    param(
        [object[]] $trafficEndpoints,
        [pscredential] $hostCred,
        [pscredential] $vmCred
    )

    $ctsTraffic = Get-DefaultCtsTrafficFile
     
    if (-not (Test-Path $ctsTraffic)) {
        throw "Initialize-VmForCtstraffic: CtsTraffic not found at $ctsTraffic"
    }    

    foreach($endpoint in $trafficEndpoints) {

        # log all properties of endpoint

        Write-TraceLog "Copy-VMFile $($endpoint.VmName) `
            Host:$($endpoint.HostName) `
            ToolPath:$script:CtsTrafficVMPath `
            SourcePath:$ctsTraffic `
            DestPath:$script:CtsTrafficVMPath `
            "

        # stop any previous running instances of ctstraffic
        Invoke-PowershellCommandOnVm -vmName $endpoint.VmName `
                                     -hostName $endpoint.HostName `
                                     -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" `
                                     -hostCred $hostCred `
                                     -vmCred $vmCred

        $ctsEnableFirewall = "`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Inbound)"" -Direction Inbound -Program $script:CtsTrafficVMPath -Action Allow"
        $ctsEnableFirewall += ";`$rule = New-NetFirewallRule -DisplayName ""Ctstraffic (Outbound)"" -Direction Outbound -Program $script:CtsTrafficVMPath -Action Allow"

        $hostCred = Get-TurnKeySdnCred
        $vmCred = Get-TurnKeySdnWorkloadVmCred

        $results = Invoke-PowershellCommandOnVm -vmName $endpoint.VmName `
                                     -hostName $endpoint.HostName `
                                     -cmd $ctsEnableFirewall `
                                     -hostCred $hostCred `
                                     -vmCred $vmCred


        # ctstraffic.exe is detected as malware by windows defender, so we need to disable it
        # it also does full FS scan which slows deployment considerably

        $disableDefender = "Set-MpPreference -DisableRealtimeMonitoring `$true;"
        $disableDefender += "Set-MpPreference -DisableBehaviorMonitoring `$true;"
        $disableDefender += "Set-MpPreference -DisableBlockAtFirstSeen `$true;"
        $disableDefender += "Add-MpPreference -ExclusionPath $script:CtsTrafficVMFolderPath;"
        Invoke-PowershellCommandOnVm -vmName $endpoint.VmName `
            -hostName $endpoint.HostName `
            -cmd $disableDefender `
            -hostCred $hostCred `
            -vmCred $vmCred
        # now copy ctstraffic to the vm
        Copy-VMFile -VMName $endpoint.VmName ` `
                                     -SourcePath $ctsTraffic `
                                     -DestinationPath $script:CtsTrafficVMPath `
                                     -FileSource Host `
                                     -ComputerName $endpoint.HostName `
                                     -CreateFullPath `
                                     -Force
    }
}



<#
Runs traffic between given endpoints
- Completes a full mesh (NXN endpoints with the traffic pattern provided), with one at a time.
- Assumes that appropriate firewall rules are enabled on the VMs (From SDN)
#>
function Start-IntraVmTraffic {
    param(
        [object[]] $trafficEndpoints,
        [TrafficPattern[]] $trafficPatterns,
        [pscredential] $hostCred,
        [pscredential] $vmCred
    )

    Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet

    # todo : (optimization) this can be sped up if we keep the destination running and go over the targets, this
    # way the init time can be saved (needs to handle different outFileNames for each target)
    foreach($trafficPattern in $trafficPatterns) {
        foreach($srcEndpoint in $trafficEndpoints) {
            foreach($dstEndpoint in $trafficEndpoints) {

                if($srcEndpoint -eq $dstEndpoint) {
                    continue
                }
                if($srcEndpoint.VmName -like "*ilb_client*" -or $dstEndpoint.VmName -like "*ilb_client*") {
                    Write-TraceLog "Start-IntraVmTraffic: ILB & clients are skipped from EW traffic tests."
                    continue
                }

                $retryCount = 0
                while($true) {
                    try 
                    {
                        [bool] $isFailed = $false
                        $guid = New-Guid 
                        Write-TraceLog "Start-IntraVmTraffic [BEGIN] [EastWestTraffic] RunId:($guid) SrcVM:$($srcEndpoint.VmName) DstVM:$($dstEndpoint.VmName)"
                        [string] $vfpDatapathTraceRoot = ""
                        if(-not [string]::IsNullOrEmpty($Env:VFPDATAPATH_LOGROOT))
                        {
                            $vfpDatapathTraceRoot = Join-Path -Path $Env:VFPDATAPATH_LOGROOT -ChildPath $guid
                            Start-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred
                        }
                        LogTrafficInfo -listenEndpoint $dstEndpoint -sendEndpoint $srcEndpoint -trafficPattern $trafficPattern

                        $outFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".csv"
                        $errorFileName = "IntraVM-"+((New-Guid).Guid.ToString()).Substring(0,8) + ".err"

                        Start-CtstrafficListener -listerningPort $dstEndpoint.Port `
                                                -vmName $dstEndpoint.VmName `
                                                -hostName $dstEndpoint.HostName `
                                                -vmCred $vmCred `
                                                -hostCred $hostCred `
                                                -trafficPattern $trafficPattern `
                                                -outFileName $outFileName `
                                                -uri $uri `
                                                -errorFileName $errorFileName `
                                                -isILB $false

                        Start-CtstrafficSender -targetIpAddress $dstEndpoint.IpAddress `
                                                -targetPort $dstEndpoint.Port `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -vmName $srcEndpoint.VmName `
                                                -hostName $srcEndpoint.HostName `
                                                -trafficPattern $trafficPattern `
                                                -outFileName $outFileName `
                                                -uri $uri `
                                                -wait $true

                        $srcWorkloadResult = Wait-ForWorloadCompletion -vmName $srcEndpoint.VmName `
                                                -hostName $srcEndpoint.HostName `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -outFileName $outFileName

                        Test-WorkloadResult -workloadResults $srcWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $srcEndpoint.VmName

                        $dstWorkloadResult = Wait-ForWorloadCompletion -vmName $dstEndpoint.VmName `
                                                -hostName $dstEndpoint.HostName `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -outFileName $outFileName `
                                                -force $true

                        Test-WorkloadResult -workloadResults $dstWorkloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dstEndpoint.VmName

                        Write-TraceLog "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) PASSED!"
                        break
                    } 
                    catch 
                    {
                        $retryCount += 1
                        $isFailed = $true
                        if ($retryCount -eq $script:MaxTrafficRetryCount) {
                            throw "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) FAILED! Error $($_.Exception.Message) after $retryCount retries"
                        }
                        Write-TraceLog "Start-IntraVmTraffic: East West Traffic ($guid) $($srcEndpoint.VmName) $($dstEndpoint.VmName) FAILED! Error $($_.Exception.Message), retrying... (retry: $retryCount)"
                    }
                    finally {
                        Remove-Item -Path $outFileName -Force -Verbose -ErrorAction SilentlyContinue | Out-Null
                        if(-not [string]::IsNullOrEmpty($vfpDatapathTraceRoot)) {
                            Stop-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred -preserveZip $isFailed
                        }
                        Write-TraceLog "Start-IntraVmTraffic [END] [EastWestTraffic] RunId:($guid) SrcVM:$($srcEndpoint.VmName) DstVM:$($dstEndpoint.VmName)"
                    }
                }
            }
        }
    }
}

<#
    Runs SLB Traffic on a given set of endpoints.
 
    The traffic pattern is defined by the following parameters:
    - RulesToTarget : The type of rules to target on the load balancer. Valid values are:
        - LoadbalancerRules : All load balancer rules
        - Inbound : All inbound NAT rules
        - Outbound : All outbound NAT rules
 
    The caller must create the metadata json which describes the topology of DIPs and VIPs before calling this method.
    Use the helper Get-EndpointsFromLoadBalancer to extract the metadata from a given loadbalancer and pass it to this method to initiate traffic.
 
    NOTE: the Get-EndpointsFromLoadBalancer does not detect if probes are down for the given endpoints. Please disable probes before calling this method.
    For inbound/lb rules the server instances are
 
#>
function Start-SLBTraffic {
    param(
        [ValidateSet("LoadbalancerRules", "Inbound", "Outbound")]
        [string] $TargetType,
        [object] $vips,
        [object] $clientEndpoint = $null,
        [TrafficPattern[]] $trafficPatterns,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $uri
    )

    $retryCount = 0

    while($true) {
        try {
            if($TargetType -eq "LoadbalancerRules") {

                $vipInfo = $vips[$LOADBALANCER_TRAFFIC_RULES]

                if($null -eq $vipInfo -or $vipInfo.Count -eq 0) {
                    throw "No loadbalancer rules found"
                }

                foreach($vipEndpoint in $vipInfo) {
                    foreach($trafficPattern in $trafficPatterns) {

                            Write-TraceLog "Start-SLBTraffic:(lb) $vipEndpoint TRAFFICPATTERN: $trafficPattern"

                            if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") {

                                Write-TraceLog "Start-SLBTraffic:(lb) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)"
                                continue;
                            }

                            Start-SLBLBRuleTraffic -vipEndpoint $vipEndpoint `
                                                -clientEndpoint $clientEndpoint `
                                                -trafficPattern $trafficPattern `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -uri $uri
                    }
                }

            }

            elseif($TargetType -eq "Inbound") {

                $vipInfo = $vips[$INBOUND_NAT_TRAFFIC_RULES]

                foreach($vipEndpoint in $vipInfo) {
                    foreach($trafficPattern in $trafficPatterns) {

                        Write-TraceLog "Start-SLBTraffic:(inbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern"

                        if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") {
                            Write-TraceLog "Start-SLBTraffic:(inbound) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)"
                            continue;
                        }

                        Start-SLBInboundTraffic -vipEndpoint $vipEndpoint `
                                                -clientEndpoint $clientEndpoint `
                                                -trafficPattern $trafficPattern `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -uri $uri
                    }
                }

            }

            elseif($TargetType -eq "Outbound") {

                $vipInfo = $vips[$OUTBOUND_NAT_TRAFFIC_RULES]

                foreach($vipEndpoint in $vipInfo) {
                    foreach($trafficPattern in $trafficPatterns) {

                        Write-TraceLog "Start-SLBTraffic:(outbound) $vipEndpoint TRAFFICPATTERN: $trafficPattern"

                        if($vipEndpoint[$VIP_PROTOCOL] -ne $trafficPattern.Protocol -and $vipEndpoint[$VIP_PROTOCOL] -ne "All") {
                            Write-TraceLog "Start-SLBTraffic:(outbound) Skipping as protocol does not match $($vipEndpoint[$VIP_PROTOCOL]) $($trafficPattern.Protocol)"
                            continue;
                        }

                        Start-SLBOutboundTraffic -vipEndpoint $vipEndpoint `
                                                -clientEndpoint $clientEndpoint `
                                                -trafficPattern $trafficPattern `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -uri $uri
                    }
                }

            } else {
                throw "Invalid TargetType parameter"
            }

            break
        } catch {
            $retryCount += 1
            if ($retryCount -eq $script:MaxTrafficRetryCount) {
                throw "Start-SLBTraffic: Error $($_.Exception.Message) after $retryCount retries"
            }
            Write-TraceLog "Start-SLBTraffic: Error $($_.Exception.Message) retrying... (retry: $retryCount)"
        }
    }
}

<#
 
 Gets a suitable endpoint to run the client traffic on, if its ILB, it will pick up a VNET location (for ILB: The ILB client NIC is used)
 for everything else, it will use the PublicIP on the ILB CLient
 
 todo: make this configurable to run the workload in different locations
 
 #>
function Get-SLBClientEndpoint {
    param(
        [object[]] $dips,
        [int] $vipType,
        [string] $uri,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $port
    )

    if($dips -eq $null -or $dips.Count -eq 0) {
        throw "Update-DipInfoIfILBEnabledDips: Dips is null or empty"
    }

    $ipConfigTokens = $dips[0][$DIP_IPCONFIG].split("/")
    $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException
    $subnetNameResourceRef = $ipConfig.properties.Subnet.resourceRef
    $subnetNameResRefTokens = $subnetNameResourceRef.split("/")
    $subnetName = $subnetNameResRefTokens[4]

    $vmName = "$($subnetName)_ilb_client"

    $nic = Get-NetworkControllerNetworkInterface -ResourceId $vmName -ConnectionUri $uri -PassInnerException
    if($nic -eq $null) {
        throw "Could not find NIC $vmName"
    }


    # for ILB cases, use the ILB VNET IP Address for traffic
    if($vipType -eq 3) {

        New-TrafficEndpoint -vmName $vmName `
            -hostName $null `
            -vmCredential $vmCred `
            -hostCredential $hostCred `
            -ipAddress $nic.properties.IpConfigurations[0].properties.privateIpAddress `
            -port 5001 `
            -endpointType $CLIENT_IP `
            -resolveHostName $true `
            -publicIp $null

    } else {
         # for other LB scenarios use the PublicIP address for traffic
        # get public ip from the ilb client nic

        $publicIpResRef = $nic.properties.IpConfigurations[0].properties.PublicIPAddress.resourceRef
        $publicIpResRefTokens = $publicIpResRef.split("/")
        $publicIpResource = Get-NetworkControllerPublicIPAddress -ResourceId $publicIpResRefTokens[2] -ConnectionUri $uri -PassInnerException
        $publicIpAddress = $publicIpResource.properties.IpAddress

            New-TrafficEndpoint -vmName $vmName `
                -hostName $null `
                -vmCredential $vmCred `
                -hostCredential $hostCred `
                -ipAddress $null `
                -port 5001 `
                -endpointType $CLIENT_IP `
                -resolveHostName $true `
                -publicIp $publicIpAddress

    }

}

function Start-SLBInboundTraffic {

    param(
        [object] $vipEndpoint,
        [object] $clientEndpoint = $null,
        [TrafficPattern] $trafficPattern,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $uri
    )

    # todo : this can be sped up if we pre resolve the host names for all the DIPs
    $dipHostName = Resolve-HostName -hostName $vipEndpoint[$DIPS][0][$DIP_VMNAME] `
        -hostCred $hostCred `
        -vmName $vipEndpoint[$DIPS][0][$DIP_VMNAME] `
        -Force $true

    # select the best client based on the VIP type
    if($null -eq $clientEndpoint) {
        $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] `
                                                -vipType $vipEndpoint[$VIP_TYPE] `
                                                -uri $uri `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred `
                                                -port $vipEndpoint[$VIP_PORT]
    }

    $guid = New-Guid

    $isFailed = $false
    $vfpDatapathTraceRoot = ""
    if(-not [string]::IsNullOrEmpty($Env:VFPDATAPATH_LOGROOT))
    {
        $vfpDatapathTraceRoot = Join-Path -Path $Env:VFPDATAPATH_LOGROOT -ChildPath $guid
        Start-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred
    }
    try
    {
        $outFileName = ($guid.ToString()).Substring(0,8) + ".csv"
        Write-TraceLog "----------------------------------------------"
        Write-TraceLog "Start-SLBInboundTraffic: Starting INBOUND Traffic ($($guid))" -Warning
        Write-TraceLog "Start-SLBInboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) "
        Write-TraceLog "Start-SLBInboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])"
        foreach($dipInfo in $vipEndpoint[$DIPS]) {
            Write-TraceLog "Start-SLBInboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" 
        }
        Write-TraceLog "Start-SLBInboundTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress) "
        Write-TraceLog "Start-SLBInboundTraffic: `t TRAFFIC : $($trafficPattern.ToString())"
        Write-TraceLog "----------------------------------------------"


        # set up dip to recv
        Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] `
                                -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] `
                                -hostName $dipHostName `
                                -vmCred $vmCred `
                                -hostCred $hostCred `
                                -outFileName $outFileName `
                                -trafficPattern $trafficPattern


        Start-CtstrafficSender  -vmName $clientEndpoint.VmName `
                                -hostName $clientEndpoint.HostName `
                                -targetIpAddress $vipEndpoint[$VIP_IP] `
                                -targetPort $vipEndpoint[$VIP_PORT] `
                                -hostCred $hostCred `
                                -vmCred $vmCred `
                                -ctsTrafficPathLocal $script:CtsTrafficVMPath `
                                -outFileName $outFileName `
                                -trafficPattern $trafficPattern

        # once the sender is complete, the receiver can be forced killed

        $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $vipEndpoint["DIPS"][0][$DIP_VMNAME] `
                                                    -hostName $dipHostName `
                                                    -hostCred $hostCred `
                                                    -vmCred $vmCred `
                                                    -outFileName $outFileName `
                                                    -force $true
        Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $vipEndpoint["DIPS"][0][$DIP_VMNAME]



        $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName  `
                                                    -hostName  $clientEndpoint.HostName `
                                                    -hostCred $hostCred `
                                                    -vmCred $vmCred `
                                                    -outFileName $outFileName
        Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName

        Write-TraceLog "Start-SLBInboundTraffic: --- END --- "
    }
    catch
    {
        $isFailed = $true
        throw $_
    }
    finally
    {
        Remove-Item -Path $outFileName -Force -Verbose -ErrorAction SilentlyContinue | Out-Null
        if(-not [string]::IsNullOrEmpty($vfpDatapathTraceRoot)) {
            Stop-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred -preserveZip $isFailed
        }
        Write-TraceLog "Start-SLBInboundTraffic [END] RunId:($guid)"
    }
}

function Test-WorkloadResult {
    param(
        [object] $workloadResults,
        [int] $maxFailedPercentage = 10,
        [string] $EndpointName
    )

    if($workloadResults -eq $null) {
        throw "Workload result is null"
    }
    if($workloadResults.count -eq 0) {
        throw "Workload result is empty"
    }

    [int] $total = 0
    [int] $failed = 0
    [int] $successful = 0

    foreach($result in $workloadResults) {

        if($result.Result -eq "Failed") {
            $failed += 1
        } else {
            $successful += 1
        }
        $total += 1
    }

    if($total -eq 0) {
        throw "Workload result is empty"
    }

    $failedPercentage = $failed / $total * 100
    if($failedPercentage -gt $maxFailedPercentage) {
        throw "Workload failed for more than 10% of the connections"
    }

    Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" -Warning
    Write-TraceLog "Test-WorkloadResult: Connections Total:$total Failed:$failed Successful:$successful Failed %:$failedPercentage" -Warning
    Write-TraceLog "Test-WorkloadResult: ----------SUMMARY for ($EndpointName)--------------------" -Warning

    return $true

}

function Start-VfpTracing
{
    param(
        [string] $traceFolderName = "",
        [pscredential] $Credential
    )
    $nodeNames = (get-clusternode).Name
    foreach($nodeName in $nodeNames) {
        Start-SdnNetshTrace -ComputerName $nodeName -OutputDirectory $traceFolderName -Capture No -Credential $Credential -Role Server
    }
}
function Stop-VfpTracing
{
    param(
        [string] $traceFolderName = "",
        [pscredential] $Credential,
        [bool] $preserveZip = $false # by default all traces will be discarded, unless this flag is set
    )
    if([string]::IsNullOREmpty($traceFolderName)) {
        return
    }

    $nodeNames = (get-clusternode).Name
    foreach($nodeName in $nodeNames) {
        Stop-SdnNetshTrace -ComputerName $nodeName -Credential $Credential
    }
    if($preserveZip)
    {
        Compress-Archive -Path $traceFolderName -DestinationPath "$traceFolderName.zip" -Force -Verbose
    }
    Remove-Item -Path $traceFolderName -Recurse -Force -Verbose
}
function Start-SLBLBRuleTraffic {

    param(
        [object] $vipEndpoint,
        [object] $clientEndpoint = $null,
        [TrafficPattern] $trafficPattern,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $uri
    )
    try
    {
        $guid = New-Guid
        [bool] $isFailed = $false
        [string] $vfpDatapathTraceRoot = ""
        if(-not [string]::IsNullOrEmpty($Env:VFPDATAPATH_LOGROOT))
        {
            $vfpDatapathTraceRoot = Join-Path -Path $Env:VFPDATAPATH_LOGROOT -ChildPath $guid
            Start-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred
        }
        # select the best client based on the VIP type
        if($null -eq $clientEndpoint) {
            $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] `
                                                    -vipType $vipEndpoint[$VIP_TYPE] `
                                                    -uri $uri `
                                                    -hostCred $hostCred `
                                                    -vmCred $vmCred `
                                                    -port $vipEndpoint[$VIP_PORT]
        }
        $outFileName = ($guid.ToString()).Substring(0,8) + ".csv"
        Write-TraceLog "----------------------------------------------"
        Write-TraceLog "Start-SLBLBRuleTraffic: Starting LB Rule Traffic ($($guid))"  -Warning
        Write-TraceLog "Start-SLBLBRuleTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) "
        Write-TraceLog "Start-SLBLBRuleTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])"
        foreach($dipInfo in $vipEndpoint[$DIPS]) {
            Write-TraceLog "Start-SLBLBRuleTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" 
        }
        Write-TraceLog "Start-SLBLBRuleTraffic: `t CLIENT : $($clientEndpoint.vmName):$($clientEndpoint.IpAddress):$($clientEndpoint.port) publicIp:$($clientEndpoint.PublicIP)"
        Write-TraceLog "----------------------------------------------"

        foreach($dipInfo in $vipEndpoint[$DIPS]) {

            Start-CtstrafficListener -listerningPort $vipEndpoint[$DIP_PORT] `
                                    -vmName $dipInfo[$DIP_VMNAME] `
                                    -hostName $null `
                                    -vmCred $vmCred `
                                    -hostCred $hostCred `
                                    -outFileName $outFileName `
                                    -trafficPattern $trafficPattern
        }
        Start-CtstrafficSender -vmName $clientEndpoint.VmName `
                            -hostName $clientEndpoint.HostName `
                            -targetIpAddress $vipEndpoint[$VIP_IP] `
                            -targetPort $vipEndpoint[$VIP_PORT] `
                            -hostCred $clientEndpoint.HostCredential `
                            -vmCred $clientEndpoint.VmCredential `
                            -outFileName $outFileName `
                            -trafficPattern $trafficPattern

        foreach($dipInfo in $vipEndpoint[$DIPS]) {

            # once the sender is complete, the receiver can be forced killed
            $workloadResultsOnDip = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME]  `
                                -hostName $null `
                                -hostCred $hostCred `
                                -vmCred $vmCred `
                                -outFileName $outFileName `
                                -force $true

            Test-WorkloadResult -workloadResults $workloadResultsOnDip -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME]
        }
        $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName `
                -hostName $clientEndpoint.HostName `
                -hostCred $clientEndpoint.HostCredential `
                -vmCred $clientEndpoint.VmCredential `
                -outFileName $outFileName
        Test-WorkloadResult -workloadResults $workloadResultClient -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName

        Write-TraceLog "Start-SLBLBRuleTraffic: LB Rule Traffic ($($guid)) PASSED"
    }
    catch
    {
        $isFailed = $true
        throw $_
    }
    finally
    {
        Remove-Item -Path $outFileName -Force -Verbose -ErrorAction SilentlyContinue | Out-Null
        if(-not [string]::IsNullOrEmpty($vfpDatapathTraceRoot)) {
            Stop-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred -preserveZip $isFailed
        }
        Write-TraceLog "Start-SLBLBRuleTraffic: [END]"
    }
}
function Start-SLBOutboundTraffic {
    param(
        [object] $vipEndpoint,
        [object] $clientEndpoint = $null,
        [TrafficPattern] $trafficPattern,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $uri
    )

    Write-TraceLog "Start-SLBOutboundTraffic: $dipInfo $vipInfo"
    try
    {
        # for Get-SLBClientEndpoint gives us the ILB client VM which has a PublicIP on it, so it should work from there
        if($null -eq $clientEndpoint) {
            $clientEndpoint = Get-SLBClientEndpoint -dips $vipEndpoint[$DIPS] `
                                                    -vipType $vipEndpoint[$VIP_TYPE] `
                                                    -uri $uri `
                                                    -hostCred $hostCred `
                                                    -vmCred $vmCred `
                                                    -port $vipEndpoint[$VIP_PORT]
        }

        $vipPort = 5001
        $guid = New-Guid
        $isFailed = $false
        $vfpDatapathTraceRoot = ""
        if(-not [string]::IsNullOrEmpty($Env:VFPDATAPATH_LOGROOT))
        {
            $vfpDatapathTraceRoot = Join-Path -Path $Env:VFPDATAPATH_LOGROOT -ChildPath $guid
            Start-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred
        }
        $outFileName = ($guid.ToString()).Substring(0,8) + ".csv"

        Write-TraceLog "----------------------------------------------"
        Write-TraceLog "Start-SLBOutboundTraffic: Starting OUTBOUND LB Rule Traffic ($($guid))"  -Warning
        Write-TraceLog "Start-SLBOutboundTraffic: `t VIP : $($vipEndpoint[$VIP_IP]):$($vipEndpoint[$VIP_PORT]) "
        Write-TraceLog "Start-SLBOutboundTraffic: `t VIPTYPE : $($vipEndpoint[$VIP_TYPE])"
        foreach($dipInfo in $vipEndpoint[$DIPS]) {
            Write-TraceLog "Start-SLBOutboundTraffic: `t `t DIP: $($dipInfo[$DIP_IP]):$($vipEndpoint[$DIP_PORT]) vm($($dipInfo[$DIP_VMNAME]))" 
        }
        Write-TraceLog "Start-SLBOutboundTraffic: `t CLIENT : $($clientEndpoint.vmName) IP $($clientEndpoint.IpAddress) PORT;$($clientEndpoint.port) PUBLICIP:$($clientEndpoint.PublicIP)"
        Write-TraceLog "----------------------------------------------"

        # set up client to recv
        Start-CtstrafficListener -listerningPort  $vipPort  `
                                -vmName $clientEndpoint.VmName `
                                -hostName $clientEndpoint.HostName `
                                -vmCred $vmCred `
                                -hostCred $hostCred `
                                -outFileName $outFileName `
                                -trafficPattern $trafficPattern

        # start senders
        foreach($dipInfo in $vipEndpoint[$DIPS]) {

            Start-CtstrafficSender -targetIpAddress $clientEndpoint.PublicIP `
                                -targetPort $vipPort `
                                -hostCred $hostCred `
                                -vmCred $vmCred `
                                -outFileName $outFileName `
                                -vmName $dipInfo[$DIP_VMNAME] `
                                -trafficPattern $trafficPattern

        }

        # wait for senders
        $workloadResults = @()
        foreach($dipInfo in $vipEndpoint[$DIPS]) {

            $workloadResult = Wait-ForWorloadCompletion -vmName $dipInfo[$DIP_VMNAME] `
                                    -hostName "" `
                                    -hostCred $hostCred `
                                    -vmCred $vmCred `
                                    -outFileName $outFileName `
                                    -force $true

            # test workload results per DIP
            Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $dipInfo[$DIP_VMNAME]
            $workloadResults += $workloadResult

        }

        # also validate the total workload results
        Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName "SLB-OUTBOUND-DIPS_SUMMARY"

        # validate workload results on the oubound receiver
        $workloadResultClient = Wait-ForWorloadCompletion -vmName $clientEndpoint.VmName `
            -hostName $clientEndpoint.HostName `
            -hostCred $clientEndpoint.HostCredential `
            -vmCred $clientEndpoint.VmCredential `
            -outFileName $outFileName `
            -force $true

        Test-WorkloadResult -workloadResults $workloadResult -maxFailedPercentage $TRAFFIC_MAX_ALLOWED_FAILURE_PERCENT -EndpointName $clientEndpoint.VmName
        Write-TraceLog "Start-SLBOutboundTraffic: Passed"
    }
    catch
    {
        $isFailed = $true
        throw $_
    }
    finally
    {
        Remove-Item -Path $outFileName -Force -Verbose -ErrorAction SilentlyContinue | Out-Null
        if(-not [string]::IsNullOREmpty($vfpDatapathTraceRoot)) {
            Stop-VfpTracing -traceFolderName $vfpDatapathTraceRoot -Credential $hostCred -preserveZip $isFailed
        }
        Write-TraceLog "Start-SLBOutboundTraffic: [END]"
    }
}

function Start-Traffic {
    param(
        [ValidateSet("Listen", "Target")]
        [string] $Role,

        [object] $trafficPattern,

        [object] $currEndpoint,
        [object] $targetEndpoint,

        [pscredential] $hostCred,
        [pscredential] $vmCred
    )

    # note:
    # currEndpoint contains context on where the instance of ctstraffic is going to be run (either listen or target)
    # targetEndpoint contains context on where the traffic is going to be sent to, its required for sender to know where to send the traffic to

    $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;"
    if($Role -eq "Listen") {
        $cmd +=  "Start-Process " + $script:CtsTrafficVMPath + " -ArgumentList ""-listen:* -port:$($targetEndpoint.Port)""";
    } else {
        $cmd +=  "Start-Process -FilePath " + $script:CtsTrafficVMPath + " -Wait -ArgumentList ""-target:$($targetEndpoint.IpAddress) -port:$($targetEndpoint.Port) -protocol:tcp -connections:$($trafficPattern.Connections) -iterations:$($trafficPattern.Iterations)""";
    }

    Write-TraceLog -Message "Start-Traffic ($Role): $cmd"

    Invoke-PowershellCommandOnVmInternal -vmName $currEndpoint.VmName `
        -hostName $currEndpoint.HostName `
        -cmd $cmd `
        -hostCredentials $hostCred `
        -vmCredentials  $vmCred
}

function Resolve-HostName {

    param(
        [string] $hostName,
        [pscredential] $hostCred,
        [string] $vmName,
        [bool] $Force = $false
    )

    if(-not [string]::IsNullOrEmpty($hostName) -and $Force -eq $false) {
        Write-TraceLog "Resolve-HostName: HostName is not null or empty, returning $hostName"
        return $hostName
    }
    if([string]::IsNullOrEmpty($vmName)) {
        Write-TraceLog "Resolve-HostName: VMName is null, returning $hostName"
        return $hostName;
    }

    $nodes = (get-clusternode).Name
    foreach($node in $nodes) {

        $vm = Get-VM -VMName $vmName -ComputerName $node -ErrorAction SilentlyContinue
        if($vm -ne $null) {

            Write-TraceLog "Resolve-HostName: Resolved VM $vmName on node $node"
            return $node
        }
    }
    throw "Could not resolve host name for vm $vmName"
}

function Wait-ForWorloadCompletion {

    param(
        [string] $vmName,
        [string] $hostName,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $outFileName,
        [int] $timeoutInSeconds = 3600,
        [string] $ctstrafficPathLocal = $script:CtsTrafficVMPath,
        [bool] $force = $false
    )

    # Write-FunctionEntryWithParams -FunctionName "Wait-ForWorloadCompletion" -BoundParameters $PSBoundParameters -UnboundArguments $args

    # todo : avoid resolution of VM to speed things up a bit
    $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $false
    Write-TraceLog "Wait-ForWorloadCompletion: host:$hostName vm:$vmName.."

    $ctsTrafficFolderPath = $ctstrafficPathLocal.Substring(0, $ctstrafficPathLocal.LastIndexOf("\"))
    $csvFilePath = Join-Path $ctsTrafficFolderPath $outFileName

    if([string]::IsNullOrEmpty($vmName) -and [string]::IsNullOrEmpty($hostName)) {

        # wait for ctstraffic.exe to complete or kill and exit
        $done = $false
        [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
        $stopwatch.Start()

        while(-not $done  -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) {

            Write-TraceLog "Waiting for ctstraffic to complete"
            $p = Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue

            if($null -eq $p) {
                $done = $true
                break
            }
        }

        if(-not $done) {
            Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue
        }
        if(-not $force -and -not $done) {
            throw "Workload did not complete in time"
        }

        $WorkloadResult = Get-Content -Path $csvFilePath | ConvertFrom-Csv
        Write-TraceLog "Wait-ForWorloadCompletion: host:$hostName vm:$vmName..done"
        return $WorkloadResult
    }

    # the following is only if we are running traffic from the host (which is running turnkey)
    $done = $false
    [System.Diagnostics.Stopwatch]$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
    $stopwatch.Start()

    while(-not $done -and $stopwatch.Elapsed.TotalSeconds -lt $timeoutInSeconds -and $force -eq $false) {

        Write-TraceLog "Waiting for ctstraffic to complete"
        $cmd = "Get-Process -Name Ctstraffic -ErrorAction SilentlyContinue"

        $p = Invoke-PowershellCommandOnVm -vmName $vmName `
                                          -hostName $hostName `
                                          -cmd $cmd `
                                          -hostCred $hostCred `
                                          -vmCred $vmCred

        if($null -eq $p) {
            $done = $true
            break
        }
    }

    if ($force -or -not $done) {

        Invoke-PowershellCommandOnVm -vmName $vmName `
                -hostName $hostName `
                -cmd "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue" `
                -hostCred $hostCred `
                -vmCred $vmCred
    }

    if (-not $done -and -not $force) {
        throw "Workload did not complete in time"
    }

    $cmd = "Get-Content -Path $csvFilePath | ConvertFrom-Csv"
    $WorkloadResult = Invoke-PowershellCommandOnVm -vmName $vmName `
                                                   -hostName $hostName `
                                                   -cmd $cmd `
                                                   -hostCred $hostCred `
                                                   -vmCred $vmCred

    return $WorkloadResult
}

function Unblock-AllPortsOnVm {
    param(
        [string] $nicResourceId,
        [string] $restName
    )

    $uri = "https://$restName";

    $vnic = Get-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -PassInnerException
    if($vnic.properties.IpConfigurations -eq $null -or $vnic.properties.IpConfigurations.Count -eq 0) {
        throw "No IP configuration found on the NIC"
    }
    if($vnic.properties.IpConfigurations.Count -gt 1) {
        throw "Multiple IP configurations found on the NIC"
    }

    # create the NICs ACL
    $nicAclResourceId = $nicResourceId + "_ACL"
    $newNicAcl = New-Object Microsoft.Windows.NetworkController.AccessControlList
    $newNicAcl.properties = New-Object Microsoft.Windows.NetworkController.AccessControlListProperties
    $newNicAcl.properties.AclRules = @()
    $newNicAclRuleIdBase = "ACL_RULE_";
    $newNicAcl.properties.AclRules  += GetAclRule -resourceId $($newNicAclRuleIdBase + 1) `
                                            -protocol "All" `
                                            -action "Allow" `
                                            -sourcePrefix "*" `
                                            -destPrefix "*" `
                                            -type "Inbound" `
                                            -priority 1000

    $newNicAcl.properties.AclRules  += GetAclRule -resourceId $($newNicAclRuleIdBase + 2) `
                                            -protocol "All" `
                                            -action "Allow" `
                                            -sourcePrefix "*" `
                                            -destPrefix "*" `
                                            -type "Outbound" `
                                            -priority 1000
                                            
    $newAcl = New-NetworkControllerAccessControlList -ResourceId $nicAclResourceId -ConnectionUri $uri -Properties $newNicAcl.properties -PassInnerException -Force
    Write-TraceLog "Created/updated allow all acl $nicAclResourceId, resourceRef : $($newAcl.resourceRef)"

    #bind this ACL to the NIC
    $vnic.properties.IpConfigurations[0].Properties.AccessControlList = $newAcl
    $nic = New-NetworkControllerNetworkInterface -ResourceId $nicResourceId -ConnectionUri $uri -Properties $vnic.properties -PassInnerException -Force
    Write-TraceLog "Applied unblock-all ports ACL for NIC $nicResourceId"
}

function GetAclRule
{
    param(
        [parameter(Mandatory=$true)] [String]$resourceId,
        [parameter(Mandatory=$true)] [String]$protocol,
        [parameter(Mandatory=$true)] [String]$action,
        [parameter(Mandatory=$true)] [String]$sourcePrefix,
        [parameter(Mandatory=$true)] [String]$destPrefix,
        [parameter(Mandatory=$true)] [String]$type,
        [parameter(Mandatory=$true)] [int]$priority
        )

    $ruleproperties = new-object Microsoft.Windows.NetworkController.AclRuleProperties
    $ruleproperties.Protocol = $protocol
    $ruleproperties.SourcePortRange = "*"
    $ruleproperties.DestinationPortRange = "*"
    $ruleproperties.Action = $action
    $ruleproperties.SourceAddressPrefix = $sourcePrefix
    $ruleproperties.DestinationAddressPrefix = $destPrefix
    $ruleproperties.Priority = $priority
    $ruleproperties.Type = $type
    $ruleproperties.Logging = "Enabled"

    $aclrule = new-object Microsoft.Windows.NetworkController.AclRule
    $aclrule.Properties = $ruleproperties
    $aclrule.ResourceId = $resourceId
    return $aclrule
}

function Get-EndpointsFromLoadBalancer  {
    param(
        [string] $lbResourceId,
        [pscredential] $hostCred,
        [pscredential] $vmCred,
        [string] $uri
    )

    $lb = Get-NetworkControllerLoadBalancer -ResourceId $lbResourceId -ConnectionUri $uri -PassInnerException
    if($lb.properties.frontendIPConfigurations -eq $null -or $lb.properties.frontendIPConfigurations.Count -eq 0) {
        return $lbEndpoints
    }
    if( $lb.properties.frontendIPConfigurations.Count -gt 1) {
        throw "Multiple frontend IP configurations found on the load balancer"
    }

    # all Traffic rules
    $inboundNATTrafficRules = @()
    $loadbalancerTrafficRules = @()
    $outboundNATTrafficRules = @()

    # parse and create workload patterns for all INBOUND NAT RULES
    foreach($inboundNatRule in $lb.properties.inboundNatRules) {
        $inboundNatTraffic = @{}
        if( $inboundNatRule.properties.backendIpConfiguration -eq $null -or $inboundNatRule.properties.backendIpConfiguration.Count -eq 0) {
            continue
        }
        $inboundNatTraffic["frontendIpConfig"] = $inboundNatRule.properties.frontendIpConfigurations[0].resourceRef
        $inboundNatTraffic["VIP_PORT"] = $inboundNatRule.properties.frontendPort
        $inboundNatTraffic["VIP_PROTOCOL"] = $inboundNatRule.properties.protocol
        $inboundNatTraffic["backendIpConfig"] = $inboundNatRule.properties.backendIpConfiguration.resourceRef
        $inboundNatTraffic["DIP_PORT"] = $inboundNatRule.properties.backendPort
        $inboundNATTrafficRules += $inboundNatTraffic
        $inboundNatTraffic["DIPS"] = @()
    }
    # resolve all DIPs to IPs
    foreach($inboundNatTraffic in $inboundNATTrafficRules) {

        $dipInfo = @{}
        $ipConfigTokens = $inboundNatTraffic["backendIpConfig"].split("/")
        $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException
        $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress
        $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2]
        $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef
        $dipInfo["DIP_PORT"] = $inboundNatTraffic["DIP_PORT"]
        $inboundNatTraffic["DIPS"] += $dipInfo
    }
    # resolve all VIPs
    foreach($inboundNatTraffic in $inboundNATTrafficRules) {

        $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $inboundNatTraffic["frontendIpConfig"] -uri $uri
        $inboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"]
        $inboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"]
    }

    # parse and create workload patterns for all LB RULES
    foreach($lbRule in $lb.properties.loadBalancingRules) {
        $lbTrafficRule = @{}
        $lbTrafficRule["frontendIpConfig"] = $lbRule.properties.frontendIpConfigurations[0].resourceRef
        $lbTrafficRule["VIP_PORT"] = $lbRule.properties.frontendPort
        $lbTrafficRule["VIP_PROTOCOL"] = $lbRule.properties.protocol
        $lbTrafficRule["DIPS"] = @()

        #resolve DIPs
        $bePoolTokens = $lbRule.properties.backendAddressPool.resourceRef.split("/")
        $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException
        foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) {

            $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/")
            $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException
            $dipInfo = @{}
            $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress
            $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2]
            $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef
            $lbTrafficRule["DIPS"] += $dipInfo

        }
        $lbTrafficRule["DIP_PORT"] = $lbRule.properties.backendPort
        $loadbalancerTrafficRules += $lbTrafficRule
    }
    # resolve all VIPs
    foreach($lbTrafficRule in $loadbalancerTrafficRules) {

        $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $lbTrafficRule["frontendIpConfig"] -uri $uri
        $lbTrafficRule["VIP_IP"] = $vipInfo["VIP_IP"]
        $lbTrafficRule["VIP_TYPE"] = $vipInfo["VIP_TYPE"]
    }

    # parse and create workload patterns for all OUTBOUND NAT RULES
    foreach($outboundNatRule in $lb.properties.outboundNatRules) {
        $outboundNatTraffic = @{}
        $outboundNatTraffic["frontendIpConfig"] = $outboundNatRule.properties.frontendIpConfigurations[0].resourceRef
        $outboundNatTraffic["VIP_PORT"] = 0
        $outboundNatTraffic["VIP_PROTOCOL"] = "TCP"
        $outboundNatTraffic["DIPS"] = @()

        # resolve be pool to ipconfigurations (DIPS)
        $bePoolTokens = $outboundNatRule.properties.backendAddressPool.resourceRef.split("/")
        $bePool = Get-NetworkControllerLoadBalancerBackendAddressPool -LoadBalancerId $bePoolTokens[2] -ResourceId $bePoolTokens[4] -ConnectionUri $uri -PassInnerException
        foreach($bePoolIpConfig in $bePool.properties.BackendIPConfigurations) {
            $ipConfigTokens = $bePoolIpConfig.resourceRef.split("/")
            $ipConfig = Get-NetworkControllerNetworkInterfaceIpConfiguration -NetworkInterfaceId $ipConfigTokens[2] -ResourceId $ipConfigTokens[4] -ConnectionUri $uri -PassInnerException
            $dipInfo = @{}
            $dipInfo["DIP_IP"] = $ipConfig.properties.PrivateIPAddress
            $dipInfo["DIP_VMNAME"] = $ipConfigTokens[2]
            $dipInfo[$DIP_IPCONFIG] = $ipConfig.resourceRef
            $outboundNatTraffic["DIPS"] += $dipInfo
        }
        $outboundNatTraffic["DIP_PORT"] = $outboundNatRule.properties.backendPort
        $outboundNATTrafficRules += $outboundNatTraffic
    }
    # resolve all VIPs
    foreach($outboundNatTraffic in $outboundNATTrafficRules) {

        $vipInfo = GetVIPInfoFromFroneEndIpConfig -feConfigResourceRef $outboundNatTraffic["frontendIpConfig"] -uri $uri
        $outboundNatTraffic["VIP_IP"] = $vipInfo["VIP_IP"]
        $outboundNatTraffic["VIP_TYPE"] = $vipInfo["VIP_TYPE"]
    }

    # Add all traffic rules to the endpoint
    $out = @{}
    $out["INBOUND_NAT_TRAFFIC_RULES"] = $inboundNATTrafficRules
    $out["LOADBALANCER_TRAFFIC_RULES"] = $loadbalancerTrafficRules
    $out["OUTBOUND_NAT_TRAFFIC_RULES"] = $outboundNATTrafficRules
    return $out
}

function GetVIPInfoFromFroneEndIpConfig {
    param(
        [ValidateNotNullOrEmpty()]
        [parameter(Mandatory=$true)]
        [string] $feConfigResourceRef,

        [ValidateNotNullOrEmpty()]
        [parameter(Mandatory=$false)]
        [string] $uri
    )

    $feConfigTokens = $feConfigResourceRef.split("/")

    $feConfig = Get-NetworkControllerLoadBalancerFrontendIpConfiguration -LoadBalancerId $feConfigTokens[2] -ResourceId $feConfigTokens[4] -ConnectionUri $uri -PassInnerException
    $vipInfo = @{}
    if($null -ne $feConfig.properties.PublicIPAddress) {

        $publicIpResRef = $feConfig.properties.PublicIPAddress.resourceRef;
        $publicIpResRef = $publicIpResRef.Substring($publicIpResRef.LastIndexOf("/") + 1)
        $publicIPResource = Get-NetworkControllerPublicIpAddress -ResourceId $publicIpResRef -ConnectionUri $uri -PassInnerException
        $vipInfo["VIP_IP"] = $publicIPResource.properties.IpAddress
        $vipInfo["VIP_TYPE"] = 2; # public IP
    } else {
        $vipInfo["VIP_IP"] = $feConfig.properties.PrivateIPAddress
        # check if the subnet is vnet or lnet
        if($feConfig.Properties.Subnet.resourceRef.StartsWith("/virtualNetworks/")) {
            $vipInfo["VIP_TYPE"] = 3 ; # ILB
        } else {
            $vipInfo["VIP_TYPE"] = 1 ; # private IP
        }
    }
    return $vipInfo
}


function Stop-CtstrafficListener {

    paramn(
        [string] $vmName,
        [string] $hostName,
        [pscredential] $vmCred,
        [pscredential] $hostCred
    )
    Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet

    # hostName will be empty for hostName as the client is the local machine
    if([string]::IsNullOrEmpty($hostName)) {
        Stop-Process -Name Ctstraffic -Verbose -Force

        Write-traceLog "Stopped ctstraffic listener on $hostName "
        return
    }

    $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName

    # run on remote machine
    $cmd = "Start-Process -FilePath $ctsTrafficPath -ArgumentList ""-listen:* -port:$($listerningPort)"""
    invoke-powershellcommandonvm -vmName $vmName `
                                -hostName $hostName `
                                -cmd $cmd `
                                -hostCred $hostCred `
                                -vmCred $vmCred

}
function Start-CtstrafficListener {

    param(
        [int] $listerningPort,
        [string] $vmName,
        [string] $hostName,
        [pscredential] $vmCred,
        [pscredential] $hostCred,
        [string] $uri,
        [string] $outFileName,
        [TrafficPattern] $trafficPattern,
        [string] $errorFileName = "error.log"
    )

    [string] $ctsTrafficPath = $script:CtsTrafficVMPath
    # Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet

    if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") {
        $ctsTrafficPath = $script:CtsTrafficVMPath
    }
    $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\"));

    if($listerningPort -eq 0) {

        # for HA ports, the port value may be set to 0 , override it in that case
        $listerningPort = $DEFAULT_CTSTRAFFICPORT
        write-TraceLog "Start-CtstrafficListener: Defaulting to port $listerningPort"
    }

    write-TraceLog "Start-CtstrafficListener: Starting ctstraffic ($ctsTrafficPath) listener on $hostName $vmName, ctsfolder : $ctsFolder port:$listerningPort"

    # use local machine if host and vmname are null
    if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) {

        if(-not [string]::IsNullOrEmpty($outFileName)) {
            $cmd = "-listen:* -port:$($listerningPort) -connectionFileName:$ctsFolder\$outFileName -transfer:$($trafficPattern.transferSizeInBytes)"
        } else {
            $cmd = "-listen:* -port:$($listerningPort) -transfer:$($trafficPattern.transferSizeInBytes)"
        }

        if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) {
            $cmd += " -pattern:$($trafficPattern.PushPattern)"
        }

        Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose
        Write-traceLog "Started ctstraffic listener on $hostName with command $cmd"
        return
    }

    if([string]::isNullOrEmpty($hostName)) {

        $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true
    }


    # build cts traffic parameter list

    $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;"
    $cmd += "del $ctsFolder\*.csv -Force;"
    $cmd += "`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();"

    $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -PassThru -ArgumentList """
    $cmd += " -listen:* -port:$($listerningPort)"

    if($trafficPattern.Protocol -eq "UDP") {
        $cmd += " -protocol:udp"
        $cmd += " -BitsPerSecond:$($trafficPattern.BitsPerSecond)"
        $cmd += " -FrameRate:$($trafficPattern.FrameRate)"
        $cmd += " -StreamLength:$($trafficPattern.StreamLength)"
        
    } else {
        $cmd += " -protocol:tcp"
        if($trafficPattern.transferSizeInBytes -gt 0) {
            $cmd += " -transfer:$($trafficPattern.transferSizeInBytes)"
        }
        if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) {
            $cmd += " -pattern:$($trafficPattern.PushPattern)"
        }
    }

    if(-not [string]::IsNullOrEmpty($outFileName)) {
        $cmd += " -connectionFileName:$ctsFolder\$outFileName"
    }
    if(-not [string]::IsNullOrEmpty($errorFileName)) {
        $cmd += " -errorFileName:$ctsFolder\$errorFileName"
    }

    $cmd += "`";" + [System.Environment]::NewLine # end the -argument list

    # detect failures in the command and throw exceptions

    # for listerner, wait for 2 seconds to see if the process has exited (bad params or something)
    $cmd += [System.Environment]::NewLine + "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalSeconds -lt 2) {Start-Sleep -Seconds 1};"
    # if process is alive, means it seems stable, proceed
    $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$false ) { return };"
    # if process is dead, even if it exited with 0 error code and capture / throw error
    # if ctstraffic does not generate the error file for some reason then throw a generic error or process return code
    $cmd += [System.Environment]::NewLine + "if(`$p.HasExited -eq `$true ) {`$c = get-content $ctsFolder\$errorFileName -ErrorAction SilentlyContinue; if(-not [string]::IsNullOrEmpty(`$c)) { throw `$c } else { throw ""No error log found. CtsTraffic Exit With code `$p.exitCode""} };"
 
    invoke-powershellcommandonvm -vmName $vmName `
                                -hostName $hostName `
                                -cmd $cmd `
                                -hostCred $hostCred `
                                -vmCred $vmCred
}

function Start-CtstrafficSender {

    param(
        [string] $targetIpAddress,
        [int] $targetPort,
        [string] $vmName,
        [string] $hostName,
        [pscredential] $vmCred,
        [pscredential] $hostCred,
        [string] $uri,
        [string] $ctsTrafficPath = $script:CtsTrafficVMPath,
        [string] $outFileName,
        [bool] $Wait = $true,
        [TrafficPattern] $trafficPattern
   )

    #Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet

    if($null -eq $ctsTrafficPath -or $ctsTrafficPath -eq "") {
        $ctsTrafficPath = $script:CtsTrafficVMPath
    }

    $ctsFolder = $ctsTrafficPath.Substring(0, $ctsTrafficPath.LastIndexOf("\"));
    $errorFileName = "$ctsFolder\error.log"

    write-TraceLog "Start-CtstrafficSender Starting ctstraffic ($ctsTrafficPath) sender on $hostName $vmName, ctsfolder : $ctsFolder"

    if($targetPort -eq 0) {
        # for HA ports, the port value may be set to 0 , override it in that case
        $targetPort = $DEFAULT_CTSTRAFFICPORT
        write-TraceLog "Start-CtstrafficSender: Defaulting to port $targetPort"
    }

    # use local machine if host and vmname are null
    if([string]::IsNullOrEmpty($hostName) -and [string]::IsNullOrEmpty($vmName)) {

        # note: this is only to test it locally, this will not be used finally
        $cmd = " -target:$($targetIpAddress) -port:$($targetPort) -protocol:$($trafficPattern.protocol) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) -transfer:$($trafficPattern.transferSizeInBytes)"
        if(-not [string]::IsNullOrEmpty($outFileName)) {
            $cmd += " -connectionFileName:$ctsFolder\$outFileName "
        }

        if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) {
            $cmd += " -pattern:$($trafficPattern.PushPattern)"
        }
        

        Start-Process -FilePath $ctsTrafficPath -ArgumentList $cmd -Verbose -Wait
        Write-traceLog "Started ctstraffic listener on local machine with command $cmd"
        return
    }

    if([string]::isNullOrEmpty($hostName)) {

        $hostName = Resolve-HostName -hostName $hostName -hostCred $hostCred -vmName $vmName -Force $true
    }

    # run on remote machine
    $cmd = "Stop-Process -Name CtsTraffic -Force -ErrorAction SilentlyContinue; Start-Sleep 5;"
    $cmd += "del $ctsFolder\*.csv -Force;`$sw = [System.Diagnostics.Stopwatch]::new();`$sw.Start();"

    $cmd += "`$p = Start-Process -FilePath $ctsTrafficPath -ArgumentList """

    if($trafficPattern.Protocol -eq "UDP") {
        $cmd += " -protocol:udp"
        $cmd += " -BitsPerSecond:$($trafficPattern.BitsPerSecond)"
        $cmd += " -FrameRate:$($trafficPattern.FrameRate)"
        $cmd += " -StreamLength:$($trafficPattern.StreamLength)"
        
    } else {
        $cmd += " -protocol:tcp"
        if($trafficPattern.transferSizeInBytes -gt 0) {
            $cmd += " -transfer:$($trafficPattern.transferSizeInBytes)"
        }
        if(-not [string]::IsNullOREmpty($trafficPattern.PushPattern)) {
            $cmd += " -pattern:$($trafficPattern.PushPattern)"
        }
    }
    $cmd += " -target:$($targetIpAddress) -port:$($targetPort) -connections:$($trafficPattern.connections) -iterations:$($trafficPattern.iterations) "

    if(-not [string]::IsNullOrEmpty($outFileName)) {
        $cmd += " -connectionFileName:$ctsFolder\$($outFileName) -errorfilename:$errorFileName"
    } 

    $cmd += """"

    $cmd += " -PassThru;"
    $cmd += "while(`$p.HasExited -eq `$false -and `$sw.Elapsed.TotalMinutes -lt 90) {Start-Sleep -Seconds 15};"
    $cmd += "if(`$p.HasExited -eq `$false ) { `$p.Kill(); throw ""Ctstraffic did not complete on time"" };"
    $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -ne 0) {`$c = get-content $errorFileName; throw `$c };"
    $cmd += "if(`$p.HasExited -eq `$true -and `$p.exitCode -eq 0) { return };"

    invoke-powershellcommandonvm -vmName $vmName `
                                -hostName $hostName `
                                -cmd $cmd `
                                -hostCred $hostCred `
                                -vmCred $vmCred

}



function Get-TrafficPattens {

    # [CmdletBinding()]
    # param (
    # [Parameter()]
    # [ValidateSet("TCP", "UDP")]
    # $Protocol,
    # [Parameter()]
    # [ValidateSet("ShortBursts", "LongRunning")
    # [string] $ProfileType
    # )
    # todo : implement different pattern types based on profile

    # for HLK , we just hardcode some default patterns for now
    $trafficPatterns = @()

    $OneKBInBytes = 1024
    $OneMBInBytes = 1048576
    $OneGBInBytes = 1073741824
    # few long runnning connections
    $trafficPatterns += New-TrafficPattern -connections 5 `
                                        -duration 60 `
                                        -iterations 2 `
                                        -TransferSizeInBytes ($OneGBInBytes * 2) `
                                        -protocol "TCP"
    # # medium set of connections
    # $trafficPatterns += New-TrafficPattern -connections 20 `
    # -duration 60 `
    # -iterations 20 `
    # -TransferSizeInBytes ($OneMBInBytes * 2) `
    # -protocol "TCP"
    
    # udp connections

    # 10mbps stream over 1000 frames
    $mbps10 = 25000000
    $fr = 60
    $trafficPatterns += New-TrafficPattern -connections 5 `
                                        -duration 60 `
                                        -iterations 2 `
                                        -protocol "UDP" `
                                        -bitsPerSecond $mbps10 `
                                        -FrameRate $fr `
                                        -BufferDepth 1 `
                                        -pushPattern "" `
                                        -streamLength 100

    return $trafficPatterns
}


<#
    Triggers a flurry of live mirgation operations of the given VMs.
    The VMs are represented by DIP endpoints, which have a VM name and IP address.
#>
function Start-SDNVmMigrationValidation {

    param(
        [int] $percentageOfVmsToMove = 50,
        [object[]] $trafficEndpoints
    )

    # calculate how many migrations need to be performed
    [int] $numOfVmsToMove = $trafficEndpoints.Count * $percentageOfVmsToMove / 100

    # track vms already attempted to move
    $vmsMoved = @()

    # track vms that failed to move
    $vmsFailedToMove = @()

    # track vms that moved successfully
    $vmsMovedSuccessfully = @()

    $done = $false

    do {

        [int] $breaker = 0
        $vmToMove = $null
        # find a target vm to move
        $vmToMove = Get-Random -InputObject $trafficEndpoints

        if($null -eq $vmToMove) {
            throw "Could not find a VM to move"
        }

        # move the vm
        try {

            # resolve host for the VM
            $hostName = Resolve-HostName -vmName $vmToMove.VmName -hostCred $vmToMove.HostCredential -vmCred $vmToMove.VmCredential
            $allHostnames = Get-ClusterNode | ?{$_.State -eq "UP" -and $_.Name -ne $hostName}

            # in test lab we might have one node, so livemigration is not possible
            # in HLK we mandate there are 2 machines through LMS
            if($allHostnames.Count -le 1) {
                return
            }

            $target = Get-Random -InputObject $allHostnames

            # for now using only live migration
            # $i = Get-Random -Maximum 4
            # if($i -eq 0) {
            # $migrationType = "Quick"
            # }
            # if($i -eq 1) {
            # $migrationType = "Live"
            # }
            # if($i -eq 2) {
            # $migrationType = "Shutdown"
            # }
            # if($i -eq 3) {
            # $migrationType = "TurnOff"
            # }

            Write-TraceLog "Start-SDNVmMigrationValidation: Moving VM $($vmToMove.VmName) from host $hostName to host $($target.Name)"

            Move-ClusterVirtualMachineRole -Name $vmToMove.VMName -Node $target.Name -MigrationType Live

            $vmsMovedSuccessfully += $vmToMov

            if($vmsMovedSuccessfully.Count -ge $numOfVmsToMove) {
                $done = $true
            }

            $trafficEndpoints = $trafficEndpoints | Where-Object {$_.VmName -ne $vmToMove.VmName}

        } catch {
            Write-TraceLog "Start-SDNVmMigrationValidation: Failed to move VM $($vmToMove.VmName) from host $hostName to host $($target.Name) Error : $_"
            $vmsFailedToMove += $vmToMove

            # give up on 2nd failure
            if($vmsFailedToMove.Count -ge 2) {
                throw "Multiple live migrations failure, please check your configuration and run the test again"
            }
            continue
        }
    }until($done)

    Write-TraceLog "Start-SDNVmMigrationValidation: Successfully moved $numOfVmsToMove VMs"
}

function Test-TenantVirtualMachine {
    param(
        [string] $vmName,
        [string] $hostName,
        [pscredential] $vmCred,
        [pscredential] $hostCred
    )
    
    Write-FunctionEntryWithParams -FunctionName $MyInvocation.MyCommand.Name -boundparameters $psboundparameters -UnboundArguments $MyINvocation.UnboundArguments -ParamSet $psCmdlet

    $cmd = "`$process = Get-Process -Name winlogon; if(`$null -ne `$process) { return `$true } else { return `$false }"

    $hostCred = Get-TurnKeySdnCred
    $vmCred = Get-TurnKeySdnWorkloadVmCred

    $retryCount = 0
    $maxRetryCount = 20

    do {
        try {
            $result = Invoke-PowershellCommandOnVm -vmName $vmName `
                                                -hostName $hostName `
                                                -cmd $cmd `
                                                -hostCred $hostCred `
                                                -vmCred $vmCred 
        } catch {
            Write-TraceLog "Test-TenantVirtualMachine: Failed to connect to VM $vmName, retrying... (retry: $retryCount)"
            Write-TraceLog "Test-TenantVirtualMachine: Error : $_"
            $result = $false
        }

        if($result -eq $true) {
            return $true
        }
        $retryCount += 1
        Start-Sleep -Seconds 60

    } while($retryCount -lt $maxRetryCount)

    if($retryCount -ge $maxRetryCount) {
        throw "Failed to connect to VM $vmName"
    }
    Write-TraceLog "Test-TenantVirtualMachine: VM $vmName is available"
}

function LogTrafficInfo {
    param(
        [TrafficEndpoint] $listenEndpoint,
        [TrafficEndpoint] $sendEndpoint,
        [TrafficPattern] $trafficPattern
    )
    Write-TraceLog  "`t listenEndpoint: $($listenEndpoint.ToString())"
    Write-TraceLog  "`t sendEndpoint: $($sendEndpoint.ToString())"
    Write-TraceLog  "`t traffic: $($trafficPattern.ToString())"
}
# SIG # Begin signature block
# MIIoQwYJKoZIhvcNAQcCoIIoNDCCKDACAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCHcbA/IX41fd/x
# 3YFU+jW2vsSf19vOYrBeE0Cgf73/Z6CCDXYwggX0MIID3KADAgECAhMzAAAEBGx0
# Bv9XKydyAAAAAAQEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTE0WhcNMjUwOTExMjAxMTE0WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQC0KDfaY50MDqsEGdlIzDHBd6CqIMRQWW9Af1LHDDTuFjfDsvna0nEuDSYJmNyz
# NB10jpbg0lhvkT1AzfX2TLITSXwS8D+mBzGCWMM/wTpciWBV/pbjSazbzoKvRrNo
# DV/u9omOM2Eawyo5JJJdNkM2d8qzkQ0bRuRd4HarmGunSouyb9NY7egWN5E5lUc3
# a2AROzAdHdYpObpCOdeAY2P5XqtJkk79aROpzw16wCjdSn8qMzCBzR7rvH2WVkvF
# HLIxZQET1yhPb6lRmpgBQNnzidHV2Ocxjc8wNiIDzgbDkmlx54QPfw7RwQi8p1fy
# 4byhBrTjv568x8NGv3gwb0RbAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU8huhNbETDU+ZWllL4DNMPCijEU4w
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMjkyMzAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAIjmD9IpQVvfB1QehvpC
# Ge7QeTQkKQ7j3bmDMjwSqFL4ri6ae9IFTdpywn5smmtSIyKYDn3/nHtaEn0X1NBj
# L5oP0BjAy1sqxD+uy35B+V8wv5GrxhMDJP8l2QjLtH/UglSTIhLqyt8bUAqVfyfp
# h4COMRvwwjTvChtCnUXXACuCXYHWalOoc0OU2oGN+mPJIJJxaNQc1sjBsMbGIWv3
# cmgSHkCEmrMv7yaidpePt6V+yPMik+eXw3IfZ5eNOiNgL1rZzgSJfTnvUqiaEQ0X
# dG1HbkDv9fv6CTq6m4Ty3IzLiwGSXYxRIXTxT4TYs5VxHy2uFjFXWVSL0J2ARTYL
# E4Oyl1wXDF1PX4bxg1yDMfKPHcE1Ijic5lx1KdK1SkaEJdto4hd++05J9Bf9TAmi
# u6EK6C9Oe5vRadroJCK26uCUI4zIjL/qG7mswW+qT0CW0gnR9JHkXCWNbo8ccMk1
# sJatmRoSAifbgzaYbUz8+lv+IXy5GFuAmLnNbGjacB3IMGpa+lbFgih57/fIhamq
# 5VhxgaEmn/UjWyr+cPiAFWuTVIpfsOjbEAww75wURNM1Imp9NJKye1O24EspEHmb
# DmqCUcq7NqkOKIG4PVm3hDDED/WQpzJDkvu4FrIbvyTGVU01vKsg4UfcdiZ0fQ+/
# V0hf8yrtq9CkB8iIuk5bBxuPMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGiMwghofAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAAQEbHQG/1crJ3IAAAAABAQwDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIPOweHnyzsqGYEADR57C/Q0l
# PGVZjyhFA/CoPgPVOAE8MEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEAN0dOmYmyhv03P6TBOmeL5d4uO707yv4ElosK1LcupIGndIXuyxfo7Z0f
# 9tCDZbHDtEobsvOlzrqLq435EHGbbcsCtXcvYol7b2O4TWP/lQhPJkHkU1hCmfZZ
# GmOkdoEizyGN8jJBdTVxGx5tbvEWmOWuvvn0yDV5PaSKAtMSxvfwtQ2OL1Kr6YlY
# pcAreMISNDCRkW9Cluf0oUV/lHhkmYY9XabqZUNHs3NUWGLxnyL6Kma/TVjo2rt4
# yzqMFlkKB4XEz8DVr+rDRIvoio2KoTt4s0b+RM/7JRFjPmmZ1nSgBynEcgghmj0o
# IS8l7gXm5vrfw6a05B2h/AWBbxWdj6GCF60wghepBgorBgEEAYI3AwMBMYIXmTCC
# F5UGCSqGSIb3DQEHAqCCF4YwgheCAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsq
# hkiG9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCBtzu/71xwLu5rsoM/CebwsfsHKlTY8JajBFEtngv95xQIGZ7Yh/JkI
# GBMyMDI1MDMyMDIwMTEyNi4xNTJaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT
# Tjo0QzFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg
# U2VydmljZaCCEfswggcoMIIFEKADAgECAhMzAAAB/xI4fPfBZdahAAEAAAH/MA0G
# CSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u
# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp
# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI0
# MDcyNTE4MzExOVoXDTI1MTAyMjE4MzExOVowgdMxCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9w
# ZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjRDMUEt
# MDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl
# MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyeiV0pB7bg8/qc/mkiDd
# JXnzJWPYgk9mTGeI3pzQpsyrRJREWcKYHd/9db+g3z4dU4VCkAZEXqvkxP5QNTtB
# G5Ipexpph4PhbiJKwvX+US4KkSFhf1wflDAY1tu9CQqhhxfHFV7vhtmqHLCCmDxh
# ZPmCBh9/XfFJQIUwVZR8RtUkgzmN9bmWiYgfX0R+bDAnncUdtp1xjGmCpdBMygk/
# K0h3bUTUzQHb4kPf2ylkKPoWFYn2GNYgWw8PGBUO0vTMKjYD6pLeBP0hZDh5P3f4
# xhGLm6x98xuIQp/RFnzBbgthySXGl+NT1cZAqGyEhT7L0SdR7qQlv5pwDNerbK3Y
# SEDKk3sDh9S60hLJNqP71iHKkG175HAyg6zmE5p3fONr9/fIEpPAlC8YisxXaGX4
# RpDBYVKpGj0FCZwisiZsxm0X9w6ZSk8OOXf8JxTYWIqfRuWzdUir0Z3jiOOtaDq7
# XdypB4gZrhr90KcPTDRwvy60zrQca/1D1J7PQJAJObbiaboi12usV8axtlT/dCeP
# C4ndcFcar1v+fnClhs9u3Fn6LkHDRZfNzhXgLDEwb6dA4y3s6G+gQ35o90j2i6am
# aa8JsV/cCF+iDSGzAxZY1sQ1mrdMmzxfWzXN6sPJMy49tdsWTIgZWVOSS9uUHhSY
# kbgMxnLeiKXeB5MB9QMcOScCAwEAAaOCAUkwggFFMB0GA1UdDgQWBBTD+pXk/rT/
# d7E/0QE7hH0wz+6UYTAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBf
# BgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3Bz
# L2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmww
# bAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29m
# dC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El
# MjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUF
# BwMIMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAgEAOSNN5MpLiyun
# m866frWIi0hdazKNLgRp3WZPfhYgPC3K/DNMzLliYQUAp6WtgolIrativXjOG1lI
# jayG9r6ew4H1n5XZdDfJ12DLjopap5e1iU/Yk0eutPyfOievfbsIzTk/G51+uiUJ
# k772nVzau6hI2KGyGBJOvAbAVFR0g8ppZwLghT4z3mkGZjq/O4Z/PcmVGtjGps2T
# CtI4rZjPNW8O4c/4aJRmYQ/NdW91JRrOXRpyXrTKUPe3kN8N56jpl9kotLhdvd89
# RbOsJNf2XzqbAV7XjV4caCglA2btzDxcyffwXhLu9HMU3dLYTAI91gTNUF7BA9q1
# EvSlCKKlN8N10Y4iU0nyIkfpRxYyAbRyq5QPYPJHGA0Ty0PD83aCt79Ra0IdDIMS
# uwXlpUnyIyxwrDylgfOGyysWBwQ/js249bqQOYPdpyOdgRe8tXdGrgDoBeuVOK+c
# RClXpimNYwr61oZ2/kPMzVrzRUYMkBXe9WqdSezh8tytuulYYcRK95qihF0irQs6
# /WOQJltQX79lzFXE9FFln9Mix0as+C4HPzd+S0bBN3A3XRROwAv016ICuT8hY1In
# yW7jwVmN+OkQ1zei66LrU5RtAz0nTxx5OePyjnTaItTSY4OGuGU1SXaH49JSP3t8
# yGYA/vorbW4VneeD721FgwaJToHFkOIwggdxMIIFWaADAgECAhMzAAAAFcXna54C
# m0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UE
# CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z
# b2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp
# Y2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMy
# MjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQH
# EwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNV
# BAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0B
# AQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51
# yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY
# 6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9
# cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN
# 7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDua
# Rr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74
# kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2
# K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5
# TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZk
# i1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9Q
# BXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3Pmri
# Lq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUC
# BBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJl
# pxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIB
# FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9y
# eS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUA
# YgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU
# 1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2Ny
# bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIw
# MTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDov
# L3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0w
# Ni0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/yp
# b+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulm
# ZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM
# 9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECW
# OKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4
# FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3Uw
# xTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPX
# fx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVX
# VAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGC
# onsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU
# 5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEG
# ahC0HVUzWLOhcGbyoYIDVjCCAj4CAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT
# Tjo0QzFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg
# U2VydmljZaIjCgEBMAcGBSsOAwIaAxUAqROMbMS8JcUlcnPkwRLFRPXFspmggYMw
# gYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE
# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD
# VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsF
# AAIFAOuG2ZMwIhgPMjAyNTAzMjAxODE0NDNaGA8yMDI1MDMyMTE4MTQ0M1owdDA6
# BgorBgEEAYRZCgQBMSwwKjAKAgUA64bZkwIBADAHAgEAAgIOrTAHAgEAAgISNjAK
# AgUA64grEwIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB
# AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBCwUAA4IBAQAJvqJTv0vjOFB1
# HLv/ljCZI+bWypmNiFbJaQ87WkxhonZNC8eGJCvZXPUysybTJgVnCp88T91rdcXa
# t4iRV7mFUfZwWTM+DVl71cWh2Z1WdZTnILqkBnBYDRo4EipI7ytsuWVO/GwMz3eM
# LHaI3o+6cO5jDi830Nr60QIVD0PozgXhjFTac091JCexiSqQm7ecOssJxYZA73dG
# 805Kl9nAaiijlT5LpN0U8EbMa/3E3kwI3phbkJatOxgHBvpnGpWWHoaDlKNR7wmK
# Pc/lKh4gXV1ggdMY1hNE06IzWDHzGCbqOlRn2Dina/hnQGLI0CEEjOdVLeY9mSsZ
# tsULcfMPMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh
# c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD
# b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw
# MTACEzMAAAH/Ejh898Fl1qEAAQAAAf8wDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqG
# SIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQxIgQgu+vC3CpK4dBq
# KsKlNLQKUoUTTT4nDnTn641696ySy3AwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHk
# MIG9BCDkMu++yQJ3aaycIuMT6vA7JNuMaVOI3qDjSEV8upyn/TCBmDCBgKR+MHwx
# CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt
# b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1p
# Y3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB/xI4fPfBZdahAAEAAAH/
# MCIEIJt0mjHI+DXDWCx3RW8SJ51fA0l7zncRW57pK5ZzKoZqMA0GCSqGSIb3DQEB
# CwUABIICADo9BFLL6dTBwCSTflQxfco/oPOZBjxcEcbk3GqTBFO8fvcxEr4/BqFo
# c+8F6DPcJTYNr/jFeym7fPJkEjxSbpY2vaq3WYJEHjKhiWIDr4MmFtEvpuIa2umQ
# tWrALDQjBVpH+PwyYPa0xNKTs0yW+7e4Yys/jDSNxePYH42NZxo0CRkAMnV5oNIu
# 0+oCNGvt5zUs/QFy0OuVj4xT+qeygyncciLLoRdYMorBTOH+Dis3nyK+9yGU3Bs3
# rOaJVT/yLYKJhvH4M8dEcq6TsgEj8aimXRoVjLfHp9AKeeIeiXU0FBmEvwrfE2b3
# Mwg4EnaDV7pjzrSbcgqPl6wFB14r4vWk6fhJAzot4zwUP0kmXCR/aCc/Y0mW9jPB
# iucvsCe+kUnjf6qqQ8DFbf+lVmR3pUtig2qGOGFWr3Wfhqmxw5jmj9KmQGjFwOn2
# Oc7QM7F4TvnJLzp4wl/cvaYYUoI2x8JVa797/59D7WjH9eMc/OnMdCYcRuRnWImp
# aQapvcpQW1MNp5LPKD1FRIYdsD79BHYC5gppmzzViq7252l4d8HJLa8SVwmKFZVa
# g0nxMwrjjPW7dhS8t6Io2UiCZiOltQ2+1EjXCL82R4a4H/Cz938hwoFKDtv786bC
# UbFMoxJ62h7xOYprGjym/dDJsmgrOleeAaN61MV+mztjV6oScCcN
# SIG # End signature block