TurnKeySdn

0.0.26

TurnKeySdnInternal.psm1

Import-Module $PSScriptRoot\ConfigManager.psm1
Import-Module $PSScriptRoot\ClusUtils.psm1
Import-Module $PSScriptRoot\Logger.psm1
Import-Module $PSScriptRoot\PsHelper.psm1
Import-Module $PSScriptRoot\Utils.psm1
Import-Module $PSScriptRoot\WinBuildUtils.psm1
Import-Module $PSScriptRoot\TrafficUtil.psm1

function Get-NewDepId {

    return @($id, $idbytes)
}

function Initialize-Credentials {
    
    $cred = Get-TurnKeySdnCred
    if ($cred -ne [pscredential]::Empty) {
        Write-TraceLog "Initialize-Credentials : Existing credential found"
        return
    }

    $cred = $null
    $infraconfig = Get-DeploymentConfig
    $domain = $infraconfig.domainName

    if (-not [String]::IsNullOrEmpty($env:TEST_USERNAME) -and -not [String]::IsNullOrEmpty($env:TEST_PASSWORD)) {
        
        Write-TraceLog "Initialize-Credentials: Credential discovered from ADO environment variables"

        if (-not [String]::IsNullOrEmpty($env:TEST_DOMAIN)) {
            $username = $env:TEST_DOMAIN + "\" + $env:TEST_USERNAME    
        }
        else {
            $username = $env:TEST_USERNAME
        }
        $cred = @{}
        $cred["username"] = $username
        $cred["password"] = $env:TEST_PASSWORD | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    } 

    if ($cred -ne $null) {
        Set-TurnKeySdnInternalConfig -configType credential -config $cred
    }
    else {
        Write-TraceLog "Initialize-Credentials: Failed to initialize test credentials, current test user [$($env:USERNAME)]" -Warning
        Write-TraceLog "Please set test credentials using Set-TurnKeySdnCredentials and retry" -Warning
        throw "Unable to read test credential for user [$($env:USERNAME)]. Please set credentials using Set-TurnKeySdnCredentials and retry."
    }
}

function Initialize-WorkloadVMCredentials {

    # initialize workload vm credentials

    $cred = @{}

    if(-not [string]::IsNullOREmpty("$Env:TURNKEY_WORKLOAD_USERNAME") -and -not [string]::IsNullOREmpty("$Env:TURNKEY_WORKLOAD_PASSWORD")) {
        $cred["username"] = $Env:TURNKEY_WORKLOAD_USERNAME
        $cred["password"] = $Env:TURNKEY_WORKLOAD_PASSWORD | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    } else {
        $userName = "administrator"
        $cred["username"] = $userName
        $cred["password"] =$userName | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    }

    Set-TurnKeySdnInternalConfig -configType workloadcredential -config $cred
}


function Initialize-DomainConfiguration {

    $infraconfig = Get-DeploymentConfig
    $domain = $infraconfig.domainName

    if ([String]::IsNullOrEmpty($domain) -and ((Get-WmiObject -Class Win32_ComputerSystem).PartOfdomain)) {
        $domain = (Get-WmiObject -Class Win32_ComputerSystem).Domain
        $infraconfig.domainName = $domain
        Write-TraceLog "Domain is set to $domain"
    }
    else {
        Write-TraceLog "Domain is not set, using local machine domain"
    }  

    Set-DeploymentConfig -deploymentConfig $infraconfig
}

function Initialize-HyperVHostConfig {
    $infraconfig = Get-DeploymentConfig

    try {
    
        if ($infraconfig.hyperVHosts -ne $null -and $infraconfig.hyperVHosts.Count -gt 0) {
            return
        }

        $infraconfig.hyperVHosts = @()

        if (Test-IsCluster) {
            (get-cluster | get-clusternode).Name | ForEach-Object { $infraconfig.hyperVHosts += $_ }
        }
        else {
            $infraconfig.hyperVHosts += (hostname)
        }

        Set-DeploymentConfig -deploymentConfig $infraconfig
    }
    finally {
        $hosts = $infraconfig.hyperVHosts -join ","
        Write-TraceLog "Initialize-HyperVHostConfig: Hosts are $hosts"
    }
}

function Initialize-VSwitchConfig {

    $infraconfig = Get-DeploymentConfig
    if ([String]::IsNullOrEmpty($infraconfig.internetSwitchName)) {
        $infraconfig.internetSwitchName = Resolve-InternetSwitch
    }

    if ([String]::IsNullOrEmpty($infraconfig.internetSwitchName)) {
     
        $iSwitchCreated = $true
        $switchName = Get-DefaultInternetSwitchName
        $infraconfig.hyperVHosts | ForEach-Object {
            Write-TraceLog "Initialize-VSwitchConfig: Attempting to auto create internet switch on host ($_)"
            $iSwitchCreated = $iSwitchCreated -band (New-InternetSwitch -computer $_ -name $switchName)
        }

        if ($iSwitchCreated) {
            $infraconfig.internetSwitchName = $switchName
        }
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $infraconfig.sdnSwitchName = Resolve-SdnSwitch
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $sdnSwitchCreated = $true
        $sdnSwitchName = Get-DefaultSdnSwitchName
        $infraconfig.hyperVHosts | ForEach-Object {
            Write-TraceLog "Initialize-VSwitchConfig: Attempting to auto create sdn switch on host ($_)"
            $sdnSwitchCreated = $sdnSwitchCreated -band (New-SdnSwitch -computer $_ -name $sdnSwitchName)
        }

        if ($sdnSwitchCreated) {
            $infraconfig.sdnSwitchName = $sdnSwitchName
        }
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $infraconfig.sdnSwitchName = $infraconfig.internetSwitchName
        Write-TraceLog "Initialize-VSwitchConfig: Using internetswitch for sdn"
    }

    Write-TraceLog "Initialize-VSwitchConfig: Internet Switch is $($infraconfig.internetSwitchName)"
    Write-TraceLog "Initialize-VSwitchConfig: Sdn Switch is $($infraconfig.sdnSwitchName)"
    Set-DeploymentConfig -deploymentConfig $infraconfig
}

function Initialize-DeploymentVhd { 

    $depConfig = Get-DeploymentConfig

    if (-not [String]::IsNullOrEmpty($depConfig.vhdPath) -and `
            -not [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        $vhd = Join-Path $depConfig.vhdPath $depConfig.vhdFile
        Write-TraceLog "Initialize-DeploymentVhd: Deployment VHD is $vhd"
        return
    }

    Write-TraceLog "Initialize-DeploymentVhd: Trying to auto discover deployment vhd"

    if (-not [String]::IsNullOrEmpty($depConfig.windowsBuild)) {
        Write-TraceLog "Initialize-DeploymentVhd: Using specified build $($depConfig.windowsBuild)"
    }
    
    $bestBuild = Get-BestBuild -SpecificBuild $depConfig.windowsBuild -WindowsBuildBranch $depConfig.windowsBuildBranch

    if ([String]::IsNullOrEmpty($bestBuild)) {
        throw "Initialize-DeploymentVhd: Failed to auto detect a build to use for deployment, please set vhd location manually using Set-TurnKeySdnDeploymentVhd"
    }

    $buildVhd = Get-BuildVersionVhd -buildLocation $bestBuild

    if ([String]::IsNullOrEmpty($buildVhd)) {
        throw "Initialize-DeploymentVhd: Vhd auto discover failed, please set vhd location manually using Set-TurnKeySdnDeploymentVhd"
    }

    $vhdPath = Join-Path (Get-VhdStore) "deploymentvhd"
    
    if (-not (Test-Path $vhdPath)) {
        New-Item $vhdPath -ItemType Directory -ErrorAction Stop | Out-Null
    } 
    
        
    $fileName = Split-Path $buildVhd -leaf   
    $dest = Join-Path $vhdPath $fileName

    if (Test-Path $dest) {
        Write-TraceLog "Initialize-DeploymentVhd: Vhd already exists at $dest, skipping copy"        
    }
    else {
        Write-TraceLog "Initialize-DeploymentVhd: Resolved vhd to $buildVhd"
        Write-TraceLog "Initialize-DeploymentVhd: Copying vhd to $vhdPath"
        #Copy-Item $buildVhd $vhdPath
        try {
            Start-BitsTransfer -Source $buildVhd -Destination $vhdPath -Description $buildVhd -DisplayName "CopyFromWinbuild"
        }
        catch {
            Copy-Item $buildVhd $vhdPath
        }
    }

    $vhdFile = $buildVhd.Split("\") | Select -Last 1
    $depConfig.vhdPath = $vhdPath
    $depConfig.vhdFile = $vhdFile

    Set-DeploymentConfig -deploymentConfig $depConfig
}

function Initialize-DeploymentPath {
    $depConfig = Get-DeploymentConfig

    $defaultPath = $depConfig.vmLocation
    $csv = Get-CSV 
    $isCsv = $csv -ne $null

    if ([String]::isnullorempty($defaultPath)) {

        if ($isCsv) {
            $defaultPath = Join-Path $csv "sdnvms"
        }
        else {
            # Use a local default path
            $defaultPath = Join-Path $env:SystemDrive "sdnvms"
        }      
    }

    $sb = {
        param ($defaultPath)
        if (-not $(Test-Path($defaultPath))) {
            try {
                New-Item $defaultPath -ItemType Directory -Force -ErrorAction Stop | Out-Null
            }
            catch {
                throw "Unable to access deployment path $defaultPath"
            }
        }
    }

    if (-not $isCsv) {
        #check all hosts
        Invoke-CmdOnInfraHosts -scriptBlock $sb -args @($defaultPath)
    }
    else {
        Invoke-Command -ScriptBlock $sb -ArgumentList @($defaultPath)
    }

    $depConfig.vmLocation = $defaultPath

    Set-DeploymentConfig -deploymentConfig $depConfig
}

function Initialize-NetworkControllerPackage {
    $sdnConfig = get-sdnconfig

    if ($sdnconfig.networkController.useRestIp -eq 'false' -and `
        [String]::IsNullOrEmpty($sdnconfig.networkController.restName) -and `
        ([String]::IsNullOrEmpty($sdnconfig.networkController.restIpAddress) -or `
    $sdnConfig.networkController.runtime -eq "FC")) {

        # Auto generate rest name
        $depId = Get-TurnKeySdnDeploymentId
        $sdnConfig.networkController.restName = "NC-$depId"
        $infraconfig = Get-DeploymentConfig    
        if (-not [String]::IsNullOrEmpty($infraconfig.domainName)) {
            $sdnConfig.networkController.restName += "." + $infraconfig.domainName
        }
    }

    Write-TraceLog "Initialize-NetworkControllerPackage: RestName $($sdnConfig.networkController.restName)"

    if ($sdnConfig.networkController.runtime -ne "FC") {
        Set-TurnKeySdnConfig -sdnConfig $sdnConfig
        return
    }

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.PowershellModuleRootPath)) {
        $sdnConfig.networkController.FC.PowershellModuleRootPath = Get-DefaultNetworkControllerPsModulePath
    }

    # Sdnexpress fetches this global var
    
    $Global:FCNC_MODULE_PATH_ROOT = $sdnConfig.networkController.FC.PowershellModuleRootPath

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.PackageLocation)) {
        $sdnConfig.networkController.FC.PackageLocation = Get-DefaultNetworkControllerPackageLocation
        #$sdnConfig.networkController.FC.PackageLocation = "C:\NetworkController"
    }

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.DatabaseLocation)) {
        $sdnConfig.networkController.FC.DatabaseLocation = Get-DefaultNetworkControllerDBLocation
    }     

    Write-TraceLog "Initialize-NetworkControllerPackage: PsModule $($sdnConfig.networkController.FC.PowershellModuleRootPath)"
    Write-TraceLog "Initialize-NetworkControllerPackage: PackageLocation $($sdnConfig.networkController.FC.PackageLocation)"
    Write-TraceLog "Initialize-NetworkControllerPackage: DatabaseLocation $($sdnConfig.networkController.FC.DatabaseLocation)"

    Set-TurnKeySdnConfig -sdnConfig $sdnConfig
}

function Initialize-SdnExpressConfig { 
    param(
        [Parameter(Mandatory = $true)] $sdnExpConfig
    )

    $infraconfig = Get-DeploymentConfig
    
    $sdnExpConfig.VMProcessorCount = Get-InfraVMProcessorCount
    $sdnExpConfig.VMMemory = Get-InfraVMMemory
    $sdnExpConfig.DisableIPv6DHCP = $true

    $sdnExpConfig.JoinDomain = $infraconfig.domainName
    $cred = Get-TurnKeySdnCred

    if ($cred -eq [pscredential]::Empty) {
        throw "Initialize-SdnExpressConfig: Test Credential not found, please set credentials using Set-TurnKeySdnCredentials and retry"
    }

    $secPass = $cred.password | ConvertFrom-SecureString

    $sdnExpConfig.DomainJoinUsername = $cred.username
    $sdnExpConfig.DomainJoinSecurePassword = $secPass
    $sdnExpConfig.LocalAdminSecurePassword = $secPass
    $sdnExpConfig.LocalAdminDomainUser = $cred.username

    $sdnExpConfig.NCUsername = $cred.username
    $sdnExpConfig.NCSecurePassword = $secPass

    $sdnExpConfig.SwitchName = $infraconfig.sdnSwitchName

    $sdnExpConfig.VHDPath = $infraconfig.vhdPath
    $sdnExpConfig.VHDFile = $infraconfig.vhdFile

    if (-not [String]::isnullorempty($infraconfig.productKey)) {
        $sdnExpConfig.ProductKey = $infraconfig.productKey
    }

    $sdnExpConfig.VMLocation = $infraconfig.vmLocation

    $defaultMacPoolConfig = Get-DefaultMacPoolConfig
    $sdnExpConfig.SDNMacPoolStart = $defaultMacPoolConfig.properties.startMacAddress
    $sdnExpConfig.SDNMacPoolEnd = $defaultMacPoolConfig.properties.endMacAddress

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $sdnExpConfig.ManagementSubnet = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix
    $sdnExpConfig.ManagementGateway = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.ManagementDNS = $mgmtNetConfig.properties.subnets[0].properties.dnsServers
    $sdnExpConfig.ManagementVLANID = $mgmtNetConfig.properties.subnets[0].properties.vlanID
    #$sdnExpConfig.ManagementRoutes = @($mgmtNetConfig.properties.subnets[0].properties.addressPrefix)
    $sdnExpConfig.ManagementRoutes = @()
    $mgmtIpPoolStart = $mgmtNetConfig.properties.subnets[0].properties.ipPools[0].properties.startIpAddress

    $paNetConfig = Get-HnvPaNetworkConfig
    $sdnExpConfig.PASubnet = $paNetConfig.properties.subnets[0].properties.addressPrefix
    $sdnExpConfig.PAGateway = $paNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.PAVLANID = $paNetConfig.properties.subnets[0].properties.vlanID
    $sdnExpConfig.PAPoolStart = $paNetConfig.properties.subnets[0].properties.ipPools[0].properties.startIpAddress
    $sdnExpConfig.PAPoolEnd = $paNetConfig.properties.subnets[0].properties.ipPools[0].properties.endIpAddress  
    
    $currentHostName = $(hostname)
    $depId = Get-TurnKeySdnDeploymentId

    $sdnConfig = get-sdnconfig

    $sdnExpConfig['RestName'] = $sdnconfig.networkController.restName
    $sdnExpConfig['RestIpAddress'] = $sdnConfig.networkController.restIpAddress

    if ([string]::IsNullOrEmpty($sdnconfig.networkController.restName) -or `
            $sdnConfig.networkController.runtime -eq "FC" -or `
            $sdnconfig.networkController.useRestIp -eq 'true') {
        # Restip auto discovery
        # For FC always generate IP
        # Otherwise use IP if restName is not set
        if ([String]::IsNullOrEmpty($sdnConfig.networkController.restIpAddress)) {
            $mgmtIpOffset = Get-NextManagementIPOffset
            $restIp = Get-IPAddressInSubnet -Subnet $mgmtIpPoolStart -Offset $mgmtIpOffset
            $prefix = $sdnExpConfig.ManagementSubnet.Split("/")[1]    
            $sdnConfig.networkController.restIpAddress = "$restIp/$prefix"
            $sdnExpConfig['RestIpAddress'] = $sdnConfig.networkController.restIpAddress
        } 
    }
    
    if ([String]::IsNullOrEmpty($sdnExpConfig['RestName'])) {
        $restName = $sdnConfig.networkController.restIpAddress.Split("/")[0]
        $sdnconfig.networkController.restName = $restName
        $sdnExpConfig['RestName'] = $restName
    }

    $sdnExpConfig['HyperVHosts'] = $infraconfig.hyperVHosts    
    $paPoolOffset = 0

    if ($sdnConfig.networkController.runtime -eq "FC") {
        $sdnExpConfig.UseFCNC = 1
        $sdnExpConfig.FCNCBins = $sdnConfig.networkController.FC.PackageLocation
        $sdnExpConfig.FCNCDBs = $sdnConfig.networkController.FC.DatabaseLocation       
    }
    else {

        $i = 0
        $sdnExpConfig['NCs'] = @()
        $NCNodeCount = $sdnConfig.networkController.SF.nodecount

        while ($i -lt $NCNodeCount) {
            $ncConfig = @{}
            $ncConfig['ComputerName'] = "$depId-NC$i"
    
            $mgmtIpOffset = Get-NextManagementIPOffset
            $ncConfig['ManagementIP'] = Get-IPAddressInSubnet -Subnet $mgmtIpPoolStart -Offset $mgmtIpOffset        

            $sdnExpConfig['NCs'] += $ncConfig
            $i++
        }
    }

    $muxNodeCount = $sdnConfig.mux.nodecount
    $i = 0
    $sdnExpConfig['Muxes'] = @()

    if ($sdnConfig.mux.enabled -eq "true") {
        while ($i -lt $muxNodeCount) {
            $mux = @{}
            $mux['ComputerName'] = "$depId-MUX$i"
            $mgmtIpOffset = Get-NextManagementIPOffset
            $mux['ManagementIP'] = Get-IPAddressInSubnet -Subnet $mgmtIpPoolStart -Offset $mgmtIpOffset
            $mux['PAIPAddress'] = Get-IPAddressInSubnet -Subnet $sdnExpConfig.PAPoolStart -Offset $paPoolOffset
            
            $paPoolOffset++

            $sdnExpConfig['Muxes'] += $mux
            $i++
        }
        
        if ($infraconfig.bypassTORForLBDataPath -eq 'true' -and $muxNodeCount -eq 1) {
            # Optimization for faster data path for certain test scenarios where validating BGP route advertisements is not required.
            # HLK tests for example.
            # Setting MUX IP as PA vnic gateway forces traffic from PA to go directly to MUX, bypassing TOR
            # This improves data path performance as TOR is not involved in the data path.
            # Only valid for single MUX scenario
            # As a side effect, any traffic that needs TOR VM/
            # - Internet access from VNET VMs will not work
            # - Host to VIP connectivity will not work
            Write-TraceLog "Initialize-SdnExpressConfig: Bypassing TOR for LB data path"
            $sdnExpConfig.PAGateway = $sdnExpConfig['Muxes'][0]['PAIPAddress']
        }
    }   

    $sdnExpConfig.ManagementRoutes = Get-MsftCorpRoutes

    $gwNodeCount = $sdnConfig.gateway.nodecount
    $i = 0
    $sdnExpConfig['Gateways'] = @()
    if ($sdnConfig.gateway.enabled -eq "true") {
        $poolType = "IPSec"

        if ($sdnConfig.gateway.gre.enabled -eq "true") {
            $gre = Get-GreVipLogicalNetworkConfig
            $sdnExpConfig['GRESubnet'] = $gre.properties.subnets[0].properties.addressPrefix
            $poolType = "All"
        }

        while ($i -lt $gwNodeCount) {
            $gw = @{}
            $gw['ComputerName'] = "$depId-GW$i"
            $mgmtIpOffset = Get-NextManagementIPOffset
            $gw['ManagementIP'] = Get-IPAddressInSubnet -Subnet $mgmtIpPoolStart -Offset $mgmtIpOffset

            $sdnExpConfig['Gateways'] += $gw
            $i++
        }

        $sdnExpConfig['GatewayPoolType'] = $poolType

        if ($gwNodeCount -eq 1) {
            $sdnExpConfig.RedundantCount = 0
        }
    }

    if ($paPoolOffset -lt 10) {
        # Add some space for future scaling
        $paPoolOffset = 10
    }

    $sdnExpConfig.PAPoolStart = Get-IPAddressInSubnet -Subnet $sdnExpConfig.PAPoolStart -Offset $paPoolOffset

    $publicVipNetwork = Get-PublicVipNetworkConfig    
    $sdnExpConfig.PublicVIPSubnet = $publicVipNetwork.properties.subnets[0].properties.addressPrefix

    $privateVipNetwork = Get-PrivateVipNetworkConfig
    $sdnExpConfig.PrivateVIPSubnet = $privateVipNetwork.properties.subnets[0].properties.addressPrefix

    $sdnExpConfig.Routers[0].RouterIPAddress = $paNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.Routers[0].RouterASN = $sdnConfig.mux.peerRouterASN
    $sdnExpConfig.SDNASN = $sdnConfig.mux.asn
    
    Set-TurnKeySdnConfig -sdnConfig $sdnConfig

    return $sdnExpConfig
}   

function Initialize-SdnWorkloadDeploymentPath {
    $workloadConfig = Get-TurnKeySdnWorkloadConfig

    $defaultPath = $workloadConfig.deploymentpath
    $csv = Get-CSV 
    $isCsv = $csv -ne $null

    if ([String]::isnullorempty($defaultPath)) {

        if ($isCsv) {
            $defaultPath = Join-Path $csv "SdnWorkload"
        }
        else {
            # Use a local default path
            $defaultPath = Join-Path $env:SystemDrive "SdnWorkload"
        }      
    }

    $sb = {
        param ($defaultPath)
        if (-not $(Test-Path($defaultPath))) {
            try {
                New-Item $defaultPath -ItemType Directory -Force -ErrorAction Stop | Out-Null
            }
            catch {
                throw "Unable to access deployment path $defaultPath"
            }
        }
    }

    if (-not $isCsv) {
        #check all hosts
        Invoke-CmdOnInfraHosts -scriptBlock $sb -args @($defaultPath)
    }
    else {
        Invoke-Command -ScriptBlock $sb -ArgumentList @($defaultPath)
    }

    $workloadConfig.deploymentpath = $defaultPath
    Set-TurnKeySdnWorkloadConfig -config $workloadConfig
}

function Invoke-CmdOnInfraHosts {
    param(
        [parameter(Mandatory = $true)][ScriptBlock] $scriptBlock,
        [parameter(Mandatory = $false)][System.Array] $args = @()
    )

    $hypervHosts = Get-TurnKeySdnHyperVHosts     

    if ($hypervHosts -eq $null -or $hypervHosts.Count -eq 0) {
        Write-TraceLog "Invoke-CmdOnInfraHosts: No hosts found, skipping."
        return
    }
    
    $cred = Get-TurnKeySdnCred

    $results = @()
    foreach ($h in $hypervHosts) {
        $result = Invoke-ReliableCommand -computerName $h -credential $cred -scriptBlock $scriptBlock -argumentList $args
        $results += $result
    }

    return $results
}

function Get-SdnWorkloadVhd {
    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory = $false)][string] $os = "Linux",
        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory = $false)][string] $osSku = "Mariner",

        [parameter(Mandatory=$false)][bool] $useDefaultWindows = $false
    )

    if ($useDefaultWindows) {

        Write-TraceLog "Get-SdnWorkloadVhd: Using default windows vhd"
        $vhdPath = Get-DeploymentConfig | Select -ExpandProperty vhdPath
        $vhdFile = Get-DeploymentConfig | Select -ExpandProperty vhdFile
        return @($vhdPath, $vhdFile)
    }

    $defaultWorkloadVhd = Join-Path (Get-DefaultWorkloadVhdShare) (Get-DefaultWorkloadVhdFile)
    $msg = "Retry after running. A default vhd can be found at $defaultWorkloadVhd."

    $workloadConfig = Get-TurnKeySdnWorkloadConfig
    $osImageConfig = $workloadConfig.imageconfigs | Where-Object { $_.osType -eq $os -and $_.osSku -eq $osSku } | Select -First 1

    if ([String]::IsNullOrEmpty($osImageConfig.vhdPath) -or [string]::IsNullOrEmpty($osImageConfig.vhdFile)) {

        throw "Vhd location for image $($osImageConfig.osType) - $($osImageConfig.osSku) is not set. $msg"
    }

    $vhd = Join-Path $osImageConfig.vhdPath $osImageConfig.vhdFile

    if (-not $(Test-Path($vhd))) {
        throw "$vhd not found. $msg"
    }

    return @($osImageConfig.vhdPath, $osImageConfig.vhdFile)
}

function Enable-Hosts {
    param (
        [parameter(Mandatory = $true)][array] $Hosts,
        [parameter(Mandatory = $false)][pscredential] $Credential,
        [parameter(Mandatory = $true)][string] $SwitchName,
        [parameter(Mandatory = $true)][string] $MgmtHostNicName,
        [parameter(Mandatory = $true)][string] $ManagementVLANID,
        [parameter(Mandatory = $true)][string] $ManagementSubnet,
        [parameter(Mandatory = $true)][string] $ManagementIPPoolStart,
        [parameter(Mandatory = $true)][string] $ManagementGateway,
        [parameter(Mandatory = $true)][array] $ManagementDNS
    )

    $prefix = $ManagementSubnet.Split("/")[1]
    $hostNameIPMap = @{}
    Disable-IPv6AutoConfig -ComputerName $Hosts -Credential $Credential
    
    $Hosts | Foreach-Object {
        $offset = Get-NextManagementIPOffset
        $hostIP = Get-IPAddressInSubnet -subnet $ManagementIPPoolStart -offset $offset
        $hostNameIPMap[$_] = $hostIP

        Write-TraceLog "Enable-Hosts: Host ($_), with IP ($hostIP)"
        Invoke-ReliableCommand -computername $_ -credential $Credential -scriptBlock {
            param(
                [string]$switchName,
                [string]$vNicName,
                [string]$vlanId,
                [string]$ip,
                [string]$prefix,
                [string]$gateway,
                [array]$dns
            )                
            $nic = Get-VMNetworkAdapter -ManagementOS -Name $vNicName -ErrorAction SilentlyContinue
                
            if ($nic -eq $null) {
                Add-VMNetworkAdapter -ManagementOS -Name $vNicName -SwitchName $switchName
                $nic = Get-VMNetworkAdapter -ManagementOS -Name $vNicName -ErrorAction SilentlyContinue
            }

            $ifIndex = (Get-NetAdapter -name "*$vNicName*").Ifindex

            Set-VMNetworkAdapterVlan -ManagementOS -VMNetworkAdapterName $vNicName -VlanId $vlanId -access | Out-Null
            $currentIP = Get-NetIPAddress -Interfaceindex $ifIndex -IPAddress $ip -ErrorAction SilentlyContinue -Verbose

            if ($currentIP -eq $null) {
                New-NetIPAddress -Interfaceindex $ifIndex -AddressFamily IPv4 -IPAddress $ip -PrefixLength $prefix | Out-Null
            }

            if ($dns -ne $null -and $dns.Count -gt 0) {
                Set-DnsClientServerAddress -ServerAddresses $dns -InterfaceIndex $ifIndex -ErrorAction SilentlyContinue
            }

            # Set Jumbo packet size to atleast 4088 for pnics
            [array]$netadapters = (Get-VMswitch -Name $switchName).NetAdapterInterfaceDescriptions
            $netadapters | ForEach-Object { 
                $jumboSetting = Get-NetAdapterAdvancedProperty -InterfaceDescription $_ -RegistryKeyword "*JumboPacket" -ErrorAction SilentlyContinue
                if ($jumboSetting -eq $null) {
                    return
                }

                $value = $jumboSetting.DisplayValue
                if ($value -lt 1660) {
                    Set-NetAdapterAdvancedProperty -InterfaceDescription $_ -RegistryKeyword "*JumboPacket" -RegistryValue "4088" -Confirm:$false 
                }
            }

            
            # Disable IPV6 auto discovery on mgmt nic
            Set-NetIpInterface -InterfaceIndex $ifIndex -RouterDiscovery Disabled

            # do not enable rsc offload https://microsoft.visualstudio.com/OS/_git/os/pullrequest/4586474
            # default is anyways false
            Set-VMSwitch -Name $switchName -EnableRscOffload $false
            # do not enable software rsc, we had multiple issues with this enabled.
            Set-VMSwitch -Name $switchName -EnableSoftwareRsc $false
        } -ArgumentList @($SwitchName, $MgmtHostNicName, $ManagementVLANID, $hostIP, $prefix, $ManagementGateway, $ManagementDNS)
    }

    Write-TraceLog "Enable-Hosts: Sleeping 5 seconds to allow network to settle"
    Start-Sleep 5
    $isDuplicate = {
        param($vnicName)
        $nic = (Get-NetIPAddress -InterfaceAlias "*$vnicName*") | where-Object { $_.AddressState -eq "Duplicate" }
        if ($nic -eq $null) {
            return $null
        }
        return $nic.IPAddress
    }
    
    $Hosts | Foreach-Object {
        $dupIP = Invoke-ReliableCommand -computername $_ -credential $Credential `
            -scriptBlock $isDuplicate -argumentList $MgmtHostNicName
        if ($dupIP -ne $null) {
            Write-TraceLog "Enable-Hosts: Duplicate IP detected on host $_, IP $dupIP" -Err
            $dupDetected = $true
        }
    }

    if ($dupDetected) {
        Write-TraceLog "Enable-Hosts: Please change randmoizer seed using Set-TurnkeySdnAddressRandmoizerSeed and retry" -Err
        throw "Duplicate IP detected on one or more hosts, please change randmoizer seed using Set-TurnkeySdnAddressRandmoizerSeed and retry"
    }

    $isCluster = Test-IsCluster

    if ($isCluster) {
        Write-TraceLog "Enable-Hosts: Setting network $ManagementSubnet to ClusterAndClient"
        Wait-ForClusterNetwork -computerName $Hosts[0] -credential $Credential -subnet $ManagementSubnet
        Set-ClusterNetworkToClient -computerName $Hosts[0] -credential $Credential -subnet $ManagementSubnet
    }
    
    Write-TraceLog "Enable-Hosts: Disabling link local nics on hosts"
    $Hosts | Foreach-Object { Disable-LinkLocalNics -computerName $_ -credential $Credential }
    Update-CorpNicDnsServerAddress -hosts $Hosts -credential $Credential -dnsServers $ManagementDNS
    Enable-HostStorageNetwork -ComputerName $Hosts -Credential $Credential -SwitchName $SwitchName

    return $hostNameIPMap
}

function New-TurnKeySdnVM {
    param(
        [parameter(Mandatory = $true)][hashtable] $CreateParams,
        [parameter(Mandatory = $false)][string] $VMCustomTag,
        [parameter(Mandatory = $false)][switch] $WaitForVM
    )
    $vmName = $CreateParams.VMName
    $computerName = $CreateParams.ComputerName
    $cred = Get-TurnKeySdnCred

    try {
        New-SDNExpressVM @CreateParams | Out-Null
    }
    finally {
        
        if ([String]::IsNullOrEmpty($VMCustomTag)) {
            $VMCustomTag = "Client=TurnKeySDN"
        }

        Invoke-ReliableCommand -ComputerName $computerName -Credential $cred -ScriptBlock {
            param($VMName, $VMCustomTag)
            $vm = Get-VM -Name $VMName -ErrorAction SilentlyContinue

            if ($vm -ne $null) {
                $vm | Set-VM -Notes $VMCustomTag
            }
        } -ArgumentList @($vmName, $VMCustomTag)
    }
    
    if ($WaitForVM.IsPresent) {
        WaitforComputerToBeReady -ComputerName $vmName -Credential $cred
    }  
}

function New-TurnKeySdnWorkloadVMUsingSdnExpress {

    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory=$false)][string] $os = "Linux",
        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory=$false)][string] $osSku = "Mariner",
        [parameter(Mandatory=$true)][string] $ncNicResourceId,
        [parameter(Mandatory=$true)][string] $ncRestEndpoint,
        [parameter(Mandatory=$false)][string] $hypervHost = $(hostname),
        [parameter(Mandatory=$false)][pscredential] $hypervHostCred = [pscredential]::Empty,
        [parameter(Mandatory=$false)][pscredential] $vmCreds = $false,
        [parameter(Mandatory=$false)][bool] $useDefaultWindows = $false,
        [parameter(Mandatory=$false)][Int64] $memoryInGB,
        [parameter(Mandatory=$false)][int] $coreCount = 2,
        [parameter(Mandatory=$false)][bool] $enableSecureBoot = $false,
        [parameter(Mandatory=$false)][string] $productKey
    )

    Write-TraceLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: Server $hypervHost, os $os, osSku $osSku, ncNicResourceId $ncNicResourceId, ncRestEndpoint $ncRestEndpointclient"

    if(-not $useDefaultWindows) {
        if([string]::IsNullOrEmpty($os) -or [string]::IsNullOrEmpty($osSku)) {
            throw "New-TurnKeySdnWorkloadVMUsingSdnExpress: os and osSku must be specified when forcing the default Vhd"
        }
    }

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"
    $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $ncNicResourceId -PassInnerException -ErrorAction SilentlyContinue

    if ($res -eq $null) {
        throw "Networkinterface $ncNicResourceId not found"
    }

    if ($res.Properties.ProvisioningState -ne "Succeeded") {
        throw "Networkinterface $ncNicResourceId provisioning state is $($res.Properties.ProvisioningState), cannot continue with deployment"
    }

    $mac = $res.Properties.PrivateMacAddress
    $ipAddress = $res.Properties.IpConfigurations[0].Properties.privateIpaddress

    $instanceId = $res.InstanceId.Guid

    if ([String]::IsNullOrEmpty($res.Tags.vmName)) {
        $vmName = $ncNicResourceId
    } else {
        $vmName = $res.Tags.vmName
    }

    $vhdConfig = Get-SdnWorkloadVhd -os $os -osSku $osSku -useDefaultWindows $useDefaultWindows
    $deploymentPath = (Get-DeploymentConfig).vmLocation
    $sdnSwitchName = Get-SdnSwitchName
    $vmLocalCredential = Get-TurnKeySdnWorkloadVmCred
    $vmPath = Join-Path $deploymentPath $vmName

    #Copy-FileToRemote -srcPath $vhdConfig[0] -srcFile $vhdConfig[1] -destMachine $hypervHost -destMachineCred $hypervHostCred -destDirectory $vmPath -ErrorAction Stop

    $vmVhdPath = Join-Path $vmPath $vhdConfig[1]
    $isCluster = Test-IsCluster


    $vmNic=@(
        @{Name="SdnNic_$vmName"; MacAddress=$mac; SwitchName=$sdnSwitchName}
    )

    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: Creating VM $vmName on host $hypervHost"
    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: VM VHD Path is $vmVhdPath"
    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: IP Address is $ipAddress"

    [bool] $isVmCreated = $false

    try {
        if($isCluster) {

            # check if VMs were deployed on any of the hosts in the cluster
            # throws if VM is not found
            $hypervHost = Resolve-HostName -hostName "" -hostCred $hypervHostCred -vmName $vmName -force $true
            $isVmCreated = $true
        } else {

            # sdn express checks if the vm is already present, so let it handle
            $isVmCreated = $false
        }
    } catch {
        Write-TraceLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: VM $vmName not found on any of the hosts in the cluster, creating new VM"
        $isVmCreated = $false
    }

    if(-not $isVmCreated) {

        New-SDNExpressVM -ComputerName $hypervHost `
            -VMLocation $vmPath `
            -VMName $vmName `
            -VHDSrcPath $vhdConfig[0] `
            -VHDName $vhdConfig[1] `
            -SwitchName $sdnSwitchName `
            -CredentialDomain $Env:TEST_DOMAIN  `
            -CredentialUserName $Env:TEST_USERNAME `
            -CredentialPassword $Env:TEST_PASSWORD `
            -JoinDomain "" `
            -LocalAdminPassword $Env:TURNKEY_WORKLOAD_PASSWORD `
            -VMProcessorCount $coreCount `
            -VMMemory $memoryInGB `
            -Nics $vmNic `
            -EnablePreDeploymentNetworkConnectionCheck $false `
            -EnableProcessorCompatibilityForLiveMigration $true `
            -ProductKey $productKey | out-null
    }

    # enable misc settings (add to cluster, guest intergation etc..)
    Invoke-Command -ComputerName $hypervHost -Credential $hypervHostCred -ScriptBlock {
        
        $vm = Get-VM -Name $using:vmName
        $vm | Get-VMIntegrationService | Enable-VMIntegrationService | Out-Null
        $vm | Set-VM -Notes "Client=TurnKeySDN;Role=Workload;IsInfra=$false;Os=Windows"

        if ($using:isCluster) {
            $vm = Get-VM -Name $using:vmName

            $currentRole = get-clustergroup -name $using:vmName -ErrorAction SilentlyContinue
            if($null -ne $currentRole -and $currentRole.State -ne "Online") {

                Remove-ClusterGroup -RemoveResources -Force -Name $using:vmName
            }

            $vm | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null

        }
    }
    Enable-SDNExpressVMPort -ComputerName $hypervHost -VMName $vmName -InstanceId "{$instanceId}" -Credential $hypervHostCred

    $trafficEndpoint = New-TrafficEndpoint -vmName $vmName `
        -hostName $hypervHost `
        -ipAddress $ipAddress `
        -port 5001 `
        -endpointType 0 `
        -vmCredential $vmLocalCredential `
        -hostCredential $hypervHostCred `
        -resolveHostName $false

    return $trafficEndpoint
}



function New-TurnKeySdnWorkloadVM {
    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory = $false)][string] $os = "Linux",
        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory = $false)][string] $osSku = "Mariner",
        [parameter(Mandatory = $true)][string] $ncNicResourceId,
        [parameter(Mandatory = $true)][string] $ncRestEndpoint,
        [parameter(Mandatory = $false)][string] $hypervHost = $(hostname),
        [parameter(Mandatory = $false)][array] $AllComputers = @(),
        [parameter(Mandatory = $false)][pscredential] $hypervHostCred = [pscredential]::Empty
    )

    Write-TraceLog "New-TurnKeySdnVM: Server $hypervHost, os $os, osSku $osSku, ncNicResourceId $ncNicResourceId, ncRestEndpoint $ncRestEndpointclient"

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"
    $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $ncNicResourceId -PassInnerException -ErrorAction SilentlyContinue

    if ($res -eq $null) {
        throw "Networkinterface $ncNicResourceId not found"
    }

    if ($res.Properties.ProvisioningState -ne "Succeeded") {
        throw "Networkinterface $ncNicResourceId provisioning state is $($res.Properties.ProvisioningState), cannot continue with deployment" 
    }

    $mac = $res.Properties.PrivateMacAddress
    $instanceId = $res.InstanceId.Guid

    if ([String]::IsNullOrEmpty($res.Tags.vmName)) {
        $vmName = $ncNicResourceId
    }
    else {
        $vmName = $res.Tags.vmName
    }

    if ($AllComputers.Count -ne 0) {
        # If the VM is already present in another, override the name
        $hostName = Get-VMHostName -Computers $AllComputers -Credential $hypervHostCred -VMName $vmName
        if (-not [String]::IsNullOrEmpty($hostName)) {
            $hypervHost = $hostName
        }
    }

    $vhdConfig = Get-SdnWorkloadVhd -os $os -osSku $osSku
    $deploymentPath = Get-SdnWorkloadDeploymentPath
    $sdnSwitchName = Get-SdnSwitchName
    $vmPath = Join-Path $deploymentPath $vmName

    Copy-FileToRemote -srcPath $vhdConfig[0] -srcFile $vhdConfig[1] -destMachine $hypervHost -destMachineCred $hypervHostCred -destDirectory $vmPath -ErrorAction Stop

    $vmVhdPath = Join-Path $vmPath $vhdConfig[1]
    $isCluster = Test-IsCluster
    Invoke-Command -ComputerName $hypervHost -Credential $hypervHostCred -ScriptBlock {
        param($sdnSwitchName, $vmVhdPath, $vmPath, $vmName, $mac, $os, $isCluster)
        $vm = Get-VM -Name $vmName -ErrorAction SilentlyContinue

        if ($vm -eq $null) {
            New-VM -SwitchName $sdnSwitchName -VHDPath $vmVhdPath -Generation 2 -BootDevice VHD -Path $vmPath -name $vmName -MemoryStartupBytes 512MB `
                -ErrorAction Stop | Out-Null    
        }
        else {
            Stop-Vm -VM $vm -TurnOff -Force
        }

        Set-VMNetworkAdapter -VMName $vmName -StaticMacAddress $mac `
            -ErrorAction Stop

        if ($os -eq "Linux") {
            Set-VMFirmware -EnableSecureBoot On -SecureBootTemplate MicrosoftUEFICertificateAuthority -VMName $vmName `
                -ErrorAction Stop
        }
        Get-VM -Name $vmName | Set-VM -Notes "Client=TurnKeySDN;Role=Workload;IsInfra=$false;Os=$os"
        Start-VM -VMName $vmName

        if ($isCluster) {
            Get-VM -Name $vmName | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null
        }
        
    } -ArgumentList @($sdnSwitchName, $vmVhdPath, $vmPath, $vmName, $mac, $os, $isCluster) -ErrorAction Stop

    Enable-SDNExpressVMPort -ComputerName $hypervHost -VMName $vmName -InstanceId "{$instanceId}" -Credential $hypervHostCred
}

function Set-VMVLANID {
    param(
        [Parameter(Mandatory = $true)][String]$VMName,
        [Parameter(Mandatory = $true)][pscredential]$Credential,
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][String]$NicName,
        [Parameter(Mandatory = $true)][int]$VLanID
    )

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock {
        param(
            [Parameter(Mandatory = $true)][String]$VMName,
            [Parameter(Mandatory = $true)][String]$NicName,
            [Parameter(Mandatory = $true)][int]$VLanID
        )

        [int]$currentVlan = (Get-VMNetworkAdapterVLan -VMName $VMName -VMNetworkAdapterName $NicName).AccessVlanId

        if ($currentVlan -eq $VLanID) {
            return
        }

        Set-VMNetworkAdapterVlan -Access -VlanId $VLanID -VMName $VMName -VMNetworkAdapterName $NicName
    } -ArgumentList @($VMName, $NicName, $VLanID)
}
function New-RouterVM {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $MgMtIP,
        [Parameter(Mandatory = $true)]
        $MgmtNicMac,  
        [Parameter(Mandatory = $true)]
        $PAIP,
        [Parameter(Mandatory = $true)]
        $PANicMac,        
        [Parameter(Mandatory = $true)]
        $CorpNicMac,        
        [switch]
        $Force
    )

    $depConfig = Get-DeploymentConfig
    $cred = Get-TurnKeySdnCred

    if ($cred -eq [pscredential]::Empty) {
        throw "New-RouterVM: Test Credential not found, please set credentials using Set-TurnKeySdnCredentials and retry"
    }

    Write-TraceLog "New-RouterVM: Creating VM $vmName"  

    $hyperVHosts = $depConfig.hyperVHosts
    $vmHost = Get-VMHostName -Computers $hyperVHosts -Credential $cred -VMName $vmName 
    
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtVlanId = $mgmtNetConfig.properties.subnets[0].properties.VLANID
    
    $paVlanId = Get-HnvPaNetworkConfig
    $paVlanId = $paVlanId.properties.subnets[0].properties.vlanID
    
    $managementNicName = "Management"
    $paNicName = "HNVPA"
    
    if ($vmHost -ne $null) {
       
        if ($Force.IsPresent) {
            Write-TraceLog "New-RouterVM: Recreating the VM"
            Remove-DomainJoinedVMs -Tag "Role=Router"            
        }
        else {
            if ($(Test-RouterVM -VMName $vmName)) {
                Write-TraceLog "New-RouterVM: VM already exists, skipping creation"
                # Reset VLAN if it changes between deployments.
                Set-VMVLANID -VMName $vmName -Credential $cred -ComputerName $vmHost -NicName $managementNicName -VLanID $mgmtVlanId
                Set-VMVLANID -VMName $vmName -Credential $cred -ComputerName $vmHost -NicName $paNicName -VLanID $paVlanId
                return                
            }

            Remove-DomainJoinedVMs -Tag "Role=Router"    
        }
    }

    $mem = Get-InfraVMMemory
    $proc = Get-InfraVMProcessorCount

    $createparams = @{
        'ComputerName'        = $(hostname);
        'VMLocation'          = $depConfig.vmLocation;
        'VMName'              = $vmName;
        'VHDSrcPath'          = $depConfig.vhdPath;
        'VHDName'             = $depConfig.vhdFile;
        'VMMemory'            = $mem;
        'VMProcessorCount'    = $proc;
        'Nics'                = @();
        'CredentialDomain'    = $cred.username.Split("\")[0];
        'CredentialUserName'  = $cred.username.Split("\")[1];
        'CredentialPassword'  = $cred.GetNetworkCredential().password;
        'JoinDomain'          = $depConfig.domainName;
        'LocalAdminPassword'  = $cred.GetNetworkCredential().password;
        'DomainAdminDomain'   = $cred.username.Split("\")[0];
        'DomainAdminUserName' = $cred.username.Split("\")[1];
        'SwitchName'          = $depConfig.sdnSwitchName;
        'ProductKey'          = $depConfig.productKey;  
        'DisableIPv6DHCP'     = $true;
    }
  
    $dnsServers = $mgmtNetConfig.properties.subnets[0].properties.dnsServers

    $createparams.Nics = @(
        @{Name = $managementNicName; MacAddress = $MgmtNicMac; VLANID = $mgmtVlanId; IPAddress = $MgMtIP; SwitchName = $depConfig.sdnSwitchName; DNS = $dnsServers; IsMuxPA = $true },
        @{Name = $paNicName; MacAddress = $PANicMac; VLANID = $paVlanId; IPAddress = $PAIP; SwitchName = $depConfig.sdnSwitchName; DNS = $dnsServers; IsMuxPA = $true }
    ) 

    $corpNic = @{Name = "Corp"; MacAddress = $CorpNicMac; SwitchName = $depConfig.internetSwitchName; IsMuxPA = $true;}
    $torVMConfig = $depConfig.torVMConfig

    if (-not [String]::IsNullOrWhiteSpace($torVMConfig.IPAddress)) {
        $corpNic["IPAddress"] = "$($torVMConfig.IPAddress)/$($torVMConfig.PrefixLength)"
        $corpNic["Gateway"] = $torVMConfig.DefaultGateway
        if ($null -ne $torVMConfig.DNSServers -and $torVMConfig.DNSServers.Count -gt 0) {
            $corpNic["DNS"] = $torVMConfig.DNSServers
        } else {
            $corpNic["DNS"] = $dnsServers
        }
        $corpNic["VLANID"] = [int]$torVMConfig.VLANID
    }
    
    $createparams.Nics += $corpNic

    $createparams.Roles = @("RemoteAccess", "RemoteAccessServer", "RemoteAccessMgmtTools", "RemoteAccessPowerShell", "RasRoutingProtocols", "Web-Application-Proxy")

    New-TurnKeySdnVM -CreateParams $createparams -VMCustomTag "Client=TurnKeySDN;Role=Router;IsInfra=$true" -WaitForVM   

    Start-Sleep 10

    #Invoke-ActivateWindows -ComputerName $vmName -Credential $cred
    Enable-DefaultFirewallRules -ComputerName $vmName -Credential $cred
    Disable-IPv6AutoConfig -ComputerName $vmName -Credential $cred
}

function Test-RouterVM {
    param(
        [Parameter(Mandatory = $true)]$VMName
    )

    $cred = Get-TurnKeySdnCred
    try {
        Invoke-ReliableCommand -ComputerName $VMName -Credential $cred -ScriptBlock {
            
        }

        #TODO Add more test
        Write-TraceLog "Test-RouterVM: Health check succeeded"
        return $true
    }
    catch {
        Write-TraceLog "Test-RouterVM: Health check failed for VM $VMName, error $_"
        return $false
    }
}
function Enable-NatOnRouter {
    param(
        [Parameter(Mandatory = $true)]$VMName,
        [Parameter(Mandatory = $true)]$MgmtNicMac,
        [Parameter(Mandatory = $true)]$CorpNicMac,
        [Parameter(Mandatory = $true)]$PANicMac
    )

    $cred = Get-TurnKeySdnCred

    Invoke-ReliableCommand -ComputerName $VMName -Credential $cred -ScriptBlock {
        param(
            [Parameter(Mandatory = $true)]$MgmtNicMac,
            [Parameter(Mandatory = $true)]$CorpNicMac,
            [Parameter(Mandatory = $true)]$PANicMac
        )
        $external = "External"
        $ext = (Get-NetAdapter -Name $External -ErrorAction SilentlyContinue)
        
        if ($ext -eq $null) {
            $corpNic = Get-NetAdapter | Where-Object { $_.MacAddress -eq $CorpNicMac }
            Rename-NetAdapter -Name $corpNic.Name -NewName $external
        }

        $mgmt = "Manangement"
        $mgmtAdapter = (Get-NetAdapter -Name $mgmt -ErrorAction SilentlyContinue)

        if ($mgmtAdapter -eq $null) {
            $mgmtAdapter = Get-NetAdapter | Where-Object { $_.MacAddress -eq $MgmtNicMac }
            Rename-NetAdapter -Name $mgmtAdapter.Name -NewName $mgmt
        }

        $hnvpa = "HNVPA"
        $hnvpaAdapter = (Get-NetAdapter -Name $hnvpa -ErrorAction SilentlyContinue)

        if ($hnvpaAdapter -eq $null) {
            $hnvpaAdapter = Get-NetAdapter | Where-Object { $_.MacAddress -eq $PANicMac }
            Rename-NetAdapter -Name $hnvpaAdapter.Name -NewName $hnvpa
        }
        
        # enable jumbo packet
        Set-NetAdapterAdvancedProperty -Name $hnvpa -RegistryKeyword "*JumboPacket" -RegistryValue "4088" -Confirm:$false
        
        #Setup NAT for internet Access
        #Install-WindowsFeature -name RemoteAccess -IncludeAllSubFeature -IncludeManagementTools
        Install-RemoteAccess -VpnType RoutingOnly

        cmd.exe /c "netsh routing ip nat uninstall" | Out-Null
        cmd.exe /c "netsh routing ip nat install" | Out-Null
        cmd.exe /c "netsh routing ip nat set global tcptimeoutmins=1 udptimeoutmins=0 loglevel=ERROR" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $external" | Out-Null
        cmd.exe /c "netsh routing ip nat set interface $external mode=full" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $mgmt" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $hnvpa" | Out-Null

        #Disable Windows Defender
        Set-MpPreference -DisableRealtimeMonitoring $true
        Set-MpPreference -DisableBehaviorMonitoring $true
    } -ArgumentList @($MgmtNicMac, $CorpNicMac, $PANicMac)
}

function Enable-BgpOnRouter {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $bgpLocalIP,
        [Parameter(Mandatory = $true)]
        $LocalASN
    )

    $cred = Get-TurnKeySdnCred
    Invoke-ReliableCommand -ComputerName $vmName -Credential $cred -ScriptBlock {

        param(
            [Parameter(Mandatory = $true)]
            $bgpLocalIP,
            [Parameter(Mandatory = $true)]
            $LocalASN
        )
        # Add-windowsfeature -name RemoteAccess -IncludeAllSubFeature -IncludeManagementTools

        Install-RemoteAccess -RoleType RoutingOnly
       
        try {
            $router = get-bgprouter -ErrorAction SilentlyContinue
        }
        catch {

        }

        if ($router -ne $null) {
            $router | Remove-BgpRouter -force
        }

        Add-BgpRouter -BgpIdentifier $bgpLocalIP -LocalASN $LocalASN

    } -ArgumentList @($bgpLocalIP, $LocalASN)
}

function Add-MuxPeerToBgp {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $bgpPeerName,
        [Parameter(Mandatory = $true)]
        $bgpLocalIP,
        [Parameter(Mandatory = $true)]
        $bgpPeerIP,
        [Parameter(Mandatory = $true)]
        $LocalASN,
        [Parameter(Mandatory = $true)]
        $PeerASN
    )

    $cred = Get-TurnKeySdnCred
    Invoke-ReliableCommand -ComputerName $vmName -Credential $cred -ScriptBlock {

        param(
            [Parameter(Mandatory = $true)]
            $bgpLocalIP,
            [Parameter(Mandatory = $true)]
            $bgpPeerName,
            [Parameter(Mandatory = $true)]
            $bgpPeerIP,
            [Parameter(Mandatory = $true)]
            $LocalASN,
            [Parameter(Mandatory = $true)]
            $PeerASN
        )
       
        Get-BgpRouter -ErrorAction Stop      
        Add-BgpPeer -PeerName $bgpPeerName -PeerIpAddress $bgpPeerIP -PeerASN $PeerASN -LocalIpAddress $bgpLocalIP

    } -ArgumentList @($bgpLocalIP, $bgpPeerName, $bgpPeerIP, $LocalASN, $PeerASN)
}


function Set-StaticRoute {
    param(
        [array] $Computer,
        [pscredential] $Credential = [pscredential]::Empty,
        [string] $SwitchName,
        [string] $NicName,
        [string] $DestinationPrefix,
        [string] $NextHop
    )

    Write-TraceLog "Set-StaticRoute: Computer $Computer Setting route for $DestinationPrefix to $NextHop, NicName $NicName"
    $Computer | Foreach-Object {
        Invoke-ReliableCommand -computerName $_ -Credential $Credential -ScriptBlock {
            param(
                [string] $SwitchName,
                [string] $NicName,
                [string] $DestinationPrefix,
                [string] $NextHop
            )

            if (-not [String]::isnullorempty($SwitchName)) {
                $hostVNic = Get-VMNetworkAdapter -ManagementOS -Name $NicName -switchName $SwitchName -ErrorAction Stop
                $adapter = Get-NetAdapter | Where-Object { $($_.MacAddress -replace "-", "") -eq $($hostVNic.MacAddress) }
            }
            else {
                $adapter = Get-NetAdapter -Name $NicName -ErrorAction Stop
            }

            $index = $adapter.ifIndex

            $route = Get-NetRoute -InterfaceIndex $index -NextHop $NextHop -DestinationPrefix $DestinationPrefix -ErrorAction SilentlyContinue

            if ($route -ne $null) {
                return
            }

            try {
                New-NetRoute -DestinationPrefix $DestinationPrefix -InterfaceIndex $index -NextHop $NextHop -ErrorAction Stop | Out-Null
            } catch {
                # workaround, for some reason, when a route is removed and added, get-netroute check above occasionally doesnt catch it
                # so we ignore the exception if the route already exists

                if ($_.Exception.Message.Trim() -ine "Instance MSFT_NetRoute already exists") {
                    throw
                }
            }
        } -ArgumentList @($SwitchName, $NicName, $DestinationPrefix, $NextHop)
    }
}

function Set-SDNVMNotes {

    $cred = Get-TurnKeySdnCred
    $hyperVHosts = Get-TurnKeySdnHyperVHosts
    $depId = Get-TurnKeySdnDeploymentId

    Write-TraceLog "Set-SDNVMNotes: Setting custom tags on SDN VMs"

    foreach ($h in $hyperVHosts) {
        Invoke-ReliableCommand -ComputerName $h -Credential $cred -ScriptBlock {
            param($depId)
            $ncPrefix = "$depId-NC*"
            
            $ncVMs = Get-VM -Name $ncPrefix -ErrorAction SilentlyContinue
            if ($ncVMs -ne $null) {
                $ncVMs | Set-VM -Notes "Client=TurnKeySDN;Role=NetworkController;IsSDN=$true"
            }

            $muxPrefix = "$depId-MUX*"
            $muxVMs = Get-VM -Name $muxPrefix -ErrorAction SilentlyContinue
            if ($muxVMs -ne $null) {
                $muxVMs | Set-VM -Notes "Client=TurnKeySDN;Role=MUX;IsSDN=$true"
            }

            $gwPrefix = "$depId-GW*"
            $gwVMs = Get-VM -Name $gwPrefix -ErrorAction SilentlyContinue
            if ($gwVMs -ne $null) {
                $gwVMs | Set-VM -Notes "Client=TurnKeySDN;Role=Gateway;IsSDN=$true"
            }

        } -ArgumentList $($depId)
    }
}

function Invoke-SDNExpress {
    param(
        [Parameter(Mandatory = $true)]$sdnExpConfig
    )

    if ($sdnExpConfig -eq $null) {
        throw "Please load the config and retry"
    }

    $sdnExpress = Get-SdnExpressScript
    $path = Resolve-Path $sdnExpress
    $parent = Split-Path $(Resolve-Path $sdnExpress)
    $maxRetry = [int]$env:SDNEXPRESS_RETRY_COUNT
    if ($maxRetry -eq 0) {
        $maxRetry = 1
    }

    $retryCount = 0
    while ($retryCount -lt $maxRetry) {
        Push-Location $parent
        try {
            & $sdnExpress -ConfigurationData $sdnExpConfig
            break
        }
        catch {
            Write-TraceLog "Invoke-SDNExpress: Sdnexpress failed, error $_"
            $retryCount++

            if ($retryCount -eq $maxRetry) {
                throw
            }

            Write-TraceLog "Invoke-SDNExpress: Sdnexpress retrying in 60seconds, RetryCount $retryCount"
            Start-Sleep 60
        }
        finally {
            Pop-Location
        }
    }
}

function Set-PostInstallConfig {
    param(
        [Parameter(Mandatory = $true)]$sdnExpConfig,
        [Parameter(Mandatory = $true)]$hostNameIPMap        
    )

    if ($sdnExpConfig.UseFCNC) {
        return
    } 

    $cred = Get-TurnKeySdnCred

    $sdnExpConfig['NCs'] | ForEach-Object {
        Write-TraceLog "Set-PostInstallConfig: [SLBMWorkaround] Updating host file on NC VM $_" -Warning
        # Froce SLBM to use the expected IP of the host. SLBM breaks if host IP changes.
        Invoke-ReliableCommand -ComputerName $_['ComputerName'] -credential $cred -ScriptBlock {

            param($hostIPs)

            function UpdateHostsFile($name, $ip, $skipResolve) {
                if (-not $skipResolve) {
                    $addr = (Resolve-DnsName $name -ErrorAction SilentlyContinue).IPAddress | `
                        Select-Object -First 1
                }

                if ($addr -ne $ip) {
                    Add-Content C:\Windows\System32\drivers\etc\hosts "$ip $name" | out-null
                    return $true
                }

                return $false
            }

            $hosts = $hostIPs.Keys
            $hosts | Foreach-Object {
                $changed = $false
                $name = $_
                $ip = $hostIPs[$name]
                $changed = UpdateHostsFile -name $name -ip $ip -skipResolve $false

                if ($changed -and (-not $name.Contains("."))) {
                    $hostfqdn = "$name.$($env:USERDNSDOMAIN)"
                    UpdateHostsFile -name $hostfqdn -ip $ip -skipResolve $true |  out-null            
                }

                if ($changed) {
                    Stop-Process -name SDNSLBM -force -ErrorAction SilentlyContinue
                }
            }

        } -ArgumentList $hostNameIPMap
    }
}

function New-DataCenterGateway {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $IPAddress,
        [Parameter(Mandatory = $true)]
        $MacAddress,
        [Parameter(Mandatory = $true)]
        $VlanID,
        [Parameter(Mandatory = $true)]
        $DefaultGateway,
        [switch]
        $Force
    )

    Write-TraceLog "New-DataCenterGateway: Creating VM $vmName"

    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts
    $cred = Get-TurnKeySdnCred

    $vmHost = Get-VMHostName -Computers $hyperVHosts -Credential $cred -VMName $vmName 

    if ($vmHost -ne $null) {
        if ($Force.IsPresent) {
            Write-TraceLog "New-DataCenterGateway: Recreating the VM"
            Invoke-Command -ComputerName $vmHost -Credential $cred -ScriptBlock {
                param($vmName)
                $vm = Get-VM -Name $vmName -ErrorAction SilentlyContinue
                $vm | Stop-VM -TurnOff -ErrorAction SilentlyContinue
                $vm | Remove-VM -Force
            } -ArgumentList $vmName
        }
        else {
            Write-TraceLog "New-DataCenterGateway: NOOP"
            return
        }
    }

    $mem = Get-InfraVMMemory
    $proc = Get-InfraVMProcessorCount
    $createparams = @{
        'ComputerName'        = $(hostname);
        'VMLocation'          = $depConfig.vmLocation;
        'VMName'              = $vmName;
        'VHDSrcPath'          = $depConfig.vhdPath;
        'VHDName'             = $depConfig.vhdFile;
        'VMMemory'            = $mem;
        'VMProcessorCount'    = $proc;
        'Nics'                = @();
        'CredentialDomain'    = $cred.username.Split("\")[0];
        'CredentialUserName'  = $cred.username.Split("\")[1];
        'CredentialPassword'  = $cred.GetNetworkCredential().password;
        'JoinDomain'          = $depConfig.domainName;
        'LocalAdminPassword'  = $cred.GetNetworkCredential().password;
        'DomainAdminDomain'   = $cred.username.Split("\")[0];
        'DomainAdminUserName' = $cred.username.Split("\")[1];
        'SwitchName'          = $depConfig.sdnSwitchName;
        'DisableIPv6DHCP'     = $true;
    }

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $dnsServers = $mgmtNetConfig.properties.subnets[0].properties.dnsServers

    $createparams.Nics = @(
        @{Name = "DCNET"; MacAddress = $MacAddress; VLANID = $VlanID; IPAddress = $IPAddress; SwitchName = $depConfig.sdnSwitchName; Gateway = $DefaultGateway; DNS = $dnsServers; IsMuxPA = $false }
    ) 

    $createparams.Roles = @("RemoteAccess", "RemoteAccessServer", "RemoteAccessMgmtTools", "RemoteAccessPowerShell", "RasRoutingProtocols", "Web-Application-Proxy")
    New-TurnKeySdnVM -CreateParams $createparams -VMCustomTag "Client=TurnKeySDN;Role=DataCenterGateway;IsInfra=$false"
}

function Test-DeploymentMachines {

    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts
    
    if ($hyperVHosts -eq $null -or $hyperVHosts.Count -eq 0) {
        $hyperVHosts = @($(hostname))
    }
    else {
        $curHost = hostname
        $isPresent = ($hyperVHosts | Where-Object { $_.StartsWith($curHost) }) -ne $null

        if (-not $isPresent) {
            $err = "Current host($(hostname)) is not part of the deployment machines, deployment machines are $hyperVHosts"
            $err += "`nSet deployment using Set-TurnKeySdnDeploymentMachines or update $(Get-DeploymentFile)"
            throw $err
        }
    }
}
function Test-CommonConfig {

    if (-not (Get-WmiObject -Class Win32_ComputerSystem).PartOfdomain) {
        throw "Only domain joined setup is supported currently, please join the machine to a domain and retry."
    }

    try {
        $depConfig = Get-DeploymentConfig
    }
    catch {
        Write-TraceLog "Failed to fetch deployment config, please run Initialize-TurnKeySdnDeployment and retry" -Err
        throw
    }

    if ([String]::IsNullOrEmpty($depConfig.domainName)) {
        throw "Invalid domain name."
    }

    $cred = Get-TurnKeySdnCred
    if ($cred -eq [pscredential]::Empty) {
        throw "Unable to read test credential for $($env:USERNAME). Please reset using Set-TurnKeySdnCredentials and retry. Error $_"
    }

    $sdnexpress = Get-SdnExpressModule

    if (-not $(Test-Path $sdnexpress -ErrorAction Stop)) {
        throw "$sdnexpress not found, Please run Set-SdnExpressPath and retry"
    }

    $sdnexpress = Get-SdnExpressScript
    if (-not $(Test-Path $sdnexpress -ErrorAction Stop)) {
        throw "$sdnexpress not found, Please run Set-SdnExpressPath and retry"
    }

    

    if ([String]::IsNullOrEmpty($depConfig.sdnSwitchName)) {
        throw "SDN Switch $($depConfig.sdnSwitchName) not configured"
    } 

    if ([String]::IsNullOrEmpty($depConfig.internetSwitchName) -and $depConfig.useRRASRouter) {
        throw "Internet Switch $($depConfig.internetSwitchName) not configured"
    } 

    Test-DeploymentMachines

    $cred = Get-TurnKeySdnCred 
    $hyperVHosts = $depConfig.hyperVHosts

    Test-VMSwitchExist -computerName $hyperVHosts -switchName $depConfig.sdnSwitchName -credential $cred

    if ($depConfig.useRRASRouter -and $($depConfig.sdnSwitchName -ne $depConfig.internetSwitchName)) {
        Test-VMSwitchExist -computerName $hyperVHosts -switchName $depConfig.internetSwitchName -credential $cred
    }
}

function Test-NetworkControllerPackage {

    $sdnConfig = Get-SDNConfig

    if ($sdnConfig.networkController.runtime -ne "FC") {
        return
    }

    if (-not (Test-Path $sdnConfig.networkController.FC.PowershellModuleRootPath)) {
        Write-TraceLog "Please run Install-WindowsFeature -name RSAT-NetworkController and retry" -Warning
        Write-TraceLog "To change package loction or update runtime please run Set-TurnKeySdnNCConfig" -Warning
        
        throw "Networkcontroller PowershellModuleRootPath not found. Configured path is $($sdnConfig.networkController.FC.PowershellModuleRootPath)"
    }

    if (-not (Test-Path $sdnConfig.networkController.FC.PackageLocation)) {
        Write-TraceLog "Please run Install-WindowsFeature -name NetworkController and retry" -Warning
        Write-TraceLog "To change package loction or update runtime please run Set-TurnKeySdnNCConfig" -Warning
        throw "Networkcontroller PackageLocation not found. Configured path is $($sdnConfig.networkController.FC.PackageLocation)"
    }
}

function Test-FreeSpace {
    $depConfig = Get-DeploymentConfig

    if (Test-IsCsvPath -path $depConfig.vmLocation) {
        # For csv skip free space check for now.
        return
    }

    $rpath = Resolve-Path $depConfig.vmLocation

    $scricptBlock = {
        param($rpath)
        $freeGb = $rpath.Drive.Free / (1024 * 1024 * 1024)
        $result = @{}
        $result[$(hostname)] = $freeGb
        return $result
    }

    $results = Invoke-CmdOnInfraHosts -scriptBlock $scricptBlock -args @($rpath)
    $fail = $false
    foreach ($r in $results) {
        $freeSpace = $r[$r.Keys[0]]
        Write-TraceLog "Test-FreeSpace: HypervHost $($r.Keys[0]), Drive $rpath, FreeSpace $freeSpace"
        if ($freeSpace -lt 50) {
            Write-TraceLog "Test-FreeSpace: HypervHost $($r.Keys[0]), has < 50GB in drive $($rpath), FreeSpace $freeSpace" -Warning
            $fail = $true
        }        
    }

    if ($fail) {
        Write-TraceLog "Test-FreeSpace: Test failed. Please free up space or change deployment path. Current deployment path is $($depConfig.vmLocation)" -Warning
        Write-TraceLog "Test-FreeSpace: To change the deployment path, please run Set-TurnKeySdnDeploymentPath and retry" -Warning
        
        throw "All hosts machines should have atleast 50GB free space."
    }
}

function Test-Vhd {
    $depConfig = Get-DeploymentConfig  

    if ([String]::IsNullOrEmpty($depConfig.vhdPath) -or [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        Write-TraceLog "Please copy a valid vhd and run Set-TurnKeySdnDeploymentVhd, Eg; Set-TurnKeySdnDeploymentVhd c:\vhd -vhdFile sdnexpress.vhdx" -Warning
        throw "Deployment vhd path is null Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }

    if ([String]::IsNullOrEmpty($depConfig.vhdPath) -or [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        Write-TraceLog "Please copy a valid vhd and run Set-TurnKeySdnDeploymentVhd, Eg; Set-TurnKeySdnDeploymentVhd c:\vhd -vhdFile sdnexpress.vhdx" -Warning
        throw "Deployment vhdFile is null. Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }

    $vhd = Join-Path $depConfig.vhdPath $depConfig.vhdFile

    if (-not $(Test-Path $vhd -ErrorAction Stop)) {
        throw "Deployment vhd ($vhd) not found. Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }
}
function Test-DeploymentConfig {       
    Test-CommonConfig
    Test-FreeSpace
    Test-Vhd
    Test-NetworkControllerPackage
}

function Test-WorkloadConfig {       
    param(
        [parameter(Mandatory = $false)][string]$workLoadConfig = [String]::Empty
    )

    Write-TraceLog "Test-WorkloadConfig: Testing workload config"
    
    Test-CommonConfig

    $restoreReplayScript = Get-RestoreReplayScript
    if (-not $(Test-Path $restoreReplayScript -ErrorAction Stop)) {
        throw "Restore replay script not found at $restoreReplayScript"
    }

    if ([String]::IsNullOrEmpty($workLoadConfig)) {
        $workLoadConfig = Get-WorkloadConfigPath
    }

    if (-not $(Test-Path $workLoadConfig -ErrorAction Stop)) {
        throw "Worloadconfig file not found at $workLoadConfig"
    }

    $jsonPath = Join-Path $workLoadConfig "jsons"

    
    if (-not $(Test-Path $jsonPath -ErrorAction Stop)) {
        throw "Jsons folder not found under $workLoadConfig"
    }

    $restName = Get-TurnKeySdnRestEndpoint

    if ([String]::IsNullOrEmpty($restName)) {
        throw "NC RestName is empty, run 'Install-TurnKeySdn' to configure SDN. If SDN is already configured, set rest name using Set-TurnKeySdnRestName"
    }

    Get-SdnWorkloadVhd -os Linux -osSku Mariner | Out-Null
}

function Test-DeploymentLogicalNetworkConfig {
    param(
        [parameter(Mandatory = $true)][PSCustomObject]$Network
    )

    if ($Network.Properties.subnets -eq $null -or $Network.Properties.subnets.Count -eq 0) {
        throw "LogicalNetwork has no subnets"
    }

    if ($Network.Properties.subnets[0].properties.addressPrefix -eq $null) {
        throw "LogicalNetwork has no addressPrefix"
    }

    if ($Network.Properties.subnets[0].properties.ipPools -eq $null -or $Network.Properties.subnets[0].properties.ipPools.Count -eq 0) {
        throw "LogicalNetwork has no ipPools"
    }
}

function Test-TorVMConfig {
    param(
        [parameter(Mandatory = $false)][PSCustomObject]$Config
    )

    if ($null -eq $Config) {
        return
    }
    
    if ([String]::IsNullOrWhiteSpace($Config.IPAddress)) {
        return 
    }

    if (-not ($Config.IPAddress -as [ipaddress] -as [bool])) {
        throw "A valid IP address is required"
    }

    $prefixLen = [int]$Config.PrefixLength
    if ($prefixLen -lt 1 -or $prefixLen -gt 32) {
        throw "A valid prefix length is required"
    }

    if (-not ($Config.DefaultGateway -as [ipaddress] -as [bool])) {
        throw "Gateway is required"
    }

    if ($Config.DNSServers.Count -eq 0) {
        throw "DNS is required"
    }

    foreach($dns in $Config.DNSServers) {
        if (-not ($dns -as [ipaddress] -as [bool])) {
            throw "A valid DNS server is required"
        }
    }

    if ($Config.VlanId -lt 0 -or $Config.VlanId -gt 4094) {
        throw "A valid VLAN ID is required"
    }
}

function Get-RouterVMName {
    (Get-TorId) + (Get-RouterVMSuffix)
}


function Enable-LogicalNetworkDefaultGateway {
    param(
        [Parameter(Mandatory = $true)]
        [string]$LogicalNetworkResourceId
    )

    <#
    .SYNOPSIS
        Adds the default gw of the logical network to the nat vm
    .PARAMETER LogicalNetworkResourceId
        Name of the logical network
    #>

    $routerVM = Get-RouterVMName
    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Adding default gateway of logical network $LogicalNetworkResourceId on $routerVM"

    $cred = Get-TurnKeySdnCred
    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts

    $vmHost = Get-VMHostName -Computers $hyperVHosts -Credential $cred -VMName $routerVM 

    if ($vmHost -eq $null) {
        throw "Nat VM $routerVM not found on any hosts"
    }

    $restName = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restName"

    $ncLNET = Get-NetworkControllerLogicalNetwork -ConnectionUri $restUrl -ResourceId $LogicalNetworkResourceId -ErrorAction SilentlyContinue

    if ($ncLNET -eq $null) {
        throw "LogicalNetwork $LogicalNetworkResourceId not found"
    }
    
    $nicsToAdd = @()
    $ncLNET.Properties.Subnets | ForEach-Object { 
        $subnetId = $_.ResourceId
        $ip = $_.Properties.DefaultGateways | Select -First 1 
        $prefixLen = $_.Properties.AddressPrefix.Split("/") | Select -Last 1
        $addressPrefix = $_.Properties.AddressPrefix
        $vlan = $_.Properties.VlanID
        $nicProperties = New-Object Microsoft.Windows.NetworkController.NetworkInterfaceProperties
        $nicProperties.privateMacAllocationMethod = "Dynamic"
        $nicProperties.IPConfigurations = @()
        $nicProperties.IPConfigurations += New-Object Microsoft.Windows.NetworkController.NetworkInterfaceIpConfiguration
        $nicProperties.IPConfigurations[0].ResourceId = $subnetId
        $nicProperties.IPConfigurations[0].Properties = New-Object Microsoft.Windows.NetworkController.NetworkInterfaceIpConfigurationProperties
        $nicProperties.IPConfigurations[0].Properties.Subnet = New-Object Microsoft.Windows.NetworkController.Subnet
        $nicProperties.IPConfigurations[0].Properties.Subnet.ResourceRef = $_.ResourceRef
        # Unmanaged, nic is not managed by NC, only for MAC address
        $nicProperties.IPConfigurations[0].Properties.PrivateIPAllocationMethod = "Unmanaged"
        $nicProperties.IPConfigurations[0].Properties.PrivateIPAddress = $ip
        $defaultGwNic = New-NetworkcontrollerNetworkinterface -ConnectionUri $restUrl -ResourceId $subnetId -Properties $nicProperties -Force -PassInnerException
        $defaultGwNic = Get-NetworkcontrollerNetworkinterface -ConnectionUri $restUrl -ResourceId $subnetId
        $mac = $defaultGwNic.Properties.PrivateMacAddress
        Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Lnet Subnet $subnetId, IP $ip, Mac $mac, PrefixLen $prefixLen, Vlan $vlan"
        $nicsToAdd += @{Name = $subnetId; MacAddress = $mac; IPAddress = $ip; PrefixLen = $prefixLen; Vlan = $vlan; AddressPrefix = $addressPrefix }
    }

    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM)
        $vm = Get-VM $routerVM
        if ($vm.Generation -lt 2) {
            Stop-VM -VMName $routerVM -Force
        }        
    } -ArgumentList @($routerVM)
    
    $sdnSwitch = Get-SdnSwitchName
    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Adding default gateway netadapter to nat vm $routerVM on switch $sdnSwitch"
    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM, $nicsToAdd, $switchName)

        foreach ($nic in $nicsToAdd) {
            Get-VMNetworkAdapter -VMName $routerVM -Name $nic.Name -ErrorAction SilentlyContinue | Remove-VMNetworkAdapter -ErrorAction SilentlyContinue -Confirm:$false
            Add-VMNetworkAdapter -VMName $routerVM -SwitchName $switchName -StaticMacAddress $nic.MacAddress -ErrorAction Stop -Name $nic.Name
            Set-VMNetworkAdapterVLan -VMName $routerVM -VMNetworkAdapterName $nic.Name -Access -VlanId $nic.Vlan -ErrorAction Stop
        }
    } -ArgumentList @($routerVM, $nicsToAdd, $sdnSwitch)

    foreach ($nic in $nicsToAdd) {
        Enable-SDNExpressVMPort -ComputerName $vmHost -VMName $routerVM -VMNetworkAdapterName $nic.Name -Credential $cred
    }

    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM)
        Start-VM -VMName $routerVM
    } -ArgumentList @($routerVM)

    WaitforComputerToBeReady -ComputerName $routerVM -Credential $cred

    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: setting ip address for default gw nics on nat vm $routerVM"
    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        param($nicsToAdd)
        foreach ($nic in $nicsToAdd) {
            $netAdapter = Get-NetAdapter | Where-Object { $($_.MacAddress.split("-") -join "") -eq $nic.MacAddress }
            if ($netAdapter -eq $null) {
                throw "Unable to find nic with mac $($nic.MacAddress)"
            }

            if ($(Get-NetAdapter -Name $nic.Name -ErrorAction SilentlyContinue) -eq $null) {
                $netAdapter | Rename-NetAdapter -NewName $nic.Name -ErrorAction SilentlyContinue
                
                #Rename-NetAdapter : {Object Exists} An attempt was made to create an object and the object name already existed.
            }
            
            # $newName = Get-Netadapter -ifIndex $netAdapter.IfIndex
            # cmd.exe /c "netsh routing ip nat add interface $newName" | Out-Null

            New-NetIPAddress -Interfaceindex $netAdapter.IfIndex -AddressFamily IPv4 -IPAddress $nic.IPAddress -PrefixLength $nic.PrefixLen | Out-Null
        }
    } -ArgumentList @($nicsToAdd)

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    # Forward lnet traffic to nat VM.
    foreach ($nic in $nicsToAdd) {
        Set-StaticRoute -Computer $hyperVHosts -Credential $cred -SwitchName $sdnSwitch -NicName $(Get-MgmtNicName) -DestinationPrefix $nic.AddressPrefix -NextHop $mgmtGw
    }

    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        Restart-Service RemoteAccess -Force
    }
}

function Add-ManagmentRouteToVirtualGateways {

    <#
    .Synopsis
        Adds the management network route to the virtual gateway resource in NC to enable connectivity from the
        host to VNET
    .PARAMETER NCVirtualGateways
        Array of virtual gateway resources in NC.
    .PARAMETER BgpConnections
        NetworkConnections with BGP configured
    #>
    param(
        [Parameter(Mandatory = $true)]
        [Array]$NCVirtualGateways,
        [Parameter(Mandatory = $true)]
        [hashtable]$BgpConnections
        
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtPrefix = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix

    $NCVirtualGateways = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    foreach ($vg in $NCVirtualGateways) {
        [array]$bgpCons = $BgpConnections[$vg.ResourceId]
        $update = $false
        foreach ($nw in $vg.Properties.NetworkConnections) {

            if ($bgpCons -ne $null -and $bgpCons.Contains($nw.ResourceId)) {
                # No need to add mgmt route to networkconnection static route if bgp is enabled.
                # Route is pushed by bgp.
                Write-TraceLog "Add-ManagmentRouteToVirtualGateways: Virtual gateway $($vg.ResourceId) nw $($nw.ResourceId) has bgp enabled, skipping static route addition"
                continue
            }

            $foundMgmt = $false
            foreach ($route in $nw.Properties.Routes) {
                if ($route.DestinationPrefix -eq $mgmtPrefix) {
                    $foundMgmt = $true
                    break
                }
            }

            if (-not $foundMgmt) {
                Write-TraceLog "Add-ManagmentRouteToVirtualGateways: Adding management network $mgmtPrefix to the routes of virtual gateway $($vg.ResourceId)"
                $mgmtRoute = New-Object Microsoft.Windows.NetworkController.RouteInfo
                $mgmtRoute.DestinationPrefix = $mgmtPrefix
                $mgmtRoute.NextHop = "0.0.0.0"
                $mgmtRoute.Metric = 10
                $mgmtRoute.Protocol = "Static"
                $nw.Properties.Routes += $mgmtRoute
                $update = $true
            }
        }

        if ($update) {
            New-NetworkControllerVirtualGateway -ResourceId $vg.ResourceId -Properties $vg.Properties -ConnectionUri $restUrl -Force -PassInnerException | Out-Null
        }
    }
}

function Get-SourceNetworkInterface {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Destination,
        [Parameter(Mandatory = $true)]
        [string]$ComputerName,
        [Parameter(Mandatory = $false)]
        [pscredential]$Credential = [pscredential]::Empty
    )

    $result = Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock {
        param($Destination)
        $result = Test-NetConnection $Destination 
        return $result.InterfaceAlias
    } -ArgumentList $($Destination)
    
    return $result
}

function Update-WorkloadSecrets {
    param(
        [Parameter(Mandatory = $true)]
        [string]$WorkloadConfigPath
    )

    $depId = Get-TurnKeySdnDeploymentId
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath

    foreach ($vgw in $vgwsConfigJson) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if ($con.Properties.connectionType -ieq "IPsec") {
                Write-TraceLog "Update-WorkloadSecrets: Updating VG $($vgwJson.resourceId) shared secret for connection $($con.resourceId)"    
                $con.Properties.IPSecConfiguration.SharedSecret = $depId
            }
        }
        $vgwJson | ConvertTo-Json -Depth 100 | Out-File $vgw.FullName -Force
    }
}
function Update-WorkloadIPAddresses {
    param(
        [Parameter(Mandatory = $true)]
        [string]$WorkloadConfigPath
    )

    $stateFile = Join-Path $WorkloadConfigPath "addressRandmoized"

    if (Test-Path $stateFile) {
        Write-TraceLog "Update-WorkloadIPAddresses: IP addresses already randomized in workload config $WorkloadConfigPath"    
        return
    }

    Write-TraceLog "Update-WorkloadIPAddresses: Randomizing workload ip addresses in workload config $WorkloadConfigPath"

    #Randmoize lnet ips
    $lnets = Get-SdnWorkloadLogicalNetworks -WorkloadConfigPath $WorkloadConfigPath

    foreach ($lnet in $lnets) {
        $lnetJson = Get-Content $lnet.FullName | ConvertFrom-Json
        $updatedNetwork = Update-LogicalNetworkAddresses -Network $lnetJson
        $updatedNetwork | ConvertTo-Json -Depth 100 | Out-File $lnet.FullName -Force
    }

    #Randmozie gw networkconnection ips
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath

    foreach ($vgw in $vgwsConfigJson) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if (-not [String]::IsNullOrEmpty($con.Properties.DestinationIPAddress)) {
                $con.Properties.DestinationIPAddress = Update-IPv4Address -Address $con.Properties.DestinationIPAddress
            }
            
            foreach ($address in $con.Properties.ipAddresses) {
                $address.ipAddress = Update-IPv4Address -Address $address.ipAddress
            }
            
            $updatedPeerAddresses = @()
            foreach ($peerAddress in $con.Properties.peerIPAddresses) {
                $updatedPeerAddresses += Update-IPv4Address -Address $peerAddress
            }
            $con.Properties.peerIPAddresses = $updatedPeerAddresses
            
            $routes = $con.Properties.Routes
            foreach ($r in $routes) {
                $r.DestinationPrefix = Update-IPv4Address -Address $r.DestinationPrefix
            }
        }

        foreach ($router in $vgwJson.Properties.bgpRouters) {
            foreach ($peer in $router.Properties.bgpPeers) {
                $peer.Properties.peerIpAddress = Update-IPv4Address -Address $peer.Properties.peerIpAddress
            }
        }

        $vgwJson | ConvertTo-Json -Depth 100 | Out-File $vgw.FullName -Force
    }

    $nics = Get-SdnWorkloadNetworkInterfaces -WorkloadConfigPath $WorkloadConfigPath

    #Randmozie lnet nic ips
    foreach ($nic in $nics) {
        $nicJson = get-content $nic.FullName | ConvertFrom-Json
        foreach ($ipconfig in $nicJson.Properties.IPConfigurations) {

            if ($ipconfig.Properties.subnet.resourceRef -ilike "/virtualNetworks/*") {
                break
            }

            if ($ipconfig.Properties.PrivateIPAddress -ne $null) {
                $ipconfig.Properties.PrivateIPAddress = Update-IPv4Address -Address $ipconfig.Properties.PrivateIPAddress    
            }
        }

        $nicJson | ConvertTo-Json -Depth 100 | Out-File $nic.FullName -Force
    }

    New-Item $stateFile -ItemType File -Force | Out-Null
}

function Add-TurneKeyVMToCluster {

    param(
        [Parameter(Mandatory = $true)]
        $Tag
    )

    Write-TraceLog "Add-TurneKeyVMToCluster: Adding VMs with tag $Tag to cluster"

    $isCluster = Test-IsCluster

    if (-not $isCluster) {
        Write-TraceLog "Add-TurneKeyVMToCluster: Not a cluster, skipping"
        return
    }

    $addVMToCluster = {       
        
        param($Tag)

        $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains $Tag }
 
        if ($vms -eq $null) {
            return
        }

        foreach ($vm in $vms) {
            $isPresent = (Get-ClusterResource -VMId $vm.VMId -ErrorAction SilentlyContinue) -ne $null

            if ($isPresent) {
                continue
            }

            $vm | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null
        }
    }

    Invoke-CmdOnInfraHosts -scriptBlock $addVMToCluster -args @($Tag)
}

function Remove-TurnKeyVMs {
    param(
        [parameter(Mandatory = $true, ParameterSetName = "Tag")][string] $Tag,        
        [parameter(Mandatory = $true, ParameterSetName = "NamePattern")][string] $NamePattern
    )

    if ($PSCmdlet.ParameterSetName -eq "NamePattern") {
        Write-TraceLog "Remove-TurnKeyVMs: Removing VMs with name pattern $NamePattern"     
    } else {
        Write-TraceLog "Remove-TurnKeyVMs: Removing VMs with tag $Tag"
    }

    $isCluster = Test-IsCluster

    $deleteVM = {       
        param($Tag, $NamePattern, $IsCluster)

        if (-not [String]::IsNullOrEmpty($Tag)) {
            $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains $Tag }
        } else {
            $vms = Get-VM | Where-Object { $_.Name -ilike "*$NamePattern*" }
        }
 
        if ($vms -ne $null) {
            if ($IsCluster) {
                foreach ($vm in $vms) {
                    Remove-ClusterGroup -VMId $vm.VMId -RemoveResources -Force -ErrorAction SilentlyContinue
                }
            }
            
            $hardDrives = ($vms | Select-Object -ExpandProperty HardDrives).path
            $vmStorage = $vms.path
            $vms | Stop-VM -TurnOff -force
            $vms | Remove-VM -Force
 
            if ($hardDrives -ne $null -and $hardDrives.Count -gt 0) {
                Remove-Item $hardDrives -ErrorAction SilentlyContinue -Force -Recurse
            }

            if ($vmStorage -ne $null -and $vmStorage.Count -gt 0) {
                Remove-Item $vmStorage -ErrorAction SilentlyContinue -Force -Recurse
            }   
        }
    }

    Invoke-CmdOnInfraHosts -scriptBlock $deleteVM -args @($Tag, $NamePattern, $isCluster)
}
function Remove-WorkloadVMs {

    try {
        Write-TraceLog "Remove-WorkloadVMs: Removing workload vms"
        Remove-DomainJoinedVMs -Tag "Role=DataCenterGateway"
        Remove-TurnKeyVMs -Tag "Role=Workload"
        Remove-TurnKeyVMs -NamePattern "workload_vnet" # workaround for hlk
    }
    catch {
        Write-TraceLog "Remove-WorkloadVMs: Failed to remove workload vms, error $_" -Warning
    }
}

function Remove-RouterVMWorkloadAdapters {

    $cleanupRouterVM = {       
        $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains "Role=Router" }
        
        if ($vms -eq $null) {
            return
        }
        
        $vms | ForEach-Object {
            Get-VMNetworkAdapter -VM $_ | Where-Object { $_.Name -ine "Management" -and $_.Name -ine "Corp" -and $_.Name -ine "HNVPA" } | Remove-VMNetworkAdapter -Confirm:$false -ErrorAction SilentlyContinue
        }
        
        $vms | Restart-VM -Force -Type Reboot
        
    }

    try {
        Write-TraceLog "Remove-RouterVMWorkloadAdapters: Removing workload adapters from router vm"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupRouterVM
    }
    catch {
        Write-TraceLog "Remove-RouterVMWorkloadAdapters: Failed to remove workload adapters from router vm, error $_" -Warning
    }
}

function Remove-DomainJoinedVMs {
    param(
        [parameter(Mandatory = $true)][string] $Tag
    )

    Write-TraceLog "Remove-DomainJoinedVMs: Removing domain joined vms with tag $Tag"

    $unjoinDomain = {
        param($DomainCredential, $Tag)
        $vms = Get-VM | Where-Object { 
            $_.Notes.Split(";") -icontains $Tag }
        
        if ($vms -ne $null) {
            $vms | ForEach-Object {
                Remove-Computer -UnjoinDomainCredential $DomainCredential -ComputerName $_.Name `
                    -LocalCredential $DomainCredential -ErrorAction SilentlyContinue -Force 
            }          
        }
    }
    
    $cred = Get-TurnKeySdnCred
    Invoke-CmdOnInfraHosts -scriptBlock $unjoinDomain -Args @($cred, $Tag)
    Remove-TurnKeyVMs -Tag $Tag
}
function Remove-SdnConfig {
    
    try {
        $sdnConfig = Get-SdnConfig
        $fcncdb = $sdnConfig.networkController.FC.DatabaseLocation

        if ([String]::IsNullOrEmpty($fcncdb)) {
            $fcncdb = Get-DefaultNetworkControllerDBLocation
        }

        Write-TraceLog "Uninstall-TurnKeySdn: Workaround - FCNC uninstall does not delete db. Deleting $fcncdb"
        Remove-Item $fcncdb -ErrorAction SilentlyContinue -Recurse -Force
    }
    catch {
        Write-TraceLog "Remove-SdnConfig: Failed to remove sdn config, error $_" -Warning
    }
}

function Remove-SdnVMs {

    try {
        Remove-DomainJoinedVMs -Tag "IsSDN=True"
    }
    catch {
        Write-TraceLog "Remove-SdnVMs: Failed to remove sdn vms, error $_" -Warning
    }

    try {
        Remove-TurnKeyVMs -NamePattern $(Get-NCVMSuffix)
    }
    catch {
        Write-TraceLog "Remove-SdnVMs: Failed to remove sdn vms, error $_" -Warning
    }

    try {
        Remove-TurnKeyVMs -NamePattern $(Get-MUXVMSuffix)
    }
    catch {
        Write-TraceLog "Remove-SdnVMs: Failed to remove sdn vms, error $_" -Warning
    }

    try {
        Remove-TurnKeyVMs -NamePattern $(Get-GWVMSuffix)
    }
    catch {
        Write-TraceLog "Remove-SdnVMs: Failed to remove sdn vms, error $_" -Warning
    }
}

function Remove-InfraVMs {
    try {
        Remove-DomainJoinedVMs -Tag "IsInfra=True"
    }
    catch {
        Write-TraceLog "Remove-InfraVMs: Failed to remove infra vms, error $_" -Warning
    }

    try {
        Remove-TurnKeyVMs -NamePattern $(Get-RouterVMSuffix)
    }
    catch {
        Write-TraceLog "Remove-InfraVMs: Failed to remove infra vms, error $_" -Warning
    }
    
    # backward compat
    try {
        Remove-TurnKeyVMs -NamePattern "-Router"
    }
    catch {
        Write-TraceLog "Remove-InfraVMs: Failed to remove infra vms, error $_" -Warning
    }
}

function Remove-PhysicalHostVnic {
    param (
        [parameter(Mandatory = $false)][switch] $KeepInfraNetwork
    )
    $cleanupScript = {
        param($MgmtHostNicName, [bool]$KeepInfraNetwork)
        
        if (-not $KeepInfraNetwork) {
            Remove-VMNetworkAdapter -ManagementOS -Name $MgmtHostNicName -ErrorAction SilentlyContinue
        }

        Remove-VMNetworkAdapter -ManagementOS -Name PA* -ErrorAction SilentlyContinue
        Remove-VMNetworkAdapter -ManagementOS -Name DR* -ErrorAction SilentlyContinue
    }

    try {
        $testNic = Get-MgmtNicName
        Write-TraceLog "Remove-PhysicalHostVnic: Removing $testNic, PA* and DR* adapters from all hosts in the deployment"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($testNic, $KeepInfraNetwork.IsPresent)
    }
    catch {
        Write-TraceLog "Remove-PhysicalHostVnic: Failed to remove from all hosts in the deployment, error $_" -Warning
    }
}

function Import-FCNCModule {
    try {
        $sdnConfig = Get-SdnConfig        
    } catch {
    }
    
    if ($sdnConfig -ne $null -and -not [string]::IsNullOrWhiteSpace($sdnConfig.networkController.FC.PowershellModuleRootPath)) {
        $privateModule = Join-Path $sdnConfig.networkController.FC.PowershellModuleRootPath -ChildPath NetworkControllerFc.psd1
        if (Test-Path $privateModule) {
            Import-Module  $privateModule -Force -Scope Global -ErrorAction SilentlyContinue
        }
        $module = Get-Module -name NetworkControllerFc
        if ($module) {
            Write-TraceLog "Remove-FCNC: NetworkControllerFc Module imported from $privateModule"    
            return $true
        }
    }

    Import-Module -name NetworkControllerFc -ErrorAction SilentlyContinue -Scope Global
    $module = Get-Module -name NetworkControllerFc
    if ($module) {
        Write-TraceLog "Remove-FCNC: NetworkControllerFc Module imported from $($module.Path)"    
        return $true
    }

    Write-TraceLog "Remove-FCNC: NetworkControllerFc Module not found"    
    return $false
}
function Remove-FCNCLogging {
    try {
        $hypervHosts = Get-TurnKeySdnHyperVHosts
        if ($hypervHosts -eq $null -or $hypervHosts.Count -eq 0) {
            Write-TraceLog "Remove-FCNCLogging: No hosts found, skipping."
            return
        }
        
        $cred = Get-TurnKeySdnCred

        $hypervHosts | ForEach-Object {
            try {
                Disable-NetworkControllerOnFailoverClusterLogging -DeviceName $_ -Type Server
            } catch {}

            Invoke-ReliableCommand -computerName $_ -Credential $cred -scriptBlock {
                Get-ScheduledTask -TaskName sdn* | Unregister-ScheduledTask -Confirm:$false -ErrorAction SilentlyContinue
                Get-ScheduledTask -TaskName FcDiagnostics* | Unregister-ScheduledTask -Confirm:$false -ErrorAction SilentlyContinue
            }            
        }        
    } catch {
        Write-TraceLog "Remove-FCNCLogging: Failed to remove logging, error $_" -Warning
    }
}
function Remove-FCNC {
    $isImported = Import-FCNCModule

    if (-not $isImported) {
        Write-TraceLog "Remove-FCNC: NetworkControllerFc Module not found, skipping uninstall"    
        return
    }

    Remove-FCNCLogging
    
    try {        
        Uninstall-NetworkControllerOnFailoverCluster
    }
    catch {
        Write-TraceLog "Remove-FCNC: Error ($_)"
    }
}

function Remove-HostRegistryKeys {
    try {
        $hypervHosts = Get-TurnKeySdnHyperVHosts
        if ($hypervHosts -eq $null -or $hypervHosts.Count -eq 0) {
            Write-TraceLog "Remove-HostRegistryKeys: No hosts found, skipping."
            return
        }
        $cred = Get-TurnKeySdnCred
        Remove-HostAgentRegistryKeys -Hosts $hypervHosts -Credential $cred
    } catch {
        Write-TraceLog "Remove-HostRegistryKeys: Failed to remove registry keys, error $_" -Warning
    }
}

function Remove-TestRoutesOnHostVnic {

    $testNic = Get-MgmtNicName
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    Write-TraceLog "Remove-TestRoutesOnHostVnic: Removing test routes from vnic $testNic on all hosts in the deployment"

    $cleanupScript = {
        param($MgmtHostNicName, $ManagementGateway)
        $nic = Get-VMNetworkAdapter -ManagementOS -Name $MgmtHostNicName -ErrorAction SilentlyContinue
        if ($nic -eq $null) {
            return
        }

        $adapter = Get-NetAdapter | Where-Object { $($_.MacAddress -replace "-", "") -eq $($nic.MacAddress) }

        if ($adapter -eq $null) {
            return
        }

        $routes = Get-NetRoute -ifIndex $adapter.ifIndex | Where-Object { $_.NextHop -eq $ManagementGateway }
        if ($routes -eq $null) {
            return
        }

        $routes | Remove-NetRoute -Confirm:$false 
    }

    Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($testNic, $mgmtGw)
}

function Remove-DeploymentCertificates {

    $cleanupScript = {
        param($depId)
        $rootCerts = Get-ChildItem Cert:\LocalMachine\Root | where-object { $_.Subject -ilike "*$depId*" }
        $rootCerts | Remove-Item -ErrorAction SilentlyContinue -Force
        $myCerts = Get-ChildItem Cert:\LocalMachine\My | where-object { $_.Subject -ilike "*$depId*" }
        $myCerts | Remove-Item -ErrorAction SilentlyContinue -Force
    }

    try {
        $depId = Get-TurnKeySdnDeploymentId
        if ([String]::IsNullOrEmpty($depId)) {
            Write-TraceLog "Remove-DeploymentCertificates: No deployment id found, skipping"
            return
        }
        
        Write-TraceLog "Remove-DeploymentCertificates: Removing certificates with subject *$depId* from all hosts in the deployment"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($depId)
    }
    catch {
        Write-TraceLog "Remove-DeploymentCertificates: Failed to remove certificates from all hosts in the deployment, error $_" -Warning
    }
}

function Get-DefaultWorkloadMarinerVhd {

    try {
        Get-SdnWorkloadVhd -os Linux -osSku Mariner | Out-Null
        return $null
    }
    catch {
        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: No workload vhd configured, trying to auto copy"
    }

    try {
        $vhdStore = Get-VhdStore
        $vhdFolder = Join-Path $vhdStore "marinerVhd"
        $vhdFile = Get-DefaultWorkloadVhdFile

        $vhd = Join-Path $vhdFolder (Get-DefaultWorkloadVhdFile)

        if (Test-Path $vhd) {
            Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Existing vhd found at $destVhd"    
            return @($vhdFolder, $vhdFile)
        }

        $share = Get-DefaultWorkloadVhdShare
        $cred = Get-WttNAOCred

        $connected = Connect-SMBShare -RemotePath $share -Credential $cred -RestartLanManServer
        
        if (-not $connected) {
            throw "Unable to connect to $share"
        }

        if (-not $(Test-Path $vhdFolder -ErrorAction Stop)) {
            New-Item $vhdFolder -ItemType Directory | Out-Null
        }    
        
        $shareVhd = Join-Path $share $vhdFile

        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Copying $shareVhd to $vhdFolder"
        try {
            Start-BitsTransfer -Source $shareVhd -Destination $vhdFolder -Description $shareVhd -DisplayName "WorkloadVHD"
        }
        catch {
            Copy-Item $shareVhd $vhdFolder
        }

        return @($vhdFolder, $vhdFile)
    }
    catch {
        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Failed to auto copy default workload vhd, error $_" -Warning
    }

    return $null
}
function Get-VirtualGatewayVnetPrefix {
    param(
        [Parameter(Mandatory = $true)]
        [string]$VirtualGatewayResourceId
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  
    $vg = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl -ResourceID $VirtualGatewayResourceId
    $subnetEle = $vg.Properties.GatewaySubnets[0].ResourceRef.Split("/", [StringSplitOptions]::RemoveEmptyEntries)
    $vnetId = $subnetEle[1]
    $vnet = Get-NetworkControllerVirtualNetwork -ResourceId $vnetId -ConnectionUri $restUrl
    return $vnet.Properties.AddressSpace.AddressPrefixes
}

function Add-StaticRoutesForVirtualGateways {
    <#
    .SYNOPSIS
        Adds static routes for vnet prefixes corresponding to virtual gateway connections on physical hosts and NAT VM.
    .PARAMETER NCVirtualGateways
        Array of virtual gateway resources in NC.
    #>
    param(
        [Parameter(Mandatory = $true)]
        [Array]$NCVirtualGateways,
        [hashtable]$BgpConnections
    )

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnSwitchName = Get-SdnSwitchName
    $cred = Get-TurnKeySdnCred
    $routerVM = Get-RouterVMName
    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts

    foreach ($vg in $NCVirtualGateways) {

        $destinationPrefixes = Get-VirtualGatewayVnetPrefix -VirtualGatewayResourceId $vg.ResourceId
        [array]$bgpCons = $BgpConnections[$vg.ResourceId]
        foreach ($connection in $vg.Properties.networkConnections) {
            if ($connection.Properties.connectionType -ieq "IPsec") {
                # Next hop for IPsec connection from Router VM is datacenter gateway VM.
                $nextHop = $connection.Properties.DestinationIPAddress 
            }
            
            if ($connection.Properties.connectionType -ieq "L3") {
                # Next hop for L3 connection from Router VM is the L3 IP in tenant compartment of NC gateway VM.
                $nextHop = $connection.Properties.ipAddresses[0].ipAddress 
            }

            $isBgpEnabled = $false
            if ($bgpCons -ne $null -and $bgpCons.Contains($connection.ResourceId)) {
                $isBgpEnabled = $true
            }

            $routerVMNic = Get-SourceNetworkInterface -ComputerName $routerVM -Credential $cred -Destination $nextHop
            Write-TraceLog "Set-StaticRoutesForVirtualGateways: Setting static route for virtual gateway $($vg.ResourceId), `
                connection $($connection.ResourceId) routerVMNic $routerVMNic, nextHop $nextHop"
            foreach ($prefix in $destinationPrefixes) {
                # Add route for each vnet IP prefix on physical host and router vm.
                # For physical host, next hop is the router VM(the default gateway of management network)
                Set-StaticRoute -Computer $hyperVHosts -Credential $cred -SwitchName $sdnSwitchName -NicName $(Get-MgmtNicName) -DestinationPrefix $prefix -NextHop $mgmtGw
                # Ideally if bgp is enabled, VNET routes should come via bgp and static route is not required.
                # But bgp connection is between datacneter gateway and NC gateway VM.
                # There is no upstream bgp btw router vm and datacenter gateway vm.
                # So static routes are still required on router vm.
                # TODO: Add upstream bgp peer to router vm.
                #if (-not $isBgpEnabled) {
                Set-StaticRoute -Computer $routerVM -Credential $cred -NicName $routerVMNic -DestinationPrefix $prefix -NextHop $nextHop
                #}
            }
        }
    }
}

function Get-IPsecSharedSecrets {
    param(
        [Parameter(Mandatory = $true)][String]$WorkloadConfigPath
    )

    $vgwsConfigJsons = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath
    if ($vgwsConfigJsons -eq $null -or $vgwsConfigJsons.Count -eq 0) {
        return @{}
    }

    $sharedSecrets = @{}
    foreach ($vgw in $vgwsConfigJsons) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if ($con.Properties.connectionType -ieq "IPsec") {
                $sharedSecrets[$con.Properties.DestinationIPAddress] = $con.Properties.IpSecConfiguration.sharedSecret
            }
        }
    }

    return $sharedSecrets
}
function Initialize-DataCenterGWVM {
    param(
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][pscredential]$Credential
    )

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -scriptBlock {
        if ((get-remoteaccess).VpnS2SStatus -ne "Installed") {
            Install-RemoteAccess -VpnType VpnS2S
        }
        else {
            Restart-Service RemoteAccess -Force
        }

        $retries = 5
        while ($retries--) {
            try {
                # Wait for RRAS to be ready
                Get-VpnS2SInterface -ErrorAction Stop | Out-Null                     
            }
            catch {
                Start-Sleep 10
                Continue
            }

            break
        }
    }
}
function Add-BgpPeerOnVM {
    param(
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][pscredential]$Credential,
        [Parameter(Mandatory = $true)][String]$RouterIP,
        [Parameter(Mandatory = $true)][String]$LocalASN,
        [Parameter(Mandatory = $true)][String]$PeerIP,
        [Parameter(Mandatory = $true)][String]$PeerASN,
        [Parameter(Mandatory = $true)][Array]$CustomRoutes
    )

    $addBgpPeer = {
        param($RouterIP, $LocalASN, $PeerIP, $PeerASN, $CustomRoutes)
        try {
            $router = Get-BgpRouter -ErrorAction SilentlyContinue
        }
        catch {}

        if ($router -eq $null) {
            Add-BgpRouter -BgpIdentifier $RouterIP -LocalASN $LocalASN | Out-Null
        }

        $bgpPeer = Get-BgpPeer -Name $PeerIP -ErrorAction SilentlyContinue

        if ($bgpPeer -ne $null) {
            Remove-BgpPeer -Name $PeerIP -Force -Confirm:$false -ErrorAction SilentlyContinue
        }

        Add-BgpPeer -Name $PeerIP -LocalIPAddress $RouterIP -PeerIPAddress $PeerIP -PeerASN $PeerASN | Out-Null
        Add-BgpCustomRoute -Network $CustomRoutes
    }

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -scriptBlock $addBgpPeer `
        -ArgumentList @($RouterIP, $LocalASN, $PeerIP, $PeerASN, $CustomRoutes)
}

function Add-IPsecVPNConnection {

    param(
        [Parameter(Mandatory = $true)][Array]$DataCenterGWs,
        [Parameter(Mandatory = $true)][hashtable]$SharedSecrets
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  

    $addVpnConnection = {
        param($connectionName, $destinationIP, $mainMode, $quickMode, $sharedSecret, $destinationPrefixes)
        
        $con = Get-VpnS2SInterface -Name $connectionName -ErrorAction SilentlyContinue

        if ($con -ne $null) {
            Set-VpnS2SInterface -Name $connectionName -AdminStatus $false -Confirm:$false -Force
            Remove-VpnS2SInterface -Name $connectionName -Force -Confirm:$false -ErrorAction SilentlyContinue
        }

        Add-VpnS2SInterface `
            -Name $connectionName `
            -CustomPolicy `
            -Destination $destinationIP `
            -Protocol IKEv2 `
            -SharedSecret $sharedSecret `
            -CipherTransformConstants $quickMode.CipherTransformationConstant `
            -AuthenticationTransformConstants $quickMode.AuthenticationTransformationConstant `
            -DHGroup $mainMode.DiffieHellmanGroup `
            -EncryptionMethod $mainMode.EncryptionAlgorithm `
            -IntegrityCheckMethod $mainMode.IntegrityAlgorithm `
            -AuthenticationMethod PSKOnly `
            -ResponderAuthenticationMethod PSKOnly `
            -Persistent

        $prefixArray = @()
        foreach ($prefix in $destinationPrefixes) {
            $prefixArray += $($prefix + ":10")
        }

        Set-VpnS2SInterface -Name $connectionName -IPv4Subnet $prefixArray
    }

    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    $cred = Get-TurnKeySdnCred

    foreach ($gw in $DataCenterGWs) {
        Initialize-DataCenterGWVM -ComputerName $gw.Name -Credential $cred

        $connection = $null
        foreach ($vg in $vgws) {
            $connection = $vg.Properties.networkConnections | Where-Object {
                $_.properties.destinationIPAddress -eq $gw.IPAddress -and $_.properties.connectionType -ieq "IPsec"
            }

            if ($connection -ne $null) {
                break
            }
        }

        if ($connection -eq $null) {
            Write-TraceLog "No virtual gateway found for datacenter gateway $($gw.Name)"
            continue
        }

        if ([String]::IsNullOrEmpty($connection.Properties.SourceIPAddress)) {
            throw "No SourceIPAddress found for datacenter gateway $($connection.ResourceId)"
        }

        $destinationPrefixes = Get-VirtualGatewayVnetPrefix -VirtualGatewayResourceId $vg.ResourceId

        $sharedSecret = $SharedSecrets[$connection.Properties.DestinationIPAddress]
        Write-TraceLog "Adding VPN connection for $($gw.Name), ConnectionName: $($connection.ResourceId), sAddr $($connection.Properties.SourceIPAddress), dAddr $($connection.Properties.DestinationIPAddress) SharedSecret: $sharedSecret, DestinationPrefixes: $destinationPrefixes"
        Invoke-ReliableCommand -computerName $gw.Name -Credential $cred -scriptBlock $addVpnConnection `
            -ArgumentList @($connection.ResourceId, $connection.Properties.SourceIPAddress, `
                $connection.Properties.IpSecConfiguration.MainMode, $connection.Properties.IpSecConfiguration.QuickMode, $sharedSecret, $destinationPrefixes)
    }
}

function Add-BgpPeerOnDataCenterGW {

    param(
        [Parameter(Mandatory = $true)][Array]$DataCenterGWs
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  

    
    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    $cred = Get-TurnKeySdnCred
    $bgpConnections = @{}
    foreach ($gw in $DataCenterGWs) {

        $connection = $null
        foreach ($vg in $vgws) {
            $connection = $vg.Properties.networkConnections | Where-Object {
                $_.properties.destinationIPAddress -eq $gw.IPAddress -and $_.properties.connectionType -ieq "IPsec"
            }
            if ($connection -ne $null) {
                $router = $vg.Properties.BgpRouters | where-object { $_.Properties.BgpPeers | Where-Object { $_.Properties.peerIpAddress -eq $gw.IPAddress } }
                break
            }
        }

        if ($router -eq $null) {
            Write-TraceLog "Add-BgpPeerOnDataCenterGW No bgp peer found for datacenter gateway $($gw.Name), skipping"
            continue
        }

        $localASN = $router.Properties.localASN
        $peerIP = $router.Properties.RouterIP[0]
        $peerASN = $router.Properties.ExtAsNumber.Split(".") | Select -Last 1
        $localPeer = $router.Properties.BgpPeers | Where-Object { $_.Properties.peerIpAddress -eq $gw.IPAddress }
        $localASN = $localPeer.Properties.AsNumber
        $routerIP = $localPeer.Properties.PeerIpAddress
        $mgmtNetConfig = Get-MgmtNetworkConfig
        $mgmtPrefix = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix

        Write-TraceLog "Add-BgpPeerOnDataCenterGW Adding BGP peer on $($gw.Name), LocalASN: $localASN, PeerIP: $peerIP, PeerASN: $peerASN"
        Add-BgpPeerOnVM -ComputerName $gw.Name -Credential $cred -RouterIP $routerIP `
            -LocalASN $localASN -PeerIP $peerIP -PeerASN $peerASN -CustomRoutes @($mgmtPrefix)

        $bgpConnections[$vg.ResourceId] = $connection.ResourceId
    }

    return $bgpConnections
}

function Add-DataCenterGatewayNetwork {

    <#
    .SYNOPSIS
    Adds datacenter gateway network with datacenter gateway VM to host IPsec connections.
    .DESCRIPTION
    IPsec connections from virtual gateways configured in SDN terminate in the datacenter gateway VM.
    This function adds the datacenter gateway VM in a logical network configured for it.
    #>

    param(
        $WorkloadConfigPath,
        [ref]$DataCenterGWVMs
    )
      
    $dcNetworkPath = Join-Path $WorkloadConfigPath "datacenternetwork"
    if (-not (Test-Path $dcNetworkPath)) {
        return
    }

    Update-WorkloadIPAddresses -WorkloadConfigPath $dcNetworkPath

    $dcLnets = Get-SdnWorkloadLogicalNetworks -WorkloadConfigPath $dcNetworkPath

    if ($dcLnets -eq $null -or $dcLnets.Count -eq 0) {
        Write-TraceLog "No datacenter logical networks found at $dcNetworkPath. IPsec VPN will not be configured" -Warning
        return
    }   

    $nics = Get-SdnWorkloadNetworkInterfaces -WorkloadConfigPath $dcNetworkPath

    if ($nics -eq $null ) {
        Write-TraceLog "No network interface json under datacenter network, $dcNetworkPath. IPsec VPN will not be configured" -Warning
        return
    }  

    $gwNics = @()
    $nics | ForEach-Object {
        $nicJson = get-content $_.FullName | ConvertFrom-Json
        if ($nicJson.Tags.Role -ieq "GatewayConnection" ) {
            $gwNics += $_
        }
    }

    if ($gwNics.Count -eq 0 ) {
        Write-TraceLog "No networkinterface json under datacenter network $dcNetworkPath with GatewayConnection tag. IPsec VPN will not be configured" -Warning
        return
    } 

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restoreReplayScript = Get-RestoreReplayScript
    & $restoreReplayScript -OperationType Put -BackupDirectoryOrZipFile $dcNetworkPath -BackupType Manual -Verbose -NCRestEndPoint $restEndpoint -Force -Confirm:$false    
    
    $dcLnets | ForEach-Object {
        $lnetJson = get-content $_.FullName | ConvertFrom-Json
        $resourceId = $lnetJson.resourceId
        Enable-LogicalNetworkDefaultGateway -LogicalNetworkResourceId $resourceId  
    }


    $dataCenterGWs = @()
    $depId = Get-TurnKeySdnDeploymentId
    $restUrl = "https://$restEndpoint"  

    $gwNics | ForEach-Object {
        $nic = $_
        $nicJson = get-content $nic.FullName | ConvertFrom-Json
        $resourceId = $nicJson.resourceId
        $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $resourceId -PassInnerException -ErrorAction Stop
        $mac = $res.Properties.PrivateMacAddress
        $ipAddress = $res.Properties.IpConfigurations[0].Properties.PrivateIPAddress
        $subnetRef = $res.Properties.IpConfigurations[0].Properties.Subnet.ResourceRef
        
        $subEle = $subnetRef.Split("/", [StringSplitOptions]::RemoveEmptyEntries)
        $lnetId = $subEle[1]
        $subnetId = $subEle[3]
        $subnet = Get-NetworkControllerLogicalSubnet -LogicalNetworkId $lnetId -ResourceId $subnetId -ConnectionUri $restUrl
        $vlanId = $subnet.Properties.VlanID
        $defaultGw = $subnet.Properties.DefaultGateways[0]
        $prefixLen = $subnet.Properties.AddressPrefix.Split("/") | Select-Object -last 1
        $ipAddressWithPrefix = $ipAddress + "/" + $prefixLen

        $vmName = "$depId-$resourceId"
        New-DataCenterGateway -vmName $vmName `
            -MacAddress $mac -IPAddress $ipAddressWithPrefix `
            -VlanId $vlanId -DefaultGateway $defaultGw

        $instanceId = $res.InstanceId.Guid            
        # Enable-SDNExpressVMPort -ComputerName (hostname) -VMName $dataCenterGW -InstanceId "{$instanceId}" -Credential $cred
        $dataCenterGWs += @{"Name" = $vmName; "IPAddress" = $ipAddress; "InstanceId" = $instanceId; "VMHost" = (hostname) }
    }

    $cred = Get-TurnKeySdnCred
    foreach ($dataCenterGW in $dataCenterGWs) {
        WaitforComputerToBeReady -ComputerName $dataCenterGW.Name -Credential $cred
        Invoke-ActivateWindows -ComputerName $dataCenterGW.Name -Credential $cred
        $instanceId = $dataCenterGW.InstanceId
        Enable-SDNExpressVMPort -ComputerName $dataCenterGW.VMHost -VMName $dataCenterGW.Name -InstanceId "{$instanceId}" -Credential $cred
        WaitforComputerToBeReady -ComputerName $dataCenterGW.Name -Credential $cred
        Enable-DefaultFirewallRules -ComputerName $dataCenterGW.Name -Credential $cred
    }

    $DataCenterGWVMs.Value = $dataCenterGWs
}
function Enable-GatewayWorkload {

    <#
    # .SYNOPSIS
    # Enables gateway workload data path
    # .DESCRIPTION
    # Creates a datacenter gateway VM for each of the virtual gateway connection configured in the workload config.
    # S2S vpn interface is added to the datacenter gateway VM for each gateway connection.
    # Network for datacenter gateway VM is configured using the jsons under workload\gateway\datacenternetwork folder.
    # The logical network resources specify the logical network to be used for the datacenter gateway.
    # Datacenter network is configured as an actual logical network in SDN.
    # This data center logical network corresponds to the destination ip address of the virtual gateway connection.
    # Under workload\gateway\datacenternetwork\jsons\networkinterfaces, a network interface must exist with the tag "GatewayConnection" for each datacenter gateway VM.
    # IP address of this networkinterface must correspond to the destination ip address of the virtual gateway connection.
    # To avoid conflict between different setups, various IP addresses are randmoized unless randomizeAddresses is set to false in the deployment config.
    # To enable datapath from physical host following static routes are added,
    # - On physical host, route to the vnet ip with next hop as management gateway(router vm)
    # - On router vm, route to the vnet ip with next hop as datacenter gateway corresponding to the vnet.
    # - Management address prefix is auto appended to the virtual gateway connection routes. This enables return traffic from the VNET VM to physical host.
    # Traffic flow -> Physical Host->Router VM->Datacenter Gateway VM->S2S VPN->SDN Gateway->VNET VM
    # For L3 connections, the logical network correpsonding to the L3 is enabled on the router VM.
    # .PARAMETER WorkloadConfigPath
    # Path to the datacenternetwork network workload config
    #>

    param(
        $WorkloadConfigPath
    )

    Write-TraceLog "Enable-GatewayWorkload: Enabling gateway workload"
    
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath
    if ($vgwsConfigJson -eq $null -or $vgwsConfigJson.Count -eq 0) {
        Write-TraceLog "Enable-GatewayWorkload: No virtual gateway configuration found. Skipping gateway workload configuration"
        return
    }

    $dataCenterGWs = @()
    Add-DataCenterGatewayNetwork -WorkloadConfigPath $WorkloadConfigPath -DataCenterGWVMs ([ref]$dataCenterGWs)
    
    $sharedSecrets = Get-IPsecSharedSecrets -WorkloadConfigPath $WorkloadConfigPath
    Add-IPsecVPNConnection -DataCenterGWs $dataCenterGWs -SharedSecrets $sharedSecrets    
    $bgpConnections = Add-BgpPeerOnDataCenterGW -DataCenterGWs $dataCenterGWs

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  
    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    Add-StaticRoutesForVirtualGateways -NCVirtualGateways $vgws -BgpConnections $bgpConnections
    Add-ManagmentRouteToVirtualGateways -NCVirtualGateways $vgws -BgpConnections $bgpConnections
    $routerVM = Get-RouterVMName
    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        Restart-Service RemoteAccess -Force
    }

    Add-TurneKeyVMToCluster -Tag "Role=DataCenterGateway"
}

function Set-VipRouteOnHosts {
    <#
    .SYNOPSIS
        Adds a static route for the public vip network on the hosts.
        Next hop is the default gateway of the management network, the router VM.
    #>

    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts
    
    $sdnSwitchName = Get-SdnSwitchName
    $cred = Get-TurnKeySdnCred

    $publicVipNetwork = Get-PublicVipNetworkConfig    
    $publicVipPrefix = $publicVipNetwork.properties.subnets[0].properties.addressPrefix

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $nextHop = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    Set-StaticRoute -Computer $hyperVHosts -SwitchName $sdnSwitchName `
        -DestinationPrefix $publicVipPrefix `
        -NextHop $nextHop `
        -NicName $(Get-MgmtNicName) -Credential $cred

    $privateVipNetwork = Get-PrivateVipNetworkConfig    
    $privateVipPrefix = $privateVipNetwork.properties.subnets[0].properties.addressPrefix

    Set-StaticRoute -Computer $hyperVHosts -SwitchName $sdnSwitchName `
        -DestinationPrefix $privateVipPrefix `
        -NextHop $nextHop `
        -NicName $(Get-MgmtNicName) -Credential $cred

}


function Invoke-ActivateWindows {
    param(
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][pscredential]$Credential
    )

    $activateWindows = {
        slmgr /ato //b
    }

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -scriptBlock $activateWindows
}

function Get-InfraVMMemory {
    $memory = Get-HostMemory

    if ($memory -gt 100) {
        return 8GB
    }

    if ($memory -gt 64) {
        return 4GB
    }

    return 2GB
}

function Get-InfraVMProcessorCount {
    $pCount = Get-HostProcessorCount

    if ($pCount -ge 40) {
        return 8
    }

    if ($pCount -ge 24) {
        return 4
    }

    return 2
}

function Set-SdnNetIntent {

    <#
    .SYNOPSIS
    Enables netatc with storage and compute intends
    #>

    param(
        [Parameter(Mandatory=$true)] [Array] $ComputerName,
        [Parameter(Mandatory=$false)] [pscredential] $Credential = [pscredential]::Empty,
        [Parameter(Mandatory=$false)] [string] $ifDescPattern = "Mellanox*",
        [Parameter(Mandatory=$false)] [Array] $storageVLAN = @(8,9),
        [Parameter(Mandatory=$false)] [switch] $Force
    )

    $namePrefix = "turnkeyPNIC"
    $renameNetAdapter = {
        
        param($ifDescPattern, $newNamePrefix)
        $nics = Get-NetAdapter -InterfaceDescription $ifDescPattern
        $newNames = @()
        $i = 0
        foreach($nic in $nics) {
            $newName = "$newNamePrefix$i"
            $nic | Rename-NetAdapter -NewName $newName
            $newNames += $newName
            $i++
        }    
    }

    foreach($c in $ComputerName) {
        Write-TraceLog "Set-SdnNetIntent: Renaming netadapter on $c" 
        $s = New-ReliablePsSession -ComputerName $c -Credential $Credential
        Invoke-Command -Session $s -ScriptBlock $renameNetAdapter -ArgumentList @($ifDescPattern, $namePrefix) | Out-Null
    }

    $intentName = "turnkeysdn"
    $configureNetAtc = {        
        param($intentName, $namePrefix, $storageVLAN, [bool]$force)
        $nics = (Get-NetAdapter $namePrefix*).Name
        if ($force) {
            Remove-NetIntent -Name $intentName -ErrorAction SilentlyContinue
        } else {
            if ($null -ne $(Get-NetIntent -Name $intentName -ErrorAction SilentlyContinue)) {
                return
            }
        }

        $adapterProperties = New-NetIntentAdapterPropertyOverrides
        $adapterProperties.JumboPacket = 4088
        # Disable software RSC, there were multiple issues with this feature.
        $s = New-NetIntentSwitchConfigurationOverrides
        $s.EnableSoftwareRsc = $false
        Add-NetIntent -Name $intentName -Compute -Storage -AdapterName $nics `
            -StorageVlans $storageVLAN -SwitchPropertyOverrides $s -AdapterPropertyOverrides $adapterProperties -Wait
    }
    
    Write-TraceLog "Set-SdnNetIntent: Configuring SDN intent" 
    Invoke-Command -ScriptBlock $configureNetAtc -ArgumentList @($intentName, $namePrefix, $storageVLAN, $Force.IsPresent)

    # Best effort to setup a corp intent if a nic with name "CORP" is found
    $configureCorpIntent = {
        param($intentName, $nicName, [bool]$force)
        $nic = Get-NetAdapter -name $nicName -ErrorAction SilentlyContinue
        if ($null -eq $nic) {
            # TODO: Validate nic exists on all hosts
            Write-TraceLog "Set-SdnNetIntent: NIC $nicName not found on $env:COMPUTERNAME, skipping corp intent"
            return
        }

        $vswitchNics = (Get-VMSwitch -ErrorAction SilentlyContinue).NetAdapterInterfaceDescriptions

        if ($null -ne $vswitchNics) {
            if ($vswitchNics -contains $nic.InterfaceDescription) {
                Write-TraceLog "Set-SdnNetIntent: NIC $nicName is already associated with another switch, skipping corp intent"
                return
            }
        }

        if ($force) {
            Remove-NetIntent -Name $intentName -ErrorAction SilentlyContinue
        } else {
            if ($null -ne $(Get-NetIntent -Name $intentName -ErrorAction SilentlyContinue)) {
                return
            }
        }

        Add-NetIntent -Name $intentName -Management -AdapterName $nicName -SkipNetworkInterfaceValidation -Wait
    }

    Write-TraceLog "Set-SdnNetIntent: Configuring management intent" 
    Invoke-Command -ScriptBlock $configureCorpIntent -ArgumentList @("turnkeycorp", "CORP", $Force.IsPresent)
}

# SIG # Begin signature block
# MIInvwYJKoZIhvcNAQcCoIInsDCCJ6wCAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAMo2ww2JkuDHzF
# +G7VDcWbFU9ycKHpEI/mF6chwLuZiqCCDXYwggX0MIID3KADAgECAhMzAAADrzBA
# DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA
# hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG
# 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN
# xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL
# go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB
# tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd
# mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ
# 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY
# 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp
# XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn
# TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT
# e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG
# OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O
# PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk
# ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx
# HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt
# CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGZ8wghmbAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIL2/SLjWcfwoahW96+pbdut1
# E9aUs5C+9iSHF3hQSHAjMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEAlHzGOIQ9d/q+d9EX2m+/IRSTOX/MngqhoWAIQ72+X5JwRkxrEHyAcP15
# D8SFDnIYAlwv/IQvPr5iN9dLXQrTqRwTZQL0kvveQgLnkHxkltzTrC9R2UkH8sqg
# OmE7pIfSEiQMUnA1d4WNJsO3XbSqGW+wnhO4erGQltxa2SOl7pvyPYH6v1Zmz+VG
# l/WjStgP33wyLHOyzv5Ul+K/y1+Z7HoxwTZIEyi7SdUB6tbVt2lERciVq+DVUAA9
# 0C0tNTDQTE1VoJ8P9uy2BAwMD6Mbyn+/U1LdXAklgh7KEqTMYxd9wh5xDbn3rT+g
# HkEi4ksGWhq4+KataoT7rvInPBvsiKGCFykwghclBgorBgEEAYI3AwMBMYIXFTCC
# FxEGCSqGSIb3DQEHAqCCFwIwghb+AgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq
# hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCDEQw4uAT1FdaMnignU4FbEmtuVrcZfQZ7i2oFZifTZVgIGZnMSLVuL
# GBMyMDI0MDcwMjIyMjYwNC43ODVaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO
# OkQwODItNEJGRC1FRUJBMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT
# ZXJ2aWNloIIReDCCBycwggUPoAMCAQICEzMAAAHcweCMwl9YXo4AAQAAAdwwDQYJ
# KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjMx
# MDEyMTkwNzA2WhcNMjUwMTEwMTkwNzA2WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV
# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv
# c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl
# cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpEMDgyLTRC
# RkQtRUVCQTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC
# AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvIsyA1sjg9kSKJzelrUWF5
# ShqYWL83amn3SE5JyIVPUC7F6qTcLphhHZ9idf21f0RaGrU8EHydF8NxPMR2KVNi
# AtCGPJa8kV1CGvn3beGB2m2ltmqJanG71mAywrkKATYniwKLPQLJ00EkXw5TSwfm
# JXbdgQLFlHyfA5Kg+pUsJXzqumkIvEr0DXPvptAGqkdFLKwo4BTlEgnvzeTfXukz
# X8vQtTALfVJuTUgRU7zoP/RFWt3WagahZ6UloI0FC8XlBQDVDX5JeMEsx7jgJDdE
# nK44Y8gHuEWRDq+SG9Xo0GIOjiuTWD5uv3vlEmIAyR/7rSFvcLnwAqMdqcy/iqQP
# MlDOcd0AbniP8ia1BQEUnfZT3UxyK9rLB/SRiKPyHDlg8oWwXyiv3+bGB6dmdM61
# ur6nUtfDf51lPcKhK4Vo83pOE1/niWlVnEHQV9NJ5/DbUSqW2RqTUa2O2KuvsyRG
# MEgjGJA12/SqrRqlvE2fiN5ZmZVtqSPWaIasx7a0GB+fdTw+geRn6Mo2S6+/bZEw
# S/0IJ5gcKGinNbfyQ1xrvWXPtXzKOfjkh75iRuXourGVPRqkmz5UYz+R5ybMJWj+
# mfcGqz2hXV8iZnCZDBrrnZivnErCMh5Flfg8496pT0phjUTH2GChHIvE4SDSk2hw
# WP/uHB9gEs8p/9Pe/mt9AgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU6HPSBd0OfEX3
# uNWsdkSraUGe3dswHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD
# VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j
# cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG
# CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw
# MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
# CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBANnrb8Ewr8eX/H1s
# Kt3rnwTDx4AqgHbkMNQo+kUGwCINXS3y1GUcdqsK/R1g6Tf7tNx1q0NpKk1JTupU
# JfHdExKtkuhHA+82lT7yISp/Y74dqJ03RCT4Q+8ooQXTMzxiewfErVLt8Wefebnc
# ST0i6ypKv87pCYkxM24bbqbM/V+M5VBppCUs7R+cETiz/zEA1AbZL/viXtHmryA0
# CGd+Pt9c+adsYfm7qe5UMnS0f/YJmEEMkEqGXCzyLK+dh+UsFi0d4lkdcE+Zq5JN
# jIHesX1wztGVAtvX0DYDZdN2WZ1kk+hOMblUV/L8n1YWzhP/5XQnYl03AfXErn+1
# Eatylifzd3ChJ1xuGG76YbWgiRXnDvCiwDqvUJevVRY1qy4y4vlVKaShtbdfgPyG
# eeJ/YcSBONOc0DNTWbjMbL50qeIEC0lHSpL2rRYNVu3hsHzG8n5u5CQajPwx9Pzp
# sZIeFTNHyVF6kujI4Vo9NvO/zF8Ot44IMj4M7UX9Za4QwGf5B71x57OjaX53gxT4
# vzoHvEBXF9qCmHRgXBLbRomJfDn60alzv7dpCVQIuQ062nyIZKnsXxzuKFb0TjXW
# w6OFpG1bsjXpOo5DMHkysribxHor4Yz5dZjVyHANyKo0bSrAlVeihcaG5F74SZT8
# FtyHAW6IgLc5w/3D+R1obDhKZ21WMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ
# mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT
# Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m
# dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh
# dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1
# WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH
# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD
# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB
# BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK
# NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg
# fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp
# rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d
# vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9
# 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR
# Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu
# qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO
# ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb
# oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6
# bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t
# AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW
# BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb
# UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz
# aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku
# aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA
# QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2
# VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu
# bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw
# LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93
# d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt
# MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q
# XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6
# U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt
# I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis
# 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp
# kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0
# sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e
# W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ
# sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7
# Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0
# dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ
# tB1VM1izoXBm8qGCAtQwggI9AgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx
# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT
# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh
# bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpE
# MDgyLTRCRkQtRUVCQTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy
# dmljZaIjCgEBMAcGBSsOAwIaAxUAHDn/cz+3yRkIUCJfSbL3djnQEqaggYMwgYCk
# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH
# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD
# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF
# AOouCgQwIhgPMjAyNDA3MDIxMzA5MjRaGA8yMDI0MDcwMzEzMDkyNFowdDA6Bgor
# BgEEAYRZCgQBMSwwKjAKAgUA6i4KBAIBADAHAgEAAgILkjAHAgEAAgIS+DAKAgUA
# 6i9bhAIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAID
# B6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAFmT2/VDbzhvRbgWxlWQ
# lFUvktqg+PvPtexYNDHJ/YznNAvYPgS4Eo17Ae6b0kZHCVeZC5N+802FE4docULf
# EdjoJS7yqeg8msatIIT9BwWslDOSsYSW2VWqxhxECp9E2dBbq7P6rwvVejVZpnag
# QFVwqcpf1/rGDd8guBNMaYg5MYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMx
# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT
# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUt
# U3RhbXAgUENBIDIwMTACEzMAAAHcweCMwl9YXo4AAQAAAdwwDQYJYIZIAWUDBAIB
# BQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQx
# IgQghV9+pm0e/nGowDSMGsSSgTYbpMDVmMHjrtKlZASPA4YwgfoGCyqGSIb3DQEJ
# EAIvMYHqMIHnMIHkMIG9BCBTpxeKatlEP4y8qZzjuWL0Ou0IqxELDhX2TLylxIIN
# NzCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw
# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x
# JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB3MHg
# jMJfWF6OAAEAAAHcMCIEIMrBcXFOs3jMRtl/r/UaAn4+JzzP7IQSyrqBM/GmNzH1
# MA0GCSqGSIb3DQEBCwUABIICAC/Ey4ezmdSAuXu3zK/NWXJSYsGAAdHsn+YEbTK9
# EADWd6Sag7h7JQSH58FsvjMu/RdA/qzmCuEznFalI5ygrpR72hAlTLTq7H5qTe57
# j8SxCULuWulUTdacbBPk1gkgGWgh4g46dV6UBeX0uH1Yi7AEbuWhoEMO5cCAhjtr
# X8VDuFWx53JB2CSUXd3JfvfK1DXuE8sml+m0vUSJFSQudIrZ3A4aThmNzCgDX/PB
# PAEWh+XMNgI3i1GQQXBOTehkpy+k2V/NAwT0DOPJLgIA4Eef0lIwulTbeaH5esk8
# K7I03Df72w0VHHSN3Rc+eyhrVWeDpO6vhNFk18PbH3fBHvzV04eMfvAUaCgSS7Nc
# SbcImLY/4COk/tiMtcNF4opBsvkS0vFFLwF5LdqwKl8jEP0MV2/ViJ5wlal9siJj
# E/FiTsXGwWjJ6yFE7r9HSywR0qipUbAd8aL0/m8Ev5wRk2uAME3cROq3b79RLwXk
# qWv7pxndL00lodL4JcOtnQzJqP165gkwnVMKVx6CXvU8zedK2aRsG/twpPHvCzNB
# pPZ9vhi351QuKfDSNV22xjkBYMvNf+H7KSzyY8PDhFBWbtg8A/T8v11V6ddRksQ5
# 11cZZdLGGuczf5GJNMaY/3oYXNOpnY5a4CMnUcGPl1TdrG/Ru/qUUOzb5PjMQI1o
# dBv7
# SIG # End signature block