TurnKeySdn

0.0.8

TurnKeySdnInternal.psm1

Import-Module $PSScriptRoot\ConfigManager.psm1
Import-Module $PSScriptRoot\ClusUtils.psm1
Import-Module $PSScriptRoot\Logger.psm1
Import-Module $PSScriptRoot\PsHelper.psm1
Import-Module $PSScriptRoot\Utils.psm1
Import-Module $PSScriptRoot\WinBuildUtils.psm1
Import-Module $PSScriptRoot\TrafficUtil.psm1

function Get-NewDepId {

    return @($id, $idbytes)
}

function Initialize-Credentials {
    try {
        Get-TurnKeySdnCred | Out-Null
        Write-TraceLog "Initialize-Credentials : Existing credential found"
        return
    }
    catch {
        Write-TraceLog "Initialize-Credentials : Credential not found, creating new credential"
    }

    $cred = $null
    $infraconfig = Get-DeploymentConfig
    $domain = $infraconfig.domainName

    if (-not [String]::IsNullOrEmpty($env:TEST_USERNAME) -and -not [String]::IsNullOrEmpty($env:TEST_PASSWORD)) {

        Write-TraceLog "Initialize-Credentials: Credential discovered from ADO environment variables"

        if (-not [String]::IsNullOrEmpty($env:TEST_DOMAIN)) {
            $username = $env:TEST_DOMAIN + "\" + $env:TEST_USERNAME    
        }
        else {
            $username = $env:TEST_USERNAME
        }
        $cred = @{}
        $cred["username"] = $username
        $cred["password"] = $env:TEST_PASSWORD | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    }
    elseif ($domain.ToUpper() -eq "CFDEV.NTTEST.MICROSOFT.COM" -and $env:USERNAME -eq "wolfpack") {        
        Write-TraceLog "Initialize-Credentials: Credential defaulted to cfdev"
        $cred = @{}
        $cred["username"] = "$domain\$env:USERNAME"
        $cred["password"] = $env:USERNAME | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    }

    if ($cred -ne $null) {
        Set-TurnKeySdnInternalConfig -configType credential -config $cred
    }
    else {
        Write-TraceLog "Initialize-Credentials: Failed to initialize credentials" -Warning
    }
}

function Initialize-WorkloadVMCredentials {

    # initialize workload vm credentials

    $cred = @{}

    if(-not [string]::IsNullOREmpty("$Env:TURNKEY_WORKLOAD_USERNAME") -and -not [string]::IsNullOREmpty("$Env:TURNKEY_WORKLOAD_PASSWORD")) {
        $cred["username"] = $Env:TURNKEY_WORKLOAD_USERNAME
        $cred["password"] = $Env:TURNKEY_WORKLOAD_PASSWORD | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    } else {
        $userName = "administrator"
        $cred["username"] = $userName
        $cred["password"] =$userName | ConvertTo-SecureString -AsPlainText -Force | ConvertFrom-SecureString
    }

    Set-TurnKeySdnInternalConfig -configType workloadcredential -config $cred
}


function Initialize-DomainConfiguration {

    $infraconfig = Get-DeploymentConfig
    $domain = $infraconfig.domainName

    if ([String]::IsNullOrEmpty($domain) -and ((Get-WmiObject -Class Win32_ComputerSystem).PartOfdomain)) {
        $domain = (Get-WmiObject -Class Win32_ComputerSystem).Domain
        $infraconfig.domainName = $domain
        Write-TraceLog "Domain is set to $domain"
    }
    else {
        Write-TraceLog "Domain is not set, using local machine domain"
    }

    Set-DeploymentConfig -deploymentConfig $infraconfig
}

function Initialize-HyperVHostConfig {
    $infraconfig = Get-DeploymentConfig

    try {

        if ($infraconfig.hyperVHosts -ne $null -and $infraconfig.hyperVHosts.Count -gt 0) {
            return
        }

        $infraconfig.hyperVHosts = @()

        if (Test-IsCluster) {
            (get-cluster | get-clusternode).Name | ForEach-Object { $infraconfig.hyperVHosts += $_ }
        }
        else {
            $infraconfig.hyperVHosts += (hostname)
        }

        Set-DeploymentConfig -deploymentConfig $infraconfig
    }
    finally {
        $hosts = $infraconfig.hyperVHosts -join ","
        Write-TraceLog "Initialize-HyperVHostConfig: Hosts are $hosts"
    }
}

function Initialize-VSwitchConfig {

    $infraconfig = Get-DeploymentConfig
    if ([String]::IsNullOrEmpty($infraconfig.internetSwitchName)) {
        $infraconfig.internetSwitchName = Resolve-InternetSwitch
    }

    if ([String]::IsNullOrEmpty($infraconfig.internetSwitchName)) {

        $iSwitchCreated = $true
        $infraconfig.hyperVHosts | ForEach-Object {
            Write-TraceLog "Initialize-VSwitchConfig: Attempting to auto create internet switch on host ($_)"
            $iSwitchCreated = $iSwitchCreated -band (New-InternetSwitch -computer $_ -name "corp")
        }

        if ($iSwitchCreated) {
            $infraconfig.internetSwitchName = "corp"
        }
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $infraconfig.sdnSwitchName = Resolve-SdnSwitch
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $sdnSwitchCreated = $true
        $infraconfig.hyperVHosts | ForEach-Object {
            Write-TraceLog "Initialize-VSwitchConfig: Attempting to auto create sdn switch on host ($_)"
            $sdnSwitchCreated = $sdnSwitchCreated -band (New-SdnSwitch -computer $_ -name "sdnswitch")
        }

        if ($sdnSwitchCreated) {
            $infraconfig.sdnSwitchName = "sdnswitch"
        }
    }

    if ([String]::IsNullOrEmpty($infraconfig.sdnSwitchName)) {
        $infraconfig.sdnSwitchName = $infraconfig.internetSwitchName
        Write-TraceLog "Initialize-VSwitchConfig: Using internetswitch for sdn"
    }

    Write-TraceLog "Initialize-VSwitchConfig: Internet Switch is $($infraconfig.internetSwitchName)"
    Write-TraceLog "Initialize-VSwitchConfig: Sdn Switch is $($infraconfig.sdnSwitchName)"
    Set-DeploymentConfig -deploymentConfig $infraconfig
}

function Initialize-DeploymentVhd {

    $depConfig = Get-DeploymentConfig

    if (-not [String]::IsNullOrEmpty($depConfig.vhdPath) -and `
            -not [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        $vhd = Join-Path $depConfig.vhdPath $depConfig.vhdFile
        Write-TraceLog "Initialize-DeploymentVhd: Deployment VHD is $vhd"
        return
    }

    Write-TraceLog "Initialize-DeploymentVhd: Trying to auto discover deployment vhd"

    if (-not [String]::IsNullOrEmpty($depConfig.windowsBuild)) {
        Write-TraceLog "Initialize-DeploymentVhd: Using specified build $($depConfig.windowsBuild)"
    }

    $bestBuild = Get-BestBuild -SpecificBuild $depConfig.windowsBuild -WindowsBuildBranch $depConfig.windowsBuildBranch

    if ([String]::IsNullOrEmpty($bestBuild)) {
        throw "Initialize-DeploymentVhd: Failed to auto detect a build to use for deployment, please set vhd location manually using Set-TurnKeySdnDeploymentVhd"
    }

    $buildVhd = Get-BuildVersionVhd -buildLocation $bestBuild

    if ([String]::IsNullOrEmpty($buildVhd)) {
        throw "Initialize-DeploymentVhd: Vhd auto discover failed, please set vhd location manually using Set-TurnKeySdnDeploymentVhd"
    }

    $vhdPath = Join-Path (Get-VhdStore) "deploymentvhd"
    
    if (-not (Test-Path $vhdPath)) {
        New-Item $vhdPath -ItemType Directory -ErrorAction Stop | Out-Null
    }


    $fileName = Split-Path $buildVhd -leaf
    $dest = Join-Path $vhdPath $fileName

    if (Test-Path $dest) {
        Write-TraceLog "Initialize-DeploymentVhd: Vhd already exists at $dest, skipping copy"        
    }
    else {
        Write-TraceLog "Initialize-DeploymentVhd: Vhd already exists at $dest, skipping copy"
    } else {
        Write-TraceLog "Initialize-DeploymentVhd: Resolved vhd to $buildVhd"
        Write-TraceLog "Initialize-DeploymentVhd: Copying vhd to $vhdPath"
        #Copy-Item $buildVhd $vhdPath
        try {
            Start-BitsTransfer -Source $buildVhd -Destination $vhdPath -Description $buildVhd -DisplayName "CopyFromWinbuild"
        }
        catch {
            Copy-Item $buildVhd $vhdPath
        }
    }

    $vhdFile = $buildVhd.Split("\") | Select -Last 1
    $depConfig.vhdPath = $vhdPath
    $depConfig.vhdFile = $vhdFile

    Set-DeploymentConfig -deploymentConfig $depConfig
}

function Initialize-DeploymentPath {
    $depConfig = Get-DeploymentConfig

    $defaultPath = $depConfig.vmLocation
    $csv = Get-CSV
    $isCsv = $csv -ne $null

    if ([String]::isnullorempty($defaultPath)) {

        if ($isCsv) {
            $defaultPath = Join-Path $csv "sdnvms"
        }
        else {
            # Use a local default path
            $defaultPath = Join-Path $env:SystemDrive "sdnvms"
        }
    }

    $sb = {
        param ($defaultPath)
        if (-not $(Test-Path($defaultPath))) {
            try {
                New-Item $defaultPath -ItemType Directory -Force -ErrorAction Stop | Out-Null
            }
            catch {
                throw "Unable to access deployment path $defaultPath"
            }
        }
    }

    if (-not $isCsv) {
        #check all hosts
        Invoke-CmdOnInfraHosts -scriptBlock $sb -args @($defaultPath)
    }
    else {
        Invoke-Command -ScriptBlock $sb -ArgumentList @($defaultPath)
    }

    $depConfig.vmLocation = $defaultPath

    Set-DeploymentConfig -deploymentConfig $depConfig
}

function Initialize-NetworkControllerPackage {
    $sdnConfig = get-sdnconfig

    if ([String]::IsNullOrEmpty($sdnconfig.networkController.restIpAddress) -and `
            [String]::IsNullOrEmpty($sdnconfig.networkController.restName)) {
        # Auto generate rest name
        $depId = Get-TurnKeySdnDeploymentId
        $sdnConfig.networkController.restName = "NC-$depId"
        $infraconfig = Get-DeploymentConfig
        if (-not [String]::IsNullOrEmpty($infraconfig.domainName)) {
            $sdnConfig.networkController.restName += "." + $infraconfig.domainName
        }
    }

    Write-TraceLog "Initialize-NetworkControllerPackage: RestName $($sdnConfig.networkController.restName)"

    if ($sdnConfig.networkController.runtime -ne "FC") {
        Set-TurnKeySdnConfig -sdnConfig $sdnConfig
        return
    }

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.PowershellModuleRootPath)) {
        $sdnConfig.networkController.FC.PowershellModuleRootPath = Get-DefaultNetworkControllerPsModulePath
    }

    # Sdnexpress fetches this global var

    $Global:FCNC_MODULE_PATH_ROOT = $sdnConfig.networkController.FC.PowershellModuleRootPath

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.PackageLocation)) {
        $sdnConfig.networkController.FC.PackageLocation = Get-DefaultNetworkControllerPackageLocation
        #$sdnConfig.networkController.FC.PackageLocation = "C:\NetworkController"
    }

    if ([String]::IsNullOrEmpty($sdnConfig.networkController.FC.DatabaseLocation)) {
        $sdnConfig.networkController.FC.DatabaseLocation = Get-DefaultNetworkControllerDBLocation
    }

    Write-TraceLog "Initialize-NetworkControllerPackage: PsModule $($sdnConfig.networkController.FC.PowershellModuleRootPath)"
    Write-TraceLog "Initialize-NetworkControllerPackage: PackageLocation $($sdnConfig.networkController.FC.PackageLocation)"
    Write-TraceLog "Initialize-NetworkControllerPackage: DatabaseLocation $($sdnConfig.networkController.FC.DatabaseLocation)"

    Set-TurnKeySdnConfig -sdnConfig $sdnConfig
}

function Initialize-SdnExpressConfig { 
    param(
        [Parameter(Mandatory = $true)] $sdnExpConfig
    )

    $infraconfig = Get-DeploymentConfig

    $sdnExpConfig.JoinDomain = $infraconfig.domainName
    $cred = Get-TurnKeySdnCred
    $secPass = $cred.password | ConvertFrom-SecureString

    $sdnExpConfig.DomainJoinUsername = $cred.username
    $sdnExpConfig.DomainJoinSecurePassword = $secPass
    $sdnExpConfig.LocalAdminSecurePassword = $secPass
    $sdnExpConfig.LocalAdminDomainUser = $cred.username

    $sdnExpConfig.NCUsername = $cred.username
    $sdnExpConfig.NCSecurePassword = $secPass

    $sdnExpConfig.SwitchName = $infraconfig.sdnSwitchName

    $sdnExpConfig.VHDPath = $infraconfig.vhdPath
    $sdnExpConfig.VHDFile = $infraconfig.vhdFile

    $sdnExpConfig.VMLocation = $infraconfig.vmLocation

    $defaultMacPoolConfig = Get-DefaultMacPoolConfig
    $sdnExpConfig.SDNMacPoolStart = $defaultMacPoolConfig.properties.startMacAddress
    $sdnExpConfig.SDNMacPoolEnd = $defaultMacPoolConfig.properties.endMacAddress

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $sdnExpConfig.ManagementSubnet = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix
    $sdnExpConfig.ManagementGateway = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.ManagementDNS = $mgmtNetConfig.properties.subnets[0].properties.dnsServers
    $sdnExpConfig.ManagementVLANID = $mgmtNetConfig.properties.subnets[0].properties.VLANID
    #$sdnExpConfig.ManagementRoutes = @($mgmtNetConfig.properties.subnets[0].properties.addressPrefix)
    $sdnExpConfig.ManagementRoutes = @()

    $paNetConfig = Get-HnvPaNetworkConfig
    $sdnExpConfig.PASubnet = $paNetConfig.properties.subnets[0].properties.addressPrefix
    $sdnExpConfig.PAGateway = $paNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.PAVLANID = $paNetConfig.properties.subnets[0].properties.vlanID
    $sdnExpConfig.PAPoolStart = $paNetConfig.properties.subnets[0].properties.ipPools[0].properties.startIpAddress
    $sdnExpConfig.PAPoolEnd = $paNetConfig.properties.subnets[0].properties.ipPools[0].properties.endIpAddress

    $currentHostName = $(hostname)
    $depId = Get-TurnKeySdnDeploymentId

    $sdnConfig = get-sdnconfig

    $sdnExpConfig['RestName'] = $sdnconfig.networkController.restName
    $sdnExpConfig['RestIpAddress'] = $sdnConfig.networkController.restIpAddress
    $mgmtIpOffset = 20

    if ([string]::IsNullOrEmpty($sdnconfig.networkController.restName) -or `
            $sdnConfig.networkController.runtime -eq "FC") {
        # Restip auto discovery
        # For FC always generate IP
        # Otherwise use IP if restName is not set
        if ([String]::IsNullOrEmpty($sdnConfig.networkController.restIpAddress)) {
            $restIp = Get-IPAddressInSubnet -Subnet $sdnExpConfig.ManagementSubnet -Offset $mgmtIpOffset
            $prefix = $sdnExpConfig.ManagementSubnet.Split("/")[1]
            $sdnExpConfig['RestIpAddress'] = "$restIp/$prefix"
            $mgmtIpOffset++
        }
    }


    $sdnExpConfig['HyperVHosts'] = $infraconfig.hyperVHosts
    $paPoolOffset = 0

    if ($sdnConfig.networkController.runtime -eq "FC") {
        $sdnExpConfig.UseFCNC = 1
        $sdnExpConfig.FCNCBins = $sdnConfig.networkController.FC.PackageLocation
        $sdnExpConfig.FCNCDBs = $sdnConfig.networkController.FC.DatabaseLocation       
    }
    else {

        $i = 0
        $sdnExpConfig['NCs'] = @()
        $NCNodeCount = $sdnConfig.networkController.SF.nodecount

        while ($i -lt $NCNodeCount) {
            $ncConfig = @{}
            $ncConfig['ComputerName'] = "$depId-NC$i"

            $ncConfig['ManagementIP'] = Get-IPAddressInSubnet -Subnet $sdnExpConfig.ManagementSubnet -Offset $mgmtIpOffset
            $mgmtIpOffset++
            $sdnExpConfig['NCs'] += $ncConfig
            $i++
        }
    }

    $muxNodeCount = $sdnConfig.mux.nodecount
    $i = 0
    $sdnExpConfig['Muxes'] = @()

    if ($sdnConfig.mux.enabled -eq "true") {
        while ($i -lt $muxNodeCount) {
            $mux = @{}
            $mux['ComputerName'] = "$depId-MUX$i"
            $mux['ManagementIP'] = Get-IPAddressInSubnet -Subnet $sdnExpConfig.ManagementSubnet -Offset $mgmtIpOffset
            $mgmtIpOffset++

            $mux['PAIPAddress'] = Get-IPAddressInSubnet -Subnet $sdnExpConfig.PAPoolStart -Offset $paPoolOffset
            $paPoolOffset++

            $sdnExpConfig['Muxes'] += $mux
            $i++
        }
    }

    $sdnExpConfig.ManagementRoutes = Get-MsftCorpRoutes

    $gwNodeCount = $sdnConfig.gateway.nodecount
    $i = 0
    $sdnExpConfig['Gateways'] = @()
    if ($sdnConfig.gateway.enabled -eq "true") {
        $poolType = "IPSec"

        if ($sdnConfig.gateway.gre.enabled -eq "true") {
            $gre = Get-GreVipLogicalNetworkConfig
            $sdnExpConfig['GRESubnet'] = $gre.properties.subnets[0].properties.addressPrefix
            $poolType = "All"
        }

        while ($i -lt $gwNodeCount) {
            $gw = @{}
            $gw['ComputerName'] = "$depId-GW$i"
            $gw['ManagementIP'] = Get-IPAddressInSubnet -Subnet $sdnExpConfig.ManagementSubnet -Offset $mgmtIpOffset
            $mgmtIpOffset++

            $sdnExpConfig['Gateways'] += $gw
            $i++
        }

        $sdnExpConfig['GatewayPoolType'] = $poolType

        if ($gwNodeCount -eq 1) {
            $sdnExpConfig.RedundantCount = 0
        }
    }

    if ($paPoolOffset -lt 10) {
        # Add some space for future scaling
        $paPoolOffset = 10
    }

    $sdnExpConfig.PAPoolStart = Get-IPAddressInSubnet -Subnet $sdnExpConfig.PAPoolStart -Offset $paPoolOffset

    $publicVipNetwork = Get-PublicVipNetworkConfig
    $sdnExpConfig.PublicVIPSubnet = $publicVipNetwork.properties.subnets[0].properties.addressPrefix

    $sdnExpConfig.Routers[0].RouterIPAddress = $paNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnExpConfig.Routers[0].RouterASN = $sdnConfig.mux.peerRouterASN
    $sdnExpConfig.SDNASN = $sdnConfig.mux.asn

    return $sdnExpConfig
}

function Initialize-SdnWorkloadDeploymentPath {
    $workloadConfig = Get-TurnKeySdnWorkloadConfig

    $defaultPath = $workloadConfig.deploymentpath
    $csv = Get-CSV
    $isCsv = $csv -ne $null

    if ([String]::isnullorempty($defaultPath)) {

        if ($isCsv) {
            $defaultPath = Join-Path $csv "SdnWorkload"
        }
        else {
            # Use a local default path
            $defaultPath = Join-Path $env:SystemDrive "SdnWorkload"
        }
    }

    $sb = {
        param ($defaultPath)
        if (-not $(Test-Path($defaultPath))) {
            try {
                New-Item $defaultPath -ItemType Directory -Force -ErrorAction Stop | Out-Null
            }
            catch {
                throw "Unable to access deployment path $defaultPath"
            }
        }
    }

    if (-not $isCsv) {
        #check all hosts
        Invoke-CmdOnInfraHosts -scriptBlock $sb -args @($defaultPath)
    }
    else {
        Invoke-Command -ScriptBlock $sb -ArgumentList @($defaultPath)
    }

    $workloadConfig.deploymentpath = $defaultPath
    Set-TurnKeySdnWorkloadConfig -config $workloadConfig
}

function Invoke-CmdOnInfraHosts {
    param(
        [parameter(Mandatory = $true)][ScriptBlock] $scriptBlock,
        [parameter(Mandatory = $false)][System.Array] $args = @()
    )

    $cred = Get-TurnKeySdnCred
    $hypervHosts = Get-TurnKeySdnHyperVHosts

    $results = @()
    foreach ($h in $hypervHosts) {
        $result = Invoke-ReliableCommand -computerName $h -credential $cred -scriptBlock $scriptBlock -argumentList $args
        $results += $result
    }

    return $results
}

function Get-SdnWorkloadVhd {

    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory = $false)][string] $os = "Linux",

        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory = $false)][string] $osSku = "Mariner",

        [parameter(Mandatory=$false)][bool] $useDefaultWindows = $false
    )

    if ($useDefaultWindows) {

        Write-TraceLog "Get-SdnWorkloadVhd: Using default windows vhd"
        $vhdPath = Get-DeploymentConfig | Select -ExpandProperty vhdPath
        $vhdFile = Get-DeploymentConfig | Select -ExpandProperty vhdFile
        return @($vhdPath, $vhdFile)
    }

    $defaultWorkloadVhd = Join-Path (Get-DefaultWorkloadVhdShare) (Get-DefaultWorkloadVhdFile)
    $msg = "Retry after running. A default vhd can be found at $defaultWorkloadVhd."

    $workloadConfig = Get-TurnKeySdnWorkloadConfig
    $osImageConfig = $workloadConfig.imageconfigs | Where-Object { $_.osType -eq $os -and $_.osSku -eq $osSku } | Select -First 1

    if ([String]::IsNullOrEmpty($osImageConfig.vhdPath) -or [string]::IsNullOrEmpty($osImageConfig.vhdFile)) {

        throw "Vhd location for image $($osImageConfig.osType) - $($osImageConfig.osSku) is not set. $msg"
    }

    $vhd = Join-Path $osImageConfig.vhdPath $osImageConfig.vhdFile

    if (-not $(Test-Path($vhd))) {
        throw "$vhd not found. $msg"
    }

    return @($osImageConfig.vhdPath, $osImageConfig.vhdFile)
}

function Enable-Hosts
{
    param (
        [parameter(Mandatory=$true)][array] $Hosts,
        [parameter(Mandatory=$false)][pscredential] $Credential,
        [parameter(Mandatory=$true)][string] $SwitchName,
        [parameter(Mandatory=$true)][string] $MgmtHostNicName,
        [parameter(Mandatory=$true)][string] $ManagementVLANID,
        [parameter(Mandatory=$true)][string] $ManagementSubnet,
        [parameter(Mandatory=$true)][string] $ManagementGateway,
        [parameter(Mandatory=$true)][array] $ManagementDNS,
        [parameter(Mandatory=$true)][bool] $IsFcNc
    )

    $offset = 10
    $prefix = $ManagementSubnet.Split("/")[1]
    $hostNameIPMap = @{}
    $Hosts | Foreach-Object {
            $hostIP = Get-IPAddressInSubnet -subnet $ManagementSubnet -offset $offset
            $hostNameIPMap[$_] = $hostIP
            $offset++
            Write-TraceLog "Enable-Hosts: Host ($_), with IP ($hostIP)"
            Invoke-ReliableCommand -computername $_ -credential $Credential -scriptBlock {
                param(
                    [string]$switchName,
                    [string]$vNicName,
                    [string]$vlanId,
                    [string]$ip,
                    [string]$prefix,
                    [string]$gateway,
                    [array]$dns
                )
                $nic = Get-VMNetworkAdapter -ManagementOS -Name $vNicName -ErrorAction SilentlyContinue

                if ($nic -eq $null)
                {
                    Add-VMNetworkAdapter -ManagementOS -Name $vNicName -SwitchName $switchName
                    $nic = Get-VMNetworkAdapter -ManagementOS -Name $vNicName -ErrorAction SilentlyContinue
                }

                $ifIndex = (Get-NetAdapter -name "*$vNicName*").Ifindex

                Set-VMNetworkAdapterVlan -ManagementOS -VMNetworkAdapterName $vNicName -VlanId $vlanId -access | Out-Null
                $currentIP = Get-NetIPAddress -Interfaceindex $ifIndex -IPAddress $ip -ErrorAction SilentlyContinue -Verbose

                if ($currentIP -eq $null)
                {
                    New-NetIPAddress -Interfaceindex $ifIndex -AddressFamily IPv4 -IPAddress $ip -PrefixLength $prefix | Out-Null
                }
                if ($dns -ne $null -and $dns.Count -gt 0) {
                    Set-DnsClientServerAddress -ServerAddresses $dns -InterfaceIndex $ifIndex -ErrorAction SilentlyContinue
                }

                Set-DnsClientServerAddress -ServerAddresses $dns -InterfaceIndex $ifIndex -ErrorAction SilentlyContinue

            } -ArgumentList @($SwitchName, $MgmtHostNicName, $ManagementVLANID, $hostIP, $prefix, $ManagementGateway, $ManagementDNS)
    }

    Write-TraceLog "Enable-Hosts: Sleeping 5 seconds to allow network to settle"
    Start-Sleep 5

    $Hosts | Foreach-Object { Disable-LinkLocalNics -computerName $_ -credential $Credential}

    if ($IsFcNc) {
        Write-TraceLog "Enable-Hosts: Setting network $ManagementSubnet to ClusterAndClient"
        Wait-ForClusterNetwork -computerName $Hosts[0] -credential $Credential -subnet $ManagementSubnet
        Set-ClusterNetworkToClient -computerName $Hosts[0] -credential $Credential -subnet $ManagementSubnet
    }

    Update-CorpNicDnsServerAddress -hosts $Hosts -credential $Credential -dnsServers $ManagementDNS
    Enable-HostStorageNetwork -ComputerName $Hosts -Credential $Credential -SwitchName $SwitchName

    return $hostNameIPMap
}


function New-TurnKeySdnVM {
    param(
        [parameter(Mandatory = $true)][hashtable] $CreateParams,
        [parameter(Mandatory = $false)][string] $VMCustomTag,
        [parameter(Mandatory = $false)][switch] $WaitForVM
    )
    $vmName = $CreateParams.VMName
    $computerName = $CreateParams.ComputerName
    $cred = Get-TurnKeySdnCred

    try {
        New-SDNExpressVM @CreateParams | Out-Null
    }
    finally {
        
        if ([String]::IsNullOrEmpty($VMCustomTag)) {
            $VMCustomTag = "Client=TurnKeySDN"
        }

        Invoke-ReliableCommand -ComputerName $computerName -Credential $cred -ScriptBlock {
            param($VMName, $VMCustomTag)
            $vm = Get-VM -Name $VMName -ErrorAction SilentlyContinue

            if ($vm -ne $null) {
                $vm | Set-VM -Notes $VMCustomTag
            }
        } -ArgumentList @($vmName, $VMCustomTag)
    }
    
    if ($WaitForVM.IsPresent) {
        WaitforComputerToBeReady -ComputerName $vmName -Credential $cred
    }  
}

function New-TurnKeySdnWorkloadVMUsingSdnExpress {

    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory=$false)][string] $os = "Linux",
        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory=$false)][string] $osSku = "Mariner",
        [parameter(Mandatory=$true)][string] $ncNicResourceId,
        [parameter(Mandatory=$true)][string] $ncRestEndpoint,
        [parameter(Mandatory=$false)][string] $hypervHost = $(hostname),
        [parameter(Mandatory=$false)][pscredential] $hypervHostCred = [pscredential]::Empty,
        [parameter(Mandatory=$false)][pscredential] $vmCreds = $false,
        [parameter(Mandatory=$false)][bool] $useDefaultWindows = $false,
        [parameter(Mandatory=$false)][int] $memoryInGB = 1,
        [parameter(Mandatory=$false)][int] $coreCount = 2,
        [parameter(Mandatory=$false)][bool] $enableSecureBoot = $false

    )

    Write-TraceLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: Server $hypervHost, os $os, osSku $osSku, ncNicResourceId $ncNicResourceId, ncRestEndpoint $ncRestEndpointclient"

    if(-not $useDefaultWindows) {
        if([string]::IsNullOrEmpty($os) -or [string]::IsNullOrEmpty($osSku)) {
            throw "New-TurnKeySdnWorkloadVMUsingSdnExpress: os and osSku must be specified when forcing the default Vhd"
        }
    }

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"
    $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $ncNicResourceId -PassInnerException -ErrorAction SilentlyContinue

    if ($res -eq $null) {
        throw "Networkinterface $ncNicResourceId not found"
    }

    if ($res.Properties.ProvisioningState -ne "Succeeded") {
        throw "Networkinterface $ncNicResourceId provisioning state is $($res.Properties.ProvisioningState), cannot continue with deployment"
    }

    $mac = $res.Properties.PrivateMacAddress
    $ipAddress = $res.Properties.IpConfigurations[0].Properties.privateIpaddress

    $instanceId = $res.InstanceId.Guid

    if ([String]::IsNullOrEmpty($res.Tags.vmName)) {
        $vmName = $ncNicResourceId
    } else {
        $vmName = $res.Tags.vmName
    }

    $vhdConfig = Get-SdnWorkloadVhd -os $os -osSku $osSku -useDefaultWindows $useDefaultWindows
    $deploymentPath = (Get-DeploymentConfig).vmLocation
    $sdnSwitchName = Get-SdnSwitchName
    $vmLocalCredential = Get-TurnKeySdnWorkloadVmCred
    $vmPath = Join-Path $deploymentPath $vmName

    #Copy-FileToRemote -srcPath $vhdConfig[0] -srcFile $vhdConfig[1] -destMachine $hypervHost -destMachineCred $hypervHostCred -destDirectory $vmPath -ErrorAction Stop

    $vmVhdPath = Join-Path $vmPath $vhdConfig[1]
    $isCluster = Test-IsCluster


    $vmNic=@(
        @{Name="SdnNic_$vmName"; MacAddress=$mac; SwitchName=$sdnSwitchName}
    )

    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: Creating VM $vmName on host $hypervHost"
    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: VM VHD Path is $vmVhdPath"
    Write-SdnExpressLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: IP Address is $ipAddress"

    [bool] $isVmCreated = $false

    try {
        if($isCluster) {

            # check if VMs were deployed on any of the hosts in the cluster
            # throws if VM is not found
            $hypervHost = Resolve-HostName -hostName "" -hostCred $hypervHostCred -vmName $vmName -force $true
            $isVmCreated = $true
        } else {

            # sdn express checks if the vm is already present, so let it handle
            $isVmCreated = $false
        }
    } catch {
        Write-TraceLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: VM $vmName not found on any of the hosts in the cluster, creating new VM"
        $isVmCreated = $false
    }

    if(-not $isVmCreated) {

        New-SDNExpressVM -ComputerName $hypervHost `
            -VMLocation $vmPath `
            -VMName $vmName `
            -VHDSrcPath $vhdConfig[0] `
            -VHDName $vhdConfig[1] `
            -SwitchName $sdnSwitchName `
            -CredentialDomain $Env:TEST_DOMAIN  `
            -CredentialUserName $Env:TEST_USERNAME `
            -CredentialPassword $Env:TEST_PASSWORD `
            -JoinDomain "" `
            -LocalAdminPassword $Env:TURNKEY_WORKLOAD_PASSWORD `
            -VMProcessorCount 4 `
            -Nics $vmNic `
            -EnablePreDeploymentNetworkConnectionCheck $false `
            -EnableProcessorCompatibilityForLiveMigration $true | out-null
    }

    # enable misc settings (add to cluster, guest intergation etc..)
    Invoke-Command -ComputerName $hypervHost -Credential $hypervHostCred -ScriptBlock {
        if ($using:isCluster) {
            $vm = Get-VM -Name $using:vmName

            $currentRole = get-clustergroup -name $using:vmName -ErrorAction SilentlyContinue
            if($null -ne $currentRole -and $currentRole.State -ne "Online") {

                Write-TraceLog "New-TurnKeySdnWorkloadVMUsingSdnExpress: VM $($using:vmName) is already part of cluster, removing old group"
                Remove-ClusterGroup -RemoveResources -Force -Name $using:vmName
            }

            $vm | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null
            $vm | Get-VMIntegrationService | Enable-VMIntegrationService | Out-Null

        } else {
            $vm = Get-VM -Name $using:vmName
            $vm | Get-VMIntegrationService | Enable-VMIntegrationService | Out-Null
        }
    }
    Enable-SDNExpressVMPort -ComputerName $hypervHost -VMName $vmName -InstanceId "{$instanceId}" -Credential $hypervHostCred

    $trafficEndpoint = New-TrafficEndpoint -vmName $vmName `
        -hostName $hypervHost `
        -ipAddress $ipAddress `
        -port 5001 `
        -endpointType 0 `
        -vmCredential $vmLocalCredential `
        -hostCredential $hypervHostCred `
        -resolveHostName $false

    return $trafficEndpoint
}



function New-TurnKeySdnWorkloadVM {
    param(
        [ValidateSet("Linux", "Windows")]
        [parameter(Mandatory=$false)][string] $os = "Linux",
        [ValidateSet("Mariner", "WindowsLatest")]
        [parameter(Mandatory=$false)][string] $osSku = "Mariner",
        [parameter(Mandatory=$true)][string] $ncNicResourceId,
        [parameter(Mandatory=$true)][string] $ncRestEndpoint,
        [parameter(Mandatory=$false)][string] $hypervHost = $(hostname),
        [parameter(Mandatory=$false)][pscredential] $hypervHostCred = [pscredential]::Empty,
        [parameter(Mandatory=$false)][bool] $useDefaultWindows = $false,
        [parameter(Mandatory=$false)][int] $memoryInGB = 1,
        [parameter(Mandatory=$false)][int] $coreCount = 2,
        [parameter(Mandatory=$false)][bool] $enableSecureBoot = $false

    )

    Write-TraceLog "New-TurnKeySdnVM: Server $hypervHost, os $os, osSku $osSku, ncNicResourceId $ncNicResourceId, ncRestEndpoint $ncRestEndpointclient"

    if(-not $useDefaultWindows) {
        if([string]::IsNullOrEmpty($os) -or [string]::IsNullOrEmpty($osSku)) {
            throw "New-TurnKeySdnVM: os and osSku must be specified when forcing the default Vhd"
        }
    }

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"
    $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $ncNicResourceId -PassInnerException -ErrorAction SilentlyContinue

    if ($res -eq $null) {
        throw "Networkinterface $ncNicResourceId not found"
    }

    if ($res.Properties.ProvisioningState -ne "Succeeded") {
        throw "Networkinterface $ncNicResourceId provisioning state is $($res.Properties.ProvisioningState), cannot continue with deployment"
    }

    $mac = $res.Properties.PrivateMacAddress
    $instanceId = $res.InstanceId.Guid

    if ([String]::IsNullOrEmpty($res.Tags.vmName)) {
        $vmName = $ncNicResourceId
    }
    else {
        $vmName = $res.Tags.vmName
    }

    if ($AllComputers.Count -ne 0) {
        # If the VM is already present in another, override the name
        $hostName = Get-VMHostName -Computers $AllComputers -Credential $hypervHostCred -VMName $vmName
        if (-not [String]::IsNullOrEmpty($hostName)) {
            $hypervHost = $hostName
        }
    }

    $vhdConfig = Get-SdnWorkloadVhd -os $os -osSku $osSku -useDefaultWindows $useDefaultWindows
    $deploymentPath = Get-SdnWorkloadDeploymentPath
    $sdnSwitchName = Get-SdnSwitchName
    $vmPath = Join-Path $deploymentPath $vmName

    Copy-FileToRemote -srcPath $vhdConfig[0] -srcFile $vhdConfig[1] -destMachine $hypervHost -destMachineCred $hypervHostCred -destDirectory $vmPath -ErrorAction Stop

    $vmVhdPath = Join-Path $vmPath $vhdConfig[1]
    $isCluster = Test-IsCluster
    Invoke-Command -ComputerName $hypervHost -Credential $hypervHostCred -ScriptBlock {

        $vm = Get-VM -Name $using:vmName -ErrorAction SilentlyContinue

        [uint64] $memoryStarupBytes = $using:memoryInGB*1024*1024*1024

        if ($vm -eq $null) {
            New-VM -SwitchName $sdnSwitchName -VHDPath $vmVhdPath -Generation 2 -BootDevice VHD -Path $vmPath -name $vmName -MemoryStartupBytes 512MB `
                -ErrorAction Stop | Out-Null    
        }
        else {
            New-VM -SwitchName $using:sdnSwitchName `
                -VHDPath $using:vmVhdPath `
                -Generation 2 `
                -BootDevice VHD `
                -Path $using:vmPath `
                -name $using:vmName `
                -MemoryStartupBytes $memoryStarupBytes `
                -ErrorAction Stop | Out-Null
        } 

        Set-VMNetworkAdapter -VMName $using:vmName -StaticMacAddress $using:mac -ErrorAction Stop

        if ($using:os -eq "Linux") {
            Set-VMFirmware -EnableSecureBoot On -SecureBootTemplate MicrosoftUEFICertificateAuthority -VMName $using:vmName `
                -ErrorAction Stop
        } elseif($disableSecureBoot) {
            Set-VMFirmware -EnableSecureBoot Off -VMName $using:vmName -ErrorAction Stop
        }

        Get-VM -Name $using:vmName | Set-VM -Notes "Client=TurnKeySDN;Role=Workload;IsInfra=$false;Os=$using:os"
        Start-VM -VMName $using:vmName

        if ($using:isCluster) {
            Get-VM -Name $using:vmName | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null
        }

    } -ErrorAction Stop

    Enable-SDNExpressVMPort -ComputerName $hypervHost -VMName $vmName -InstanceId "{$instanceId}" -Credential $hypervHostCred
}

function Set-VMVLANID {
    param(
        [Parameter(Mandatory = $true)][String]$VMName,
        [Parameter(Mandatory = $true)][pscredential]$Credential,
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][String]$NicName,
        [Parameter(Mandatory = $true)][int]$VLanID
    )

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock {
        param(
            [Parameter(Mandatory = $true)][String]$VMName,
            [Parameter(Mandatory = $true)][String]$NicName,
            [Parameter(Mandatory = $true)][int]$VLanID
        )

        [int]$currentVlan = (Get-VMNetworkAdapterVLan -VMName $VMName -VMNetworkAdapterName $NicName).AccessVlanId

        if ($currentVlan -eq $VLanID) {
            return
        }

        Set-VMNetworkAdapterVlan -Access -VlanId $VLanID -VMName $VMName -VMNetworkAdapterName $NicName
    } -ArgumentList @($VMName, $NicName, $VLanID)
}
function New-RouterVM {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $MgMtIP,
        [Parameter(Mandatory = $true)]
        $MgmtNicMac,  
        [Parameter(Mandatory = $true)]
        $PAIP,
        [Parameter(Mandatory = $true)]
        $PANicMac,        
        [Parameter(Mandatory = $true)]
        $CorpNicMac,        
        [switch]
        $Force
    )

    $depConfig = Get-DeploymentConfig
    $cred = Get-TurnKeySdnCred

    Write-TraceLog "New-RouterVM: Creating VM $vmName"  

    $hyperVHosts = $depConfig.hyperVHosts
    $vmHost = Get-VMHostName -Computers $hyperVHosts -Credential $cred -VMName $vmName 
    
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtVlanId = $mgmtNetConfig.properties.subnets[0].properties.VLANID
    
    $paVlanId = Get-HnvPaNetworkConfig
    $paVlanId = $paVlanId.properties.subnets[0].properties.vlanID
    
    $managementNicName = "Management"
    $paNicName = "HNVPA"
    
    if ($vmHost -ne $null) {
       
        if ($Force.IsPresent) {
            Write-TraceLog "New-RouterVM: Recreating the VM"
            Remove-DomainJoinedVMs -Tag "Role=Router"            
        }
        else {
            Write-TraceLog "New-RouterVM: VM already exists, skipping creation"
            # Reset VLAN if it changes between deployments.
            Set-VMVLANID -VMName $vmName -Credential $cred -ComputerName $vmHost -NicName $managementNicName -VLanID $mgmtVlanId
            Set-VMVLANID -VMName $vmName -Credential $cred -ComputerName $vmHost -NicName $paNicName -VLanID $paVlanId
            return
        }
    }

    $createparams = @{
        'ComputerName'        = $(hostname);
        'VMLocation'          = $depConfig.vmLocation;
        'VMName'              = $vmName;
        'VHDSrcPath'          = $depConfig.vhdPath;
        'VHDName'             = $depConfig.vhdFile;
        'VMMemory'            = 4GB;
        'VMProcessorCount'    = 4;
        'Nics'                = @();
        'CredentialDomain'    = $cred.username.Split("\")[0];
        'CredentialUserName'  = $cred.username.Split("\")[1];
        'CredentialPassword'  = $cred.GetNetworkCredential().password;
        'JoinDomain'          = $depConfig.domainName;
        'LocalAdminPassword'  = $cred.GetNetworkCredential().password;
        'DomainAdminDomain'   = $cred.username.Split("\")[0];
        'DomainAdminUserName' = $cred.username.Split("\")[1];
        'SwitchName'          = $depConfig.sdnSwitchName;
    }
  
    $dnsServers = $mgmtNetConfig.properties.subnets[0].properties.dnsServers

    $createparams.Nics = @(
        @{Name = $managementNicName; MacAddress = $MgmtNicMac; VLANID = $mgmtVlanId; IPAddress = $MgMtIP; SwitchName = $depConfig.sdnSwitchName; DNS = $dnsServers; IsMuxPA = $true },
        @{Name = "Corp"; MacAddress = $CorpNicMac; SwitchName = $depConfig.internetSwitchName; IsMuxPA = $true },
        @{Name = $paNicName; MacAddress = $PANicMac; VLANID = $paVlanId; IPAddress = $PAIP; SwitchName = $depConfig.sdnSwitchName; DNS = $dnsServers; IsMuxPA = $true }
    ) 

    $createparams.Roles = @("RemoteAccess", "RemoteAccessServer", "RemoteAccessMgmtTools", "RemoteAccessPowerShell", "RasRoutingProtocols", "Web-Application-Proxy")

    New-TurnKeySdnVM -CreateParams $createparams -VMCustomTag "Client=TurnKeySDN;Role=Router;IsInfra=$true" -WaitForVM   

    Start-Sleep 10

    Enable-DefaultFirewallRules -ComputerName $vmName -Credential $cred
}

function Enable-NatOnRouter {
    param(
        [Parameter(Mandatory = $true)]$VMName,
        [Parameter(Mandatory = $true)]$MgmtNicMac,
        [Parameter(Mandatory = $true)]$CorpNicMac,
        [Parameter(Mandatory = $true)]$PANicMac
    )

    $cred = Get-TurnKeySdnCred

    Invoke-ReliableCommand -ComputerName $VMName -Credential $cred -ScriptBlock {
        param(
            [Parameter(Mandatory = $true)]$MgmtNicMac,
            [Parameter(Mandatory = $true)]$CorpNicMac,
            [Parameter(Mandatory = $true)]$PANicMac
        )
        $external = "External"
        $ext = (Get-NetAdapter -Name $External -ErrorAction SilentlyContinue)
        
        if ($ext -eq $null) {
            $corpNic = Get-NetAdapter | Where-Object { $_.MacAddress -eq $CorpNicMac }
            Rename-NetAdapter -Name $corpNic.Name -NewName $external
        }

        $mgmt = "Manangement"
        $mgmtAdapter = (Get-NetAdapter -Name $mgmt -ErrorAction SilentlyContinue)

        if ($mgmtAdapter -eq $null) {
            $mgmtAdapter = Get-NetAdapter | Where-Object { $_.MacAddress -eq $MgmtNicMac }
            Rename-NetAdapter -Name $mgmtAdapter.Name -NewName $mgmt
        }

        $hnvpa = "HNVPA"
        $hnvpaAdapter = (Get-NetAdapter -Name $hnvpa -ErrorAction SilentlyContinue)

        if ($hnvpaAdapter -eq $null) {
            $hnvpaAdapter = Get-NetAdapter | Where-Object { $_.MacAddress -eq $PANicMac }
            Rename-NetAdapter -Name $hnvpaAdapter.Name -NewName $hnvpa
        }

        #Setup NAT for internet Access
        Install-WindowsFeature -name RemoteAccess -IncludeAllSubFeature -IncludeManagementTools
        Install-RemoteAccess -VpnType RoutingOnly

        cmd.exe /c "netsh routing ip nat uninstall" | Out-Null
        cmd.exe /c "netsh routing ip nat install" | Out-Null
        cmd.exe /c "netsh routing ip nat set global tcptimeoutmins=1 udptimeoutmins=0 loglevel=ERROR" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $external" | Out-Null
        cmd.exe /c "netsh routing ip nat set interface $external mode=full" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $mgmt" | Out-Null
        cmd.exe /c "netsh routing ip nat add interface $hnvpa" | Out-Null
    } -ArgumentList @($MgmtNicMac, $CorpNicMac, $PANicMac)
}

function Enable-BgpOnRouter {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $bgpLocalIP,
        [Parameter(Mandatory = $true)]
        $LocalASN
    )

    $cred = Get-TurnKeySdnCred
    Invoke-ReliableCommand -ComputerName $vmName -Credential $cred -ScriptBlock {

        param(
            [Parameter(Mandatory = $true)]
            $bgpLocalIP,
            [Parameter(Mandatory = $true)]
            $LocalASN
        )
        Add-windowsfeature -name RemoteAccess -IncludeAllSubFeature -IncludeManagementTools

        Install-RemoteAccess -RoleType RoutingOnly
       
        try {
            $router = get-bgprouter -ErrorAction SilentlyContinue
        }
        catch {

        }

        if ($router -ne $null) {
            $router | Remove-BgpRouter -force
        }

        Add-BgpRouter -BgpIdentifier $bgpLocalIP -LocalASN $LocalASN

    } -ArgumentList @($bgpLocalIP, $LocalASN)
}

function Add-MuxPeerToBgp {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $bgpPeerName,
        [Parameter(Mandatory = $true)]
        $bgpLocalIP,
        [Parameter(Mandatory = $true)]
        $bgpPeerIP,
        [Parameter(Mandatory = $true)]
        $LocalASN,
        [Parameter(Mandatory = $true)]
        $PeerASN
    )

    $cred = Get-TurnKeySdnCred
    Invoke-ReliableCommand -ComputerName $vmName -Credential $cred -ScriptBlock {

        param(
            [Parameter(Mandatory = $true)]
            $bgpLocalIP,
            [Parameter(Mandatory = $true)]
            $bgpPeerName,
            [Parameter(Mandatory = $true)]
            $bgpPeerIP,
            [Parameter(Mandatory = $true)]
            $LocalASN,
            [Parameter(Mandatory = $true)]
            $PeerASN
        )
       
        Get-BgpRouter -ErrorAction Stop      
        Add-BgpPeer -PeerName $bgpPeerName -PeerIpAddress $bgpPeerIP -PeerASN $PeerASN -LocalIpAddress $bgpLocalIP
    } -ArgumentList @($bgpLocalIP, $bgpPeerName, $bgpPeerIP, $LocalASN, $PeerASN)
}


function Set-StaticRoute {
    param(
        [array] $Computer,
        [pscredential] $Credential = [pscredential]::Empty,
        [string] $SwitchName,
        [string] $NicName,
        [string] $DestinationPrefix,
        [string] $NextHop
    )

    Write-TraceLog "Set-StaticRoute: Computer $Computer Setting route for $DestinationPrefix to $NextHop, NicName $NicName"
    $Computer | Foreach-Object {
        Invoke-ReliableCommand -computerName $_ -Credential $Credential -ScriptBlock {
            param(
                [string] $SwitchName,
                [string] $NicName,
                [string] $DestinationPrefix,
                [string] $NextHop
            )

            if (-not [String]::isnullorempty($SwitchName)) {
                $hostVNic = Get-VMNetworkAdapter -ManagementOS -Name $NicName -switchName $SwitchName -ErrorAction Stop
                $adapter = Get-NetAdapter | Where-Object { $($_.MacAddress -replace "-", "") -eq $($hostVNic.MacAddress) }
            }
            else {
                $adapter = Get-NetAdapter -Name $NicName -ErrorAction Stop
            }

            $index = $adapter.ifIndex

            $route = Get-NetRoute -InterfaceIndex $index -NextHop $NextHop -DestinationPrefix $DestinationPrefix -ErrorAction SilentlyContinue

            if ($route -ne $null) {
                return
            }

            New-NetRoute -DestinationPrefix $DestinationPrefix -InterfaceIndex $index -NextHop $NextHop -ErrorAction Stop | Out-Null
        } -ArgumentList @($SwitchName, $NicName, $DestinationPrefix, $NextHop)
    }
}

function Set-SDNVMNotes {

    $cred = Get-TurnKeySdnCred
    $hyperVHosts = Get-TurnKeySdnHyperVHosts
    $depId = Get-TurnKeySdnDeploymentId

    Write-TraceLog "Set-SDNVMNotes: Setting custom tags on SDN VMs"

    foreach ($h in $hyperVHosts) {
        Invoke-ReliableCommand -ComputerName $h -Credential $cred -ScriptBlock {
            param($depId)
            $ncPrefix = "$depId-NC*"

            $ncVMs = Get-VM -Name $ncPrefix -ErrorAction SilentlyContinue
            if ($ncVMs -ne $null) {
                $ncVMs | Set-VM -Notes "Client=TurnKeySDN;Role=NetworkController;IsSDN=$true"
            }

            $muxPrefix = "$depId-MUX*"
            $muxVMs = Get-VM -Name $muxPrefix -ErrorAction SilentlyContinue
            if ($muxVMs -ne $null) {
                $muxVMs | Set-VM -Notes "Client=TurnKeySDN;Role=MUX;IsSDN=$true"
            }

            $gwPrefix = "$depId-GW*"
            $gwVMs = Get-VM -Name $gwPrefix -ErrorAction SilentlyContinue
            if ($gwVMs -ne $null) {
                $gwVMs | Set-VM -Notes "Client=TurnKeySDN;Role=Gateway;IsSDN=$true"
            }

        } -ArgumentList $($depId)
    }
}

function Invoke-SDNExpress {
    param(
        [Parameter(Mandatory = $true)]$sdnExpConfig
    )

    if ($sdnExpConfig -eq $null) {
        throw "Please load the config and retry"
    }

    $sdnExpress = Get-SdnExpressScript
    $path = Resolve-Path $sdnExpress
    $parent = Split-Path $(Resolve-Path $sdnExpress)
    $retryCount = 0
    while ($retryCount -lt 2) {
        Push-Location $parent
        try {
            & $sdnExpress -ConfigurationData $sdnExpConfig
            break
        }
        catch {
            Write-TraceLog "Invoke-SDNExpress: Sdnexpress failed, error $_"
            $retryCount++

            if ($retryCount -eq 2) {
                throw
            }

            Write-TraceLog "Invoke-SDNExpress: Sdnexpress retrying in 60seconds, RetryCount $retryCount"
            Start-Sleep 60
        }
        finally {
            Pop-Location
        }
    }
}

function Set-PostInstallConfig {
    param(
        [Parameter(Mandatory = $true)]$sdnExpConfig,
        [Parameter(Mandatory = $true)]$hostNameIPMap        
    )

    if ($sdnExpConfig.UseFCNC) {
        return
    }

    $cred = Get-TurnKeySdnCred

    $sdnExpConfig['NCs'] | ForEach-Object {
        Write-TraceLog "Set-PostInstallConfig: [SLBMWorkaround] Updating host file on NC VM $_" -Warning
        # Froce SLBM to use the expected IP of the host. SLBM breaks if host IP changes.
        Invoke-ReliableCommand -ComputerName $_['ComputerName'] -credential $cred -ScriptBlock {

            param($hostIPs)

            function UpdateHostsFile($name, $ip, $skipResolve) {
                if (-not $skipResolve) {
                    $addr = (Resolve-DnsName $name -ErrorAction SilentlyContinue).IPAddress | `
                        Select-Object -First 1
                }

                if ($addr -ne $ip) {
                    Add-Content C:\Windows\System32\drivers\etc\hosts "$ip $name" | out-null
                    return $true
                }

                return $false
            }

            $hosts = $hostIPs.Keys
            $hosts | Foreach-Object {
                $changed = $false
                $name = $_
                $ip = $hostIPs[$name]
                $changed = UpdateHostsFile -name $name -ip $ip -skipResolve $false

                if ($changed -and (-not $name.Contains("."))) {
                    $hostfqdn = "$name.$($env:USERDNSDOMAIN)"
                    UpdateHostsFile -name $hostfqdn -ip $ip -skipResolve $true |  out-null
                }

                if ($changed) {
                    Stop-Process -name SDNSLBM -force -ErrorAction SilentlyContinue
                }
            }

        } -ArgumentList $hostNameIPMap
    }
}

function New-DataCenterGateway {
    param(
        [Parameter(Mandatory = $true)]
        $vmName,
        [Parameter(Mandatory = $true)]
        $IPAddress,
        [Parameter(Mandatory = $true)]
        $MacAddress,
        [Parameter(Mandatory = $true)]
        $VlanID,
        [Parameter(Mandatory = $true)]
        $DefaultGateway,
        [switch]
        $Force
    )

    Write-TraceLog "New-OnPremiseGw: Creating VM $vmName"

    $vm = Get-VM -Name $vmName -ErrorAction SilentlyContinue

    if ($vm -ne $null) {
        if ($Force.IsPresent) {
            Write-TraceLog "New-OnPremiseGw: Recreating the VM"
            $vm | Stop-VM -TurnOff -ErrorAction SilentlyContinue
            $vm | Remove-VM -Force
        }
        else {
            Write-TraceLog "New-OnPremiseGw: NOOP"
            return
        }
    }

    $depConfig = Get-DeploymentConfig
    $cred = Get-TurnKeySdnCred

    $createparams = @{
        'ComputerName'        = $(hostname);
        'VMLocation'          = $depConfig.vmLocation;
        'VMName'              = $vmName;
        'VHDSrcPath'          = $depConfig.vhdPath;
        'VHDName'             = $depConfig.vhdFile;
        'VMMemory'            = 4GB;
        'VMProcessorCount'    = 4;
        'Nics'                = @();
        'CredentialDomain'    = $cred.username.Split("\")[0];
        'CredentialUserName'  = $cred.username.Split("\")[1];
        'CredentialPassword'  = $cred.GetNetworkCredential().password;
        'JoinDomain'          = $depConfig.domainName;
        'LocalAdminPassword'  = $cred.GetNetworkCredential().password;
        'DomainAdminDomain'   = $cred.username.Split("\")[0];
        'DomainAdminUserName' = $cred.username.Split("\")[1];
        'SwitchName'          = $depConfig.sdnSwitchName;
    }

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $dnsServers = $mgmtNetConfig.properties.subnets[0].properties.dnsServers

    $createparams.Nics = @(
        @{Name = "DCNET"; MacAddress = $MacAddress; VLANID = $VlanID; IPAddress = $IPAddress; SwitchName = $depConfig.sdnSwitchName; Gateway = $DefaultGateway; DNS = $dnsServers; IsMuxPA = $false }
    ) 


    $createparams.Roles = @("RemoteAccess", "RemoteAccessServer", "RemoteAccessMgmtTools", "RemoteAccessPowerShell", "RasRoutingProtocols", "Web-Application-Proxy")
    New-TurnKeySdnVM -CreateParams $createparams -VMCustomTag "Client=TurnKeySDN;Role=DataCenterGateway;IsInfra=$false"
}

function Test-DeploymentMachines {

    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts

    if ($hyperVHosts -eq $null -or $hyperVHosts.Count -eq 0) {
        $hyperVHosts = @($(hostname))
    }
    else {
        $curHost = hostname
        $isPresent = ($hyperVHosts | Where-Object { $_.StartsWith($curHost) }) -ne $null

        if (-not $isPresent) {
            $err = "Current host($(hostname)) is not part of the deployment machines, deployment machines are $hyperVHosts"
            $err += "`nSet deployment using Set-TurnKeySdnDeploymentMachines or update $(Get-DeploymentFile)"
            throw $err
        }
    }
}
function Test-CommonConfig {

    if (-not (Get-WmiObject -Class Win32_ComputerSystem).PartOfdomain) {
        throw "Only domain joined setup is supported currently, please join the machine to a domain and retry."
    }

    try {
        $depConfig = Get-DeploymentConfig
    }
    catch {
        Write-TraceLog "Failed to fetch deployment config, please run Initialize-TurnKeySdnDeployment and retry" -Err
        throw
    }

    if ([String]::IsNullOrEmpty($depConfig.domainName)) {
        throw "Invalid domain name."
    }

    try {
        Get-TurnKeySdnCred | Out-Null
    }
    catch {
        throw "Unable to read test credential. Please reset using Set-TurnKeySdnCredentials and retry. Error $_"
    }

    $sdnexpress = Get-SdnExpressModule

    if (-not $(Test-Path $sdnexpress -ErrorAction Stop)) {
        throw "$sdnexpress not found, Please run Set-SdnExpressPath and retry"
    }

    $sdnexpress = Get-SdnExpressScript
    if (-not $(Test-Path $sdnexpress -ErrorAction Stop)) {
        throw "$sdnexpress not found, Please run Set-SdnExpressPath and retry"
    }



    if ([String]::IsNullOrEmpty($depConfig.sdnSwitchName)) {
        throw "SDN Switch $($depConfig.sdnSwitchName) not configured"
    }

    if ([String]::IsNullOrEmpty($depConfig.internetSwitchName) -and $depConfig.useNatNetwork) {
        throw "Internet Switch $($depConfig.internetSwitchName) not configured"
    }

    Test-DeploymentMachines

    $cred = Get-TurnKeySdnCred
    $hyperVHosts = $depConfig.hyperVHosts

    Test-VMSwitchExist -computerName $hyperVHosts -switchName $depConfig.sdnSwitchName -credential $cred

    if ($depConfig.useNatNetwork -and $($depConfig.sdnSwitchName -ne $depConfig.internetSwitchName)) {
        Test-VMSwitchExist -computerName $hyperVHosts -switchName $depConfig.internetSwitchName -credential $cred
    }
}

function Test-NetworkControllerPackage {

    $sdnConfig = Get-SDNConfig

    if ($sdnConfig.networkController.runtime -ne "FC") {
        return
    }

    if (-not (Test-Path $sdnConfig.networkController.FC.PowershellModuleRootPath)) {
        Write-TraceLog "Please run Install-WindowsFeature -name RSAT-NetworkController and retry" -Warning
        Write-TraceLog "To change package loction or update runtime please run Set-TurnKeySdnNCConfig" -Warning

        throw "Networkcontroller PowershellModuleRootPath not found. Configured path is $($sdnConfig.networkController.FC.PowershellModuleRootPath)"
    }

    if (-not (Test-Path $sdnConfig.networkController.FC.PackageLocation)) {
        Write-TraceLog "Please run Install-WindowsFeature -name NetworkController and retry" -Warning
        Write-TraceLog "To change package loction or update runtime please run Set-TurnKeySdnNCConfig" -Warning
        throw "Networkcontroller PackageLocation not found. Configured path is $($sdnConfig.networkController.FC.PackageLocation)"
    }
}

function Test-FreeSpace {
    $depConfig = Get-DeploymentConfig

    if (Test-IsCsvPath -path $depConfig.vmLocation) {
        # For csv skip free space check for now.
        return
    }

    $rpath = Resolve-Path $depConfig.vmLocation

    $scricptBlock = {
        param($rpath)
        $freeGb = $rpath.Drive.Free / (1024 * 1024 * 1024)
        $result = @{}
        $result[$(hostname)] = $freeGb
        return $result
    }

    $results = Invoke-CmdOnInfraHosts -scriptBlock $scricptBlock -args @($rpath)
    $fail = $false
    foreach ($r in $results) {
        $freeSpace = $r[$r.Keys[0]]
        Write-TraceLog "Test-FreeSpace: HypervHost $($r.Keys[0]), Drive $rpath, FreeSpace $freeSpace"
        if ($freeSpace -lt 50) {
            Write-TraceLog "Test-FreeSpace: HypervHost $($r.Keys[0]), has < 50GB in drive $($rpath), FreeSpace $freeSpace" -Warning
            $fail = $true
        }
    }

    if ($fail) {
        Write-TraceLog "Test-FreeSpace: Test failed. Please free up space or change deployment path. Current deployment path is $($depConfig.vmLocation)" -Warning
        Write-TraceLog "Test-FreeSpace: To change the deployment path, please run Set-TurnKeySdnDeploymentPath and retry" -Warning

        throw "All hosts machines should have atleast 50GB free space."
    }
}

function Test-Vhd {
    $depConfig = Get-DeploymentConfig

    if ([String]::IsNullOrEmpty($depConfig.vhdPath) -or [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        Write-TraceLog "Please copy a valid vhd and run Set-TurnKeySdnDeploymentVhd, Eg; Set-TurnKeySdnDeploymentVhd c:\vhd -vhdFile sdnexpress.vhdx" -Warning
        throw "Deployment vhd path is null Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }

    if ([String]::IsNullOrEmpty($depConfig.vhdPath) -or [String]::IsNullOrEmpty($depConfig.vhdFile)) {
        Write-TraceLog "Please copy a valid vhd and run Set-TurnKeySdnDeploymentVhd, Eg; Set-TurnKeySdnDeploymentVhd c:\vhd -vhdFile sdnexpress.vhdx" -Warning
        throw "Deployment vhdFile is null. Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }

    $vhd = Join-Path $depConfig.vhdPath $depConfig.vhdFile

    if (-not $(Test-Path $vhd -ErrorAction Stop)) {
        throw "Deployment vhd ($vhd) not found. Please set a deployment vhd using Set-TurnKeySdnDeploymentVhd and retry."
    }
}
function Test-DeploymentConfig
{
    Test-CommonConfig
    Test-FreeSpace
    Test-Vhd
    Test-NetworkControllerPackage
}

function Test-WorkloadConfig {       
    param(
        [parameter(Mandatory = $false)][string]$workLoadConfig = [String]::Empty
    )

    Write-TraceLog "Test-WorkloadConfig: Testing workload config"

    Test-CommonConfig

    $restoreReplayScript = Get-RestoreReplayScript
    if (-not $(Test-Path $restoreReplayScript -ErrorAction Stop)) {
        throw "Restore replay script not found at $restoreReplayScript"
    }

    if ([String]::IsNullOrEmpty($workLoadConfig)) {
        $workLoadConfig = Get-WorkloadConfigPath
    }

    if (-not $(Test-Path $workLoadConfig -ErrorAction Stop)) {
        throw "Worloadconfig file not found at $workLoadConfig"
    }

    $jsonPath = Join-Path $workLoadConfig "jsons"

    
    if (-not $(Test-Path $jsonPath -ErrorAction Stop)) {
        throw "Jsons folder not found under $workLoadConfig"
    }

    $restName = Get-TurnKeySdnRestEndpoint

    if ([String]::IsNullOrEmpty($restName)) {
        throw "NC RestName is empty, run 'Install-TurnKeySdn' to configure SDN. If SDN is already configured, set rest name using Set-TurnKeySdnRestName"
    }

    Get-SdnWorkloadVhd -os Linux -osSku Mariner | Out-Null
}

function Test-DeploymentLogicalNetworkConfig {

    param(
        [parameter(Mandatory = $true)][PSCustomObject] $Network
    )
    if ($Network.Properties.subnets -eq $null -or $Network.Properties.subnets.Count -eq 0) {
        throw "LogicalNetwork has no subnets"
    }

    if ($Network.Properties.subnets[0].properties.addressPrefix -eq $null) {
        throw "LogicalNetwork has no addressPrefix"
    }

    if ($Network.Properties.subnets[0].properties.ipPools -eq $null -or $Network.Properties.subnets[0].properties.ipPools.Count -eq 0) {
        throw "LogicalNetwork has no ipPools"
    }
}

function Get-InfraVmPrefix {
    if (-not $(Test-IsCluster)) {
        return $(Get-TurnKeySdnDeploymentId)
    }

    $clusterName = (Get-Cluster).Name

    if ($clusterName.Length -lt 8) {
        return $clusterName
    }

    return ($clusterName.Substring(0, 8))
}
function Get-RouterVMName {
    (Get-InfraVmPrefix) + "-Router"
}

function Get-BgpVMName {
    (Get-InfraVmPrefix) + "-BGP"
}

function Get-OnPremiseGWVMName {
    (Get-InfraVmPrefix) + "-OGW"
}


function Enable-LogicalNetworkDefaultGateway {
    param(
        [Parameter(Mandatory = $true)]
        [string]$LogicalNetworkResourceId
    )

    <#
    .SYNOPSIS
        Adds the default gw of the logical network to the nat vm
    .PARAMETER LogicalNetworkResourceId
        Name of the logical network
    #>

    $routerVM = Get-RouterVMName
    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Adding default gateway of logical network $LogicalNetworkResourceId on $routerVM"

    $cred = Get-TurnKeySdnCred
    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts

    $vmHost = Get-VMHostName -Computers $hyperVHosts -Credential $cred -VMName $routerVM 

    if ($vmHost -eq $null) {
        throw "Nat VM $routerVM not found on any hosts"
    }

    $restName = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restName"

    $ncLNET = Get-NetworkControllerLogicalNetwork -ConnectionUri $restUrl -ResourceId $LogicalNetworkResourceId -ErrorAction SilentlyContinue

    if ($ncLNET -eq $null) {
        throw "LogicalNetwork $LogicalNetworkResourceId not found"
    }

    $nicsToAdd = @()
    $ncLNET.Properties.Subnets | ForEach-Object {
        $subnetId = $_.ResourceId
        $ip = $_.Properties.DefaultGateways | Select -First 1
        $prefixLen = $_.Properties.AddressPrefix.Split("/") | Select -Last 1
        $addressPrefix = $_.Properties.AddressPrefix
        $vlan = $_.Properties.VlanID
        $nicProperties = New-Object Microsoft.Windows.NetworkController.NetworkInterfaceProperties
        $nicProperties.privateMacAllocationMethod = "Dynamic"
        $nicProperties.IPConfigurations = @()
        $nicProperties.IPConfigurations += New-Object Microsoft.Windows.NetworkController.NetworkInterfaceIpConfiguration
        $nicProperties.IPConfigurations[0].ResourceId = $subnetId
        $nicProperties.IPConfigurations[0].Properties = New-Object Microsoft.Windows.NetworkController.NetworkInterfaceIpConfigurationProperties
        $nicProperties.IPConfigurations[0].Properties.Subnet = New-Object Microsoft.Windows.NetworkController.Subnet
        $nicProperties.IPConfigurations[0].Properties.Subnet.ResourceRef = $_.ResourceRef
        # Unmanaged, nic is not managed by NC, only for MAC address
        $nicProperties.IPConfigurations[0].Properties.PrivateIPAllocationMethod = "Unmanaged"
        $nicProperties.IPConfigurations[0].Properties.PrivateIPAddress = $ip
        $defaultGwNic = New-NetworkcontrollerNetworkinterface -ConnectionUri $restUrl -ResourceId $subnetId -Properties $nicProperties -Force -PassInnerException
        $defaultGwNic = Get-NetworkcontrollerNetworkinterface -ConnectionUri $restUrl -ResourceId $subnetId
        $mac = $defaultGwNic.Properties.PrivateMacAddress
        Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Lnet Subnet $subnetId, IP $ip, Mac $mac, PrefixLen $prefixLen, Vlan $vlan"
        $nicsToAdd += @{Name = $subnetId; MacAddress = $mac; IPAddress = $ip; PrefixLen = $prefixLen; Vlan = $vlan; AddressPrefix = $addressPrefix }
    }

    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM)
        $vm = Get-VM $routerVM
        if ($vm.Generation -lt 2) {
            Stop-VM -VMName $routerVM -Force
        }        
    } -ArgumentList @($routerVM)
    
    $sdnSwitch = Get-SdnSwitchName
    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: Adding default gateway netadapter to nat vm $routerVM on switch $sdnSwitch"
    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM, $nicsToAdd, $switchName)

        foreach ($nic in $nicsToAdd) {
            Get-VMNetworkAdapter -VMName $routerVM -Name $nic.Name -ErrorAction SilentlyContinue | Remove-VMNetworkAdapter -ErrorAction SilentlyContinue -Confirm:$false
            Add-VMNetworkAdapter -VMName $routerVM -SwitchName $switchName -StaticMacAddress $nic.MacAddress -ErrorAction Stop -Name $nic.Name
            Set-VMNetworkAdapterVLan -VMName $routerVM -VMNetworkAdapterName $nic.Name -Access -VlanId $nic.Vlan -ErrorAction Stop
        }
    } -ArgumentList @($routerVM, $nicsToAdd, $sdnSwitch)

    foreach ($nic in $nicsToAdd) {
        Enable-SDNExpressVMPort -ComputerName $vmHost -VMName $routerVM -VMNetworkAdapterName $nic.Name -Credential $cred
    }

    Invoke-ReliableCommand -computerName $vmHost -Credential $cred -scriptBlock {
        param($routerVM)
        Start-VM -VMName $routerVM
    } -ArgumentList @($routerVM)

    WaitforComputerToBeReady -ComputerName $routerVM -Credential $cred

    Write-TraceLog "Enable-LogicalNetworkDefaultGateway: setting ip address for default gw nics on nat vm $routerVM"
    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        param($nicsToAdd)
        foreach ($nic in $nicsToAdd) {
            $netAdapter = Get-NetAdapter | Where-Object { $($_.MacAddress.split("-") -join "") -eq $nic.MacAddress }
            if ($netAdapter -eq $null) {
                throw "Unable to find nic with mac $($nic.MacAddress)"
            }

            if ($(Get-NetAdapter -Name $nic.Name -ErrorAction SilentlyContinue) -eq $null) {
                $netAdapter | Rename-NetAdapter -NewName $nic.Name -ErrorAction SilentlyContinue
                
                #Rename-NetAdapter : {Object Exists} An attempt was made to create an object and the object name already existed.
            }
            
            # $newName = Get-Netadapter -ifIndex $netAdapter.IfIndex
            # cmd.exe /c "netsh routing ip nat add interface $newName" | Out-Null

            New-NetIPAddress -Interfaceindex $netAdapter.IfIndex -AddressFamily IPv4 -IPAddress $nic.IPAddress -PrefixLength $nic.PrefixLen | Out-Null
        }
    } -ArgumentList @($nicsToAdd)

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    # Forward lnet traffic to nat VM.
    foreach ($nic in $nicsToAdd) {
        Set-StaticRoute -Computer $hyperVHosts -Credential $cred -SwitchName $sdnSwitch -NicName $(Get-MgmtNicName) -DestinationPrefix $nic.AddressPrefix -NextHop $mgmtGw
    }

    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        Restart-Service RemoteAccess -Force
    }
}

function Add-ManagmentRouteToVirtualGateways {

    <#
    .Synopsis
        Adds the management network route to the virtual gateway resource in NC to enable connectivity from the
        host to VNET
    .PARAMETER NCVirtualGateways
        Array of virtual gateway resources in NC.
    .PARAMETER BgpConnections
        NetworkConnections with BGP configured
    #>
    param(
        [Parameter(Mandatory = $true)]
        [Array]$NCVirtualGateways,
        [Parameter(Mandatory = $true)]
        [hashtable]$BgpConnections
        
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtPrefix = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix

    $NCVirtualGateways = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    foreach ($vg in $NCVirtualGateways) {
        [array]$bgpCons = $BgpConnections[$vg.ResourceId]
        $update = $false
        foreach ($nw in $vg.Properties.NetworkConnections) {

            if ($bgpCons -ne $null -and $bgpCons.Contains($nw.ResourceId)) {
                # No need to add mgmt route to networkconnection static route if bgp is enabled.
                # Route is pushed by bgp.
                Write-TraceLog "Add-ManagmentRouteToVirtualGateways: Virtual gateway $($vg.ResourceId) nw $($nw.ResourceId) has bgp enabled, skipping static route addition"
                continue
            }

            $foundMgmt = $false
            foreach ($route in $nw.Properties.Routes) {
                if ($route.DestinationPrefix -eq $mgmtPrefix) {
                    $foundMgmt = $true
                    break
                }
            }

            if (-not $foundMgmt) {
                Write-TraceLog "Add-ManagmentRouteToVirtualGateways: Adding management network $mgmtPrefix to the routes of virtual gateway $($vg.ResourceId)"
                $mgmtRoute = New-Object Microsoft.Windows.NetworkController.RouteInfo
                $mgmtRoute.DestinationPrefix = $mgmtPrefix
                $mgmtRoute.NextHop = "0.0.0.0"
                $mgmtRoute.Metric = 10
                $mgmtRoute.Protocol = "Static"
                $nw.Properties.Routes += $mgmtRoute
                $update = $true
            }
        }

        if ($update) {
            New-NetworkControllerVirtualGateway -ResourceId $vg.ResourceId -Properties $vg.Properties -ConnectionUri $restUrl -Force -PassInnerException | Out-Null
        }
    }
}

function Get-SourceNetworkInterface {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Destination,
        [Parameter(Mandatory = $true)]
        [string]$ComputerName,
        [Parameter(Mandatory = $false)]
        [pscredential]$Credential = [pscredential]::Empty
    )

    $result = Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock {
        param($Destination)
        $result = Test-NetConnection $Destination
        return $result.InterfaceAlias
    } -ArgumentList $($Destination)

    return $result
}

function Update-WorkloadSecrets {
    param(
        [Parameter(Mandatory = $true)]
        [string]$WorkloadConfigPath
    )

    $depId = Get-TurnKeySdnDeploymentId
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath

    foreach ($vgw in $vgwsConfigJson) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if ($con.Properties.connectionType -ieq "IPsec") {
                Write-TraceLog "Update-WorkloadSecrets: Updating VG $($vgwJson.resourceId) shared secret for connection $($con.resourceId)"    
                $con.Properties.IPSecConfiguration.SharedSecret = $depId
            }
        }
        $vgwJson | ConvertTo-Json -Depth 100 | Out-File $vgw.FullName -Force
    }
}
function Update-WorkloadIPAddresses {
    param(
        [Parameter(Mandatory = $true)]
        [string]$WorkloadConfigPath
    )

    $stateFile = Join-Path $WorkloadConfigPath "addressRandmoized"

    if (Test-Path $stateFile) {
        Write-TraceLog "Update-WorkloadIPAddresses: IP addresses already randomized in workload config $WorkloadConfigPath"
        return
    }

    Write-TraceLog "Update-WorkloadIPAddresses: Randomizing workload ip addresses in workload config $WorkloadConfigPath"

    #Randmoize lnet ips
    $lnets = Get-SdnWorkloadLogicalNetworks -WorkloadConfigPath $WorkloadConfigPath

    foreach ($lnet in $lnets) {
        $lnetJson = Get-Content $lnet.FullName | ConvertFrom-Json
        $updatedNetwork = Update-LogicalNetworkAddresses -Network $lnetJson
        $updatedNetwork | ConvertTo-Json -Depth 100 | Out-File $lnet.FullName -Force
    }

    #Randmozie gw networkconnection ips
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath

    foreach ($vgw in $vgwsConfigJson) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if (-not [String]::IsNullOrEmpty($con.Properties.DestinationIPAddress)) {
                $con.Properties.DestinationIPAddress = Update-IPv4Address -Address $con.Properties.DestinationIPAddress
            }
            
            foreach ($address in $con.Properties.ipAddresses) {
                $address.ipAddress = Update-IPv4Address -Address $address.ipAddress
            }
            
            $updatedPeerAddresses = @()
            foreach ($peerAddress in $con.Properties.peerIPAddresses) {
                $updatedPeerAddresses += Update-IPv4Address -Address $peerAddress
            }
            $con.Properties.peerIPAddresses = $updatedPeerAddresses
            
            $routes = $con.Properties.Routes
            foreach ($r in $routes) {
                $r.DestinationPrefix = Update-IPv4Address -Address $r.DestinationPrefix
            }
        }

        foreach ($router in $vgwJson.Properties.bgpRouters) {
            foreach ($peer in $router.Properties.bgpPeers) {
                $peer.Properties.peerIpAddress = Update-IPv4Address -Address $peer.Properties.peerIpAddress
            }
        }

        $vgwJson | ConvertTo-Json -Depth 100 | Out-File $vgw.FullName -Force
    }

    $nics = Get-SdnWorkloadNetworkInterfaces -WorkloadConfigPath $WorkloadConfigPath

    #Randmozie lnet nic ips
    foreach ($nic in $nics) {
        $nicJson = get-content $nic.FullName | ConvertFrom-Json
        foreach ($ipconfig in $nicJson.Properties.IPConfigurations) {

            if ($ipconfig.Properties.subnet.resourceRef -ilike "/virtualNetworks/*") {
                break
            }

            if ($ipconfig.Properties.PrivateIPAddress -ne $null) {
                $ipconfig.Properties.PrivateIPAddress = Update-IPv4Address -Address $ipconfig.Properties.PrivateIPAddress    
            }
        }

        $nicJson | ConvertTo-Json -Depth 100 | Out-File $nic.FullName -Force
    }

    New-Item $stateFile -ItemType File -Force | Out-Null
}

function Add-TurneKeyVMToCluster {

    param(
        [Parameter(Mandatory = $true)]
        $Tag
    )

    Write-TraceLog "Add-TurneKeyVMToCluster: Adding VMs with tag $Tag to cluster"

    $isCluster = Test-IsCluster

    if (-not $isCluster) {
        Write-TraceLog "Add-TurneKeyVMToCluster: Not a cluster, skipping"
        return
    }

    $addVMToCluster = {

        param($Tag)

        $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains $Tag }

        if ($vms -eq $null) {
            return
        }

        foreach ($vm in $vms) {
            $isPresent = (Get-ClusterResource -VMId $vm.VMId -ErrorAction SilentlyContinue) -ne $null

            if ($isPresent) {
                continue
            }

            $vm | Add-ClusterVirtualMachineRole -ErrorAction SilentlyContinue | Out-Null
        }
    }

    Invoke-CmdOnInfraHosts -scriptBlock $addVMToCluster -args @($Tag)
}

function Remove-TurnKeyVMs {
    param(
        [parameter(Mandatory = $true)][string] $Tag
    )

    Write-TraceLog "Remove-TurnKeyVMs: Removing VMs with tag $Tag"
    $isCluster = Test-IsCluster

    $deleteVM = {       
        param($Tag, $IsCluster)
        $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains $Tag }
 
        if ($vms -ne $null) {
            if ($IsCluster) {
                foreach ($vm in $vms) {
                    Remove-ClusterGroup -VMId $vm.VMId -RemoveResources -Force -ErrorAction SilentlyContinue
                }
            }
            
            $hardDrives = ($vms | Select-Object -ExpandProperty HardDrives).path
            $vmStorage = $vms.path
            $vms | Stop-VM -TurnOff -force
            $vms | Remove-VM -Force
 
            if ($hardDrives -ne $null -and $hardDrives.Count -gt 0) {
                Remove-Item $hardDrives -ErrorAction SilentlyContinue -Force -Recurse
            }

            if ($vmStorage -ne $null -and $vmStorage.Count -gt 0) {
                Remove-Item $vmStorage -ErrorAction SilentlyContinue -Force -Recurse
            }   
        }
    }

    Invoke-CmdOnInfraHosts -scriptBlock $deleteVM -args @($Tag, $isCluster)
}
function Remove-WorkloadVMs {

    try {
        Write-TraceLog "Remove-WorkloadVMs: Removing workload vms"
        Remove-DomainJoinedVMs -Tag "Role=DataCenterGateway"
        Remove-TurnKeyVMs -Tag "Role=Workload"
    }
    catch {
        Write-TraceLog "Remove-WorkloadVMs: Failed to remove workload vms, error $_" -Warning
    }
}

function Remove-RouterVMWorkloadAdapters {

    $cleanupRouterVM = {       
        $vms = Get-VM | Where-Object { $_.Notes.Split(";") -icontains "Role=Router" }
        
        if ($vms -eq $null) {
            return
        }
        
        $vms | ForEach-Object {
            Get-VMNetworkAdapter -VM $_ | Where-Object { $_.Name -ine "Management" -and $_.Name -ine "Corp" -and $_.Name -ine "HNVPA" } | Remove-VMNetworkAdapter -Confirm:$false -ErrorAction SilentlyContinue
        }
        
        $vms | Restart-VM -Force -Type Reboot
        
    }

    try {
        Write-TraceLog "Remove-RouterVMWorkloadAdapters: Removing workload adapters from router vm"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupRouterVM
    }
    catch {
        Write-TraceLog "Remove-RouterVMWorkloadAdapters: Failed to remove workload adapters from router vm, error $_" -Warning
    }
}

function Remove-DomainJoinedVMs {
    param(
        [parameter(Mandatory = $true)][string] $Tag
    )

    Write-TraceLog "Remove-DomainJoinedVMs: Removing domain joined vms with tag $Tag"

    $unjoinDomain = {
        param($DomainCredential, $Tag)
        $vms = Get-VM | Where-Object { 
            $_.Notes.Split(";") -icontains $Tag }
        
        if ($vms -ne $null) {
            $vms | ForEach-Object {
                Remove-Computer -UnjoinDomainCredential $DomainCredential -ComputerName $_.Name `
                    -LocalCredential $DomainCredential -ErrorAction SilentlyContinue -Force 
            }          
        }
    }
    
    $cred = Get-TurnKeySdnCred
    Invoke-CmdOnInfraHosts -scriptBlock $unjoinDomain -Args @($cred, $Tag)
    Remove-TurnKeyVMs -Tag $Tag
}
function Remove-SdnConfig {

    try {
        $sdnConfig = Get-SdnConfig
        $fcncdb = $sdnConfig.networkController.FC.DatabaseLocation

        if ([String]::IsNullOrEmpty($fcncdb)) {
            $fcncdb = Get-DefaultNetworkControllerDBLocation
        }

        Write-TraceLog "Uninstall-TurnKeySdn: Workaround - FCNC uninstall does not delete db. Deleting $fcncdb"
        Remove-Item $fcncdb -ErrorAction SilentlyContinue -Recurse -Force
    }
    catch {
        Write-TraceLog "Remove-SdnConfig: Failed to remove sdn config, error $_" -Warning
    }
}

function Remove-SdnVMs {
    try {
        Remove-DomainJoinedVMs -Tag "IsSDN=True"
    }
    catch {
        Write-TraceLog "Remove-SdnVMs: Failed to remove sdn vms, error $_" -Warning
    }
}

function Remove-InfraVMs {
    try {
        Remove-DomainJoinedVMs -Tag "IsInfra=True"
    }
    catch {
        Write-TraceLog "Remove-InfraVMs: Failed to remove infra vms, error $_" -Warning
    }
}

function Remove-PhysicalHostVnic {

    $cleanupScript = {
        param($MgmtHostNicName)
        Remove-VMNetworkAdapter -ManagementOS -Name $MgmtHostNicName -ErrorAction SilentlyContinue
        Remove-VMNetworkAdapter -ManagementOS -Name PA* -ErrorAction SilentlyContinue
        Remove-VMNetworkAdapter -ManagementOS -Name DR* -ErrorAction SilentlyContinue
    }

    try {
        $testNic = Get-MgmtNicName
        Write-TraceLog "Remove-PhysicalHostVnic: Removing $testNic, PA* and DR* adapters from all hosts in the deployment"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($testNic)
    }
    catch {
        Write-TraceLog "Remove-PhysicalHostVnic: Failed to remove from all hosts in the deployment, error $_" -Warning
    }
}

function Remove-TestRoutesOnHostVnic {

    $testNic = Get-MgmtNicName
    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    Write-TraceLog "Remove-TestRoutesOnHostVnic: Removing test routes from vnic $testNic on all hosts in the deployment"

    $cleanupScript = {
        param($MgmtHostNicName, $ManagementGateway)
        $nic = Get-VMNetworkAdapter -ManagementOS -Name $MgmtHostNicName -ErrorAction SilentlyContinue
        if ($nic -eq $null) {
            return
        }

        $adapter = Get-NetAdapter | Where-Object { $($_.MacAddress -replace "-", "") -eq $($nic.MacAddress) }

        if ($adapter -eq $null) {
            return
        }

        $routes = Get-NetRoute -ifIndex $adapter.ifIndex | Where-Object { $_.NextHop -eq $ManagementGateway }
        if ($routes -eq $null) {
            return
        }

        $routes | Remove-NetRoute -Confirm:$false
    }

    Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($testNic, $mgmtGw)
}

function Remove-DeploymentCertificates {

    $cleanupScript = {
        param($depId)
        $rootCerts = Get-ChildItem Cert:\LocalMachine\Root | where-object { $_.Subject -ilike "*$depId*" }
        $rootCerts | Remove-Item -ErrorAction SilentlyContinue -Force
        $myCerts = Get-ChildItem Cert:\LocalMachine\My | where-object { $_.Subject -ilike "*$depId*" }
        $myCerts | Remove-Item -ErrorAction SilentlyContinue -Force
    }

    try {
        $depId = Get-TurnKeySdnDeploymentId
        Write-TraceLog "Remove-DeploymentCertificates: Removing certificates with subject *$depId* from all hosts in the deployment"
        Invoke-CmdOnInfraHosts -scriptBlock $cleanupScript -args @($depId)
    }
    catch {
        Write-TraceLog "Remove-DeploymentCertificates: Failed to remove certificates from all hosts in the deployment, error $_" -Warning
    }
}

function Get-DefaultWorkloadMarinerVhd {

    try {
        Get-SdnWorkloadVhd -os Linux -osSku Mariner | Out-Null
        return $null
    }
    catch {
        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: No workload vhd configured, trying to auto copy"
    }

    try {
        $vhdStore = Get-VhdStore
        $vhdFolder = Join-Path $vhdStore "marinerVhd"
        $vhdFile = Get-DefaultWorkloadVhdFile

        $vhd = Join-Path $vhdFolder (Get-DefaultWorkloadVhdFile)

        if (Test-Path $vhd) {
            Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Existing vhd found at $destVhd"
            return @($vhdFolder, $vhdFile)
        }

        $share = Get-DefaultWorkloadVhdShare
        $cred = Get-WttNAOCred

        $connected = Connect-SMBShare -RemotePath $share -Credential $cred -RestartLanManServer

        if (-not $connected) {
            throw "Unable to connect to $share"
        }

        if (-not $(Test-Path $vhdFolder -ErrorAction Stop)) {
            New-Item $vhdFolder -ItemType Directory | Out-Null
        }

        $shareVhd = Join-Path $share $vhdFile

        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Copying $shareVhd to $vhdFolder"
        try {
            Start-BitsTransfer -Source $shareVhd -Destination $vhdFolder -Description $shareVhd -DisplayName "WorkloadVHD"
        }
        catch {
            Copy-Item $shareVhd $vhdFolder
        }

        return @($vhdFolder, $vhdFile)
    }
    catch {
        Write-TraceLog "Get-DefaultWorkloadMarinerVhd: Failed to auto copy default workload vhd, error $_" -Warning
    }

    return $null
}
function Get-VirtualGatewayVnetPrefix {
    param(
        [Parameter(Mandatory = $true)]
        [string]$VirtualGatewayResourceId
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  
    $vg = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl -ResourceID $VirtualGatewayResourceId
    $subnetEle = $vg.Properties.GatewaySubnets[0].ResourceRef.Split("/", [StringSplitOptions]::RemoveEmptyEntries)
    $vnetId = $subnetEle[1]
    $vnet = Get-NetworkControllerVirtualNetwork -ResourceId $vnetId -ConnectionUri $restUrl
    return $vnet.Properties.AddressSpace.AddressPrefixes
}

function Add-StaticRoutesForVirtualGateways {
    <#
    .SYNOPSIS
        Adds static routes for vnet prefixes corresponding to virtual gateway connections on physical hosts and NAT VM.
    .PARAMETER NCVirtualGateways
        Array of virtual gateway resources in NC.
    #>
    param(
        [Parameter(Mandatory = $true)]
        [Array]$NCVirtualGateways,
        [hashtable]$BgpConnections
    )

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $sdnSwitchName = Get-SdnSwitchName
    $cred = Get-TurnKeySdnCred
    $routerVM = Get-RouterVMName
    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts

    foreach ($vg in $NCVirtualGateways) {

        $destinationPrefixes = Get-VirtualGatewayVnetPrefix -VirtualGatewayResourceId $vg.ResourceId
        [array]$bgpCons = $BgpConnections[$vg.ResourceId]
        foreach ($connection in $vg.Properties.networkConnections) {
            if ($connection.Properties.connectionType -ieq "IPsec") {
                # Next hop for IPsec connection from Router VM is datacenter gateway VM.
                $nextHop = $connection.Properties.DestinationIPAddress 
            }
            
            if ($connection.Properties.connectionType -ieq "L3") {
                # Next hop for L3 connection from Router VM is the L3 IP in tenant compartment of NC gateway VM.
                $nextHop = $connection.Properties.ipAddresses[0].ipAddress 
            }

            $isBgpEnabled = $false
            if ($bgpCons -ne $null -and $bgpCons.Contains($connection.ResourceId)) {
                $isBgpEnabled = $true
            }

            $routerVMNic = Get-SourceNetworkInterface -ComputerName $routerVM -Credential $cred -Destination $nextHop
            Write-TraceLog "Set-StaticRoutesForVirtualGateways: Setting static route for virtual gateway $($vg.ResourceId), `
                connection $($connection.ResourceId) routerVMNic $routerVMNic, nextHop $nextHop"
            foreach ($prefix in $destinationPrefixes) {
                # Add route for each vnet IP prefix on physical host and router vm.
                # For physical host, next hop is the router VM(the default gateway of management network)
                Set-StaticRoute -Computer $hyperVHosts -Credential $cred -SwitchName $sdnSwitchName -NicName $(Get-MgmtNicName) -DestinationPrefix $prefix -NextHop $mgmtGw
                # Ideally if bgp is enabled, VNET routes should come via bgp and static route is not required.
                # But bgp connection is between datacneter gateway and NC gateway VM.
                # There is no upstream bgp btw router vm and datacenter gateway vm.
                # So static routes are still required on router vm.
                # TODO: Add upstream bgp peer to router vm.
                #if (-not $isBgpEnabled) {
                Set-StaticRoute -Computer $routerVM -Credential $cred -NicName $routerVMNic -DestinationPrefix $prefix -NextHop $nextHop
                #}
            }
        }
    }
}

function Get-IPsecSharedSecrets {
    param(
        [Parameter(Mandatory = $true)][String]$WorkloadConfigPath
    )

    $vgwsConfigJsons = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath
    if ($vgwsConfigJsons -eq $null -or $vgwsConfigJsons.Count -eq 0) {
        return @{}
    }

    $sharedSecrets = @{}
    foreach ($vgw in $vgwsConfigJsons) {
        $vgwJson = get-content $vgw.FullName | ConvertFrom-Json
        foreach ($con in $vgwJson.Properties.networkConnections) {
            if ($con.Properties.connectionType -ieq "IPsec") {
                $sharedSecrets[$con.Properties.DestinationIPAddress] = $con.Properties.IpSecConfiguration.sharedSecret
            }
        }
    }

    return $sharedSecrets
}
function Initialize-DataCenterGWVM {
    param(
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][pscredential]$Credential
    )

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -scriptBlock {
        if ((get-remoteaccess).VpnS2SStatus -ne "Installed") {
            Install-RemoteAccess -VpnType VpnS2S
        }
        else {
            Restart-Service RemoteAccess -Force
        }

        $retries = 5
        while ($retries--) {
            try {
                # Wait for RRAS to be ready
                Get-VpnS2SInterface -ErrorAction Stop | Out-Null                     
            }
            catch {
                Start-Sleep 10
                Continue
            }

            break
        }
    }
}
function Add-BgpPeerOnVM {
    param(
        [Parameter(Mandatory = $true)][String]$ComputerName,
        [Parameter(Mandatory = $true)][pscredential]$Credential,
        [Parameter(Mandatory = $true)][String]$RouterIP,
        [Parameter(Mandatory = $true)][String]$LocalASN,
        [Parameter(Mandatory = $true)][String]$PeerIP,
        [Parameter(Mandatory = $true)][String]$PeerASN,
        [Parameter(Mandatory = $true)][Array]$CustomRoutes
    )

    $addBgpPeer = {
        param($RouterIP, $LocalASN, $PeerIP, $PeerASN, $CustomRoutes)
        try {
            $router = Get-BgpRouter -ErrorAction SilentlyContinue
        }
        catch {}

        if ($router -eq $null) {
            Add-BgpRouter -BgpIdentifier $RouterIP -LocalASN $LocalASN | Out-Null
        }

        $bgpPeer = Get-BgpPeer -Name $PeerIP -ErrorAction SilentlyContinue

        if ($bgpPeer -ne $null) {
            Remove-BgpPeer -Name $PeerIP -Force -Confirm:$false -ErrorAction SilentlyContinue
        }

        Add-BgpPeer -Name $PeerIP -LocalIPAddress $RouterIP -PeerIPAddress $PeerIP -PeerASN $PeerASN | Out-Null
        Add-BgpCustomRoute -Network $CustomRoutes
    }

    Invoke-ReliableCommand -ComputerName $ComputerName -Credential $Credential -scriptBlock $addBgpPeer `
        -ArgumentList @($RouterIP, $LocalASN, $PeerIP, $PeerASN, $CustomRoutes)
}

function Add-IPsecVPNConnection {

    param(
        [Parameter(Mandatory = $true)][Array]$DataCenterGWs,
        [Parameter(Mandatory = $true)][hashtable]$SharedSecrets
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  

    $addVpnConnection = {
        param($connectionName, $destinationIP, $mainMode, $quickMode, $sharedSecret, $destinationPrefixes)
        
        $con = Get-VpnS2SInterface -Name $connectionName -ErrorAction SilentlyContinue

        if ($con -ne $null) {
            Set-VpnS2SInterface -Name $connectionName -AdminStatus $false -Confirm:$false -Force
            Remove-VpnS2SInterface -Name $connectionName -Force -Confirm:$false -ErrorAction SilentlyContinue
        }

        Add-VpnS2SInterface `
            -Name $connectionName `
            -CustomPolicy `
            -Destination $destinationIP `
            -Protocol IKEv2 `
            -SharedSecret $sharedSecret `
            -CipherTransformConstants $quickMode.CipherTransformationConstant `
            -AuthenticationTransformConstants $quickMode.AuthenticationTransformationConstant `
            -DHGroup $mainMode.DiffieHellmanGroup `
            -EncryptionMethod $mainMode.EncryptionAlgorithm `
            -IntegrityCheckMethod $mainMode.IntegrityAlgorithm `
            -AuthenticationMethod PSKOnly `
            -ResponderAuthenticationMethod PSKOnly `
            -Persistent

        $prefixArray = @()
        foreach ($prefix in $destinationPrefixes) {
            $prefixArray += $($prefix + ":10")
        }

        Set-VpnS2SInterface -Name $connectionName -IPv4Subnet $prefixArray
    }

    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    $cred = Get-TurnKeySdnCred

    foreach ($gw in $DataCenterGWs) {
        Initialize-DataCenterGWVM -ComputerName $gw.Name -Credential $cred

        $connection = $null
        foreach ($vg in $vgws) {
            $connection = $vg.Properties.networkConnections | Where-Object {
                $_.properties.destinationIPAddress -eq $gw.IPAddress -and $_.properties.connectionType -ieq "IPsec"
            }

            if ($connection -ne $null) {
                break
            }
        }

        if ($connection -eq $null) {
            Write-TraceLog "No virtual gateway found for datacenter gateway $($gw.Name)"
            continue
        }

        $destinationPrefixes = Get-VirtualGatewayVnetPrefix -VirtualGatewayResourceId $vg.ResourceId

        $sharedSecret = $SharedSecrets[$connection.Properties.DestinationIPAddress]
        Write-TraceLog "Adding VPN connection for $($gw.Name), ConnectionName: $($connection.ResourceId), $($connection.Properties.DestinationIPAddress) SharedSecret: $sharedSecret, DestinationPrefixes: $destinationPrefixes"
        Invoke-ReliableCommand -computerName $gw.Name -Credential $cred -scriptBlock $addVpnConnection `
            -ArgumentList @($connection.ResourceId, $connection.Properties.SourceIPAddress, `
                $connection.Properties.IpSecConfiguration.MainMode, $connection.Properties.IpSecConfiguration.QuickMode, $sharedSecret, $destinationPrefixes)
    }
}

function Add-BgpPeerOnDataCenterGW {

    param(
        [Parameter(Mandatory = $true)][Array]$DataCenterGWs
    )

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  

    
    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    $cred = Get-TurnKeySdnCred
    $bgpConnections = @{}
    foreach ($gw in $DataCenterGWs) {

        $connection = $null
        foreach ($vg in $vgws) {
            $connection = $vg.Properties.networkConnections | Where-Object {
                $_.properties.destinationIPAddress -eq $gw.IPAddress -and $_.properties.connectionType -ieq "IPsec"
            }
            if ($connection -ne $null) {
                $router = $vg.Properties.BgpRouters | where-object { $_.Properties.BgpPeers | Where-Object { $_.Properties.peerIpAddress -eq $gw.IPAddress } }
                break
            }
        }

        if ($router -eq $null) {
            Write-TraceLog "Add-BgpPeerOnDataCenterGW No bgp peer found for datacenter gateway $($gw.Name), skipping"
            continue
        }

        $localASN = $router.Properties.localASN
        $peerIP = $router.Properties.RouterIP[0]
        $peerASN = $router.Properties.ExtAsNumber.Split(".") | Select -Last 1
        $localPeer = $router.Properties.BgpPeers | Where-Object { $_.Properties.peerIpAddress -eq $gw.IPAddress }
        $localASN = $localPeer.Properties.AsNumber
        $routerIP = $localPeer.Properties.PeerIpAddress
        $mgmtNetConfig = Get-MgmtNetworkConfig
        $mgmtPrefix = $mgmtNetConfig.properties.subnets[0].properties.addressPrefix

        Write-TraceLog "Add-BgpPeerOnDataCenterGW Adding BGP peer on $($gw.Name), LocalASN: $localASN, PeerIP: $peerIP, PeerASN: $peerASN"
        Add-BgpPeerOnVM -ComputerName $gw.Name -Credential $cred -RouterIP $routerIP `
            -LocalASN $localASN -PeerIP $peerIP -PeerASN $peerASN -CustomRoutes @($mgmtPrefix)

        $bgpConnections[$vg.ResourceId] = $connection.ResourceId
    }

    return $bgpConnections
}

function Add-DataCenterGatewayNetwork {

    <#
    # .SYNOPSIS
    # Enables gateway workload data path
    # .DESCRIPTION
    # Creates a datacenter gateway VM for each of the virtual gateway connection configured in the workload config.
    # S2S vpn interface is added to the datacenter gateway VM for each gateway connection.
    # Network for datacenter gateway VM is configured using the jsons under workload\gateway\datacenternetwork folder.
    # The logical network resources specify the logical network to be used for the datacenter gateway.
    # Datacenter network is configured as an actual logical network in SDN.
    # This data center logical network corresponds to the destination ip address of the virtual gateway connection.
    # Under workload\gateway\datacenternetwork\jsons\networkinterfaces, a network interface must exist with the tag "GatewayConnection" for each datacenter gateway VM.
    # IP address of this networkinterface must correspond to the destination ip address of the virtual gateway connection.
    # To avoid conflict between different setups, various IP addresses are randmoized unless randomizeAddresses is set to false in the deployment config.
    # To enable datapath from physical host following static routes are added,
    # - On physical host, route to the vnet ip with next hop as management gateway(nat vm)
    # - On nat vm, route to the vnet ip with next hop as datacenter gateway corresponding to the vnet.
    # - Management address prefix is auto appended to the virtual gateway connection routes. This enables return traffic from the VNET VM to physical host.
    # Traffic flow -> Physical Host->Nat VM->Datacenter Gateway VM->S2S VPN->SDN Gateway->VNET VM
    # .PARAMETER WorkloadConfigPath
    # Path to the datacenternetwork network workload config
    #>

    param(
        $WorkloadConfigPath
    )

    Write-TraceLog "Enable-GatewayWorkload: Enabling gateway workload"

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $mgmtGw =  $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]
    $depId = Get-TurnKeySdnDeploymentId
    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"

    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath
    if ($vgwsConfigJson -eq $null -or $vgwsConfigJson.Count -eq 0) {
        Write-TraceLog "No virtual gateway configuration found. Cannot configure gateway connections."
        return
    }

    $dcNetworkPath = Join-Path $WorkloadConfigPath "datacenternetwork"
    if (-not (Test-Path $dcNetworkPath)) {
        return @()
    }

    Update-WorkloadIPAddresses -WorkloadConfigPath $dcNetworkPath

    $dcLnets = Get-SdnWorkloadLogicalNetworks -WorkloadConfigPath $dcNetworkPath

    if ($dcLnets -eq $null -or $dcLnets.Count -eq 0) {
        Write-TraceLog "No datacenter logical networks found at $dcNetworkPath. IPsec VPN will not be configured" -Warning
        return @()
    }   

    $nics = Get-SdnWorkloadNetworkInterfaces -WorkloadConfigPath $dcNetworkPath

    if ($nics -eq $null ) {
        Write-TraceLog "No network interface json under datacenter network, $dcNetworkPath. IPsec VPN will not be configured" -Warning
        return @()
    }  

    $gwNics = @()
    $nics | ForEach-Object {
        $nicJson = get-content $_.FullName | ConvertFrom-Json
        if ($nicJson.Tags.Role -ieq "GatewayConnection" ) {
            $gwNics += $_
        }
    }

    if ($gwNics.Count -eq 0 ) {
        Write-TraceLog "No networkinterface json under datacenter network $dcNetworkPath with GatewayConnection tag. IPsec VPN will not be configured" -Warning
        return @()
    } 

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restoreReplayScript = Get-RestoreReplayScript
    & $restoreReplayScript -OperationType Put -BackupDirectoryOrZipFile $dcNetworkPath -BackupType Manual -Verbose -NCRestEndPoint $restEndpoint -Force -Confirm:$false    
    
    $dcLnets | ForEach-Object {
        $lnetJson = get-content $_.FullName | ConvertFrom-Json
        $resourceId = $lnetJson.resourceId
        Enable-LogicalNetworkDefaultGateway -LogicalNetworkResourceId $resourceId  
    }


    $dataCenterGWs = @()
    $depId = Get-TurnKeySdnDeploymentId
    $restUrl = "https://$restEndpoint"  

    $gwNics | ForEach-Object {
        $nic = $_
        $nicJson = get-content $nic.FullName | ConvertFrom-Json
        $resourceId = $nicJson.resourceId
        $res = Get-NetworkControllerNetworkInterface -ConnectionUri $restUrl -ResourceId $resourceId -PassInnerException -ErrorAction Stop
        $mac = $res.Properties.PrivateMacAddress
        $ipAddress = $res.Properties.IpConfigurations[0].Properties.PrivateIPAddress
        $subnetRef = $res.Properties.IpConfigurations[0].Properties.Subnet.ResourceRef

        $subEle = $subnetRef.Split("/", [StringSplitOptions]::RemoveEmptyEntries)
        $lnetId = $subEle[1]
        $subnetId = $subEle[3]
        $subnet = Get-NetworkControllerLogicalSubnet -LogicalNetworkId $lnetId -ResourceId $subnetId -ConnectionUri $restUrl
        $vlanId = $subnet.Properties.VlanID
        $defaultGw = $subnet.Properties.DefaultGateways[0]
        $prefixLen = $subnet.Properties.AddressPrefix.Split("/") | Select-Object -last 1
        $ipAddressWithPrefix = $ipAddress + "/" + $prefixLen

        $vmName = "$depId-$resourceId"
        New-DataCenterGateway -vmName $vmName `
            -MacAddress $mac -IPAddress $ipAddressWithPrefix `
            -VlanId $vlanId -DefaultGateway $defaultGw

        $instanceId = $res.InstanceId.Guid
        # Enable-SDNExpressVMPort -ComputerName (hostname) -VMName $dataCenterGW -InstanceId "{$instanceId}" -Credential $cred
        $dataCenterGWs += @{"Name" = $vmName; "IPAddress" = $ipAddress; "InstanceId" = $instanceId; "VMHost" = (hostname) }
    }

    $cred = Get-TurnKeySdnCred
    foreach ($dataCenterGW in $dataCenterGWs) {
        WaitforComputerToBeReady -ComputerName $dataCenterGW.Name -Credential $cred
        $instanceId = $dataCenterGW.InstanceId
        Enable-SDNExpressVMPort -ComputerName $dataCenterGW.VMHost -VMName $dataCenterGW.Name -InstanceId "{$instanceId}" -Credential $cred
        WaitforComputerToBeReady -ComputerName $dataCenterGW.Name -Credential $cred
        Enable-DefaultFirewallRules -ComputerName $dataCenterGW.Name -Credential $cred
    }

    return $dataCenterGWs
}
function Enable-GatewayWorkload {

    <#
    # .SYNOPSIS
    # Enables gateway workload data path
    # .DESCRIPTION
    # Creates a datacenter gateway VM for each of the virtual gateway connection configured in the workload config.
    # S2S vpn interface is added to the datacenter gateway VM for each gateway connection.
    # Network for datacenter gateway VM is configured using the jsons under workload\gateway\datacenternetwork folder.
    # The logical network resources specify the logical network to be used for the datacenter gateway.
    # Datacenter network is configured as an actual logical network in SDN.
    # This data center logical network corresponds to the destination ip address of the virtual gateway connection.
    # Under workload\gateway\datacenternetwork\jsons\networkinterfaces, a network interface must exist with the tag "GatewayConnection" for each datacenter gateway VM.
    # IP address of this networkinterface must correspond to the destination ip address of the virtual gateway connection.
    # To avoid conflict between different setups, various IP addresses are randmoized unless randomizeAddresses is set to false in the deployment config.
    # To enable datapath from physical host following static routes are added,
    # - On physical host, route to the vnet ip with next hop as management gateway(router vm)
    # - On router vm, route to the vnet ip with next hop as datacenter gateway corresponding to the vnet.
    # - Management address prefix is auto appended to the virtual gateway connection routes. This enables return traffic from the VNET VM to physical host.
    # Traffic flow -> Physical Host->Router VM->Datacenter Gateway VM->S2S VPN->SDN Gateway->VNET VM
    # For L3 connections, the logical network correpsonding to the L3 is enabled on the router VM.
    # .PARAMETER WorkloadConfigPath
    # Path to the datacenternetwork network workload config
    #>

    param(
        $WorkloadConfigPath
    )

    Write-TraceLog "Enable-GatewayWorkload: Enabling gateway workload"
    
    $vgwsConfigJson = Get-SdnWorkloadVirtualGateways -WorkloadConfigPath $WorkloadConfigPath
    if ($vgwsConfigJson -eq $null -or $vgwsConfigJson.Count -eq 0) {
        Write-TraceLog "Enable-GatewayWorkload: No virtual gateway configuration found. Skipping gateway workload configuration"
        return
    }

    $dataCenterGWs = Add-DataCenterGatewayNetwork -WorkloadConfigPath $WorkloadConfigPath
    
    $sharedSecrets = Get-IPsecSharedSecrets -WorkloadConfigPath $WorkloadConfigPath
    Add-IPsecVPNConnection -DataCenterGWs $dataCenterGWs -SharedSecrets $sharedSecrets    
    $bgpConnections = Add-BgpPeerOnDataCenterGW -DataCenterGWs $dataCenterGWs

    $restEndpoint = Get-TurnKeySdnRestEndpoint
    $restUrl = "https://$restEndpoint"  
    $vgws = Get-NetworkControllerVirtualGateway -ConnectionUri  $restUrl
    Add-StaticRoutesForVirtualGateways -NCVirtualGateways $vgws -BgpConnections $bgpConnections
    Add-ManagmentRouteToVirtualGateways -NCVirtualGateways $vgws -BgpConnections $bgpConnections
    $routerVM = Get-RouterVMName
    Invoke-ReliableCommand -computerName $routerVM -Credential $cred -scriptBlock {
        Restart-Service RemoteAccess -Force
    }

    Add-TurneKeyVMToCluster -Tag "Role=DataCenterGateway"
}

function Set-PublicVipRouteOnHosts {
    <#
    .SYNOPSIS
        Adds a static route for the public vip network on the hosts.
        Next hop is the default gateway of the management network, the router VM.
    #>

    $depConfig = Get-DeploymentConfig
    $hyperVHosts = $depConfig.hyperVHosts
    
    $sdnSwitchName = Get-SdnSwitchName
    $cred = Get-TurnKeySdnCred

    $publicVipNetwork = Get-PublicVipNetworkConfig    
    $publicVipPrefix = $publicVipNetwork.properties.subnets[0].properties.addressPrefix

    $mgmtNetConfig = Get-MgmtNetworkConfig
    $nextHop = $mgmtNetConfig.properties.subnets[0].properties.defaultGateways[0]

    Set-StaticRoute -Computer $hyperVHosts -SwitchName $sdnSwitchName `
        -DestinationPrefix $publicVipPrefix `
        -NextHop $nextHop `
        -NicName $(Get-MgmtNicName) -Credential $cred

}