AksHciNcCompanion

0.4

AksHciNcCompanion.psm1

Import-Module "$((Get-InstalledModule AksHci).InstalledLocation)\common.psm1" 

$script:HostVNetNicName = "SDN_VNET"
$script:clsuterGroupName = "clustergroup"
$script:MocConfig = $null
$script:MocGroup = $null

$script:InternetUrl = "http://www.msftncsi.com/ncsi.txt"
$script:DnsResolutionName = "bing.com"
$script:NCSIResponse = "Microsoft NCSI"

$script:LinuxUserName = "clouduser"
$script:WindowsUserName = "Administrator"

$script:CloudAgentPort = "65000"
$script:CloudAgentCurlResponse = "curl: (52) Empty reply from server"

$script:VmInboundAccessState = @{}

$script:TestOutputDirectory = "C:\Windows\Tracing\AksHci"

enum AksHciNcResourceType
{
    NetworkInterface
    LoadBalancer
    VirtualNetwork
}

function Write-TestResults($results)
{
    if (-not (Test-Path $script:TestOutputDirectory )) {
        New-Item -ItemType Directory -Path $script:TestOutputDirectory  -Force
    }
    $output = $results | ConvertTo-Json -Depth 10
    $fileName = "Test-AksHciSdnNetwork-$((get-date).Ticks).Json"
    $file = Join-Path $script:TestOutputDirectory $fileName
    $output | Out-File -FilePath $file -Force
    Write-Host "Results written to file $file" -ForegroundColor Green
}

function Write-Fail {

    param (
        [string]$testName,
        [string]$msg,
        [string]$err
    )
    
    #Write-Status -moduleName $moduleName -Verbose -msg "Test Connection for $scenario, Status Succeeded"
    $out = "FAIL: $testName $msg"

    if (-not [string]::IsNullOrWhiteSpace($err))
    {
        $out += "`nERROR $err"
    }

    Write-Host $out -ForegroundColor Red
}
function Write-Pass {
    param (
        [string]$testName,
        [string]$msg
    ) 

    Write-Host "PASS: $testName $msg" -ForegroundColor Green
}
function Write-Msg {
    param (
        [string]$msg,
        [switch]$Warning
    )
    
    #Write-Status -moduleName $moduleName -Verbose -msg "Test Connection for $scenario, Status Succeeded"
    if ($Warning.IsPresent) {
        Write-Warning $msg
        return
    }

    Write-Host $msg
}

function Get-Result {
        param(
        [string]$scenario,
        [bool]$isSuccess,
        [string]$detailedStatus
    )
    
    # return @{
    # "Scenario" = $scenario;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # }

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name Scenario -Value $scenario
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus

    return $result
}

function Get-VMTestResult {
    param(
        [string]$vmName,
        [string]$testName,
        [string]$sshInfo,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$results
    )

    $vmResult = New-Object -TypeName psobject
    $vmResult | Add-Member -MemberType NoteProperty -Name VMName -Value $vmName
    $vmResult | Add-Member -MemberType NoteProperty -Name TestName -Value $testName
    $vmResult | Add-Member -MemberType NoteProperty -Name SSHInfo -Value $sshInfo
    $vmResult | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $vmResult | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $vmResult | Add-Member -MemberType NoteProperty -Name Results -Value $results

    return $vmResult

    # return @{
    # "VMName" = $vmName;
    # "SSHInfo" = $sshInfo;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # "TestName" = $testName
    # "Results" = $results;
    # }
}

function Get-TestResult {
    param(
        [string]$testName,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$testResults
    )

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name TestName -Value $testName
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $result | Add-Member -MemberType NoteProperty -Name TestResults -Value $testResults
    return $result

    # return @{
    # "TestName" = $testName;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # "TestResults" = $testResults;
    # }
}

function Get-ClusterTestResult {
        param(
        [string]$clusterName,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$testResults
    )

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name ClusterName -Value $clusterName
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $result | Add-Member -MemberType NoteProperty -Name TestResults -Value $testResults
    return $result

    # return @{
    # "ClusterName" = $clusterName;
    # "Passed" = $isSuccess;
    # "TestResults" = $testResults;
    # "DetailedStatus" = $detailedStatus;
    # }
}

function Invoke-WebRequestWithRetries {
    param(
        [System.Collections.IDictionary] $Headers,
        [string] $ContentType,
        [Microsoft.PowerShell.Commands.WebRequestMethod] $Method,
        [System.Uri] $Uri,
        [object] $Body,
        [Switch] $DisableKeepAlive,
        [Switch] $UseBasicParsing,
        [Parameter(mandatory=$false)]
        [bool] $shouldRetry = $true
    )
        
    $params = @{
        'Headers'=$headers;
        'ContentType'=$content;
        'Method'=$method;
        'uri'=$uri;
        'ErrorAction'='Stop';
        }
    
    if($Body -ne $null) 
    {
        $params.Add('Body', $Body)
    }

    if($DisableKeepAlive.IsPresent) 
    {
        $params.Add('DisableKeepAlive', $true)
    }

    if($UseBasicParsing.IsPresent) 
    {
        $params.Add('UseBasicParsing', $true)
    }

    if ($script:DefaultCredParaSet -eq $true)
    {
        $params.Add('UseDefaultCredentials', $true)
    }
    elseif($script:NetworkControllerCred -ne [System.Management.Automation.PSCredential]::Empty)
    {
        $params.Add('Credential', $script:NetworkControllerCred)
    }
        
    $retryIntervalInSeconds = 30
    $maxRetry = 6
    $retryCounter = 0
    
    do {
        try {
            $result = $null
            $result = Invoke-WebRequest @params
            break
        }
        catch {

          
            if($_.Exception.Response.StatusCode.value__ -eq 404)
            {
                #Dont retry on Not Found
                break
            }

            $retryCounter++
            if($retryCounter -le $maxRetry) {
                Start-Sleep -Seconds $retryIntervalInSeconds
            }
            else {
                # last retry still fails, so throw the exception
                throw $_
            }
        }
    } while ($shouldRetry -and ($retryCounter -le $maxRetry)) 
    
    return $result       
}

function Initialize-AksHciNcCompanion
{
    $winFeatures = @("RSAT-NetworkController")
    
    foreach ($feature in $winFeatures)
    {
        Write-Verbose "Check install state of $feature"
        if ((Get-WindowsFeature -Name $feature).InstallState -ne "Installed")
        {
            Write-Verbose "Installing $feature"
            Add-WindowsFeature -Name $feature -IncludeAllSubFeature -IncludeManagementTools
        }
    }

}

function Get-NCResource
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $Uri
    )

    $result = Invoke-WebRequestWithRetries -Uri $Uri -DisableKeepAlive -UseBasicParsing -Method "Get"      

    if(-not $result)
    {
        return $null
    }            

    $toplevel = convertfrom-json $result.Content
    if ($toplevel.value -eq $null)
    {
        $jsonOut = $toplevel
    } 
    else
    {
        $jsonOut = $toplevel.value
    }

    return $jsonOut
}

function Get-NCNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $resourceId
    )

    Write-Verbose "Fetching nic $resourceId from NC"

    $mocConfig = Get-MocConfigCache     
    $ncRestEndPoint = $mocConfig["networkControllerFqdnOrIpAddress"]
    $uri = "https://$ncRestEndPoint/networking/v1/networkinterfaces/$resourceId"

    $nic = Get-NCResource -Uri $uri

    if ($nic -eq $null)
    {
        $nics = (Invoke-WebRequest -Uri "https://$ncRestEndPoint/networking/v1/networkinterfaces/" -UseBasicParsing).content | ConvertFrom-Json
        $nic = $nics.value | Where-Object {$_.resourceMetadata.resourceName -eq "$resourceId"}
    }

    if ($nic -eq $null)
    {
        Write-Warning "Nic with resource id $resourceId not found in NC($ncRestEndPoint)"
    }
    return $nic
}

function Get-NCNicInstanceId
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $resourceId
    )

    $nic = Get-NCNic -resourceId $resourceId

    return $nic.instanceId
}

function Get-MocConfigCache
{
    if ($null -eq $script:MocConfig)
    {
        $script:MocConfig = Get-MocConfig
    }

    return $script:MocConfig
}

function Get-MocGroupCache
{
    if ($null -eq $script:MocGroup)
    {
        $script:MocGroup = Get-MocGroup -location MocLocation
    }

    return $script:MocGroup
}

function Clear-MocCache
{
    $script:MocConfig = $null
    $script:MocGroup = $null
}

function GetClusterGroup
{
    return $script:clsuterGroupName
}

function ValidateState
{ 
    $config = Get-AksHciConfig
    
    $aksHciConfig = $config["AksHci"]
    if ($aksHciConfig["installState"] -eq [InstallState]::NotInstalled)
    {
        Write-Error("UnsupportedConfiguration: AKS-HCI is not installed") -ErrorAction Stop
    }

    $mocConfig = Get-MocConfigCache

    if (-not $mocConfig["UseNetworkController"])
    {
        Write-Error("UnsupportedConfiguration: SDN intergration is not present") -ErrorAction Stop
    }
}

function CleanupMoc
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Remove-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue
}

function CleanupHost
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    $mocConfig = Get-MocConfigCache    
    $vswitchName = $mocConfig["vswitchName"]

    $nic = Get-VMNetworkAdapter -ManagementOS -Name $nicName -SwitchName $vswitchName -ErrorAction SilentlyContinue

    if ($nic -eq $null)
    {
        return
    }

    Remove-VMNetworkAdapter -ManagementOS -Name $nicName -SwitchName $vswitchName -ErrorAction Stop
}

function CreateHostVNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $name,
        [parameter(Mandatory=$true)]
        [string] $mac,
        [parameter(Mandatory=$true)]
        [string] $ipAddress,
        [parameter(Mandatory=$true)]
        [string] $prefix,
        [parameter(Mandatory=$true)]
        [string] $vswitchName
    )

    Write-Verbose "Creating a new host vnic $name, MAC $mac, IP $ipAddress/$prefix on host $(hostname)"
    $nic = Get-VMNetworkAdapter -ManagementOS -Name $name -SwitchName $vswitchName -ErrorAction SilentlyContinue

    if ($nic -ne $null -and $nic.MacAddress -eq $mac)
    {
        Write-Verbose "Reusing existing VNIC "
    }
    else
    {
        CleanupHost -nicName $name
        Start-Sleep 5

        $nic = Add-VMNetworkAdapter -ManagementOS -Name $name -SwitchName $vswitchName -StaticMacAddress $mac -ErrorAction stop
        Start-Sleep 30
    }

    $ifIndex = (Get-NetAdapter -Name "*$name*").ifIndex

    $nic = Get-NetIPAddress -IPAddress $ipAddress -ErrorAction SilentlyContinue

    if ($nic -ne $null)
    {

        if ($nic.InterfaceIndex -ne $ifIndex)
        {
            Write-Error "Fatal error, Cannot connect host, $ipAddress is already present on another interface on the host" -ErrorAction Stop
        }

        return
    }

    Remove-NetIPAddress -InterfaceIndex $ifIndex -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
    New-NetIPAddress -PrefixLength $prefix -IPAddress $ipAddress -InterfaceIndex $ifIndex -ErrorAction Stop | Out-Null
}


function CreateMocNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    $mocConfig = Get-MocConfigCache
    $vnetName = $mocConfig["vnetName"]
    $location = $mocConfig["cloudLocation"]
    $retry = $true

    while($retry)
    {
        try
        {
            New-MocNetworkInterface -name $nicName -virtualNetworkName $vnetName -group $(GetClusterGroup) -ErrorAction Stop
            break
        }
        catch
        {
            $e = $_

            $retry = $e.Exception.Message.Contains("PrivateIPAddressInUse") -eq $true

            if (-not $retry)
            {
                Write-Warning "Failed to create Nic in Moc, Error $_"
            }
            else 
            {
                Write-Verbose "IPAddress conflict, retry nic creation"
                throw
            }
        }
    }
}

function Get-MocNic
{
    
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Write-Verbose "Fetching nic $nicName from MOC"
    $vnetMocNic = Get-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue

    if ($vnetMocNic -eq $null)
    {
        Write-Verbose "Creating new nic $nicName in MOC"
        CreateMocNic -nicName $nicName
    }

    return Get-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue
}

function SetPortProfile
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName,
        [parameter(Mandatory=$true)]
        [string] $profileId
    )

    Write-Verbose "Setting port profile for $nicName on $(hostName), ProfileId $profileId"

    $vmNic = Get-VMNetworkAdapter -Name $nicName -ManagementOS -ErrorAction Stop

    $FeatureId = "9940cd46-8b06-43bb-b9d5-93d50381fd56"

    $CurrentFeature = Get-VMSwitchExtensionPortFeature -FeatureId $FeatureId -VMNetworkAdapter $vmNic

    if ($CurrentFeature -eq $null)
    {
        $Feature = Get-VMSystemSwitchExtensionPortFeature -FeatureId $FeatureId
        $Feature.SettingData.ProfileId = "{$profileId}"
        $Feature.SettingData.NetCfgInstanceId = "{56785678-a0e5-4a26-bc9b-c0cba27311a3}"
        $Feature.SettingData.CdnLabelString = "TestCdn"
        $Feature.SettingData.CdnLabelId = 1111
        $Feature.SettingData.ProfileName = "Testprofile"
        $Feature.SettingData.VendorId = "{1FA41B39-B444-4E43-B35A-E1F7985FD548}"
        $Feature.SettingData.VendorName = "NetworkController"
        $Feature.SettingData.ProfileData = 1
        Add-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $Feature -VMNetworkAdapter $vmNic
    }
    else
    {
        $CurrentFeature.SettingData.ProfileId = "{$profileId}"
        $CurrentFeature.SettingData.ProfileData = 1
        Set-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $CurrentFeature -VMNetworkAdapter $vmNic
    }

    Write-Verbose "Successfully set port profile for $nicName on $(hostName)"
}

function Remove-PortProfile
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Write-Verbose "Removing port profile on $nicName"

    $vmNic = Get-VMNetworkAdapter -Name $nicName -ManagementOS -ErrorAction SilentlyContinue

    if ($vmNic  -eq $null)
    {
        return
    }

    $FeatureId = "9940cd46-8b06-43bb-b9d5-93d50381fd56"

    $CurrentFeature = Get-VMSwitchExtensionPortFeature -FeatureId $FeatureId -VMNetworkAdapter $vmNic

    if ($CurrentFeature -eq $null)
    {
        return
    }

    Remove-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $CurrentFeature -VMNetworkAdapterName $vmNic

}

function Connect-AksHciSdnVnet
{

    ValidateState

    Write-Verbose "Connecting host $(hostname) to the Aks-HCI control plane virtual network"

    $mocNic = Get-MocNic -nicName $script:HostVNetNicName
    $ncNic = Get-NCNic -resourceId $script:HostVNetNicName

    if ($ncNic  -eq $null)
    {
        Write-Error "Failed to create a VNIC" -ErrorAction Stop
    }

    $vswitchName = (Get-MocConfigCache)["vswitchName"]
    $prefix = $mocNic.properties.ipConfigurations[0].properties.prefixlength
    $ip = $mocNic.properties.ipConfigurations[0].properties.privateIPAddress
    CreateHostVNic -name $script:HostVNetNicName -mac $ncNic.properties.privateMacAddress -vswitchName $vswitchName -ipAddress $ip -prefix $prefix

    SetPortProfile -nicName $script:HostVNetNicName -profileId $ncNic.instanceId
}

function Disconnect-AksHciSdnVnet
{
    param([switch] $RetainResources)   

    ValidateState

    Write-Verbose "Disconnecting host $(hostname) from the Aks-HCI control plane virtual network"

    if ($RetainResources.IsPresent)
    {
        Remove-PortProfile -nicName $script:HostVNetNicName
        return
    }
    
    CleanupHost -nicName $script:HostVNetNicName
    CleanupMoc -nicName $script:HostVNetNicName
}

function Remove-AksHciNetworkControllerResources
{
    param(
        [Parameter(Mandatory = $true, ParameterSetName = 'All')]
        [switch] $All,
        [Parameter(Mandatory = $true, ParameterSetName = 'Leaked')]
        [switch] $Leaked,
        [Parameter(Mandatory = $true, ParameterSetName = 'All')]
        [string] $NCRestEndPoint,
        [switch] $Force
    )

    Initialize-AksHciNcCompanion
    Clear-MocCache

    if ([string]::IsNullOrWhiteSpace($NCRestEndPoint))
    {
        if ((Get-MocConfigCache)["installState"] -eq [InstallState]::NotInstalled)
        {
            throw "Moc is not installed and no NCRestEndPoint was provided"
        }
        $NCRestEndPoint = (Get-MocConfigCache)["networkControllerFqdnOrIpAddress"]
    }

    # Check that the $NCRestEndPoint is correct. If it is, the test will populate the uri variable
    $uri = "https://$NCRestEndPoint"
    if (-not (Test-NetworkControllerFqdnOrIpAddress $uri))
    {
        throw "Cannot connect to Network Controller $NCRestEndPoint"
    }

    if ($Leaked.IsPresent)
    {
        # Check if MocGroup is configured
        if ((Get-MocGroupCache).Length -eq 0)
        {
            throw "No MocGroup found, cannot find resources in Moc to compare to NetworkController"
        }
        if ((Get-MocConfigCache)["installState"] -eq [InstallState]::NotInstalled)
        {
            throw "AksHci is not in Installed state, cannot access Moc to compare to NetworkController"
        }
        Remove-AksHciNcResourcesLeaked -ConnectionUri $uri -Force:$Force
    }
    elseif ($All.IsPresent)
    {
        if ((Get-MocConfigCache)["installState"] -ne [InstallState]::NotInstalled)
        {
            throw "AksHci is not in NotInstalled state, cannot delete all NC resources"
        }
        Remove-AksHciNcResourcesAll -ConnectionUri $uri -Force:$Force
    }

}

function Get-MocResourceName
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type
    )

    # Get all the moc groups
    $mocGroup = Get-MocGroupCache
    # Return an array of string with the group names
    $mocGroup = $mocGroup | Select-Object -ExpandProperty name

    $mocResources = @()
    switch ($Type)
    {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocNetworkInterface -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocLoadBalancer -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocVirtualNetwork -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        Default
        {
            $mocResources = @()
        }
    }
    
    return $mocResources
}

function Get-AksHciNcResourceNameAndId
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type,

        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    # Query all Nc resources from the rest api
    switch ($Type) {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            $ncResponse = Get-NetworkControllerNetworkInterface -ConnectionUri $ConnectionUri
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            $ncResponse = Get-NetworkControllerLoadBalancer -ConnectionUri $ConnectionUri
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            $ncResponse = Get-NetworkControllerVirtualNetwork -ConnectionUri $ConnectionUri
        }
        Default
        {
            $ncResponse = @()
        }
    }

    # Remove entries that don't have ResourceMetadata
    $ncResponse = $ncResponse | Where-Object -Property ResourceMetadata -ne nil
    # Re-format object to extract ResourceId, ResourceMetadata.Client, ResourceMetadata.ResourceName
    $ncResponse = $ncResponse | Select-Object ResourceId, @{N="Client";E={$_.ResourceMetadata.Client}}, @{N="ResourceName";E={$_.ResourceMetadata.ResourceName}}
    # Only select the AksHci Client Objects
    $ncResponse = $ncResponse | Where-Object -Property Client -eq "AksHci"

    return $ncResponse
}

function Remove-NCResource
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type,

        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri,
        
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ResourceId
    )

    switch ($Type) {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            Remove-NetworkControllerNetworkInterface -ConnectionUri $uri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            Remove-NetworkControllerLoadBalancer -ConnectionUri $uri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            Remove-NetworkControllerVirtualNetwork -ConnectionUri $uri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        Default
        {
            Write-Error "Network Controller Resource type unknown, cannot delete"
        }
    }
}

function Remove-AksHciNcResourcesLeaked
{
    param(
        [switch] $Force,
        [Parameter(Mandatory = $true)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    # go over each resource type and compare Moc to NC
    foreach ($resType in [AksHciNcResourceType].GetEnumValues())
    {
        $mocResources = Get-MocResourceName -Type $resType
        if ($mocResources.Length -eq 0)
        {
            Write-Error "No $resType resources found in Moc, cannot compare with NetworkController"
            continue
        }

        $ncResources = Get-AksHciNcResourceNameAndId -ConnectionUri $ConnectionUri -Type $resType
        if ($ncResources.Length -eq 0)
        {
            Write-Error "No $resType resources found in NetworkController, nothing to remove"
            continue
        }

        # Compare the objects
        $diff = Compare-Object -ReferenceObject $mocResources -DifferenceObject ($ncResources | Select-Object -ExpandProperty ResourceName)

        # NetworkController resources not found in Moc
        $ncOnly = $diff | Where-Object -Property SideIndicator -eq "=>" | Select-Object -ExpandProperty InputObject
        if ($ncOnly.Length -gt 0)
        {
            $ncOnlyResourceName = $ncOnly
            # Expand ncOnly from ResourceName to include ResourceId. This may be a slow operation, for each resource in $ncOnly,
            # it will search through $ncNetworkInterfaces to find the full object
            $ncOnly = $ncOnly | ForEach-Object { $ncResources | Where-Object -Property ResourceName -eq $_ }

            # Prompt user to delete
            $decision = 1
            if (!$Force.IsPresent)
            {
                $title = "Delete NetworkController$($resType) Resources"
                $question = "Are you sure you want to delete the following resources:`n$($ncOnlyResourceName -join "`n")"
                $choices = @('&Yes', '&No', '&Suspend')
                $decision = $Host.UI.PromptForChoice($title, $question, $choices, 1)
            }

            # Delete the Network Controller resources
            if ($decision -eq 0 -or $Force.IsPresent)
            {
                foreach ($n in $ncOnly)
                {
                    Remove-NCResource -ConnectionUri $ConnectionUri -Type $resType -ResourceId $n.ResourceId
                }
            }
            # Suspend, exit immediatly
            elseif ($decision -eq 2)
            {
                return
            }
        }
        else
        {
            Write-Host "No Leaked NetworkController$($resType) resources found"
        }
    }
}

function Remove-AksHciNcResourcesAll
{
    param(
        [switch] $Force,
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    foreach ($resType in [AksHciNcResourceType].GetEnumValues())
    {
        $ncResources = Get-AksHciNcResourceNameAndId -ConnectionUri $ConnectionUri -Type $resType

        # Prompt user to delete
        $decision = 1
        if (!$Force.IsPresent -and $ncResources.Length -ne 0)
        {
            $title = "Delete NetworkController$($resType) Resources"
            $question = "Are you sure you want to delete the following resources:`n$(($ncResources | Select-Object -ExpandProperty ResourceName) -join "`n")"
            $choices = @('&Yes', '&No', '&Suspend')
            $decision = $Host.UI.PromptForChoice($title, $question, $choices, 1)
        }

        # Delete the Network Controller resources
        if ($decision -eq 0 -or $Force.IsPresent)
        {
            foreach ($n in $ncResources)
            {
                Remove-NCResource -ConnectionUri $ConnectionUri -Type $resType -ResourceId $n.ResourceId
            }
        }
        # Suspend, exit immediatly
        elseif ($decision -eq 2)
        {
            return
        }
    }
}

function Test-AksHciNetworkControllerConfig
{
    # By default, we will use the values in Moc to test NC
    # If 1 parameter is manually set, all parameters must be supplied and moc will not be used
    [CmdletBinding(DefaultParameterSetName = 'Moc')]
    param(
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerFqdnOrIpAddress,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerLbSubnetRef,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerLnetRef,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [VipPoolSettings] $vipPool
    )

    # ----- Setup -----
    $useMoc = $PSCmdlet.ParameterSetName -eq "Moc"
    $ret = $true
    Clear-MocCache

    # Read all values from Moc
    if ($useMoc)
    {
        $mocConfig = Get-MocConfigCache

        # Check that Moc has been initialized
        if ($mocConfig["installState"] -eq [InstallState]::NotInstalled)
        {
            Write-Error "Moc is not in an installed state. Cannot read NetworkController config in Moc"
            $ret = $false
            # Cannot continue any more tests without Moc
            return $ret
        }

        # Get networkControllerFqdnOrIpAddress
        $networkControllerFqdnOrIpAddress = $mocConfig["networkControllerFqdnOrIpAddress"]
        # Get networkControllerLbSubnetRef
        $networkControllerLbSubnetRef = $mocConfig["networkControllerLbSubnetRef"]
        # Get networkControllerLnetRef
        $networkControllerLnetRef = $mocConfig["networkControllerLnetRef"]
        # Get vipPool
        $vipPoolDefaultName = $mocConfig["defaultvippoolname"]
        $vipPoolDefault = Get-MocVipPool -location MocLocation -name $vipPoolDefaultName
        $vipPool = [VipPoolSettings]::new($vipPoolDefault.name, $vipPoolDefault.properties.startIp, $vipPoolDefault.properties.endIp)
    }

    # ----- Tests -----
    # Check networkControllerFqdnOrIpAddress
    Write-Verbose "Check networkControllerFqdnOrIpAddress"
    $uri = "https://$networkControllerFqdnOrIpAddress"
    $result = Test-NetworkControllerFqdnOrIpAddress $uri
    if (-not $result)
    {
        Write-Error "Cannot connect to NetworkController with the networkControllerFqdnOrIpAddress $networkControllerFqdnOrIpAddress"
        $ret = $false
        # Cannot test any more without the Network Controller Uri
        return $ret
    }

    # Check networkControllerLnetRef
    Write-Verbose "Check networkControllerLnetRef"
    $result = Test-NetworkControllerReference $uri $networkControllerLnetRef
    if (-not $result)
    {
        Write-Error "networkControllerLnetRef ($networkControllerLnetRef) cannot be found in the Network Controller"
        $ret = $false
    }

    # Check networkControllerLbSubnetRef
    Write-Verbose "Check networkControllerLbSubnetRef"
    $lbSubnetWebResult = $null
    $result = Test-NetworkControllerReference $uri $networkControllerLbSubnetRef ([ref]$lbSubnetWebResult)
    if (-not $result)
    {
        Write-Error "networkControllerLbSubnetRef ($networkControllerLbSubnetRef) cannot be found in the Network Controller"
        $ret = $false
    }

    # Check the vipPool -- uses previous networkControllerLbSubnetRef result
    Write-Verbose "Check vippool setting"
    if ($null -ne $lbSubnetWebResult)
    {
        $result = Test-NetworkControllerDefaultVipPool ($lbSubnetWebResult.Content) $vipPool
        if (-not $result)
        {
            Write-Error "AksHci Vippool ip range is not within the NetworkController Vippool range"
            $ret = $false
        }
    }
    else
    {
        Write-Warning "Skipping Vippool Test, lb Subnet Ref not found in Network Controller"
    }

    return $ret
}

function Test-NetworkControllerFqdnOrIpAddress([string]$ConnectionUri)
{
    try {
        $result = Invoke-WebRequest -UseBasicParsing -Uri "$ConnectionUri/networking/discovery"
    }
    catch {
        $result = $null
    }

    if ($null -eq $result)
    {
        return $false
    }
    return $result.StatusCode -eq 200
}

function Test-NetworkControllerReference([string]$ConnectionUri, [string]$subRef, [ref]$webResult)
{
    $uri = "$ConnectionUri/networking/v1$subRef"
    $result = Invoke-WebRequest -UseBasicParsing -Uri $uri
    if ($null -eq $result -or $result.StatusCode -ne 200)
    {
        return $false
    }

    # set the webResult if it was passed
    if ($null -ne $webResult)
    {
        $webResult.Value = $result
    }
    return $true
}

function Test-NetworkControllerDefaultVipPool([string]$lbSubnetContent, [VipPoolSettings]$mocVipPool)
{
    $ncVippool = (ConvertFrom-Json -InputObject $lbSubnetContent).Properties.ipPools.Properties
    if ([string]::IsNullOrWhiteSpace($ncVippool))
    {
        return $false
    }

    # check that the IP addresses are in the following order
    # NCVippoolStart -> MocVippoolStart -> MocVippoolEnd -> NCVippoolEnd
    return (
        [AKSHCI.IPUtilities]::ValidateRange($ncVippool.startIpAddress, $vipPool.VipPoolStart) -and
        [AKSHCI.IPUtilities]::ValidateRange($vipPool.VipPoolStart, $vipPool.VipPoolEnd) -and
        [AKSHCI.IPUtilities]::ValidateRange($vipPool.VipPoolEnd, $ncVippool.endIpAddress)
    )
}
function Get-FirstNcVmName()
{
    $networkControllerVMName = $null
    try 
    {
        if (Test-MultiNodeDeployment)
        {
            Get-ClusterNode -ErrorAction Stop | ForEach-Object {
                
                if ([string]::IsNullOrWhiteSpace($networkControllerVMName))
                {
                    Write-Msg "Checking $_"
                    $networkControllerVMName = Invoke-Command -ComputerName $_ -ScriptBlock {
                        (Get-VM | Get-VMNetworkAdapter | where-object { $_.IPAddresses.Contains($((Get-NetTCPConnection -RemotePort 8571).RemoteAddress)) }).VMName
                    }
                }
            }
        }
        else 
        {
            $networkControllerVMName = (Get-VM | Get-VMNetworkAdapter | where-object { $_.IPAddresses.Contains($((Get-NetTCPConnection -RemotePort 8571).RemoteAddress)) }).VMName            
        }        
    }
    catch 
    {
        Write-Error("Unable to detect NC VM name")
    }

    return $networkControllerVMName
}

function Get-NatRuleMap
{
    <#
    .DESCRIPTION
        Fetches the ssh nat lb rules for the group and caches it.
        Returns a map of all ssh nat rules for the given group
    .PARAMETER groupName
        Group name
    #>

     param (
        [Parameter()]
        [String] $groupName
    )

    if ($script:natRuleCache -ne $null)
    {
        $ruleMap = $script:natRuleCache[$groupName]
        if ($ruleMap -ne $null) 
        {
            return $ruleMap
        }
    }
    else 
    {
        $script:natRuleCache = @{}
    }

    $ruleMap = @{}
    $lbs = Get-MocLoadBalancer -group $groupName

    if ($lbs -eq $null) 
    {
        $script:natRuleCache[$groupName] = $ruleMap
        return $ruleMap
    }
    
    
    $inboundLb = $lbs | where-object { $_.properties.inboundNatRules -ne $null }
    foreach($lb in $inboundLb) 
    {
        $sshNatRules = $lb.properties.inboundNatRules | `
            Where-Object { $_.Properties.backendPort -eq 22 }
        if($sshNatRules -ne $null)
        {
            $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress                
            foreach($natRule in $sshNatRules)
            {
                $port = $natRule.properties.frontendPort
                $sshData = @{}
                $sshData["ip"] = $feIP
                $sshData["port"] = $port
                $ruleMap[$natRule.Name] = $sshData
            }
        }
    }

    $natRuleCache[$groupName] = $ruleMap
    return $ruleMap
}

function Get-NatEndpoint
{
    <#
    .DESCRIPTION
        Fetches the ssh frontend nat ip and port for the given nic. When SDN integration is configured,
        ssh access to the node is via the inbound nat rule
 
    .PARAMETER nicName
        Networkinterface name
    .PARAMETER groupName
        Group name
    #>
     param (
        [Parameter()]
        [String] $nicName,

        [Parameter()]
        [String] $groupName
    )
   
    $nic = Get-MocNetworkInterface -group $groupName -name $nicName
    $natRuleName = $nic.properties.ipConfigurations[0].properties.loadBalancerInboundNatRules[0].name

    if ($natRuleName -eq $null -or $natRuleName[0] -eq $null)
    {
        return $null
    }

    $ruleMap = Get-NatRuleMap -groupName $groupName
    return $ruleMap[$natRuleName]
}

function Get-SdnLogs([string]$networkControllerVMName)
{

    <#
    .DESCRIPTION
        Collects SDN logs.
    .PARAMETER networkControllerVMName
        NetworkController VM name. If there are more than one VM, use any one of the VM name.
    #>

    if ([string]::IsNullOrWhiteSpace($networkControllerVMName))
    {
        $networkControllerVMName = Get-FirstNcVmName
    }

    if ([string]::IsNullOrWhiteSpace($networkControllerVMName))
    {
        Write-Error("Unable to detect NC VM name, please use -networkControllerVMName and provide one of the NC vm name") -ErrorAction Stop
    }


    $environmentDetails = Get-SdnInfrastructureInfo -NetworkController $vmName
    Install-SdnDiagnostics -ComputerName ($environmentDetails.FabricNodes | where {$_ -ne $null})
    Start-SdnDataCollection -NetworkController $environmentDetails.NetworkController[0] -Role Gateway,NetworkController,Server,LoadBalancerMux -IncludeLogs -IncludeNetView    
}

function Test-IsAksHciInstalled()
{
    Import-Module AksHci
    return Test-IsProductInstalled -moduleName AksHci
}
function Write-ConnectionResult([string] $scenario, $result)
{
    $status = $true
    $result | Foreach-Object {
        Write-Msg "Test Connection Result - IP $($_.IP), Port $($_.Port), Status $($_.Status)"
        if ($($_.Status) -eq "Failed") {
            $status = $false
        }
    }  

    if ($status) {
        Write-Msg "Test Connection for $scenario, Status Succeeded"
    }

    return $status
}
function Test-Connectivity([string]$ip, [string]$port)
{
    $ProgressPreference = "SilentlyContinue"
    try {
        $result = Test-NetConnection -ComputerName $ip -Port $port -ErrorAction SilentlyContinue
        Write-Msg -msg "$($MyInvocation.MyCommand) IP [$ip] Port [$port]"
        if ($result -eq $null -or $result.TcpTestSucceeded -ne $true)
        {
            return $false
        }
    }
    finally {
        $ProgressPreference = "Continue"
    }

    return $true
}
function Test-AllLoadbalancingRules([string]$lbName, [string]$group, [bool]$includeSsh)
{
    $testResult = Get-TestResult -testName "LBConnectivity" -testResults @()

    try {
        $lb = Get-MocLoadBalancer -name "$lbName" -group $group
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress
        $fePorts = @()
        $lb.properties.loadBalancingRules | foreach-object { 
            $fePorts += $_.Properties.FrontendPort
        }

        if ($includeSsh)
        {
            $fePorts += "22"
        }
        $overallStatus = $true
        $fePorts | foreach-object {            
            $status = Test-Connectivity -ip $feIP -port $_   
            $detailedStatus = "LB [$lbName] feIP [$feIP] fePort [$_] status "
            if ($status) {
                $detailedStatus += "[Success]"
            } else {
                $detailedStatus += "[Failed]"
            }

            $result = Get-Result -scenario "LBConnectivity" -isSuccess $status -detailedStatus $detailedStatus
            $testResult.TestResults += $result
            $overallStatus =  $overallStatus -and $result.Passed
        }
    } catch {
        $testResult.DetailedStatus = Convert-Error $_
        $overallStatus = $false
    }

    $testResult.Passed = $overallStatus
    return $testResult
}

function Test-ManagementControlPlaneConnectivity()
{
    $testResult = Get-TestResult -testName "ManagementControlPlaneConnectivity" -testResults @()
    try {        
        Write-Msg -msg "$($MyInvocation.MyCommand)"
        $kvaName = (Get-KvaConfig).kvaname
        $lbName = "$kvaName-load-balancer"
        $lbTestResult = Test-AllLoadbalancingRules -lbName $lbName -group "clustergroup" -includeSsh $false
        $status = $lbTestResult.Passed
        $testResult.TestResults += $lbTestResult
        $testResult.Passed = $lbTestResult.Passed

        if (-not $status) {
            throw "Management controlplane connectivity test failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand)"
    }
    catch {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -err $_
        $testResult.Passed = $false
        $testResult.DetailedStatus = Convert-Error $_
    }

    return $testResult
}


function Test-TargetControlPlaneConnectivity()
{
    param (
        [string]$clusterName
    )

    $testName = "$($MyInvocation.MyCommand)"
    $testResult = Get-TestResult -testName $testName -testResults @() -isSuccess $true
    try {
        Write-Msg -msg "$($MyInvocation.MyCommand)"
        $status = $true
        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $($_.Name)
            Write-Msg -msg "$($MyInvocation.MyCommand) Cluster [$name]"
            $lbName = "$($_.Name)-load-balancer"
            $lbTestResult = Test-AllLoadbalancingRules -lbName $lbName -group "clustergroup-$name" -includeSsh $false
            $lbTestResult.TestName = "$testName-[$name]"
            $status = $status -and $lbTestResult.Passed
            $testResult.TestResults += $lbTestResult
            $testResult.Passed =  $testResult.Passed -and $lbTestResult.Passed
        }

        if (-not $status) {
            throw "TargetControlPlaneConnectivity test failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand)"
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -err $_
        $testResult.Passed = $false
        $testResult.DetailedStatus = Convert-Error $_
    }

    return $testResult
}

function Reset-Key
{
     param (
        [string]$sshIP,
        [string]$sshPort
     )

    if ($sshPort -eq "22") {
        $keyGenAddress = $sshIP    
    } else {
        $keyGenAddress = "[$sshIP]`:$sshPort"    
    }

    (& ssh-keygen -R $keyGenAddress) 2>&1>$null
}

function Run-SshCommand
{
       param (
        [string]$sshIP,
        [string]$sshPort,
        [string]$command,
        [bool]$windows = $false
    )
    
    Reset-Key -sshIP $sshIP -sshPort $sshPort 
    $sshKey = (Get-MocConfigCache).sshPrivateKey
    if ($windows) {
        $bytes = [System.Text.Encoding]::Unicode.GetBytes($command)
        $encodedCommand = [Convert]::ToBase64String($bytes)
        $response = (& ssh -oBatchMode=yes -i $sshKey -o LogLevel=ERROR -o StrictHostKeyChecking=no "$script:WindowsUserName@$sshIP" -p $sshPort powershell.exe -NonInteractive -encodedCommand $encodedCommand 2>$null)

    } else {
        $response = (& ssh -oBatchMode=yes -i $sshKey -o LogLevel=ERROR -o StrictHostKeyChecking=no "$script:LinuxUserName@$sshIP" -p $sshPort $command)
    }
    
    return $response
}

function Test-CloudAgentConnectivity
{
    param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )
    
    Reset-Key -sshIP $sshIP -sshPort $sshPort

    $sshKey = (Get-MocConfigCache).sshPrivateKey
    $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
    $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
    $status = $false

    if ($windows) {
        $command ="
        invoke-command -scriptblock {
            try {
                    (Test-NetConnection $cloudAgentAddress -port $script:CloudAgentPort -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).TcpTestSucceeded
                }
                catch {
                }
            }"       
        $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
        $status = ($response -ieq "True")
    } else {
        $cloudAgentEndpoint = $cloudAgentAddress + ":" + $script:CloudAgentPort
        $command = "curl --connect-timeout 5 $cloudAgentEndpoint --stderr - `| grep curl:"
        $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
        $status = $response -ieq $script:CloudAgentCurlResponse
    }  

    if ($status) {
        $msg = "Cloudagent connectivity to [$cloudAgentAddress] succeeded"
    } else {
        $msg = "Cloudagent connectivity to [$cloudAgentAddress] failed"
    }
    
    $result = Get-Result -scenario "cloudagent" -isSuccess $status -detailedStatus $msg
    return $result
}

function Test-InternetConnectivity
{
    param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )
    Reset-Key -sshIP $sshIP -sshPort $sshPort
    $sshKey = (Get-MocConfigCache).sshPrivateKey

    $status = $false
    $response = $null

    if ($windows) {
        $command ="
        invoke-command -scriptblock {
            try {
                    (Invoke-WebRequest -UseBasicParsing -Uri $($script:InternetUrl) -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).Content
                }
                catch {
                }
            }"
    } else {
        $command = "curl --connect-timeout 5 $($script:InternetUrl) -s -f"
    }

    $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
    $status = $response -ieq $script:NCSIResponse

    if ($status) {
        $msg = "Internet connectivity to [$script:InternetUrl] succeeded"
    } else {
        $msg = "Internet connectivity to [$script:InternetUrl] failed"
    }
    
    $result = Get-Result -scenario "internet" -isSuccess $status -detailedStatus $msg
    return $result 
}

function Test-DnsResolution
{
        param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )

    Write-Verbose "$($MyInvocation.MyCommand) Query [$script:DnsResolutionName]"
    if ($windows) {
        $command ="
        invoke-command -scriptblock {
            try {
                    (Resolve-DnsName -Type A -DnsOnly $script:DnsResolutionName -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).IPAddress
                }
                catch {
                }
            }"
    } else {
        $command = "dig +short A $script:DnsResolutionName | grep -v ';' "
    }

    $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
    Write-Verbose "$($MyInvocation.MyCommand) Response [$response]"
    $status = (-not [String]::IsNullOrWhiteSpace($response)) -and $response.Count -gt 0 

    if ($status) {
        $msg = "DNS resolution to [$script:DnsResolutionName] succeeded"
    } else {
         $msg = "DNS resolution to [$script:DnsResolutionName] failed"
    }
    
    $result = Get-Result -scenario "dns" -isSuccess $status -detailedStatus $msg
    return $result 
}

function Test-ManagementClusterConnectivity
{
    param (
        [ValidateSet("cloudagent", "dns", "internet", "inbound")]
        [string]$scenario
    )

    try {
        
        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario]"
        $kvaName = (Get-KvaConfig).kvaname
        $lbName = "$kvaName-load-balancer"
        $lb = Get-MocLoadBalancer -name "$lbName" -group $(GetClusterGroup)
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress

        if ($scenario -ne "inbound" -and $script:VmInboundAccessState["managementControlPlane"] -eq $false) {
            Write-Msg -msg "$($MyInvocation.MyCommand) [SKIP NO INBOUND ACCESS] Scenario [$scenario]" -Warning
            throw "Test skipped, no inbound access to the management control plane vm"
        }

        switch ($scenario) {
            "inbound" {
                $status = $(Test-Connectivity  -ip $feIP -port 22)
                $script:VmInboundAccessState["managementControlPlane"] = $result
                if ($status) {
                    $msg = "Inbound connectivity to $feIP succeeded"
                } else {
                    $msg = "Inbound connectivity to $feIP failed"
                }

                $result = Get-Result -scenario "inbound" -isSuccess $status -detailedStatus $msg

                break
            }
            "cloudagent" {
                $result = $(Test-CloudAgentConnectivity -sshIP $feIP  -sshPort "22" -windows $false)
                break
            }
            "dns" {
                $result = $(Test-DnsResolution -sshIP $feIP  -sshPort "22" -windows $false)
                break                
            }
            "internet" {
                $result  = $(Test-InternetConnectivity -sshIP $feIP -sshPort "22" -windows $false)
                break
            }
        }                             
        $status = $result.Passed
        if (-not $status) {
            throw "Test-ManagementClusterConnectivity Scenario [$scenario] failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario]"
        return $result
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario]" -err $_
        return $false
    }
}

function Convert-Error($e)
{
    return $e | Select-Object @{N="Message"; E={$_.Exception.Message}}, @{N="Stack"; E={$_.ScriptStackTrace.Split("`n")[0]}}
}
function Test-VMConnectivity
{
    param (
        [ValidateSet("inbound","cloudagent", "dns", "internet")]
        [string]$scenario,
        [string]$vmName,
        [string]$group
    )

    $result = $false

    try {
        $vmTestResult = Get-VMTestResult -vmName $vmName -isSuccess $result -testName $scenario

        if ($scenario -ne "inbound" -and $script:VmInboundAccessState["$vmName"] -eq $false) {
            $msg = "$($MyInvocation.MyCommand) [SKIP NO INBOUND ACCESS] Scenario [$scenario] Group [$group] VM [$vmName]"
            Write-Msg -msg $msg -Warning
            $vmTestResult.DetailedStatus = $msg
            return $vmTestResult
        }

        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]"
        $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
        $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
        $sshKey = $(Get-MocConfigCache).sshPrivateKey
        $lbName = "$name-load-balancer"
        $lb = Get-MocLoadBalancer -name $lbName -group $group
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress
        $vm = Get-MocVirtualMachine -group $group -name $vmName
        $nicName = $_.virtualmachineproperties.networkprofile.networkinterfaces[0].id
        $isWindows = $_.virtualmachineproperties.osprofile.osType -eq "Windows"
        $endpoint = Get-NatEndpoint -groupName $group -nicName $nicName
            
        if ($endpoint -eq $null) {
            $msg = "$($MyInvocation.MyCommand) Unable to find ssh endpoint for vm $($_.Name) in cluster $name"
            $vmTestResult.DetailedStatus = $msg
            Write-Fail -msg $msg
            return
        }

        $sshInfo = "[$($endpoint.ip)`:$($endpoint.port)]"
        $vmTestResult.SSHInfo = $sshInfo

        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName], SSH [$($endpoint.ip)`:$($endpoint.port)]" 
        switch ($scenario) {
            "inbound" {
                $status = $(Test-Connectivity  -ip $($endpoint.ip) -port $($endpoint.port))
                $script:VmInboundAccessState["$vmName"] = $status
                if ($status) {
                    $msg = "Inbound connectivity to $sshInfo succeeded"
                } else {
                    $msg = "Inbound connectivity to $sshInfo failed"
                }
    

                $result = Get-Result -scenario "inbound" -isSuccess $status -detailedStatus $msg
                break
            }
            "cloudagent" {
                $result = $(Test-CloudAgentConnectivity -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break
            }
            "dns" {
                $result = $(Test-DnsResolution -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break                
            }
            "internet" {
                $result = $(Test-InternetConnectivity -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break
            }
        }              

        $vmTestResult.Results = @($result)
        $vmTestResult.Passed = $result.Passed

        if (-not $result.Passed) {
            throw "Test-VMConnectivity Scenario [$scenario] VM [$vmName] failed "
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]"
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]" -err $_
        $vmTestResult.DetailedStatus = Convert-Error $_
        $vmTestResult.Passed = $false
    }

    return $vmTestResult
}

function Test-TargetClusterConnectivity
{
    param (
        [ValidateSet("inbound", "cloudagent", "dns", "internet")]
        [string]$scenario,
        [string]$clusterName
    )

    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()
    try {
        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario]"
        $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
        $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
        $sshKey = $(Get-MocConfigCache).sshPrivateKey
        $status = $true

        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $_.Name
            $group = "clustergroup-$name"
            Write-Msg -msg "$($MyInvocation.MyCommand) Cluster [$name]"
            $lbName = "$name-load-balancer"
            $lb = Get-MocLoadBalancer -name $lbName -group $group
            $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress
            $vms = Get-MocVirtualMachine -group $group
            $vms | ForEach-Object {
                $result = Test-VMConnectivity -scenario $scenario -vmName $_.Name -group $group
                $testResult.TestResults += $result
                $status = $status -and $result.Passed
            }
        }

        if (-not $status) {
            throw "Test-TargetClusterConnectivity [$scenario] failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario]"
    }
    catch  {
        $status = $false
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario]" -err $_
        $testResult.DetailedStatus = Convert-Error $_
    } finally  {
        $testResult.Passed = $status
    }
    return $testResult
}

function Test-ManagementClusterNetwork()
{
    param (
        [ValidateSet("cloudagent", "dns", "internet", "inbound", "all")]
        [string]$scenario = "all"
    )

    $status = $true
    $mgmtClusterResult = Get-ClusterTestResult -clusterName "Management" -testResults @()
    try {
        switch($scenario) {
            {$_ -in "inbound", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "inbound"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }
            {$_ -in "controlplane", "all"} {
                $result = Test-ManagementControlPlaneConnectivity
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "cloudagent", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "cloudagent"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "dns", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "dns"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "internet", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "internet"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }
        }
    } catch {
        $status = $false
        $mgmtClusterResult.DetailedStatus = Convert-Error $_
    }

    $mgmtClusterResult.Passed = $status
    return $mgmtClusterResult
}

function Test-TargetClusterNetwork()
{
    param (
        [ValidateSet("controlplane","cloudagent", "dns", "internet", "inbound", "all")]
        [string]$scenario="all",
        [string]$clusterName
    )

    $status = $true
    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()

    try {
        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $_.Name
            $clusterTestResult = Get-ClusterTestResult -clusterName $name -isSuccess $true -testResults @()
            $testResult.TestResults += $clusterTestResult
            switch($scenario) {
                {$_ -in "inbound", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "inbound"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }
                {$_ -in "controlplane", "all"} {
                    $result = Test-TargetControlPlaneConnectivity -clusterName $name 
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "cloudagent", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "cloudagent"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "dns", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "dns"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "internet", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "internet"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }
            }
        }
    } catch {
        $status = $false
        $testResult.DetailedStatus = Convert-Error $_
    } finally {
        $testResult.Passed = $status
    }

    return $testResult

}

function Test-AksHciSdnNetwork()
{
    if (-not $(Test-IsAksHciInstalled)) {
        Write-Warning "$($MyInvocation.MyCommand) AksHci is not installed, skipping validation"
        return
    }

    $status = $true
    $results = @()

    $result = Test-ManagementClusterNetwork
    $results += $result
    $status = $status -and $result.Passed

    $result = Test-TargetClusterNetwork
    $results += $result
    $status = $status -and $result.Passed

    if ($status) {
        Write-Pass -testName "$($MyInvocation.MyCommand)"
    }
    else {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -msg "One or more tests failed"
    }

    Write-TestResults $results

    return $results
}