Kva.psm1
|
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # Kva Module # ######################################################################################### #requires -runasadministrator using module .\Common.psm1 #region Module Constants $moduleName = "Kva" $moduleVersion = "1.0.0" #endregion #region Download catalog constants # Defaults until KVA has it's own catalog $catalogName = "aks-hci-stable-catalogs-ext" $ringName = "stable" $productName = "kva" $productInfoMapName = "cloudop-product-information" $productInfoMapNamespace = "cloudop-system" #endregion #region Script Constants $defaultContainerRegistryServer = "ecpacr.azurecr.io" $defaultContainerRegistryU = "1516df5a-f1cc-4a6a-856c-03d127b02d05" $defaultContainerRegistryP = "92684690-48b5-4dce-856d-ef4cccb54f22" $logFilePath = $($env:ProgramData + "\kva\kva.log") $kubeconfigMgmtFile = "kubeconfig-mgmt" $script:defaultTokenExpiryDays = 60 $kvaBinaries = @( $global:kvactlBinary, $global:kubectlBinary ) $kvaBinariesMap = @{ $global:kvactlBinary = $global:kvaCtlFullPath; $global:kubectlBinary = $global:kubeCtlFullPath; } if (!$global:config) { $global:config = @{} } # azure $global:azureCloud = "AzureCloud" $global:azureChinaCloud = "AzureChinaCloud" $global:azureUSGovernment = "AzureUSGovernment" $global:azureGermanCloud = "AzureGermanCloud" $global:azurePPE = "AzurePPE" $global:graphEndpointResourceIdAzureCloud = "https://graph.windows.net/" $global:graphEndpointResourceIdAzurePPE = "https://graph.ppe.windows.net/" $global:graphEndpointResourceIdAzureChinaCloud = "https://graph.chinacloudapi.cn/" $global:graphEndpointResourceIdAzureUSGovernment = "https://graph.windows.net/" $global:graphEndpointResourceIdAzureGermancloud = "https://graph.cloudapi.de/" #endregion #region # Install Event Log New-ModuleEventLog -moduleName $moduleName #endregion #region Private Function function Initialize-KvaConfiguration { <# .DESCRIPTION Initialize Kva Configuration. Wipes off any existing cached configuration #> if ($global:config.ContainsKey($moduleName)) { $global:config.Remove($moduleName) } $global:config += @{ $moduleName = @{ "cloudAgentAuthorizerPort" = 0 "cloudAgentPort" = 0 "cloudLocation" = "" "controlplaneVmSize" = [VmSize]::Default "dnsservers" = "" "gateway" = "" "group" = "" "imageDir" = "" "insecure" = $false "installationPackageDir" = "" "installState" = [InstallState]::NotInstalled "ipaddressprefix" = "" "k8snodeippoolstart" = "" "k8snodeippoolend" = "" "kubeconfig" = "" "kvaconfig" = "" "kvaK8sVersion" = "" "kvaName" = "" "kvaPodCidr" = "" "macPoolEnd" = "" "macpoolname" = "" "macPoolStart" = "" "manifestCache" = [io.path]::GetTempFileName() "moduleVersion" = $moduleVersion "proxyServerCertFile" = "" "proxyServerHTTP" = "" "proxyServerHTTPS" = "" "proxyServerNoProxy" = "" "proxyServerPassword" = "" "proxyServerUsername" = "" "skipUpdates" = $false "stagingShare" = "" "tokenExpiryDays" = 0 "containerRegistryServer" = "" "containerRegistryUser" = "" "containerRegistryPass" = "" "useStagingShare" = $false "version" = "" "vlanid" = 0 "vnetName" = "" "vswitchName" = "" "vnetvippoolend" = "" "vnetvippoolstart" = "" "workingDir" = "" "catalog" = "" "ring" = "" "identity" = "" "operatorTokenValidity" = 0 "addonTokenValidity" = 0 }; } } #endregion #region global config Initialize-KvaConfiguration #endregion #region Exported Functions function Install-Kva { <# .DESCRIPTION Uses KVACTL to deploy a management cluster. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -createConfigIfNotPresent -activity $activity $curState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($curState) { switch ($curState) { ([InstallState]::Installed) { Write-Status -moduleName $moduleName "KVA is already be installed" Write-SubStatus -moduleName $moduleName "Please use Restart-Kva to reinstall or Uninstall-Kva to uninstall." return } ([InstallState]::Installing) { Write-Status -moduleName $moduleName "Kva is currently being installed. If thats not the case, please run Uninstall-Kva and try again" return break } ([InstallState]::NotInstalled) { # Fresh install break } Default { # Cleanup partial installs from previous attempts Uninstall-Kva -activity $activity } } } try { Install-KvaInternal -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Uninstall-Kva -SkipConfigCleanup:$True -activity $activity throw $_ } Write-Status -moduleName $moduleName "Done." } function Restart-Kva { <# .DESCRIPTION Cleans up an existing KVA deployment and reinstalls everything. This isn't equivalent to executing 'Uninstall-Kva' followed by 'Install-Kva' as Restart-Kva will preserve existing configuration settings and any downloaded images. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # Skip the config cleanup to reinstall Uninstall-Kva -SkipConfigCleanup:$True -activity $activity Install-KvaInternal -activity $activity Write-Status -moduleName $moduleName "Done." } function Uninstall-Kva { <# .DESCRIPTION Removes a KVA deployment. .PARAMETER SkipConfigCleanup skips removal of the configurations after uninstall. After uninstall, you have to Set-KvaConfig to install again. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Switch]$SkipConfigCleanup, [String]$activity = $MyInvocation.MyCommand.Name ) try { Initialize-KvaEnvironment -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } Set-KvaConfigValue -name "installState" -value ([InstallState]::Uninstalling) try { # Do not use Get-KvaConfigYaml, which can generate one, if not already there. if (Test-Path $kvaCtlFullPath) { $yamlFile = $global:config[$modulename]["kvaconfig"] if ($yamlFile -and (Test-Path $yamlFile)) { $kvaRegistration = Get-KvaRegistration try { Test-KvaAzureConnection # If Test-KvaAzureConnection worked, do the cleanup for the connectedcluster azure resource Invoke-KvaCtlWithAzureContext -arguments "delete --configfile $yamlFile" -showOutputAsProgress -activity $activity } catch { Write-Status -moduleName $moduleName -Verbose -msg "Uninstalling KVA without Azure Connection may result in leaked Arc Connected Clusters, Please clean up resources in portal." Invoke-KvaCtl -arguments "delete --configfile $yamlFile" -showOutput -activity $activity } } } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } try { $kubeconfig = $global:config[$moduleName]["kubeconfig"] if ($kubeconfig -and (Test-Path $kubeconfig)) { Remove-Item $kubeconfig -ErrorAction SilentlyContinue } $kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"] if ($kubernetesVersion) { $imageGalleryName = Get-KubernetesGalleryImageName -imagetype "Linux" -k8sVersion $kubernetesVersion try { Remove-MocGalleryImage -name $imageGalleryName -location $global:config[$modulename]["cloudLocation"] | Out-Null } catch { if (-not ($_.Exception.Message -like "*connection closed*")) { Write-SubStatus -moduleName $moduleName $("Warning: " + $_.Exception.Message) } } } # 1. Update the binaries if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Continue | ForEach-Object { Uninstall-KvaBinaries -nodeName $_.Name } } else { Uninstall-KvaBinaries -nodeName ($env:computername) } # 2. Remove KVA Identity try { $clusterName = $($global:config[$modulename]["kvaName"]) Remove-MocIdentity -name $clusterName | Out-Null } catch { if (-not ($_.Exception.Message -like "*connection closed*")) { Write-SubStatus -moduleName $moduleName $("Warning: " + $_.Exception.Message) } } # Clean CloudConfig Remove-Item -Path $global:kvaMetadataDirector -Force -Recurse -ErrorAction SilentlyContinue } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } Set-KvaConfigValue -name "installState" -value ([InstallState]::NotInstalled) if (!$SkipConfigCleanup.IsPresent) { Reset-Configuration -moduleName $moduleName } Write-Status -moduleName $moduleName "Done." } function New-KvaCluster { <# .DESCRIPTION Adds a worker cluster to the deployment. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneNodeCount The number of control plane (master) nodes .PARAMETER linuxNodeCount The number of Linux worker nodes .PARAMETER windowsNodeCount The number of Windows worker nodes .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER activity Activity name to use when writing progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $kubernetesVersion = $global:defaultTargetK8Version, [Parameter()] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount = 1, [Parameter()] [ValidateRange(0,250)] [int] $linuxNodeCount = 1, [Parameter()] [ValidateRange(0,250)] [int] $windowsNodeCount = 0, [Parameter()] [String] $controlplaneVmSize = $global:defaultControlPlaneVmSize, [Parameter()] [String] $loadBalancerVmSize = $global:defaultLoadBalancerVmSize, [Parameter()] [String] $linuxNodeVmSize = $global:defaultWorkerVmSize, [Parameter()] [String] $windowsNodeVmSize = $global:defaultWorkerVmSize, [Parameter()] [Switch] $enableADAuth, [Parameter()] [String] $activity, [Parameter()] [ValidateScript({return $true})]#Note: ValidateScript automatically constructs the NetworkPlugin object, therefore validates the parameter [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [VirtualNetwork]$vnet ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Validating cluster configuration..." Test-ValidClusterName -Name $Name | Out-Null $capiCluster = Invoke-Kubectl -arguments $("get akshciclusters/$Name") -ignoreError if ($null -ne $capiCluster) { throw $("The specified cluster name '$Name' already exists.") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Ensure MOC is available..." $isCAAvailable = Wait-ForCloudAgentEndpoint -activity $activity if (-not $isCAAvailable) { throw $("MOC is unreachable") } $group = $("$global:cloudGroupPrefix-$Name") New-KvaClusterInternal -Name $Name -group $group -kubernetesVersion $kubernetesVersion ` -controlPlaneReplicas $controlPlaneNodeCount -linuxWorkerReplicas $linuxNodeCount ` -windowsWorkerReplicas $windowsNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize -linuxNodeVmSize $linuxNodeVmSize ` -windowsNodeVmSize $windowsNodeVmSize -enableADAuth:$enableADAuth.IsPresent ` -activity $activity -primaryNetworkPlugin $primaryNetworkPlugin -vnet $vnet Write-Status -moduleName $moduleName "Done." } function New-KvaClusterInternal { <# .DESCRIPTION Internal function to call KVACTL to create a capicluster. .PARAMETER Name Name of the cluster .PARAMETER group Cloudagent group .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneReplicas The number of control plane (master) replicas .PARAMETER linuxWorkerReplicas The number of Linux worker replicas .PARAMETER windowsWorkerReplicas The number of Windows worker replicas .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER activity Activity name to use when writing progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$group, [Parameter(Mandatory=$true)] [String]$kubernetesVersion, [Parameter(Mandatory=$true)] [int] $controlPlaneReplicas, [Parameter(Mandatory=$true)] [int] $linuxWorkerReplicas, [Parameter(Mandatory=$true)] [int] $windowsWorkerReplicas, [Parameter(Mandatory=$true)] [VmSize] $controlplaneVmSize, [Parameter(Mandatory=$true)] [VmSize] $loadBalancerVmSize, [Parameter(Mandatory=$true)] [VmSize] $linuxNodeVmSize, [Parameter(Mandatory=$true)] [VmSize] $windowsNodeVmSize, [Parameter()] [Switch]$enableADAuth, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter()] [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [VirtualNetwork]$vnet ) Add-GalleryImage -imageType Linux -k8sVersion $kubernetesVersion -activity $activity if ($windowsWorkerReplicas -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $kubernetesVersion -activity $activity } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Creating workload cluster '$Name'") $publicKey = Get-Content -Path (Get-SshPublicKey) $publicKey = $publicKey.Split(" ") $publicKey = $($publicKey[0]+" "+$publicKey[1]) if (-not $vnet) { $vnet = Get-VNetConfiguration -module $moduleName } $yaml = @" name: $Name version: $kubernetesVersion controlplane: nodepool: replicas: $controlPlaneReplicas azurestackhcinodepool: hardwareprofile: vmsize: $controlplaneVmSize osprofile: ostype: Linux ssh: publickeys: - keydata: $publicKey servicecidr: $($global:workloadServiceCidr) podcidr: $($global:workloadPodCidr) azurestackhcicluster: storageconfiguration: storagecontainer: $($global:cloudStorageContainer) dynamic: true containernetworkconfiguration: primaryconfiguration: $($primaryNetworkPlugin.Name) loadbalancer: vmsize: $loadBalancerVmSize location: $($global:config[$modulename]["cloudLocation"]) group: $group virtualnetwork: name: "$($vnet.Name)" "@ $yaml += " additionalfeatures: - featurename: secrets-encryption - featurename: rotate-certificates" # TODO add switch to kvactl if ($enableADAuth.IsPresent) { $yaml += " - featurename: ad-auth-webhook" } $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster create --clusterconfig $yamlFile --kubeconfig $kubeconfig" -showOutput -activity $activity $linuxNodePoolYaml = @" name: $Name-default-linux-nodepool replicas: $linuxWorkerReplicas azurestackhcinodepool: hardwareprofile: vmsize: $linuxNodeVmSize osprofile: ostype: Linux ssh: publickeys: - keydata: $publicKey "@ $linuxNodePoolYamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-default-linux-nodepool.yaml") Set-Content -Path $linuxNodePoolYamlFile -Value $linuxNodePoolYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KvaCtl -arguments "cluster nodepool create --clustername $Name --nodepoolconfig $linuxNodePoolYamlFile --kubeconfig $kubeconfig" -showOutput -activity $activity $windowsNodePoolYaml = @" name: $Name-default-windows-nodepool replicas: $windowsWorkerReplicas azurestackhcinodepool: hardwareprofile: vmsize: $windowsNodeVmSize osprofile: ostype: Windows ssh: publickeys: - keydata: $publicKey "@ $windowsNodePoolYamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-default-windows-nodepool.yaml") Set-Content -Path $windowsNodePoolYamlFile -Value $windowsNodePoolYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KvaCtl -arguments "cluster nodepool create --clustername $Name --nodepoolconfig $windowsNodePoolYamlFile --kubeconfig $kubeconfig" -showOutput -activity $activity } function Get-Kva { <# .DESCRIPTION Get the Kva management cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Retrieving KVA configuration file" $yamlFile = Get-KvaConfigYaml $kubeconfig = Get-KvaCredential -activity $activity if (!(Test-Path $kubeconfig)) { # Retrieve the kubeconfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Retrieving KVA credentials" Invoke-KvaCtl -arguments "retrieve --configfile $yamlFile --outfile $kubeconfig" -showOutput -activity $activity } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Retrieving KVA deployment status" $capiCluster = $null $kvaStatus = $null try{ $kvaStatus = Invoke-KvaCtl -arguments "status --configfile $yamlFile --kubeconfig $kubeconfig" -activity $activity | ConvertFrom-Json } catch {} if ($kvaStatus) { # Verify that the appliance is deployed and reachable before requesting cluster information if (($kvaStatus.phase -ine "NotDeployed") -and ($kvaStatus.phase -ine "WaitingForAPIServer") -and ($kvaStatus.phase -ine "Failed")) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Retrieving management cluster details" $capiCluster = Get-KvaCluster -Name $global:config[$moduleName]["kvaName"] -activity $activity } } $status = [PSCustomObject]@{ CapiCluster = $capiCluster KvaStatus = $kvaStatus } return $status } function Get-KvaClusters { <# .DESCRIPTION Get the all kva clusters .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity return Get-TargetCapiClusters } function Get-KvaCluster { <# .DESCRIPTION Validates the requested cluster name and ensures that the cluster exists. Returns the cluster object. .PARAMETER Name Name of the cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Write-StatusWithProgress -activity $activity -status "Retrieving cluster..." -moduleName $moduleName return Get-KvaClusterInternal -Name $Name } function Get-KvaClusterInternal { <# .DESCRIPTION Validates the requested cluster name and ensures that the cluster exists. Returns the cluster object. .PARAMETER Name Name of the cluster #> param ( [Parameter(Mandatory=$true)] [String]$Name ) Test-ValidClusterName -Name $Name | Out-Null return Get-CapiCluster -Name $Name } function Get-KvaClusterUpgrades { <# .DESCRIPTION Gets the upgrades available for a kva cluster. .PARAMETER Name Name of the cluster. .PARAMETER activity Activity name to use when updating progress. #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $("Unable to proceed due to missing kubeconfig file: $kubeconfig") } $cmdArgs = "cluster get-upgrades --clustername $Name --kubeconfig $kubeconfig" $upgrades = Invoke-KvaCtl -arguments $cmdArgs -activity $activity | ConvertFrom-Json return $upgrades } function Wait-ForClusterUpgrade { <# .DESCRIPTION Waits for all nodes of a cluster to be running the specified kubernetes version. .PARAMETER kubeconfigFile Path to a kubeconfig file for the cluster .PARAMETER expectedVersion The version that we expect all nodes of the cluster to be running .PARAMETER sleepDuration Duration to sleep for between attempts #> param ( [String] $kubeconfigFile, [String] $expectedVersion, [int] $sleepDuration=60 ) Write-Status $("Monitoring cluster upgrade progress") -moduleName $moduleName while ($true) { $results = @() $allNodesUpgraded = $true $nodes = (Invoke-Kubectl -ignoreError -kubeconfig $kubeconfigFile -arguments "get nodes --request-timeout=1m -o name") 2>$null if (-not ($nodes)) { Write-SubStatus $("Unable to retrieve cluster node information. Will retry in $sleepDuration seconds...`n") -moduleName $moduleName Start-Sleep $sleepDuration continue } foreach ($node in $nodes) { $nodeJson = (Invoke-Kubectl -ignoreError -kubeconfig $kubeconfigFile -arguments $("get $node --request-timeout=1m -o json") | ConvertFrom-Json) 2>$null if ($nodeJson) { $result = New-Object -TypeName PsObject -Property @{Name = $($nodeJson.metadata.name); Version = $($nodeJson.status.nodeInfo.kubeletVersion); Upgraded = $true} if ($nodeJson.status.nodeInfo.kubeletVersion -ine $expectedVersion) { $result.upgraded = $false $allNodesUpgraded = $false } $results += $result } else { $allNodesUpgraded = $false } } Write-SubStatus $("Current status of cluster nodes:`n") -moduleName $moduleName $results | Format-Table * if ($allNodesUpgraded) { break } Write-SubStatus $("Cluster nodes are still upgrading. Will check again in $sleepDuration seconds...`n") -moduleName $moduleName Start-Sleep $sleepDuration } Write-SubStatus $("All cluster nodes are running the expected version.") -moduleName $moduleName } function Update-KvaCluster { <# .DESCRIPTION Updates the kubernetes version of a cluster by performing an upgrade. .PARAMETER Name Name of the cluster. .PARAMETER nextVersion desired kubernetes version. .PARAMETER operatingSystem If specified, akshcicluster will only attempt an operating system upgrade. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter()] [String]$nextVersion, [Parameter()] [Switch]$operatingSystem, [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $cluster = Get-KvaCluster -Name $Name -activity $activity $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $("Unable to proceed due to missing kubeconfig file: $kubeconfig") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Determining upgrade path..." $cmdArgs = "cluster upgrade --clustername $($cluster.metadata.name) --plan --kubeconfig $kubeconfig" if (($null -ne $nextVersion) -and ($nextVersion -ne "")) { $cmdArgs += " --kubernetes-version $nextVersion" } if ($operatingSystem.IsPresent) { $cmdArgs += " --operating-system" } $upgradePlan = Invoke-KvaCtl -arguments $cmdArgs -activity $activity $upgradePlan = $upgradePlan -join "`n" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $upgradePlan Write-Status -moduleName $moduleName -Verbose -msg "`n$upgradePlan`n" if ($upgradePlan -like "*no upgrade found*") { return } if (-not $PSCmdlet.ShouldProcess($Name, "Update the managed Kubernetes cluster")) { return } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Provisioning image gallery..." Write-SubStatus -moduleName $moduleName $("The cluster is currently on Kubernetes version: " + $cluster.spec.clusterConfiguration.kubernetesVersion) # We may need to re-add the same image, to pick up just OS update Add-GalleryImage -imageType Linux -k8sVersion $cluster.spec.clusterConfiguration.kubernetesVersion -activity $activity if ($cluster.windowsWorkerReplicas -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $cluster.spec.clusterConfiguration.kubernetesVersion -activity $activity } if ($nextVersion) { Add-GalleryImage -imageType Linux -k8sVersion $nextVersion -activity $activity if ($cluster.windowsWorkerReplicas -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $nextVersion -activity $activity } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Performing cluster upgrade..." $cmdArgs = "cluster upgrade --clustername $($cluster.metadata.name) --kubeconfig $kubeconfig" if (($null -ne $nextVersion) -and ($nextVersion -ne "")) { $cmdArgs += " --kubernetes-version $nextVersion" } if ($operatingSystem.IsPresent) { $cmdArgs += " --operating-system" } Invoke-KvaCtl -arguments $cmdArgs -showOutput -activity $activity try { $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $cluster.metadata.name -outputLocation $targetClusterKubeconfig -activity $activity if ($nextVersion) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Waiting for upgrade to complete..." Wait-ForClusterUpgrade -expectedVersion $nextVersion -kubeconfigFile $targetClusterKubeconfig } } finally { Remove-Item -Path $targetClusterKubeconfig -Force -ErrorAction SilentlyContinue } } function Set-KvaClusterNodeCount { <# .DESCRIPTION Sets the configuration of a cluster. Currently used to scale the cluster control plane or to scale the worker nodes. .PARAMETER Name Name of the cluster .PARAMETER controlPlaneNodeCount The number of control plane nodes to scale to .PARAMETER linuxNodeCount The number of Linux worker nodes to scale to .PARAMETER windowsNodeCount The number of Windows worker nodes to scale to .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='controlplane')] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $linuxNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $windowsNodeCount, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $cluster = Get-KvaCluster -Name $Name -activity $activity $controlPlane = $cluster.spec.controlPlaneConfiguration.replicas $linuxWorker = $cluster.linuxWorkerReplicas $windowsWorker = $cluster.windowsWorkerReplicas $kubeconfig = Get-KvaCredential -activity $activity if ($PSCmdlet.ParameterSetName -ieq "controlplane") { if ($controlPlaneNodeCount -lt 1) { throw $("Cluster '$($cluster.metadata.name)' cannot be scaled to less than 1 control plane node.") } if (($cluster.spec.controlPlaneConfiguration.replicas -gt 1) -and ($controlPlaneNodeCount -lt 3)) { throw $("Cluster '$($cluster.metadata.name)' is a highly available control plane and cannot be scaled to less than 3 nodes.") } $controlPlane = $controlPlaneNodeCount $cmdControlPlaneArgs = "cluster scale --clustername=$Name --controlplane=$controlPlane --kubeconfig $kubeconfig" Invoke-KvaCtl -arguments $cmdControlPlaneArgs -showOutput -activity $activity } elseif ($PSCmdlet.ParameterSetName -ieq "worker") { if ($windowsNodeCount -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $cluster.spec.clusterConfiguration.kubernetesVersion -activity $activity } $linuxWorker = $linuxNodeCount $windowsWorker = $windowsNodeCount $cmdLinuxWorkerArgs = "cluster nodepool scale --clustername=$Name --nodepoolname=$Name-default-linux-nodepool --replicas=$linuxWorker --kubeconfig $kubeconfig" $cmdWindowsWorkerArgs = "cluster nodepool scale --clustername=$Name --nodepoolname=$Name-default-windows-nodepool --replicas=$windowsWorker --kubeconfig $kubeconfig" Invoke-KvaCtl -arguments $cmdLinuxWorkerArgs -showOutput -activity $activity Invoke-KvaCtl -arguments $cmdWindowsWorkerArgs -showOutput -activity $activity } Write-Status -moduleName $moduleName "Done." } function Remove-KvaCluster { <# .DESCRIPTION Removes a cluster from the deployment. .PARAMETER Name Name of the cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Ensure MOC is available..." $isCAAvailable = Wait-ForCloudAgentEndpoint -activity $activity if (-not $isCAAvailable) { throw $("CloudAgent is unreachable") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Removing cluster..." Write-SubStatus -moduleName $moduleName "Cluster removal is in progress. This can take several minutes to complete..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster delete --clustername $Name --kubeconfig $kubeconfig" -showOutput -activity $activity $kubeconfig = [io.Path]::Combine($global:config[$modulename]["installationPackageDir"], "$Name-kubeconfig") if (Test-Path $kubeconfig) { Remove-Item $kubeconfig -ErrorAction SilentlyContinue } Write-Status -moduleName $moduleName "Done." } function Update-Kva { <# .DESCRIPTION Update a the kva .PARAMETER version Optional version to udate to .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$version, [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $currentVersion = Get-KvaVersion $currentKvaK8sVersion = $global:config[$moduleName]["kvaK8sVersion"] Write-SubStatus -moduleName $moduleName "Current KVA Version: $currentVersion" # If no version is specified, try to move to the latest if (!$version) { # If no version is specified, use the latest $release = Get-LatestRelease -moduleName $moduleName $version = $release.Version Set-KvaConfigValue -name "version" -value $version $release = Get-ProductRelease -Version $version -module $moduleName } else { if ($version -eq $currentVersion) { Write-SubStatus -moduleName $moduleName "Already in the expected version $version" return } $release = Get-ProductRelease -Version $version -module $moduleName Set-KvaConfigValue -name "version" -value $version } $workingDir = $global:config[$moduleName]["workingDir"] Write-StatusWithProgress -activity $activity -module $moduleName -status $("Updating to version $version") try { # Set the new K8s version Set-KvaConfigValue -name "kvaK8sVersion" -value ("v" +$release.CustomData.ManagementNodeImageK8sVersion) Set-KvaConfigValue -name "installationPackageDir" -value $([io.Path]::Combine($workingDir, $version)) Get-KvaConfigYaml | Out-Null Update-KvaInternal } catch { Set-KvaConfigValue -name "kvaK8sVersion" -value $currentKvaK8sVersion Set-KvaConfigValue -name "version" -value $currentVersion Set-KvaConfigValue -name "installationPackageDir" -value $([io.Path]::Combine($workingDir, $currentVersion)) Get-KvaConfigYaml | Out-Null # Revert Write-StatusWithProgress -activity $activity -module $moduleName -status $("Reverting to version $currentVersion") Update-KvaInternal throw } } function Update-KvaInternal { <# .DESCRIPTION Update KVA .PARAMETER activity Activity name to use when updating progress #> param ( [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status "Updating KVA to $(Get-KvaVersion)" # 1. Invalidate all gallery images # Gallery images are downloaded and managed by KVA. So it is okay to reset it completely for now. # If this assumption changes, change the logic here to just remove what KVA brings in Reset-GalleryImage New-Item -ItemType Directory -Force -Path $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:yamlDirectoryName)) | Out-Null # 2. Update the binaries Get-KvaRelease -version $(Get-KvaVersion) -activity $activity if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Install-KvaBinaries -nodeName $_.Name } } else { Install-KvaBinaries -nodeName ($env:computername) } # 3. Upgrade the management appliance $cluster = Get-KvaClusterInternal -Name $global:config[$moduleName]["kvaName"] Write-Status -moduleName $moduleName "Provisioning image gallery" Write-SubStatus -moduleName $moduleName $("The cluster is currently on Kubernetes version: " + $cluster.spec.clusterConfiguration.kubernetesVersion) $nextVersion = $global:config[$moduleName]["kvaK8sVersion"] Add-GalleryImage -imageType Linux -k8sVersion $nextVersion -activity $activity $kubeconfig = Get-KvaCredential -activity $activity $updateConfig = Get-KvaUpdateConfigYaml $deploymentManifestLocation = $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:cloudOperatorYaml)) $privateKey = Get-SshPrivateKey Invoke-KvaCtl -arguments "upgrade --configfile $updateConfig --kubeconfig $kubeconfig --cloudop $deploymentManifestLocation --sshprivatekey $privateKey" -showOutput -activity $activity } function Add-KvaGalleryImage { <# .DESCRIPTION Downloads an image and adds it to the cloud gallery. .PARAMETER kubernetesVersion Optional, override the default version of kubernetes that the image will use. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $kubernetesVersion, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity if (-not $kubernetesVersion) { $kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"] } Add-GalleryImage -imagetype "Linux" -k8sVersion $kubernetesVersion -activity $activity Write-Status -moduleName $moduleName "Done." } function Get-KvaVersion { <# .DESCRIPTION Get the current KVA version .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity return $global:config[$modulename]["version"] } function Get-KvaConfig { <# .DESCRIPTION Loads and returns the current KVA configuration. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Import-KvaConfig -activity $activity Write-Status -moduleName $moduleName "Getting configuration for $moduleName" $global:config[$modulename]["installState"] = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" $global:config[$modulename]["controlplaneVmSize"] = Get-ConfigurationValue -module $moduleName -type ([Type][VmSize]) -name "controlplaneVmSize" return $global:config[$modulename] } function Import-KvaConfig { <# .DESCRIPTION Loads a configuration from persisted storage. If no configuration is present then a default configuration can be optionally generated and persisted. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Switch] $createIfNotPresent, [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status "Importing Configuration" if (Test-Configuration -moduleName $moduleName) { Import-Configuration -moduleName $moduleName } else { throw "This machine does not appear to be configured for deployment." } Write-StatusWithProgress -activity $activity -module $moduleName -status "Importing Configuration Completed" } function Set-KvaRegistration { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. .PARAMETER azureResourceGroup azureResourceGroup is the name of the azure resource group to place arc resources. .PARAMETER azureLocation azureLocation is the name of the azure location where the resource group lives. #> [CmdletBinding()] param ( [String] $azureResourceGroup, [String] $azureLocation ) $kvaRegistration = Get-KvaRegistration if ([string]::IsNullOrWhiteSpace($kvaRegistration.azureResourceGroup)) { Set-KvaConfigValue -name "azureResourceGroup" -value $azureResourceGroup Set-KvaConfigValue -name "azureLocation" -value $azureLocation } } function Get-KvaRegistration { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. #> $obj = New-Object -TypeName psobject $obj | Add-Member -MemberType NoteProperty -Name azureResourceGroup -Value (Get-KvaConfigValue -name "azureResourceGroup") $obj | Add-Member -MemberType NoteProperty -Name azureLocation -Value (Get-KvaConfigValue -name "azureLocation") return $obj } function Set-KvaConfig { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. #> [CmdletBinding()] param ( [string] $activity = $MyInvocation.MyCommand.Name, [String] $kvaName = (New-Guid).Guid, [String] $workingDir = $global:defaultWorkingDir, [String] $imageDir, [String] $version = $global:version, [String] $stagingShare = $global:defaultStagingShare, [String] $cloudLocation = $global:defaultCloudLocation, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet, [VmSize] $controlplaneVmSize = $global:defaultMgmtControlPlaneVmSize, [String] $kvaPodCIDR = $global:defaultPodCidr, [Switch] $kvaSkipWaitForBootstrap, [ProxySettings] $proxySettings = $null, [Switch] $skipUpdates, [Switch] $skipHostLimitChecks, [Switch] $insecure, [String] $macPoolStart, [String] $macPoolEnd, [switch] $useStagingShare, [ContainerRegistry] $containerRegistry = $null, [String] $catalog = $script:catalogName, [String] $ring = $script:ringName, [int] $cloudAgentPort = $global:defaultCloudAgentPort, [int] $cloudAgentAuthorizerPort = $global:defaultCloudAuthorizerPort, [String] $deploymentId = [Guid]::NewGuid().ToString(), [int] $tokenExpiryDays = $script:defaultTokenExpiryDays, [parameter(DontShow)] [int] $operatorTokenValidity = $global:operatorTokenValidity, [parameter(DontShow)] [int] $addonTokenValidity = $global:addonTokenValidity ) # Import the existing config, if any try { Import-KvaConfig -activity $activity } catch {} $currentState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($currentState) { switch ($currentState) { ([InstallState]::NotInstalled) { # Fresh install break } Default { Write-Status -moduleName $moduleName "Kva is currently in $currentState state" throw "Cannot set new $moduleName configuration when in this state [$currentState]" } } } Confirm-Configuration -useStagingShare:$useStagingShare.IsPresent -stagingShare $stagingShare Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName Set-ContainerRegistryConfiguration -containerRegistry $containerRegistry # If okay to proceed, overwrite Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Creating configuration for $moduleName" Set-KvaConfigValue -name "workingDir" -value $workingDir Set-KvaConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, "$catalog.json")) New-Item -ItemType Directory -Force -Path $workingDir | Out-Null if (!$imageDir) { $imageDir = [io.Path]::Combine($workingDir, $global:imageDirectoryName) New-Item -ItemType Directory -Force -Path $workingDir | Out-Null } Set-KvaConfigValue -name "imageDir" -value $imageDir Set-KvaConfigValue -name "stagingShare" -value $stagingShare Set-KvaConfigValue -name "group" -value "clustergroup" Set-KvaConfigValue -name "cloudLocation" -value $cloudLocation Set-KvaConfigValue -name "moduleVersion" -value $moduleVersion Set-KvaConfigValue -name "catalog" -value $catalog Set-KvaConfigValue -name "ring" -value $ring Set-KvaConfigValue -name "deploymentId" -value $deploymentId Set-KvaConfigValue -name "operatorTokenValidity" -value $operatorTokenValidity Set-KvaConfigValue -name "addonTokenValidity" -value $addonTokenValidity Set-VNetConfiguration -module $moduleName -vnet $vnet Set-KvaConfigValue -name "controlplaneVmSize" -value $controlplaneVmSize Set-KvaConfigValue -name "skipUpdates" -value $skipUpdates.IsPresent Set-KvaConfigValue -name "insecure" -value $insecure.IsPresent Set-KvaConfigValue -name "useStagingShare" -value $useStagingShare.IsPresent Set-KvaConfigValue -name "macPoolStart" -value $macPoolStart Set-KvaConfigValue -name "macPoolEnd" -value $macPoolEnd Set-KvaConfigValue -name "kvaName" -value $kvaName Set-KvaConfigValue -name "kvaPodCidr" -value $kvaPodCIDR Set-KvaConfigValue -name "kvaSkipWaitForBootstrap" -value $kvaSkipWaitForBootstrap.IsPresent Set-KvaConfigValue -name "cloudAgentPort" -value $cloudAgentPort Set-KvaConfigValue -name "cloudAgentAuthorizerPort" -value $cloudAgentAuthorizerPort Set-KvaConfigValue -name "tokenExpiryDays" -value $tokenExpiryDays if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if ($version) { $release = Get-ProductRelease -Version $version -module $moduleName } else { # If no version is specified, use the latest from the product catalog $release = Get-LatestRelease -moduleName $moduleName $version = $release.Version Set-KvaConfigValue -name "version" -value $version } } else { Get-LatestCatalog -moduleName $moduleName | Out-Null # This clears the cache $release = Get-ProductRelease -Version $version -module $moduleName Set-KvaConfigValue -name "version" -value $version } $installationPackageDir = ([io.Path]::Combine($workingDir, $version)) Set-KvaConfigValue -name "installationPackageDir" -value $installationPackageDir New-Item -ItemType Directory -Force -Path $installationPackageDir | Out-Null if (-not $release.CustomData.ManagementNodeImageK8sVersion) { throw "Unable to determine management cluster kubernetes version" } Set-KvaConfigValue -name "kvaK8sVersion" -value ("v" +($release.CustomData.ManagementNodeImageK8sVersion)) Set-KvaConfigValue -name "kubeconfig" -value $([io.Path]::Combine($workingDir, $version, $kubeconfigMgmtFile)) # Check if this is rehydration of an already running deployment if ((Test-Path $global:kvaCtlFullPath)) { try { Get-Kva | Out-Null Set-KvaConfigValue -name "installState" -value ([InstallState]::Installed) Write-SubStatus -moduleName $moduleName "Existing configuration for module $moduleName has been loaded`n" return } catch { Write-Verbose -Message $_ } } Set-KvaConfigValue -name "installState" -value ([InstallState]::NotInstalled) Save-ConfigurationDirectory -moduleName $moduleName -WorkingDir $workingDir Save-Configuration -moduleName $moduleName Write-SubStatus -moduleName $moduleName "New configuration for module $moduleName has been saved`n" } function Confirm-Configuration { <# .DESCRIPTION Validate if the configuration can be used for the deployment .PARAMETER useStagingShare Requests a staging share to be used for downloading binaries and images (for private testing) .PARAMETER stagingShare The staging share endpoint to use when useStagingShare is requested #> param ( [Switch] $useStagingShare, [String] $stagingShare ) if ($useStagingShare.IsPresent -and [string]::IsNullOrWhiteSpace($stagingShare)) { throw "-useStagingShare was requested, but no staging share was specified" } } function Set-KvaConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Object] $value ) Set-ConfigurationValue -name $name -value $value -module $moduleName } function Get-KvaConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value #> param ( [String] $name ) return Get-ConfigurationValue -name $name -module $moduleName } function Get-KvaConfigYaml { <# .DESCRIPTION Sets Configuration for the management appliance. .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER group The name of the group in which the vault resides #> param ( [String]$kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"], [String]$group = $global:config[$modulename]["group"] ) $installationPackageDir = $global:config[$moduleName]["installationPackageDir"] $yamlFile = $($installationPackageDir+"\yaml\appliance.yaml") if (-not (Test-Path $yamlFile)) { New-Item -ItemType File -Force -Path $yamlFile | Out-Null } # ACL the yaml so that it is only readable by administrator Set-SecurePermissionFile -Path $yamlFile $publicKey = Get-Content -Path (Get-SshPublicKey) $publicKey = $publicKey.Split(" ") $publicKey = $($publicKey[0]+" "+$publicKey[1]) $containerRegistryUser = $global:config[$moduleName]["containerRegistryUser"] $containerRegistryPass = $global:config[$moduleName]["containerRegistryPass"] if(-not [String]::IsNullOrEmpty($containerRegistryPass)) { $securePass = $containerRegistryPass | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $containerRegistryUser, $securePass $containerRegistryPass = $credential.GetNetworkCredential().Password } $deploymentManifestLocation = $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:cloudOperatorYaml)) $cloudFqdn = Get-CloudFqdn $clusterName = $($global:config[$modulename]["kvaName"]) $cloudPort = $global:config[$moduleName]["cloudAgentPort"].ToString() $cloudAuthPort = $global:config[$moduleName]["cloudAgentAuthorizerPort"].ToString() $kvaIdentity = $global:config[$moduleName]["identity"] $vnet = Get-VNetConfiguration -module $moduleName $operatorTokenValidity = $global:config[$moduleName]["operatorTokenValidity"] $addonTokenValidity = $global:config[$moduleName]["addonTokenValidity"] # Decide whether to use the macpool or not $useMacPool = (-not [string]::IsNullOrWhiteSpace( $global:config[$modulename]["macPoolStart"]) -and -not [string]::IsNullOrWhiteSpace($global:config[$modulename]["macPoolEnd"])) $macPoolName = "" if ($useMacPool) { $macPoolName = $vnet.MacPoolName } $dnsserver = "" $addressallocation = "Dynamic" if(-Not [string]::IsNullOrEmpty($vnet.IpAddressPrefix)) { $addressallocation = "Static" foreach ($dns in $vnet.DnsServers) { if(-Not [string]::IsNullOrEmpty($dns)) { $dnsserver += "`n - " + $dns } } } $kvaRegistration = Get-KvaRegistration if (($kvaRegistration.azureResourceGroup -ne "") -and ($kvaRegistration.azureLocation -ne "")) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } Write-Status -moduleName $moduleName "Creating KVA configuration" $waitForBootstrap = -not (Get-ConfigurationValue -module $moduleName -Name "kvaSkipWaitForBootstrap" -type ([Type][System.Boolean])) $proxySettings = Get-ProxyConfiguration -moduleName $moduleName $configMaps = Get-KvaConfigMaps $isHyperThreadingEnabled = Get-MocHyperThreadingEnabled $yaml = @" clustername: $clusterName kubernetesversion: $kubernetesVersion sshauthorizedkey: $publicKey lowprivilegekubeconfig: false encryptsecrets: true waitforbootstrapcompletion: $waitForBootstrap containerregistry: name: $($global:config[$moduleName]["containerRegistryServer"]) username: $containerRegistryUser password: $containerRegistryPass networking: controlplanecidr: $($global:mgmtControlPlaneCidr) clustercidr: $($global:mgmtClusterCidr) podcidr: $($global:config[$moduleName]["kvaPodCidr"]) proxy: http: "$($proxySettings.HTTP)" https: "$($proxySettings.HTTPS)" noproxy: "$($proxySettings.NoProxy)" certfilename: "$($proxySettings.CertName)" deploymentmanifest: cloudoperatormanifestpath: $deploymentManifestLocation cni: type: calico applianceagents: onboardingagent: subscriptionid: $subscriptionid tenantid: $tenantid resourcegroup: $resourcegroup location: $location infrastructure: azure_stack_hci billingagent: hyperthreading: $isHyperThreadingEnabled azurestackhciprovider: cloudagent: address: $cloudFqdn port: $cloudPort authenticationport: $cloudAuthPort loginconfig: $kvaIdentity insecure: $($global:config[$moduleName]["insecure"]) appliancevm: imagename: "" vmsize: $(Get-ConfigurationValue -module $moduleName -name "controlplaneVmSize" -type ([Type][VmSize])) loadbalancer: imagename: "" location: $($global:config[$modulename]["cloudLocation"]) group: $group storagecontainer: $($global:cloudStorageContainer) virtualnetwork: name: "$($vnet.Name)" vswitchname: "$($vnet.VswitchName)" type: "Transparent" macpoolname: $macPoolName vlanid: $($vnet.VlanID) ipaddressprefix: $($vnet.IpAddressPrefix) addressallocation: $addressallocation gateway: $($vnet.Gateway) dnsservers: $dnsserver vippoolstart: $($vnet.VipPoolStart) vippoolend: $($vnet.VipPoolEnd) k8snodeippoolstart: $($vnet.K8snodeIPPoolStart) k8snodeippoolend: $($vnet.K8snodeIPPoolEnd) tokenvalidities: addontokenvalidity: $addonTokenValidity operatortokenvalidity: $operatorTokenValidity "@ if ($proxySettings.CertName) { $yaml += @" `ncertificates: - filename: "$($proxySettings.CertName)" contentb64: "$($proxySettings.CertContent)" "@ } if ($configMaps) { $yaml += "`n$configMaps" } Set-Content -Path $yamlFile -Value $yaml if ($null -ne $err -and $err.count -gt 0) { throw $err } Set-KvaConfigValue -name "kvaconfig" -value $yamlFile return $yamlFile } function Get-KvaUpdateConfigYaml { <# .DESCRIPTION Returns a configuration file for appliance upgrade. This configuration is merged with the existing config stored within the appliance. It is mainly used to provide updated version information and config maps. #> $installationPackageDir = $global:config[$moduleName]["installationPackageDir"] $yamlFile = $($installationPackageDir+"\yaml\appliance-update.yaml") if (-not (Test-Path $yamlFile)) { New-Item -ItemType File -Force -Path $yamlFile | Out-Null } Write-Status -moduleName $moduleName "Creating KVA update configuration" $yaml = "" $configMaps = Get-KvaConfigMaps if ($configMaps) { $yaml += $configMaps } Set-Content -Path $yamlFile -Value $yaml if ($null -ne $err -and $err.count -gt 0) { throw $err } return $yamlFile } function Get-KvaLogs { <# .DESCRIPTION Collects all the logs from the deployment .PARAMETER Path Path to store the logs .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String]$path, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $logDir = [io.Path]::Combine($path, $moduleName) New-Item -ItemType Directory -Force -Path $logDir | Out-Null Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Collecting configuration..." $global:config[$moduleName] > $logDir"\KvaConfig.txt" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Collecting module information..." Get-Command -Module Kva | Sort-Object -Property Source > $($logDir+"\moduleinfo.txt") if (Test-Path -Path $script:logFilePath) { Copy-Item -Path $script:logFilePath -Destination $logDir } if (Test-Path -Path $global:config[$moduleName]["kubeconfig"]) { Write-SubStatus -moduleName $moduleName "Collecting Kubernetes Cluster Logs" try { $clusters = Invoke-Kubectl -arguments $("get akshciclusters -o json") | ConvertFrom-Json foreach ($cluster in $clusters.items) { $clusterName = $cluster.metadata.name Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Collecting logs for cluster '$clusterName'...") if ($clusterName -ine $global:config[$moduleName]["kvaName"]) { $rando = Get-Random $kubeconfigFileLocation = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $clusterName -outputLocation $kubeconfigFileLocation } else { $kubeconfigFileLocation = Get-KvaCredential } Write-SubStatus -moduleName $moduleName "Collecting cluster $clusterName logs" try{ Invoke-Kubectl -kubeconfig $kubeconfigFileLocation -arguments "get all,pvc,pv,sc -A -o yaml" > "${logDir}\worker_objects_$clusterName.json" $clusterLogDirectory = [io.Path]::Combine($logDir, "clusterlogs_$clusterName") Invoke-Kubectl -kubeconfig $kubeconfigFileLocation -arguments $("cluster-info dump -A --output-directory=`"$clusterLogDirectory`"") | Out-Null }catch{} if ($clusterName -ine $global:config[$moduleName]["kvaName"]) { Remove-Item -Path $kubeconfigFileLocation -Force -ErrorAction SilentlyContinue } } } catch [Exception] { Write-Status -moduleName $moduleName -msg "Exception caught!!!" Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() } } else { Write-SubStatus -moduleName $moduleName "Skipping Kubernetes Cluster Log collection as management cluster kubeconfig is not present." } Write-Status -moduleName $moduleName "Done." } function Get-KvaEventLog { <# .DESCRIPTION Gets all the event logs from Kva Module #> Get-WinEvent -ProviderName $moduleName -ErrorAction SilentlyContinue } function Repair-KvaCluster { <# .DESCRIPTION Attempts to repair failed TLS on a cluster by reprovisioning control plane certs .PARAMETER Name Name of the node to reprovision certs on .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER fixCertificates If specified, will attempt to reprovision control plane certs to fix TLS errors .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $sshPrivateKeyFile, [Parameter()] [Switch] $fixCertificates, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity if ($fixCertificates.IsPresent) { $kvaConfig = Get-KvaConfigYaml Invoke-KvaCtl -arguments "cluster addons certs repair --cluster-name $Name --sshprivatekey $sshPrivateKeyFile --configfile $kvaConfig" -showOutput -activity $activity } } function Set-ContainerRegistryConfiguration { <# .DESCRIPTION Sets the container registry configuration .PARAMETER containerRegistry Container registry settings #> param ( [Parameter()] [ContainerRegistry] $containerRegistry ) $server = "" $user = "" $pass = "" if ($containerRegistry) { $server = $containerRegistry.Server if ($containerRegistry.Credential.Username) { $user = $containerRegistry.credential.UserName } if ($containerRegistry.Credential.Password) { $pass = $containerRegistry.credential.Password | ConvertFrom-SecureString -Key $global:credentialKey } } else { $server = $script:defaultContainerRegistryServer $user = $script:defaultContainerRegistryU $ss = ConvertTo-SecureString $script:defaultContainerRegistryP -AsPlainText -Force $pass = $ss | ConvertFrom-SecureString -Key $global:credentialKey } Set-ConfigurationValue -name "containerRegistryServer" -value $server -module $moduleName Set-ConfigurationValue -name "containerRegistryUser" -value $user -module $moduleName Set-ConfigurationValue -name "containerRegistryPass" -value $pass -module $moduleName } #endregion #region Billing function Sync-KvaBilling { <# .DESCRIPTION Helper function to sync billing. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity try { $syncResult = Invoke-KvaCtl -arguments "cluster addons billing sync --kubeconfig $kubeconfig" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $("Sync billing failed. $_") } return $syncResult } function Get-KvaBillingRecords { <# .DESCRIPTION Helper function to list billing records. .PARAMETER outputformat Output format of result .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $outputformat = "json", [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity try { $recordsResult = Invoke-KvaCtl -arguments "cluster addons billing get-records --kubeconfig $kubeconfig --outputformat=$outputformat" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $("Get records failed. $_") } return $recordsResult } function Get-KvaBillingStatus { <# .DESCRIPTION Helper function to get billing status. .PARAMETER outputformat Output format of result .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $outputformat = "json", [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $yamlFile = $global:config[$moduleName]["kubeconfig"] try { $statusResult = Invoke-KvaCtl -arguments "cluster addons billing get-status --kubeconfig $yamlFile --outputformat=$outputformat" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $("Get billing status failed. $_") } return $statusResult } #endregion #region Installation and Provisioning functions function Install-KvaInternal { <# .DESCRIPTION The main deployment method for KVA. This function is responsible for provisioning files, deploying the agents. .PARAMETER activity Activity name to use when writing progress #> param ( [String] $activity = $MyInvocation.MyCommand.Name ) Set-KvaConfigValue -name "installState" -value ([InstallState]::Installing) try { Get-KvaRelease -version $(Get-KvaVersion) -activity $activity if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Install-KvaBinaries -nodeName $_.Name } } else { Install-KvaBinaries -nodeName ($env:computername) } if (-not $global:config[$modulename]["insecure"]) { Write-Status -moduleName $moduleName "Creating KVA identity" $clusterName = $($global:config[$modulename]["kvaName"]) $kvaIdentity = New-MocIdentity -name $clusterName -validityDays $global:config[$moduleName]["tokenExpiryDays"] -fqdn $cloudFqdn -location $global:config[$modulename]["cloudLocation"] -port $cloudPort -authport $cloudAuthPort -encode Set-KvaConfigValue -name "identity" -value $kvaIdentity New-MocRoleAssignment -identityName $clusterName -roleName "IdentityContributor" New-MocRoleAssignment -identityName $clusterName -roleName "RoleContributor" New-MocRoleAssignment -identityName $clusterName -roleName "CertificateReader" New-MocRoleAssignment -identityName $clusterName -roleName "VipPoolReader" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "GroupContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "KeyVaultContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "VirtualNetworkContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "LBContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "NetworkInterfaceContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "VMContributor" -location $global:config[$modulename]["cloudLocation"] New-MocRoleAssignment -identityName $clusterName -roleName "SecretContributor" -location $global:config[$modulename]["cloudLocation"] } $yamlFile = Get-KvaConfigYaml -kubernetesVersion $global:config[$modulename]["kvaK8sVersion"] -group $global:config[$modulename]["group"] $kubeconfig = $global:config[$modulename]["kubeconfig"] Invoke-KvaCtlWithAzureContext -arguments "create --configfile $yamlFile --outfile $kubeconfig" -showOutputAsProgress -activity $activity } catch { Set-KvaConfigValue -name "installState" -value ([InstallState]::InstallFailed) throw $_ } Set-SecurePermissionFile -Path $kubeconfig Set-KvaConfigValue -name "installState" -value ([InstallState]::Installed) Write-Status -moduleName $moduleName "KVA installation is complete!" } function Get-KvaRelease { <# .DESCRIPTION Download a KVA release (e.g. package, images, etc) .PARAMETER version Release version .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status "Discovering Kva release" $productRelease = Get-ProductRelease -version $version -moduleName $moduleName if (-not $productRelease.CustomData.ManagementNodeImageK8sVersion) { throw "Unable to determine required management cluster kubernetes version" } $k8sVersion = $productRelease.CustomData.ManagementNodeImageK8sVersion Write-SubStatus -moduleName $moduleName "Determined that kubernetes version $k8sVersion is required for the management cluster" Add-GalleryImage -imageType "Linux" -k8sVersion $k8sVersion -activity $activity Write-SubStatus -moduleName $moduleName "The management cluster image has been provisioned" Get-ReleaseContent -name $script:productName -version $version -activity $activity if (-not ($global:config[$modulename]["useStagingShare"])) { Test-AuthenticodeBinaries -workingDir $global:config[$moduleName]["installationPackageDir"] -binaries $script:kvaBinaries } } function Get-KvaConfigMaps { <# .DESCRIPTION Retrieve KVA config maps #> Write-Status -moduleName $moduleName "Get KVA config maps" $productRelease = Get-ProductRelease -version $global:config[$modulename]["version"] -moduleName $moduleName $yaml = @" configmaps: - name: $script:productInfoMapName namespace: $script:productInfoMapNamespace data: offer: "$($productRelease.ProductName)" version: "$($global:config[$modulename]["version"])" catalog: "$($global:config[$modulename]["catalog"])" audience: "$($global:config[$modulename]["ring"])" deploymentid: "$($global:config[$modulename]["deploymentId"])" "@ return $yaml } function Get-ReleaseContent { <# .DESCRIPTION Download all required files and packages for the specified release .PARAMETER name Release name .PARAMETER version Release version .PARAMETER destination Destination directory for the content .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [string] $destination = $global:config[$moduleName]["installationPackageDir"], [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $destination = $destination -replace "\/", "\" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Discovering $name release content") $productRelease = Get-ProductRelease -version $version -moduleName $moduleName # find requested release foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { if ($subProductRelease.ProductName -ieq $name) { $versionManifestPath = [io.Path]::Combine($destination, $("$name-release.json")) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Downloading $name release content to $destination") $downloadParams = Get-ReleaseDownloadParameters -name $subProductRelease.ProductStream -version $subProductRelease.Version -destination $destination -parts 3 -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if (-not ($global:config[$moduleName]["useStagingShare"])) { if ($releaseInfo.Files.Count -ne 1) { throw $("Unexpected $name release content files downloaded. Expected 1 file, but received " + $releaseInfo.Files.Count) } $packagename = $releaseInfo.Files[0] -replace "\/", "\" # Temporary until cross-platform signing is available $auth = Get-AuthenticodeSignature -filepath $packagename if (($global:expectedAuthResponse.status -ne $auth.status) -or ($auth.SignatureType -ne $global:expectedAuthResponse.SignatureType)) { throw $("$name release content failed authenticode verification. Expected status=$($global:expectedAuthResponse.status) and type=$($global:expectedAuthResponse.SignatureType) but received status=$($auth.status) and type=$($auth.SignatureType)") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Expanding $name package $packagename into $destination") $expandoutput = expand.exe -r $packagename $destination -f:* Write-SubStatus -moduleName $moduleName "Expand result: $expandoutput" } $versionJson = $subProductRelease | ConvertTo-Json -depth 100 set-content -path $versionManifestPath -value $versionJson -encoding UTF8 return } } } throw "Unable to get $name release content for version $version" } function Initialize-KvaEnvironment { <# .DESCRIPTION Executes steps to prepare the environment for day 0 operations. .PARAMETER createConfigIfNotPresent Whether the call should create a new deployment configuration if one is not already present. .PARAMETER activity Activity name to use when updating progress #> param ( [Switch]$createConfigIfNotPresent, [String]$activity = $MyInvocation.MyCommand.Name ) Get-MocConfig -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -module $moduleName -status "Discovering configuration" Import-KvaConfig -createIfNotPresent:($createConfigIfNotPresent.IsPresent) -activity $activity Write-StatusWithProgress -activity $activity -module $moduleName -status "Applying configuration" Initialize-Environment -checkForUpdates:$false -moduleName $script:moduleName } function Install-KvaBinaries { <# .DESCRIPTION Copies KVA binaries to a node .PARAMETER nodeName The node to execute on. #> param ( [Parameter(Mandatory=$true)] [String]$nodeName ) Install-Binaries -module $moduleName -nodeName $nodeName -binariesMap $kvaBinariesMap } function Uninstall-KvaBinaries { <# .DESCRIPTION Copies KVA binaries to a node .PARAMETER nodeName The node to execute on. #> param ( [Parameter(Mandatory=$true)] [String]$nodeName ) Uninstall-Binaries -module $moduleName -nodeName $nodeName -binariesMap $kvaBinariesMap } #endregion #region Verification Functions function Test-KvaConfiguration { <# .DESCRIPTION A basic sanity test to make sure that a node is ready to deploy kubernetes. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) Test-KvaInstallation -nodeName $nodeName } function Test-KvaInstallation { <# .DESCRIPTION Sanity checks an installation to make sure that all expected binaries are present. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) Write-SubStatus -moduleName $moduleName "Testing for expected binaries" Test-Binary -nodeName $nodeName -binaryName $global:kvaCtlFullPath Test-Binary -nodeName $nodeName -binaryName $global:kubeCtlFullPath } function Get-KvaLatestVersion { <# .DESCRIPTION Get the current KVA release version .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $("Discovering latest version") $catalog = Get-LatestCatalog -moduleName $moduleName $productRelease = $catalog.ProductStreamRefs[0].ProductReleases[0] # find latest kva release foreach($subProductStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $subProductStream.ProductReleases) { if ($subProductRelease.ProductName -ieq $script:productName) { return $subProductRelease.Version } } } throw "Unable to determine KVA latest version" } function Reset-GalleryImage { <# .DESCRIPTION Resets all gallery image #> Write-Status -moduleName $moduleName "Resetting Gallery image" # 1. Invalidate all gallery images # Gallery images are downloaded and managed by KVA. So it is okay to reset it completely for now. # If this assumption changes, change the logic here to just remove what KVA brings in Reset-MocGalleryImage -location $global:config[$modulename]["cloudLocation"] # 2. Cleanup all vhd files $imagepath = $global:config[$moduleName]["imageDir"] Get-ChildItem -Path $imagepath -Filter *.vhdx | ForEach-Object { $tmpFile = $_.FullName Write-SubStatus -moduleName $moduleName "Removing image $tmpFile" Remove-Item -Path $tmpFile -Force -ErrorAction SilentlyContinue } # 3. Cleanup all json manifest cache files Get-ChildItem -Path $imagepath -Filter *.json | ForEach-Object { $tmpFile = $_.FullName Write-SubStatus -moduleName $moduleName "Removing image cache $tmpFile" Remove-Item -Path $tmpFile -Force -ErrorAction SilentlyContinue } } function Add-GalleryImage { <# .DESCRIPTION Checks the gallery to see if the requested image is present. If it is missing, this function downloads the missing image and adds it to the gallery. .PARAMETER imageName Name of the gallery image to provision .PARAMETER activity Activity name to use when updating progress #> param ( # [Parameter(Mandatory=$true)] # [String] $imageName, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String] $imageType, [Parameter(Mandatory=$true)] [String] $k8sVersion, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Provisioning Gallery image..." $imageName = Get-KubernetesGalleryImageName -imagetype $imageType -k8sVersion $k8sVersion Write-SubStatus -moduleName $moduleName $("Requested image is '$imageName'") Write-SubStatus -moduleName $moduleName $("Checking for existing gallery image ...") $galleryImage = $null $mocLocation = $global:config[$modulename]["cloudLocation"] try { $galleryImage = Get-MocGalleryImage -name $imageName -location $mocLocation } catch {} if ($null -ine $galleryImage) { Write-SubStatus -moduleName $moduleName $("Image is already present in the gallery.") return } Write-StatusWithProgress -activity $activity -module $moduleName -status "Obtaining download information for image $imageName" $imageRelease = Get-ImageReleaseManifest -imageVersion $(Get-KvaVersion) -operatingSystem $imageType -k8sVersion $k8sVersion Write-StatusWithProgress -activity $activity -module $moduleName -status "Downloading image $imageName" $result = Get-ImageRelease -imageRelease $imageRelease -imageDir $global:config[$moduleName]["imageDir"] Write-StatusWithProgress -activity $activity -module $moduleName -status "Adding image to cloud gallery ($imageName)" New-MocGalleryImage -name $imageName -location $mocLocation -imagePath $result -container $global:cloudStorageContainer | Out-Null } function Invoke-KvaCtl { <# .DESCRIPTION Executes a KVACTL command. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER activity Activity name to use when updating progress. #> param ( [string] $arguments, [switch] $ignoreError, [switch] $showOutput, [string] $activity = $MyInvocation.MyCommand.Name ) return Invoke-CommandLine -command $kvaCtlFullPath -arguments $("$arguments") -ignoreError:$ignoreError.IsPresent -showOutputAsProgress:$showOutput.IsPresent -progressActivity $activity -moduleName $moduleName } function Invoke-KvaCtlWithAzureContext { <# .DESCRIPTION Executes a command and optionally ignores errors. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER showOutputAsProgress Optionally, show output from the executing command as progress bar updates. .PARAMETER credential credential is a PSCredential holding a user's Service Principal. .PARAMETER activity The activity name to display when showOutputAsProgress was requested. #> [CmdletBinding()] param ( [Parameter()] [String]$command, [Parameter()] [String]$arguments, [Parameter()] [Switch]$ignoreError, [Parameter()] [Switch]$showOutput, [Parameter()] [Switch]$showOutputAsProgress, [Parameter()] [PSCredential] $credential, [Parameter()] [string] $activity = $MyInvocation.MyCommand.Name ) $obj = New-Object -TypeName psobject if ($null -ne $credential) { $obj | Add-Member -MemberType NoteProperty -Name clientId -Value $credential.GetNetworkCredential().UserName $obj | Add-Member -MemberType NoteProperty -Name clientSecret -Value $credential.GetNetworkCredential().Password } else { $armAccessToken = Get-AzAccessToken $graphAccessToken = Get-GraphAccessToken $obj | Add-Member -MemberType NoteProperty -Name armAccessToken -Value $armAccessToken.Token $obj | Add-Member -MemberType NoteProperty -Name graphAccessToken -Value $graphAccessToken.Token $azContext = Get-AzContext if ($azContext.Account.Type -eq "ServicePrincipal") { $obj | Add-Member -MemberType NoteProperty -Name clientId -Value $azContext.Account.Id $obj | Add-Member -MemberType NoteProperty -Name clientSecret -Value $azContext.Account.ExtendedProperties.ServicePrincipalSecret } } $arguments += " --azure-creds-stdin" try { if ($showOutputAsProgress.IsPresent) { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") | ForEach-Object { $status = $_ -replace "`t"," - "; Write-StatusWithProgress -activity $activity -moduleName $moduleName -Status $status }) 2>&1 } elseif ($showOutput.IsPresent) { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") | Out-Default) 2>&1 } else { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") 2>&1) } } catch { if ($ignoreError.IsPresent) { return } throw } $out = $result | Where-Object {$_.gettype().Name -ine "ErrorRecord"} # On a non-zero exit code, this may contain the error #$outString = ($out | Out-String).ToLowerInvariant() if ($LASTEXITCODE) { if ($ignoreError.IsPresent) { return } $err = $result | Where-Object {$_.gettype().Name -eq "ErrorRecord"} throw "$command $arguments returned a non zero exit code $LASTEXITCODE [$err]" } return $out } function Get-KvaClusterCredential { <# .DESCRIPTION Obtains the credentials for a cluster as a kubeconfig and outputs it to the requested location (by default the location is the current users .kube directory). .PARAMETER Name Name of the cluster to obtain the credential/kubeconfig for. .PARAMETER outputLocation Location to output the credential/kubeconfig file to. .PARAMETER adAuth To get the Active Directory SSO version of the kubeconfig. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $outputLocation = $($env:USERPROFILE+"\.kube\config"), [Parameter(Mandatory=$false)] [Switch] $adAuth, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity #TODO Fix this from the kvasdk layer if ($Name -eq $global:config[$moduleName]["kvaName"]) { throw "Get-KvaClusterCredential is only valid for target clusters" } $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $("Unable to proceed due to missing kubeconfig file: $kubeconfig") } # TODO move directory check to kva # # The following line will only return true if the passed in location # is a valid directory. If the path doesn't exist or if it points a # file it will return false. # # In the case where the user passes in an invalid path, we will fail at a later point as # its hard to tell whether they are passing valid location to a file to be written or an # path that doesn't exist. if (Test-Path -Path $outputLocation -PathType Container) { $outputLocation = [io.Path]::Combine($outputLocation, "kubeconfig-$Name") } # Ensure that the cluster exists $cluster = Get-KvaCluster -Name $Name -activity $activity if ($adAuth.IsPresent) { $exePath = [io.Path]::Combine($global:installDirectory, "kubectl-adsso.exe") if (-not [System.IO.File]::Exists($exePath)) { Get-ClientCredPluginRelease -version $(Get-KvaVersion) -destinationDir $global:installDirectory -activity $activity } Invoke-KvaCtl -arguments "cluster retrieve --clustername $Name --kubeconfig $kubeconfig --outfile ""$outputLocation"" --adauth" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $("AD kubeconfig was be written to: $outputLocation") } else { Invoke-KvaCtl -arguments "cluster retrieve --clustername $Name --kubeconfig $kubeconfig --outfile ""$outputLocation""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $("kubeconfig will be written to: $outputLocation") } Set-SecurePermissionFile -Path $outputLocation } function Get-KvaCredential { <# .DESCRIPTION Obtains the credentials for kva as a kubeconfig and outputs it to the default location). .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name)" } $kubeconfigPath = $global:config[$moduleName]["kubeconfig"] if (Test-Path $kubeconfigPath) { return $kubeconfigPath } $yamlFile = Get-KvaConfigYaml Invoke-KvaCtl -arguments "retrieve --configfile $yamlFile --outfile $kubeconfigPath" -showOutput -activity $activity Set-SecurePermissionFile -Path $kubeconfigPath return $kubeconfigPath } function Get-ClientCredPluginRelease { <# .DESCRIPTION Download the client cred plugin release content .PARAMETER Version Version .PARAMETER destinationDir Destination directory .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter(Mandatory=$true)] [string] $destinationDir, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Get-ReleaseContent -name "client-cred-plugin" -version $version -activity $activity $srcPath = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "kubectl-adsso.exe") Copy-Item $srcPath -Destination $destinationDir } function Wait-ForKubernetesSecret { <# .DESCRIPTION Polls the vault for a secret to become available and returns the secret value. .PARAMETER name Name of the secret .PARAMETER sleepDuration Duration to sleep for between attempts to retrieve the secret #> param ( [String]$name, [int]$sleepDuration=20 ) Write-SubStatus $("Polling cluster for secret '$name' to be available") -moduleName $moduleName while($true) { try { $clusterSecret = Invoke-Kubectl -arguments $("get secrets/$name") } catch {} if ($null -ne $clusterSecret) { break } Sleep $sleepDuration } return $clusterSecret } function Set-KvaGMSAWebhook { <# .DESCRIPTION Internal helper function that installs gMSA webhook for a cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity # ensures cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null try { # Schedules a gmsa webhook deployment onto the cluster master node $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: gmsa-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: gmsa-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-gmsa-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("apply -f $yamlFile") $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # wait for gmsa-webhook to be ready $sleepDuration = 5 $namespace = 'kube-system' $selector = 'app=gmsa-webhook' Write-SubStatus $("Waiting for gmsa-webhook pod to be ready...") -moduleName $moduleName while($true) { $result = (Invoke-Kubectl -ignoreError -kubeconfig $targetClusterKubeconfig -arguments $("wait --for=condition=Ready --timeout=5m -n $namespace pod -l $selector")) if ($null -ne $result) { break } Start-Sleep $sleepDuration } Write-SubStatus $("Pod '$podFriendlyName' is ready.`n") -moduleName $moduleName } finally { Remove-Item -Path $targetClusterKubeconfig } } function Reset-KvaGMSAWebhook { <# .DESCRIPTION Internal helper function that uninstalls gMSA webhook for a cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity # ensures cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null if (-not (Invoke-Kubectl -arguments $("get addon gmsa-webhook-$Name") -ignoreError)) { throw "Addon gmsa-webhook-$($Name) not installed" } try { $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: gmsa-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: gmsa-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-gmsa-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("delete -f $yamlFile") } finally { Remove-Item $yamlFile } } function Set-KvaGMSACredentialSpec { <# .DESCRIPTION Helper function to install setup for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecFilePath File Path of the JSON cred spec file .PARAMETER credSpecName Name of the k8s credential spec object the user would like to designate .PARAMETER secretName Name of the Kubernetes secret object storing the Active Directory user credentials and gMSA domain .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER serviceAccount Name of the K8s service account assigned to read the k8s gMSA credspec object .PARAMETER clusterRoleName Name of the K8s clusterrole assigned to use the k8s gMSA credspec object .PARAMETER overwrite Overwrites existing gMSA setup .PARAMETER activity Activity name to use when writing progress #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$credSpecFilePath, [Parameter(Mandatory=$true)] [String]$credSpecName, [Parameter(Mandatory=$true)] [String]$secretName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [switch]$overwrite, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity try { $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # Converting credSpecName, secretName, serviceAccount, and clusterRoleName to lowercase to conform with RFC1123 $credSpecName = $credSpecName.ToLower() $secretName = $secretName.ToLower() $serviceAccount = $serviceAccount.ToLower() $clusterRoleName = $clusterRoleName.ToLower() $roleBindingName = 'allow-' + $serviceAccount + '-svc-account-read-on-' + $credSpecName # check if namespace is created if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get namespace $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get namespace $secretNamespace", $secretNamespace, "namespace check")) { throw "Namespace $secretNamespace not created. Please run kubectl create namespace $secretNamespace to create the namespace." } } # check if a gmsa webhook pod is running $webhookPodRunning = (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get pods -n kube-system") -ignoreError) | Select-String -Pattern 'gmsa-webhook.*1/1.*Running' if (-not $webhookPodRunning) { throw "GMSA webhook not installed. Please install the gMSA webhook." } # check if serviceaccount exists if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get serviceaccount $serviceAccount -n $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get serviceaccount $serviceAccount -n $secretNamespace", "$serviceAccount in namespace $secretNamespace", "service account check")) { throw "$serviceAccount in namespace $secretNamespace not found. Please create the service account $serviceAccount in namespace $secretNamespace." } } # check if any of the resources already exist # if overwrite, delete them # if not overwrite, throw error if ($overwrite.IsPresent) { if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete GMSACredentialSpec $credSpecName") } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete clusterRole $clusterRoleName") } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete roleBinding $roleBindingName -n $secretNamespace") } } else { if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { throw "The specified $credSpecName already exists. Rerun the cmdlet with -overwrite flag to update the credspec." } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { throw "The specified $clusterRoleName already exists. Rerun the cmdlet with -overwrite flag to update the cluster role." } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { throw "The specified $roleBindingName in $secretNamespace exists. Rerun the cmdlet with -overwrite flag to update the rolebinding." } } # check if credspec file exists If (-not (Test-Path -Path $credSpecFilePath)) { throw "File path does not exist: $credSpecFilePath" } # catch errors when get-content or convertfrom-json $credSpecJsonRaw = Get-Content $credSpecFilePath -raw | ConvertFrom-Json $adServiceAccount = $credSpecJsonRaw.DomainJoinConfig.MachineAccountName $domainName = $credSpecJsonRaw.DomainJoinConfig.DNSName $netbios = $credSpecJsonRaw.DomainJoinConfig.NetBiosName $gmsaSid = $credSpecJsonRaw.DomainJoinConfig.Sid $gmsaGuid = $credSpecJsonRaw.DomainJoinConfig.Guid if (-not ($adServiceAccount -and $domainName -and $netbios -and $gmsaSid -and $gmsaGuid)) { throw "The credential spec JSON file $credSpecFilePath is invalid." } # create credspec yaml $credSpecYaml = @" apiVersion: windows.k8s.io/v1alpha1 kind: GMSACredentialSpec metadata: name: $credSpecName credspec: ActiveDirectoryConfig: GroupManagedServiceAccounts: - Name: $adServiceAccount Scope: $netbios - Name: $adServiceAccount Scope: $domainName HostAccountConfig: PortableCcgVersion: "1" PluginGUID: "{FF4E3124-6831-44A0-8C34-06EC66E148C6}" PluginInput: SecretName=$($secretName);SecretNamespace=$($secretNamespace) CmsPlugins: - ActiveDirectory DomainJoinConfig: DnsName: $domainName DnsTreeName: $domainName Guid: $gmsaGuid MachineAccountName: $adServiceAccount NetBiosName: $netbios Sid: $gmsaSid "@ # apply and remove credspec yaml if ($PSCmdlet.ShouldProcess("$credspecYaml", $credSpecName, "credspec creation")) { $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$credSpecName-$rando.yaml") Set-Content -Path $yamlFile -Value $credSpecYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f $yamlFile") Remove-Item $yamlFile } # create cluster role yaml $clusterRoleYaml = @" apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: $clusterRoleName rules: - apiGroups: ["windows.k8s.io"] resources: ["gmsacredentialspecs"] verbs: ["use"] resourceNames: [$($credSpecName)] "@ # apply and remove cluster role yaml if ($PSCmdlet.ShouldProcess("$clusterRoleYaml", $clusterRoleName, "clusterrole creation")) { $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$clusterRoleName-clusterrole-$rando.yaml") Set-Content -Path $yamlFile -Value $clusterRoleYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f $yamlFile") Remove-Item $yamlFile } # create rolebinding yaml $roleBindingYaml = @" apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: allow-$($serviceAccount)-svc-account-read-on-$($credSpecName) namespace: $($secretNamespace) subjects: - kind: ServiceAccount name: $($serviceAccount) namespace: $($secretNamespace) roleRef: kind: ClusterRole name: $clusterRoleName apiGroup: rbac.authorization.k8s.io "@ # apply and remove rolebinding yaml if ($PSCmdlet.ShouldProcess("$roleBindingYaml", "allow-$($serviceAccount)-svc-account-read-on-$($credSpecName)", "clusterrolebinding creation")) { # apply yaml file and delete $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$clusterRoleName-rolebinding-$rando.yaml") Set-Content -Path $yamlFile -Value $roleBindingYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f $yamlFile") Remove-Item $yamlFile } } finally { Remove-Item -Path $targetClusterKubeconfig -ErrorAction SilentlyContinue } } function Reset-KvaGMSACredentialSpec { <# .DESCRIPTION Helper function to uninstall setup for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecName Name of the k8s credential spec object the user would like to designate .PARAMETER serviceAccount K8s service account assigned to read the k8s gMSA credspec object .PARAMETER clusterRoleName Name of the K8s clusterrole assigned to use the k8s gMSA credspec object .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER activity Activity name to use when writing progress #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$credSpecName, [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity try { $noDeleteCount = 0 $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # check if namespace is created if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get namespace $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get namespace $secretNamespace", $secretNamespace, "namespace check")) { throw "Namespace $secretNamespace not created. Please run kubectl create namespace $secretNamespace to create the namespace." } } # delete GMSACredentialSpec object if exists if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete GMSACredentialSpec $credSpecName", $credSpecName, "credspec deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete GMSACredentialSpec $credSpecName") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName "GMSACredentialSpec $credSpecName does not exist, skipping deletion" } # delete clusterrole object if exists if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete $clusterRoleName", $clusterRoleName, "clusterrole deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete clusterRole $clusterRoleName") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName "ClusterRole $clusterRoleName does not exist, skipping deletion" } # delete rolebinding object if exists $roleBindingName = 'allow-' + $serviceAccount + '-svc-account-read-on-' + $credSpecName if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete $roleBindingName -n $secretNamespace", "allow-$($serviceAccount)-svc-account-read-on-$($credSpecName)", "clusterrolebinding deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete roleBinding $roleBindingName -n $secretNamespace") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName "Rolebinding $roleBindingName in $secretNamespace does not exist, skipping deletion" } # give feedback on no objects were deleted if ($noDeleteCount -eq 3) { Write-SubStatus -moduleName $moduleName "GMSACredentialSpec $credSpecName, clusterrole $clusterRole, and rolebinding $rolebindingName do not exist, no objects were deleted" } } finally { Remove-Item -Path $targetClusterKubeconfig -ErrorAction SilentlyContinue } } #endregion #region Image functions function Get-ImageReleaseManifest { <# .DESCRIPTION Discovers the requested image and returns the release manifest for it .PARAMETER imageVersion Image release version to target .PARAMETER operatingSystem Image operating system to target .PARAMETER k8sVersion Kubernetes version to target #> param( [parameter(Mandatory=$true)] [String] $imageVersion, [parameter(Mandatory=$true)] [String] $operatingSystem, [Parameter(Mandatory=$true)] [String] $k8sVersion ) $k8sVersion = $k8sVersion.TrimStart("v") $productRelease = Get-ProductRelease -version $imageVersion -moduleName $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { $vhdInfo = Get-ImageReleaseVhdInfo -release $subProductRelease if (-not $vhdInfo) { continue } if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ine $operatingSystem) { continue } foreach($pkg in $vhdInfo.CustomData.K8SPackages) { if ($pkg.Version -ieq $k8sVersion) { return $subProductRelease } } } } throw "Unable to locate a image release with Version: $imageVersion OS: $operatingSystem K8sVersion: $k8sVersion" } function Get-ImageReleaseVhdInfo { <# .DESCRIPTION Discovers and returns vhd information for the specified image release .PARAMETER imageRelease Image release manifest #> param( [parameter(Mandatory=$True)] [PSCustomObject] $release ) foreach ($fileRelease in $release.ProductFiles) { if ($fileRelease.CustomData.Type -ieq "vhd") { return $fileRelease } } return $null } function Get-ImageRelease { <# .DESCRIPTION Download the specified image release. .PARAMETER imageRelease Image release manifest .PARAMETER imageDir Directory for local image store. #> param( [parameter(Mandatory=$True)] [PSCustomObject] $imageRelease, [parameter(Mandatory=$True)] [String] $imageDir ) $downloadpath = "$imageDir\$([System.IO.Path]::GetRandomFileName().Split('.')[0])" New-Item -ItemType Directory -Force -Confirm:$false -Path $downloadpath | Out-Null try { $vhdInfo = Get-ImageReleaseVhdInfo -release $imageRelease $k8sVersion = $vhdInfo.CustomData.K8SPackages[0].Version $imageGalleryName = Get-KubernetesGalleryImageName -imagetype $vhdInfo.CustomData.BaseOSImage.OperatingSystem -k8sVersion $k8sVersion $imageVersionCurrent = $imageRelease.Version $imageVersionManifestPath = "$ImageDir\$imageGalleryName.json" $destinationpath = $("$ImageDir\$imageGalleryName.vhdx" -replace "\/", "\") if (Test-Path $imageVersionManifestPath) { $OldImageData = get-content $imageVersionManifestPath | ConvertFrom-Json $OldImageVersion = $OldImageData.Version Write-SubStatus -moduleName $moduleName "Existing image $imageVersionManifestPath has version $OldImageVersion . Requested version is $imageVersionCurrent" if ($imageVersionCurrent -ieq $OldImageVersion) { if (Test-Path -Path $destinationpath) { Write-SubStatus -moduleName $moduleName "Existing image is present and up to date. Skipping download." return $destinationpath } Write-SubStatus -moduleName $moduleName "Existing image is not present. Proceeding to download..." } } Write-Status -moduleName $moduleName $("Downloading $($imageRelease.ProductStream) (version $imageVersionCurrent) to $downloadPath") $downloadParams = Get-ReleaseDownloadParameters -name $imageRelease.ProductStream -version $imageVersionCurrent -destination $downloadPath -parts 10 -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if ($global:config[$moduleName]["useStagingShare"]) { $imageFile = $releaseInfo.Files[0] -replace "\/", "\" } else { $imageFile = Expand-SfsImage -files $releaseInfo.Files -destination $downloadPath -workingDirectory $downloadPath } $imageJson = $imageRelease | ConvertTo-Json -depth 100 Set-Content -path $imageversionManifestPath -value $imageJson -encoding UTF8 -Confirm:$false if (test-path $destinationpath) { Remove-Item $destinationpath -Force -Confirm:$false } Write-SubStatus -moduleName $moduleName "Moving image from $imageFile to $destinationpath" Move-Item -Path $imageFile -Destination $destinationpath -Confirm:$false } finally { Remove-Item -Force $downloadpath -Recurse -Confirm:$false } return $destinationpath } function Expand-SfsImage { <# .DESCRIPTION Expand and verify a SFS image download using authenticode. NOTE: This is temporary until cross-platform signing is available in Download SDK. .PARAMETER files The downloaded image files (expected to be a image file zip and a companion cab). .PARAMETER destination Destination for the expanded image file. .PARAMETER workingDirectory Working directory to use for zip and cab file expansion. #> param( [Parameter(Mandatory=$True)] [String[]] $files, [Parameter(Mandatory=$True)] [String] $destination, [Parameter(Mandatory=$True)] [String] $workingDirectory ) Write-Status -moduleName $moduleName "Verifying image companion file download" [String[]]$cabfile = $files | Where-Object { $_ -match "\.cab$"} if (($null -eq $cabfile) -or ($cabfile.count -ne 1)) { throw $("Unexpected number of .cab files were downloaded - count: " + $cabfile.count) } Write-SubStatus -moduleName $moduleName $("Verifying authenticode signature for $cabFile") $auth = Get-AuthenticodeSignature -filepath $cabfile if (($global:expectedAuthResponse.status -ne $auth.status) -or ($global:expectedAuthResponse.SignatureType -ne $auth.SignatureType)) { throw $("KVA image companion file failed authenticode verification. Expected status=$($global:expectedAuthResponse.status) and type=$($global:expectedAuthResponse.SignatureType) but received status=$($auth.status) and type=$($auth.SignatureType)") } Write-Status -moduleName $moduleName "Expanding image companion file" $expandDir = "$WorkingDirectory\expand_" + [System.IO.Path]::GetRandomFileName().Split('.')[0] New-Item -ItemType Directory -Force -Confirm:$false -Path $expandDir | Out-Null $expandoutput = expand.exe $cabfile $expandDir Write-SubStatus -moduleName $moduleName $("Expand output: $expandoutput") $manifest = Get-ChildItem $expandDir | select-object -first 1 $manifestcontents = get-content $manifest.fullname | convertfrom-json $packageAlgo = $manifestcontents.PackageVerification.VerificationDescriptor.Algorithm $packageHash = $manifestcontents.PackageVerification.VerificationDescriptor.FileHash $packageName = $manifestcontents.PackageVerification.VerificationDescriptor.Filename Write-Status -moduleName $moduleName "Verifying image file download (compressed archive)" Write-SubStatus -moduleName $moduleName $("Companion file requests verification of package: $packageName using algorithm: $packageAlgo and hash: $packageHash") [string[]]$imagezip = $files | Where-Object { $_ -match "$packageName$"} if ($null -eq $imagezip) { throw $("Unable to locate downloaded image file archive: $packageName") } Write-SubStatus -moduleName $moduleName $("Calculating $packageAlgo hash for image file archive: " + $imagezip[0]) $hash = Get-Base64Hash -file $imagezip[0] -algorithm $packageAlgo if ($packageHash -ne $hash) { throw "KVA image file archive has an unexpected hash. Expected hash: $packageHash but the downloaded file $($imagezip[0]) has hash: $hash" } Write-Status -moduleName $moduleName "Expanding image file archive" $contentsDir = "$expandDir\contents" New-Item -ItemType Directory -Force -Confirm:$false -Path $contentsdir | Out-Null Expand-Archive -path $imagezip[0] -destinationpath $contentsDir -Confirm:$false | Out-Null Remove-Item $imagezip[0] -Confirm:$false [System.IO.FileInfo[]]$workimage = Get-ChildItem -r $contentsDir | Where-Object { (-not $_.psiscontainer) -and ($_.name -match "\.vhdx$")} if ($workimage.count -ne 1) { throw $("Expected 1 image file after expansion but found $($workimage.count)") } $content0 = $manifestcontents.ContentVerification[0].VerificationDescriptor Write-SubStatus -moduleName $moduleName $("Calculating $packageAlgo hash for image file: " + $workimage[0].fullname) $hash = Get-Base64Hash -file $workimage[0].fullname -algorithm $content0.algorithm if ($content0.FileHash -ne $hash) { throw "KVA image file has an unexpected hash. Expected hash: $($content0.FileHash) but the downloaded file $($workimage[0].fullname) has hash: $hash" } $image = $("$destination\" + $workimage[0].name) Write-SubStatus -moduleName $moduleName $("Moving image file to destination: $image") Move-item -Path $workimage[0].fullname -Destination $image -Confirm:$false return $image } function Get-Base64Hash { <# .DESCRIPTION Obtain the base64 byte hash of the specified file. Used to verify SFS binaries .PARAMETER file File to generate the base64 hash for .PARAMETER algorithm Hashing algorithm to use #> param ( [Parameter(Mandatory=$True)] [string] $file, [Parameter(Mandatory=$True)] [string] $algorithm ) Write-SubStatus -moduleName $moduleName $("Generating base64 hash of $file using algorithm $algorithm") $diskhash_hex = Get-FileHash -algo $algorithm -path $file [byte[]] $diskhash_bin = for ([int]$i = 0; $i -lt $diskhash_hex.hash.length; $i += 2) { [byte]::parse($diskhash_hex.hash.substring($i, 2), [System.Globalization.NumberStyles]::HexNumber) } return [convert]::ToBase64String($diskhash_bin) } function Get-GraphAccessToken { <# .DESCRIPTION Gets the Graph Access Token #> $azContext = Get-AzContext $graphTokenItemResource = $global:graphEndpointResourceIdAzureCloud if($azContext.Environment -eq $global:azureChinaCloud) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureChinaCloud } elseif($azContext.Environment -eq $global:azureUSGovernment) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureUSGovernment } elseif($azContext.Environment -eq $global:azureGermanCloud) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureGermancloud } elseif($azContext.Environment -eq $global:azurePPE) { $graphTokenItemResource = $global:graphEndpointResourceIdAzurePPE } return Get-AzAccessToken -ResourceUrl $graphTokenItemResource } function New-KvaArcConnection { <# .DESCRIPTION Helper function to add the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } # Ensure that the cluster exists Get-KvaCluster -Name $Name | Out-Null $kubeconfig = Get-KvaCredential -activity $activity $arguments = "cluster addons arc install" $arguments += " --cluster-name $Name" $arguments += " --kubeconfig $kubeconfig" $arguments += " --subscription-id $subscriptionId" $arguments += " --tenant-id $tenantId" $arguments += " --resource-group $resourceGroup" $arguments += " --location $location" Invoke-KvaCtlWithAzureContext -arguments $arguments -showOutputAsProgress -activity $activity -credential $credential Write-SubStatus -moduleName $moduleName "Arc Onboarding Agent has been installed to the cluster" Write-SubStatus -moduleName $moduleName "To watch progress for the Arc Agents Onboarding run: kubectl logs job/azure-arc-onboarding -n azure-arc-onboarding --follow" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName } function Remove-KvaArcConnection { <# .DESCRIPTION Helper function to add the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } # Ensure that the cluster exists Get-KvaCluster -Name $Name | Out-Null $kubeconfig = Get-KvaCredential -activity $activity $arguments = "cluster addons arc uninstall" $arguments += " --cluster-name $Name" $arguments += " --kubeconfig $kubeconfig" $arguments += " --subscription-id $subscriptionId" $arguments += " --tenant-id $tenantId" $arguments += " --resource-group $resourceGroup" $arguments += " --location $location" Invoke-KvaCtlWithAzureContext -arguments $arguments -showOutputAsProgress -activity $activity -credential $credential Write-SubStatus -moduleName $moduleName "Arc Onboarding Agent has been uninstalled from the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName } function Set-KvaCsiSmb { <# .DESCRIPTION Installs csi smb plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing csi smb to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb install --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi smb installation complete" Write-Status -moduleName $moduleName "Done." } function Set-KvaCsiNfs { <# .DESCRIPTION Installs csi nfs plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing csi nfs to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs install --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi nfs installation complete" Write-Status -moduleName $moduleName "Done." } function Reset-KvaCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling csi smb to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb uninstall --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi smb uninstallation complete" Write-Status -moduleName $moduleName "Done." } function Reset-KvaCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling csi nfs to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs uninstall --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi nfs uninstallation complete" Write-Status -moduleName $moduleName "Done." } function Set-KvaCsiSmb { <# .DESCRIPTION Installs csi smb plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing csi smb to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb install --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi smb installation complete" Write-Status -moduleName $moduleName "Done." } function Set-KvaCsiNfs { <# .DESCRIPTION Installs csi nfs plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing csi nfs to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs install --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi nfs installation complete" Write-Status -moduleName $moduleName "Done." } function Reset-KvaCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling csi smb to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb uninstall --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi smb uninstallation complete" Write-Status -moduleName $moduleName "Done." } function Reset-KvaCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling csi nfs to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs uninstall --cluster-name $ClusterName --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Csi nfs uninstallation complete" Write-Status -moduleName $moduleName "Done." } function Set-KvaHciMonitoring { <# .DESCRIPTION Installs monitoring infrastructure on AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER storageSizeGB Amount of storage for Prometheus in GB .PARAMETER retentionTimeHours metrics retention time in hours. (min 2 hours, max 876000 hours(100 years)) .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [int] $storageSizeGB, [Parameter(Mandatory=$true)] [int] $retentionTimeHours, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $hundredYearsInHours = 876000 if (($retentionTimeHours -lt 2) -or ($retentionTimeHours -gt $hundredYearsInHours)) { throw "Please provide retentionTimeHours in range" } if ($storageSizeGB -lt 1) { throw "storageSizeGB value should be greater than zero" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing monitoring to cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons akshci-monitoring install --cluster-name $Name --kubeconfig $kubeconfig --storageSizeGB $storageSizeGB --retentionTimeHours $retentionTimeHours" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Monitoring installation complete" Write-Status -moduleName $moduleName "Done." } function Reset-KvaHciMonitoring { <# .DESCRIPTION Uninstalls monitoring infrastructure from AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling monitoring from cluster..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons akshci-monitoring uninstall --cluster-name $Name --kubeconfig $kubeconfig" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName "Monitoring Uninstallation complete" Write-Status -moduleName $moduleName "Done." } function Test-KvaAzureConnection { <# .DESCRIPTION Tests the connection to Azure. #> $azContext = Get-AzContext if([string]::IsNullOrEmpty($azContext.Subscription.Id)) { throw "Connection to Azure failed. Please run 'Set-AksHciRegistration' and try again." } try { Get-AzSubscription -SubscriptionId $azContext.Subscription.Id -ErrorAction Stop | Out-Null } catch [Exception] { throw "Connection to Azure failed. Please run 'Set-AksHciRegistration' and try again." } $aksHciRegistration = Get-AksHciRegistration if ([string]::IsNullOrWhiteSpace($aksHciRegistration.azureResourceGroup)) { throw "No Azure Resource Group Found. Please run 'Set-AksHciRegistration' and try again." } } #endregion #region Cluster Networking commands function New-KvaClusterNetwork { <# .DESCRIPTION Create network settings to be used for the target clusters. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNetDHCP = New-AksHciClusterNetwork -name e1 -vswitchName External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE $clusterVNetStatic = New-AksHciClusterNetwork -name e1 -vswitchName External -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Validating network configuration..." $vnet = New-VirtualNetwork -name $name -vswitchName $vswitchName -vlanID $vlanID -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend $dnsserver = "" if(-Not [string]::IsNullOrEmpty($vnet.IpAddressPrefix)) { foreach ($dns in $vnet.DnsServers) { if(-Not [string]::IsNullOrEmpty($dns)) { $dnsserver += "`n - " + $dns } } } $yaml = @" virtualnetwork: name: "$($vnet.Name)" vswitchname: "$($vnet.VswitchName)" type: "Transparent" vlanid: $($vnet.VlanID) ipaddressprefix: $($vnet.IpAddressPrefix) gateway: $($vnet.Gateway) dnsservers: $dnsserver vippoolstart: $($vnet.VipPoolStart) vippoolend: $($vnet.VipPoolEnd) k8snodeippoolstart: $($vnet.K8snodeIPPoolStart) k8snodeippoolend: $($vnet.K8snodeIPPoolEnd) "@ $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$name.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Creating network..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "network create --networkconfig $yamlFile --kubeconfig $kubeconfig" -showOutput -activity $activity Write-Status -moduleName $moduleName "Done." return $vnet } function Remove-KvaClusterNetwork { <# .DESCRIPTION Remove a virtual network object for a target cluster .PARAMETER name The name of the vnet .NOTES The cmdlet will throw an exception if the network is still being used. The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE Remove-AksHciClusterNetwork -name e1 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Removing network..." Write-SubStatus -moduleName $moduleName "Netowrk removal is in progress..." $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "network delete --networkname $name --kubeconfig $kubeconfig" -showOutput -activity $activity Write-Status -moduleName $moduleName "Done." } function Get-KvaClusterNetwork { <# .DESCRIPTION Gets the VirtualNetwork object for a target cluster given either the vnet name or the cluster name. If no parameter is given, all vnet's are returned. .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Retrieving network..." -moduleName $moduleName Write-Status $("Retrieving configuration for network '$name'") -moduleName $global:KvaModule $retrievedVnet = Get-KvaClusterNetworkInternal -name $name -clusterName $clusterName -activity $activity Write-SubStatus "Successfully retrieved network information." -moduleName $global:KvaModule return $retrievedVnet } function Get-KvaClusterNetworkInternal { <# .DESCRIPTION This function is similar to Get-KvaNetworkCluster excpet it does not update the powershell status. It is intended to be called by other cmdlets that do update the status .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) $kubeconfig = Get-KvaCredential -activity $activity $queryParam = "" if ($name) { $queryParam = "--networkname $name" } elseif ($clusterName) { $queryParam = "--clustername $clusterName" } $kvaVnets = Invoke-KvaCtl -arguments "network get $queryParam --kubeconfig $kubeconfig --outputformat json" -activity $activity | ConvertFrom-Json $vnet = @() foreach ($kvaVnet in $kvaVnets) { $vnet += [VirtualNetwork]::new($kvaVnet.virtualnetwork.name, $kvaVnet.virtualnetwork.vswitchname, $kvaVnet.virtualnetwork.ipaddressprefix, $kvaVnet.virtualnetwork.gateway, $kvaVnet.virtualnetwork.dnsservers, $kvaVnet.virtualnetwork.macpoolname, $kvaVnet.virtualnetwork.vlanid, $kvaVnet.virtualnetwork.vippoolstart, $kvaVnet.virtualnetwork.vippoolend, $kvaVnet.virtualnetwork.k8snodeippoolstart, $kvaVnet.virtualnetwork.k8snodeippoolend) } return $vnet } #endregion # SIG # Begin signature block # MIIjhQYJKoZIhvcNAQcCoIIjdjCCI3ICAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCB5vhYIWCr7ftax # 05z3NkpwUCZGAkqE6zCw+JYq8gGP5aCCDYEwggX/MIID56ADAgECAhMzAAAB32vw # LpKnSrTQAAAAAAHfMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjAxMjE1MjEzMTQ1WhcNMjExMjAyMjEzMTQ1WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC2uxlZEACjqfHkuFyoCwfL25ofI9DZWKt4wEj3JBQ48GPt1UsDv834CcoUUPMn # s/6CtPoaQ4Thy/kbOOg/zJAnrJeiMQqRe2Lsdb/NSI2gXXX9lad1/yPUDOXo4GNw # PjXq1JZi+HZV91bUr6ZjzePj1g+bepsqd/HC1XScj0fT3aAxLRykJSzExEBmU9eS # yuOwUuq+CriudQtWGMdJU650v/KmzfM46Y6lo/MCnnpvz3zEL7PMdUdwqj/nYhGG # 3UVILxX7tAdMbz7LN+6WOIpT1A41rwaoOVnv+8Ua94HwhjZmu1S73yeV7RZZNxoh # EegJi9YYssXa7UZUUkCCA+KnAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUOPbML8IdkNGtCfMmVPtvI6VZ8+Mw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDYzMDA5MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAnnqH # tDyYUFaVAkvAK0eqq6nhoL95SZQu3RnpZ7tdQ89QR3++7A+4hrr7V4xxmkB5BObS # 0YK+MALE02atjwWgPdpYQ68WdLGroJZHkbZdgERG+7tETFl3aKF4KpoSaGOskZXp # TPnCaMo2PXoAMVMGpsQEQswimZq3IQ3nRQfBlJ0PoMMcN/+Pks8ZTL1BoPYsJpok # t6cql59q6CypZYIwgyJ892HpttybHKg1ZtQLUlSXccRMlugPgEcNZJagPEgPYni4 # b11snjRAgf0dyQ0zI9aLXqTxWUU5pCIFiPT0b2wsxzRqCtyGqpkGM8P9GazO8eao # mVItCYBcJSByBx/pS0cSYwBBHAZxJODUqxSXoSGDvmTfqUJXntnWkL4okok1FiCD # Z4jpyXOQunb6egIXvkgQ7jb2uO26Ow0m8RwleDvhOMrnHsupiOPbozKroSa6paFt # VSh89abUSooR8QdZciemmoFhcWkEwFg4spzvYNP4nIs193261WyTaRMZoceGun7G # CT2Rl653uUj+F+g94c63AhzSq4khdL4HlFIP2ePv29smfUnHtGq6yYFDLnT0q/Y+ # Di3jwloF8EWkkHRtSuXlFUbTmwr/lDDgbpZiKhLS7CBTDj32I0L5i532+uHczw82 # oZDmYmYmIUSMbZOgS65h797rj5JJ6OkeEUJoAVwwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVWjCCFVYCAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAd9r8C6Sp0q00AAAAAAB3zAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgkL0D9B4J # C8glMNz0oR5pvrM6Ic/j38hNtEKl6WmG6MMwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQACPfqjF5JlSZCVfvl2Vm+dZ4IeVurXAqRz4ZxZwlpa # GK314TlX65WUWVeeQ/Unoro2lR5ljwks2jViopRD/CkUfEhie96SsuBkQhUdPdoc # /TQCIyCl58pT/+JE1ay0UGbo8gq+fnX8bg7oNuajkeF644IxXuQ17AvyBMjoVXrR # F2sF5AJi7UNaGYTbfW3aGJQX1HcSBVUpeTN9xx6ir1CEBq23Z1LNHaXo5ob0raJ/ # H58m2ca9Gr28oPb67+2Ir6cH+tC1qYjkJLXY54ardf36ZTrRmrgwDYl+GBf1fEdZ # j+T27awwWTlI7U1oFxCMcSkKpCvksIzrjA+6+HAsAvnLoYIS5DCCEuAGCisGAQQB # gjcDAwExghLQMIISzAYJKoZIhvcNAQcCoIISvTCCErkCAQMxDzANBglghkgBZQME # AgEFADCCAVEGCyqGSIb3DQEJEAEEoIIBQASCATwwggE4AgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIOEbfEMLP7VHcBX6B4D2xGG3ab4rUDyPjmd2QrU5 # bilvAgZgichXoWQYEzIwMjEwNTIxMjAzMDQ0LjQzOFowBIACAfSggdCkgc0wgcox # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1p # Y3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOkU1QTYtRTI3Qy01OTJFMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNloIIOOzCCBPEwggPZoAMCAQICEzMAAAFHnY/x5t4xg1kAAAAAAUcw # DQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcN # MjAxMTEyMTgyNTU1WhcNMjIwMjExMTgyNTU1WjCByjELMAkGA1UEBhMCVVMxEzAR # BgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1p # Y3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2Eg # T3BlcmF0aW9uczEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046RTVBNi1FMjdDLTU5 # MkUxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtBQNM6X32KFk/BJ8YaprfzEt6Lj34 # G+VLjzgfEgOGSVd1Mu7nCphK0K4oyPrzItgNRjB4gUiKq6GzgxdDHgZPgTEvm57z # sascyGrybWkf3VVr8bqf2PIgGvwKDNEgVcygsEbuWwXz9Li6M7AOoD4TB8fl4ATm # +L7b4+lYDUMJYMLzpiJzM745a0XHiriUaOpYWfkwO9Hz6uf+k2Hq7yGyguH8naPL # MnYfmYIt2PXAwWVvG4MD4YbjXBVZ14ueh7YlqZTMua3n9kT1CZDsHvz+o58nsoam # XRwRFOb7LDjVV++cZIZLO29usiI0H79tb3fSvh9tU7QC7CirNCBYagNJAgMBAAGj # ggEbMIIBFzAdBgNVHQ4EFgQUtPjcb95koYZXGy9DPxN49dSCsLowHwYDVR0jBBgw # FoAU1WM6XIoxkPNDe3xGG8UzaFqFbVUwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDov # L2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljVGltU3RhUENB # XzIwMTAtMDctMDEuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0 # cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNUaW1TdGFQQ0FfMjAx # MC0wNy0wMS5jcnQwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDCDAN # BgkqhkiG9w0BAQsFAAOCAQEAUMQOyjV+ea2kEtXqD0cOfD2Z2PFUIy5kLkGU53RD # GcfhlzIR9QlTgZLqTEhgLLuCSy6jcma+nPg7e5Xg1oqCZcZJRwtRPzS1F6/M6YR3 # 5H3brN0maVnPrmrQ91kkfsNqDTtuWDiAIBfkNEgCpQZCb4OV3HMu5L8eZzg5dUaJ # 7XE+LBuphJSLFJtabxYt4fkCQxnTD2z50Y32ZuXiNmFFia7qVq+3Yc3mmW02+/KW # H8P1HPiobJG8crGYgSEkxtkUXGdoutwGWW88KR9RRcM/4GKLqt2OQ8AWEQb7shgM # 8pxNvu30TxejRApa4WAfOAejTG4+KzBm67XjVZ2IlXAPkjCCBnEwggRZoAMCAQIC # CmEJgSoAAAAAAAIwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRp # ZmljYXRlIEF1dGhvcml0eSAyMDEwMB4XDTEwMDcwMTIxMzY1NVoXDTI1MDcwMTIx # NDY1NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNV # BAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQG # A1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggEiMA0GCSqGSIb3 # DQEBAQUAA4IBDwAwggEKAoIBAQCpHQ28dxGKOiDs/BOX9fp/aZRrdFQQ1aUKAIKF # ++18aEssX8XD5WHCdrc+Zitb8BVTJwQxH0EbGpUdzgkTjnxhMFmxMEQP8WCIhFRD # DNdNuDgIs0Ldk6zWczBXJoKjRQ3Q6vVHgc2/JGAyWGBG8lhHhjKEHnRhZ5FfgVSx # z5NMksHEpl3RYRNuKMYa+YaAu99h/EbBJx0kZxJyGiGKr0tkiVBisV39dx898Fd1 # rL2KQk1AUdEPnAY+Z3/1ZsADlkR+79BL/W7lmsqxqPJ6Kgox8NpOBpG2iAg16Hgc # sOmZzTznL0S6p/TcZL2kAcEgCZN4zfy8wMlEXV4WnAEFTyJNAgMBAAGjggHmMIIB # 4jAQBgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQU1WM6XIoxkPNDe3xGG8UzaFqF # bVUwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1Ud # EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186aGMQwVgYD # VR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwv # cHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEB # BE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9j # ZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwgaAGA1UdIAEB/wSBlTCB # kjCBjwYJKwYBBAGCNy4DMIGBMD0GCCsGAQUFBwIBFjFodHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vUEtJL2RvY3MvQ1BTL2RlZmF1bHQuaHRtMEAGCCsGAQUFBwICMDQe # MiAdAEwAZQBnAGEAbABfAFAAbwBsAGkAYwB5AF8AUwB0AGEAdABlAG0AZQBuAHQA # LiAdMA0GCSqGSIb3DQEBCwUAA4ICAQAH5ohRDeLG4Jg/gXEDPZ2joSFvs+umzPUx # vs8F4qn++ldtGTCzwsVmyWrf9efweL3HqJ4l4/m87WtUVwgrUYJEEvu5U4zM9GAS # inbMQEBBm9xcF/9c+V4XNZgkVkt070IQyK+/f8Z/8jd9Wj8c8pl5SpFSAK84Dxf1 # L3mBZdmptWvkx872ynoAb0swRCQiPM/tA6WWj1kpvLb9BOFwnzJKJ/1Vry/+tuWO # M7tiX5rbV0Dp8c6ZZpCM/2pif93FSguRJuI57BlKcWOdeyFtw5yjojz6f32WapB4 # pm3S4Zz5Hfw42JT0xqUKloakvZ4argRCg7i1gJsiOCC1JeVk7Pf0v35jWSUPei45 # V3aicaoGig+JFrphpxHLmtgOR5qAxdDNp9DvfYPw4TtxCd9ddJgiCGHasFAeb73x # 4QDf5zEHpJM692VHeOj4qEir995yfmFrb3epgcunCaw5u+zGy9iCtHLNHfS4hQEe # gPsbiSpUObJb2sgNVZl6h3M7COaYLeqN4DMuEin1wC9UJyH3yKxO2ii4sanblrKn # QqLJzxlBTeCG+SqaoxFmMNO7dDJL32N79ZmKLxvHIa9Zta7cRDyXUHHXodLFVeNp # 3lfB0d4wwP3M5k37Db9dT+mdHhk4L7zPWAUu7w2gUDXa7wknHNWzfjUeCLraNtvT # X4/edIhJEqGCAs0wggI2AgEBMIH4oYHQpIHNMIHKMQswCQYDVQQGEwJVUzETMBEG # A1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWlj # cm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBP # cGVyYXRpb25zMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpFNUE2LUUyN0MtNTky # RTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcG # BSsOAwIaAxUAq6fBtEENocNASMqL03zGJS0wZd2ggYMwgYCkfjB8MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQg # VGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIFAORR78UwIhgPMjAy # MTA1MjExNjM3MjVaGA8yMDIxMDUyMjE2MzcyNVowdjA8BgorBgEEAYRZCgQBMS4w # LDAKAgUA5FHvxQIBADAIAgEAAgMAgtIwCAIBAAIDAVwQMAoCBQDkU0FFAgEAMDYG # CisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEA # AgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAolbZnYJjGVQjjEkJdiZ9Mp4/sNjVA/FY # Xl6wI9fQP+CHdJaTwG/SV7T6wEDt369Z/omZOuX3tQPp8t+prGOW4VhUqwRWFbjn # YHPy0Bkd3Q956Z3YDN2thJV5I4AyzyxbPV56bNrZ317osIFB3xP7ztL3qzarL3mj # GQaPTMOxMLoxggMNMIIDCQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0Eg # MjAxMAITMwAAAUedj/Hm3jGDWQAAAAABRzANBglghkgBZQMEAgEFAKCCAUowGgYJ # KoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCA6qN+DGv6z # Y91rwZ+bJt6c+1+Y9BEz8cV9NCvbRj/FizCB+gYLKoZIhvcNAQkQAi8xgeowgecw # geQwgb0EIHvbPBIDlM+6BsiJk7/YfWGuKwBUi3DMOxxvRaqKGOmFMIGYMIGApH4w # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAFHnY/x5t4xg1kAAAAA # AUcwIgQgp/uc1xBrIue3RFiedzfEHhNxybK46o64VHo1r241HG0wDQYJKoZIhvcN # AQELBQAEggEAFNT2wb+8Z7/ZMikR5ysQXhKXj+1gBeMo+bCTuExIG6sVkyTN4w5e # qvIg/eJothw79BCX9ZJlRp0iPQP+kSyEMxY+g2FFJEv+B5GNlZ0YV5y+6BFDOYNT # kYdFrsGpgXynUKXup2U0CzvYOtI4oMmCqHwZBmSWktgnot8lrnQTP3oVHDxyGrLd # FrhNI8BLO0aNu0imhMUj3LkD0SmWKmpRc3SG5CcvRNRLvF0S8cTJ1CcYTVwIDto7 # AtSoqcjEhuqtYW/qC1UHDLBN3nanZ9ATEaS1zuF2BQsd2xhBWKH2cRoVnSpeczAT # w6D2uDk0l++KgRHU5hPOJi2zSwelgznZUg== # SIG # End signature block |