Kva.psm1
|
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # Kva Module # ######################################################################################### #requires -runasadministrator using module .\Common.psm1 #region Module Constants $moduleName = "Kva" $moduleVersion = "1.0.21" #endregion #region Download catalog constants # Defaults until KVA has it's own catalog $catalogName = "aks-hci-stable-catalogs-ext" $ringName = "stable" $productName = "kva" $productInfoMapName = "cloudop-product-information" $productInfoMapNamespace = "cloudop-system" #endregion #region Script Constants $defaultContainerRegistryServer = "ecpacr.azurecr.io" $defaultContainerRegistryU = "1516df5a-f1cc-4a6a-856c-03d127b02d05" $defaultContainerRegistryP = "92684690-48b5-4dce-856d-ef4cccb54f22" $logFilePath = $($env:ProgramData + "\kva\kva.log") $kubeconfigMgmtFile = "kubeconfig-mgmt" $script:defaultTokenExpiryDays = 60 $kvaBinaries = @( $global:kvactlBinary, $global:kubectlBinary ) $kvaBinariesMap = @{ $global:kvactlBinary = $global:kvaCtlFullPath; $global:kubectlBinary = $global:kubeCtlFullPath; } if (!$global:config) { $global:config = @{} } $nodePoolYamlTemplate = @" azurestackhcinodepool: hardwareprofile: vmsize: {2} osprofile: ostype: {3} ssh: publickeys: - keydata: {4} name: {0} replicas: {1} "@ # azure $global:azureCloud = "AzureCloud" $global:azureChinaCloud = "AzureChinaCloud" $global:azureUSGovernment = "AzureUSGovernment" $global:azureGermanCloud = "AzureGermanCloud" $global:azurePPE = "AzurePPE" $global:graphEndpointResourceIdAzureCloud = "https://graph.windows.net/" $global:graphEndpointResourceIdAzurePPE = "https://graph.ppe.windows.net/" $global:graphEndpointResourceIdAzureChinaCloud = "https://graph.chinacloudapi.cn/" $global:graphEndpointResourceIdAzureUSGovernment = "https://graph.windows.net/" $global:graphEndpointResourceIdAzureGermancloud = "https://graph.cloudapi.de/" #endregion #region # Install Event Log New-ModuleEventLog -moduleName $moduleName #endregion Import-LocalizedData -BindingVariable "GenericLocMessage" -FileName commonLocalizationMessages Import-LocalizedData -BindingVariable "KvaLocMessage" -FileName KvaLocalizationMessages #region Private Function function Initialize-KvaConfiguration { <# .DESCRIPTION Initialize Kva Configuration. Wipes off any existing cached configuration #> if ($global:config.ContainsKey($moduleName)) { $global:config.Remove($moduleName) } $global:config += @{ $moduleName = @{ "cloudAgentAuthorizerPort" = 0 "cloudAgentPort" = 0 "cloudLocation" = "" "controlplaneVmSize" = [VmSize]::Default "dnsservers" = "" "gateway" = "" "group" = "" "imageDir" = "" "insecure" = $false "installationPackageDir" = "" "installState" = [InstallState]::NotInstalled "ipaddressprefix" = "" "k8snodeippoolstart" = "" "k8snodeippoolend" = "" "kubeconfig" = "" "kvaconfig" = "" "kvaK8sVersion" = "" "kvaName" = "" "kvaPodCidr" = "" "macPoolEnd" = "" "macpoolname" = "" "macPoolStart" = "" "manifestCache" = "" "moduleVersion" = $moduleVersion "proxyServerCertFile" = "" "proxyServerHTTP" = "" "proxyServerHTTPS" = "" "proxyServerNoProxy" = "" "proxyServerPassword" = "" "proxyServerUsername" = "" "skipUpdates" = $false "stagingShare" = "" "tokenExpiryDays" = 0 "containerRegistryServer" = "" "containerRegistryUser" = "" "containerRegistryPass" = "" "useStagingShare" = $false "version" = "" "vlanid" = 0 "vnetName" = "" "vswitchName" = "" "vnetvippoolend" = "" "vnetvippoolstart" = "" "workingDir" = "" "catalog" = "" "ring" = "" "identity" = "" "operatorTokenValidity" = 0 "addonTokenValidity" = 0 }; } } #endregion #region global config Initialize-KvaConfiguration #endregion #region Exported Functions function Install-Kva { <# .DESCRIPTION Uses KVACTL to deploy a management cluster. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -createConfigIfNotPresent -activity $activity $curState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($null -ne $curState) { switch ($curState) { ([InstallState]::Installed) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_installed) Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_reinstall_uninstall) return } ([InstallState]::Installing) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_installing) return break } ([InstallState]::NotInstalled) { # Fresh install break } Default { # Cleanup partial installs from previous attempts Uninstall-Kva -activity $activity } } } try { Install-KvaInternal -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $_ } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Restart-Kva { <# .DESCRIPTION Cleans up an existing KVA deployment and reinstalls everything. This isn't equivalent to executing 'Uninstall-Kva' followed by 'Install-Kva' as Restart-Kva will preserve existing configuration settings and any downloaded images. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # Skip the config cleanup to reinstall Uninstall-Kva -SkipConfigCleanup:$True -activity $activity Install-KvaInternal -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Uninstall-Kva { <# .DESCRIPTION Removes a KVA deployment. .PARAMETER SkipConfigCleanup skips removal of the configurations after uninstall. After uninstall, you have to Set-KvaConfig to install again. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Switch]$SkipConfigCleanup, [String]$activity = $MyInvocation.MyCommand.Name ) try { Initialize-KvaEnvironment -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } Set-KvaConfigValue -name "installState" -value ([InstallState]::Uninstalling) try { # Do not use Get-KvaConfigYaml, which can generate one, if not already there. if (Test-Path $kvaCtlFullPath) { $yamlFile = $global:config[$modulename]["kvaconfig"] if ($yamlFile -and (Test-Path $yamlFile)) { $kvaRegistration = Get-KvaRegistration try { Test-KvaAzureConnection # If Test-KvaAzureConnection worked, do the cleanup for the connectedcluster azure resource Invoke-KvaCtlWithAzureContext -arguments "delete --configfile ""$yamlFile""" -showOutputAsProgress -activity $activity } catch { Write-Status -moduleName $moduleName -Verbose -msg $($KvaLocMessage.kva_leaked_arc_connected_clusters) Invoke-KvaCtl -arguments "delete --configfile ""$yamlFile""" -showOutput -activity $activity } } } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } try { $kubeconfig = $global:config[$moduleName]["kubeconfig"] if ($kubeconfig -and (Test-Path $kubeconfig)) { Remove-Item $kubeconfig -ErrorAction Ignore } $kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"] if ($kubernetesVersion) { $imageGalleryName = Get-KubernetesGalleryImageName -imagetype "Linux" -k8sVersion $kubernetesVersion try { Remove-MocGalleryImage -name $imageGalleryName -location $global:config[$modulename]["cloudLocation"] | Out-Null } catch { if (-not ($_.Exception.Message -like "*connection closed*")) { Write-SubStatus -moduleName $moduleName $("Warning: " + $_.Exception.Message) } } } # 1. Update the binaries if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Continue | ForEach-Object { Uninstall-KvaBinaries -nodeName $_.Name } } else { Uninstall-KvaBinaries -nodeName ($env:computername) } # 2. Remove KVA Identity try { $clusterName = $($global:config[$modulename]["kvaName"]) Remove-MocIdentity -name $clusterName | Out-Null } catch { if (-not ($_.Exception.Message -like "*connection closed*")) { Write-SubStatus -moduleName $moduleName $("Warning: " + $_.Exception.Message) } } # Clean CloudConfig Remove-Item -Path $global:kvaMetadataDirectory -Force -Recurse -ErrorAction Ignore } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } Set-KvaConfigValue -name "installState" -value ([InstallState]::NotInstalled) if (!$SkipConfigCleanup.IsPresent) { Reset-Configuration -moduleName $moduleName } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function New-KvaCluster { <# .DESCRIPTION Adds a worker cluster to the deployment. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneNodeCount The number of control plane (master) nodes .PARAMETER linuxNodeCount The number of Linux worker nodes .PARAMETER windowsNodeCount The number of Windows worker nodes .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER nodePoolName The name of the node pool .PARAMETER nodeCount The number of worker nodes in the node pool .PARAMETER nodeMaxPodCount The maximum number of pods that can run on a worker node .PARAMETER taints A list of taints to put on each worker node .PARAMETER nodeVmSize The VM size to use for the worker nodes in the node pool .PARAMETER osType The OS type for the worker nodes in the node pool .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER activity Activity name to use when writing progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". .PARAMETER clusterStorageContainer Storage container that is associated to the Cluster. .PARAMETER loadBalancerSettings Load Balancer Settings. #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName = 'twonodepools')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [String] $kubernetesVersion = $global:defaultTargetK8Version, [Parameter()] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [ValidateRange(0,250)] [int] $linuxNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [ValidateRange(0,250)] [int] $windowsNodeCount = 0, [Parameter()] [String] $controlplaneVmSize = $global:defaultControlPlaneVmSize, [Parameter()] [String] $loadBalancerVmSize = $global:defaultLoadBalancerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $linuxNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $windowsNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $nodePoolName = $global:defaultNodePoolName, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeCount = $global:defaultWorkerNodeCount, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeMaxPodCount = 0, [Parameter(ParameterSetName = 'onenodepool')] [String[]] $taints, [Parameter(ParameterSetName = 'onenodepool')] [VmSize] $nodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [ValidateSet("Windows", "Linux")] [OsType] $osType = $global:defaultWorkerNodeOS, [Parameter()] [Switch] $enableADAuth, [Parameter()] [String] $activity, [Parameter()] [ValidateScript({return $true})]#Note: ValidateScript automatically constructs the NetworkPlugin object, therefore validates the parameter [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [VirtualNetwork]$vnet, [Parameter()] [string]$clusterStorageContainer = $global:cloudStorageContainer, [Parameter(mandatory=$true)] [LoadBalancerSettings]$loadBalancerSettings ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.comm_validating_cluster_configuration) $capiCluster = Invoke-Kubectl -arguments $("get akshciclusters/$Name") -ignoreError if ($null -ne $capiCluster) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_exists, $Name)) } $group = $("$global:cloudGroupPrefix-$Name") if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { New-KvaClusterInternal ` -Name $Name -group $group -kubernetesVersion $kubernetesVersion ` -controlPlaneReplicas $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -linuxWorkerReplicas $linuxNodeCount -linuxNodeVmSize $linuxNodeVmSize ` -windowsWorkerReplicas $windowsNodeCount -windowsNodeVmSize $windowsNodeVmSize ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings } elseif ($PSCmdlet.ParameterSetName -ieq "onenodepool") { New-KvaClusterInternal ` -Name $Name -group $group -kubernetesVersion $kubernetesVersion ` -controlPlaneReplicas $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -nodePoolName $nodePoolName -nodeCount $nodeCount -nodeMaxPodCount $nodeMaxPodCount -taints $taints ` -nodeVmSize $nodeVmSize -osType $osType ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function New-KvaClusterInternal { <# .DESCRIPTION Internal function to call KVACTL to create a capicluster. .PARAMETER Name Name of the cluster .PARAMETER group Cloudagent group .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneReplicas The number of control plane (master) replicas .PARAMETER linuxWorkerReplicas The number of Linux worker replicas .PARAMETER windowsWorkerReplicas The number of Windows worker replicas .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER nodePoolName The name of the node pool .PARAMETER nodeCount The number of worker nodes in the node pool .PARAMETER nodeMaxPodCount The maximum number of pods that can run on a worker node .PARAMETER taints A list of taints to put on each worker node .PARAMETER nodeVmSize The VM size to use for the worker nodes in the node pool .PARAMETER osType The OS type for the worker nodes in the node pool .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER activity Activity name to use when writing progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". .PARAMETER clusterStorageContainer Storage container that is associated to the Cluster. .PARAMETER loadBalancerSettings Load Balancer settings #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName = 'twonodepools')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$Name, [Parameter(Mandatory=$true)] [String]$group, [Parameter(Mandatory=$true)] [String]$kubernetesVersion, [Parameter(Mandatory=$true)] [int] $controlPlaneReplicas, [Parameter(Mandatory=$true, ParameterSetName = 'twonodepools')] [int] $linuxWorkerReplicas, [Parameter(Mandatory=$true, ParameterSetName = 'twonodepools')] [int] $windowsWorkerReplicas, [Parameter(Mandatory=$true)] [VmSize] $controlplaneVmSize, [Parameter(Mandatory=$true)] [VmSize] $loadBalancerVmSize, [Parameter(Mandatory=$true, ParameterSetName = 'twonodepools')] [VmSize] $linuxNodeVmSize, [Parameter(Mandatory=$true, ParameterSetName = 'twonodepools')] [VmSize] $windowsNodeVmSize, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepool')] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $nodePoolName, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepool')] [int] $nodeCount, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeMaxPodCount, [Parameter(ParameterSetName = 'onenodepool')] [String[]] $taints, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepool')] [VmSize] $nodeVmSize, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepool')] [OsType] $osType, [Parameter()] [Switch]$enableADAuth, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter()] [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [VirtualNetwork]$vnet, [Parameter()] [String]$clusterStorageContainer = $global:cloudStorageContainer, [Parameter()] [LoadBalancerSettings]$loadBalancerSettings ) Add-GalleryImage -imageType Linux -k8sVersion $kubernetesVersion -activity $activity if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { if ($windowsWorkerReplicas -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $kubernetesVersion -activity $activity } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_creating_workload_cluster, $Name)) $publicKey = Get-Content -Path (Get-SshPublicKey) $publicKey = $publicKey.Split(" ") $publicKey = $($publicKey[0]+" "+$publicKey[1]) if (-not $vnet) { $vnet = Get-VNetConfiguration -module $moduleName } if (-not $($loadBalancerSettings.VmSize) -or $($loadBalancerSettings.VmSize) -eq 0) { $loadBalancerVmSize = $global:defaultLoadBalancerVmSize } $yamlDir = ($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName) $yaml = @" name: $Name version: $kubernetesVersion controlplane: nodepool: replicas: $controlPlaneReplicas azurestackhcinodepool: hardwareprofile: vmsize: $controlplaneVmSize osprofile: ostype: Linux ssh: publickeys: - keydata: $publicKey servicecidr: $($global:workloadServiceCidr) podcidr: $($global:workloadPodCidr) azurestackhcicluster: storageconfiguration: storagecontainer: $clusterStorageContainer dynamic: true containernetworkconfiguration: primaryconfiguration: $($primaryNetworkPlugin.Name) loadbalancer: vmsize: $loadBalancerVmSize type: $($global:loadBalancerTypeStr[$($loadBalancerSettings.LoadBalancerSku)]) replicas: $($loadBalancerSettings.loadBalancerCount) servicesloadbalancer: vmsize: $loadBalancerVmSize type: $($global:loadBalancerTypeStr[$($loadBalancerSettings.ServicesLoadBalancerSku)]) replicas: $($loadBalancerSettings.loadBalancerCount) location: $($global:config[$modulename]["cloudLocation"]) group: $group virtualnetwork: name: "$($vnet.Name)" "@ $yaml += " additionalfeatures: - featurename: secrets-encryption - featurename: rotate-certificates" # TODO add switch to kvactl if ($enableADAuth.IsPresent) { $yaml += " - featurename: ad-auth-webhook" } $yamlFile = $($yamlDir+"\$Name.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster create --clusterconfig ""$yamlFile"" --kubeconfig ""$kubeconfig""" -showOutput -activity $activity if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { $defaultLinuxNodepoolName = GetDefaultLinuxNodepoolName -clusterName $Name $defaultWindowsNodepoolName = GetDefaultWindowsNodepoolName -clusterName $Name $linuxNodePoolYaml = $nodePoolYamlTemplate -f $defaultLinuxNodepoolName, $linuxWorkerReplicas, $linuxNodeVmSize, "Linux", $publicKey $linuxNodePoolYamlFile = $($yamlDir+"\$defaultLinuxNodepoolName.yaml") Set-Content -Path $linuxNodePoolYamlFile -Value $linuxNodePoolYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KvaCtl -arguments "cluster nodepool create --clustername $Name --nodepoolconfig ""$linuxNodePoolYamlFile"" --kubeconfig ""$kubeconfig""" -showOutput -activity $activity $windowsNodePoolYaml = $nodePoolYamlTemplate -f $defaultWindowsNodepoolName, $windowsWorkerReplicas, $windowsNodeVmSize, "Windows", $publicKey $windowsNodePoolYamlFile = $($yamlDir+"\$defaultWindowsNodepoolName.yaml") Set-Content -Path $windowsNodePoolYamlFile -Value $windowsNodePoolYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KvaCtl -arguments "cluster nodepool create --clustername $Name --nodepoolconfig ""$windowsNodePoolYamlFile"" --kubeconfig ""$kubeconfig""" -showOutput -activity $activity } elseif ($PSCmdlet.ParameterSetName -ieq "onenodepool") { New-KvaClusterNodePool ` -ClusterName $Name -Name $nodePoolName ` -MaxPodCount $nodeMaxPodCount -Taints $taints -Count $nodeCount ` -OSType $osType -VMSize $nodeVmSize } } function Get-Kva { <# .DESCRIPTION Get the Kva management cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # Check the status of Installation $curState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($null -ne $curState) { switch ($curState) { ([InstallState]::Installing) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_installing) return break } ([InstallState]::NotInstalled) { throw $($KvaLocMessage.kva_not_installed) } } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_retrieving_config_file) $yamlFile = $global:config[$modulename]["kvaconfig"] if (-not (Test-Path $yamlFile)) { # For some reason, this file doest exist - Generate one $yamlFile = Get-KvaConfigYaml } $kubeconfig = Get-KvaCredential -activity $activity if (!(Test-Path $kubeconfig)) { # Retrieve the kubeconfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_retrieving_credentials) Invoke-KvaCtl -arguments "retrieve --configfile ""$yamlFile"" --outfile ""$kubeconfig"" " -showOutput -activity $activity } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_retrieving_deployment_status) $capiCluster = $null $kvaStatus = $null try{ $kvaStatus = Invoke-KvaCtl -arguments "status --configfile ""$yamlFile"" --kubeconfig ""$kubeconfig""" -activity $activity | ConvertFrom-Json } catch {} if ($kvaStatus) { # Verify that the appliance is deployed and reachable before requesting cluster information if (($kvaStatus.phase -ine "NotDeployed") -and ($kvaStatus.phase -ine "WaitingForAPIServer") -and ($kvaStatus.phase -ine "Failed")) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_retrieving_management_cluster_details) $capiCluster = Get-KvaCluster -Name $global:config[$moduleName]["kvaName"] -activity $activity } } $status = [PSCustomObject]@{ CapiCluster = $capiCluster KvaStatus = $kvaStatus } return $status } function Get-KvaCluster { <# .DESCRIPTION Validates the requested cluster name and ensures that the cluster exists. Returns the cluster object. .PARAMETER Name Name of the cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String]$Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Write-StatusWithProgress -activity $activity -status $($KvaLocMessage.kva_retrieving_cluster) -moduleName $moduleName return Get-KvaClusterInternal -Name $Name } function Get-KvaClusterInternal { <# .DESCRIPTION Validates the requested cluster name and ensures that the cluster exists. Returns the cluster object. .PARAMETER Name Name of the cluster #> param ( [Parameter()] [String]$Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $kubeconfig = Get-KvaCredential -activity $activity $migrationNeeded = $false try { $clusters = Get-KvaCapiCluster -Name $Name foreach($cluster in $clusters) { if ($null -eq $cluster.kubernetesversion) { $migrationNeeded = $true throw } } } catch { # Migration code needed for upgrades from before Sept release due to kubernetesversion being missing from Cluster object. # Can be removed alongside Get-CapiCluster once August is no longer supported. if (($_.Exception.Message -like "*unknown flag: *") -or ($migrationNeeded)) { if (![string]::IsNullOrEmpty($name)) { $clusters = @() $clusters += Get-CapiCluster -Name $Name } else { $clusters = Get-CapiClusters } $result = @() foreach($cluster in $clusters) { $props = [ordered]@{ 'ProvisioningState' = $($cluster.status.phase); 'KubernetesVersion' = $($cluster.spec.clusterConfiguration.kubernetesVersion); 'PackageVersion' = $($cluster.spec.packageVersion); 'NodePools' = $( $($cluster.nodepools.items) | %{ $_.metadata.name } ); 'WindowsNodeCount' = $($cluster.windowsWorkerReplicas); 'LinuxNodeCount' = $($cluster.linuxWorkerReplicas); 'ControlPlaneNodeCount' = $($cluster.spec.controlPlaneConfiguration.replicas); 'Name' = $($cluster.metadata.name); 'IsManagement' = $($cluster.spec.management); } $result += New-Object -TypeName PsObject -Property $props } return $result } # Filter Not Found exception if (![string]::IsNullOrEmpty($name) -and ($_.Exception.Message -like "*akshciclusters.msft.microsoft ""$name"" not found*")) { throw $([System.Exception]::new([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_no_cluster_found , $Name), $_.Exception)) } throw $_ } $result = @() foreach($cluster in $clusters) { $clusterName = $cluster.name $cmdArgs = "cluster status --clustername $clusterName --kubeconfig ""$kubeconfig""" $status = Invoke-KvaCtl -arguments $cmdArgs -activity $activity | ConvertFrom-Json # For ease, calculate default linux and windows nodepool replica counts and append them as a member of the returned object $linuxWorkerReplicas = 0 $windowsWorkerReplicas = 0 $nodePools = $cluster.nodepools if ($null -ne $nodePools) { $linuxNodePools = ($nodePools | Where-Object { $_.azurestackhcinodepool.osprofile.ostype -eq "Linux" }) foreach ($np in $linuxNodePools) { $linuxWorkerReplicas += $(if ($np | Get-Member replicas) { $np.replicas } else { 0 }) } $windowsNodePools = ($nodePools | Where-Object { $_.azurestackhcinodepool.osprofile.ostype -eq "Windows" }) foreach ($np in $windowsNodePools) { $windowsWorkerReplicas += $(if ($np | Get-Member replicas) { $np.replicas } else { 0 }) } } $statusProps = [ordered]@{ 'ProvisioningState' = $($status.phase); 'Details' = $($status.details); } if ($null -ne $status.lasterror) { $statusProps.Insert($statusProps.Count, 'Error', $status.lasterror) if ($null -ne $status.reason) { $statusProps.Insert($statusProps.Count, 'Reason', $status.reason) } } $props = [ordered]@{ 'Status' = $($statusProps); 'ProvisioningState' = $($status.phase); 'KubernetesVersion' = $($cluster.kubernetesversion); 'PackageVersion' = $($cluster.version); 'NodePools' = $( $($cluster.nodepools) | %{ $_.name } ); 'WindowsNodeCount' = $($windowsWorkerReplicas); 'LinuxNodeCount' = $($linuxWorkerReplicas); 'ControlPlaneNodeCount' = $($cluster.controlplane.nodepool.replicas); 'Name' = $($clusterName); } $result += New-Object -TypeName PsObject -Property $props } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) return $result } function Get-KvaCapiCluster { <# .DESCRIPTION Validates the requested cluster name and ensures that the cluster exists. Returns the cluster object. .PARAMETER Name Name of the cluster #> param ( [Parameter()] [String]$Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity $msg="" $cmdArgs="" if (![string]::IsNullOrEmpty($name)) { Test-ValidClusterName -Name $Name | Out-Null $msg = "Retrieving cluster $name..." $cmdArgs = "cluster get --clustername=$name --kubeconfig=""$kubeconfig""" } else { $msg = "Retrieving AksHciClusters..." $cmdArgs = "cluster list --kubeconfig=""$kubeconfig""" } Write-StatusWithProgress -activity $activity -status $msg -moduleName $moduleName $clusters = Invoke-KvaCtl -arguments $cmdArgs -activity $activity | ConvertFrom-Json return $clusters } function Get-TargetClusterKubernetesVersions { <# .DESCRIPTION Get the Kubernetes Versions used for the target clusters .PARAMETER Version Version #> $tmp = Get-TargetClusterKubernetesReferences return $tmp.Keys } function Get-TargetClusterKubernetesReferences { <# .DESCRIPTION Get the Kubernetes Versions used for the target clusters #> $k8sversionsInUse = @{} Get-KvaClusterInternal | ForEach-Object { $tmp = $_ if (!($tmp -contains 'IsManagement') -or ($null -eq $tmp.IsManagement)) { $k8sversion = $tmp.KubernetesVersion if (!$k8sversionsInUse.ContainsKey($k8sversion)) { $k8sversionsInUse += @{$k8sversion = @();} } # Add references to the target cluster name $k8sversionsInUse[$k8sversion] += ($tmp.Name) } } return $k8sversionsInUse } function Get-KvaClusterUpgrades { <# .DESCRIPTION Gets the upgrades available for a kva cluster. .PARAMETER Name Name of the cluster. .PARAMETER activity Activity name to use when updating progress. #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_missing_kubeconfig, $kubeconfig)) } $cmdArgs = "cluster get-upgrades --clustername $Name --kubeconfig ""$kubeconfig""" $upgrades = Invoke-KvaCtl -arguments $cmdArgs -activity $activity | ConvertFrom-Json return $upgrades } function Wait-ForClusterUpgrade { <# .DESCRIPTION Waits for all nodes of a cluster to be running the specified kubernetes version. .PARAMETER kubeconfigFile Path to a kubeconfig file for the cluster .PARAMETER expectedVersion The version that we expect all nodes of the cluster to be running .PARAMETER sleepDuration Duration to sleep for between attempts #> param ( [String] $kubeconfigFile, [String] $expectedVersion, [int] $sleepDuration=60 ) Write-Status $($KvaLocMessage.kva_monitoring_cluster_upgrade) -moduleName $moduleName while ($true) { $results = @() $allNodesUpgraded = $true $nodes = (Invoke-Kubectl -ignoreError -kubeconfig $kubeconfigFile -arguments "get nodes --request-timeout=1m -o name") 2>$null if (-not ($nodes)) { Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_unable_to_retrive_cluster_node_information, $sleepDuration)) -moduleName $moduleName Start-Sleep $sleepDuration continue } foreach ($node in $nodes) { $nodeJson = (Invoke-Kubectl -ignoreError -kubeconfig $kubeconfigFile -arguments $("get $node --request-timeout=1m -o json") | ConvertFrom-Json) 2>$null if ($nodeJson) { $result = New-Object -TypeName PsObject -Property @{Name = $($nodeJson.metadata.name); Version = $($nodeJson.status.nodeInfo.kubeletVersion); Upgraded = $true} if ($nodeJson.status.nodeInfo.kubeletVersion -ine $expectedVersion) { $result.upgraded = $false $allNodesUpgraded = $false } $results += $result } else { $allNodesUpgraded = $false } } Write-SubStatus $($KvaLocMessage.kva_current_status_of_cluster_nodes) -moduleName $moduleName $results | Format-Table * if ($allNodesUpgraded) { break } Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_nodes_still_upgrading, $sleepDuration)) -moduleName $moduleName Start-Sleep $sleepDuration } Write-SubStatus $($KvaLocMessage.kva_cluster_nodes_running_expected_version) -moduleName $moduleName } function Update-KvaCluster { <# .DESCRIPTION Updates the kubernetes version of a cluster by performing an upgrade. .PARAMETER Name Name of the cluster. .PARAMETER nextVersion desired kubernetes version. .PARAMETER operatingSystem If specified, akshcicluster will only attempt an operating system upgrade. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter()] [String]$nextVersion, [Parameter()] [Switch]$operatingSystem, [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $cluster = Get-KvaCluster -Name $Name -activity $activity $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_missing_kubeconfig, $kubeconfig)) } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_determining_upgrade_path) $cmdArgs = "cluster upgrade --clustername $($cluster.Name) --plan --kubeconfig ""$kubeconfig""" if (($null -ne $nextVersion) -and ($nextVersion -ne "")) { $cmdArgs += " --kubernetes-version $nextVersion" } if ($operatingSystem.IsPresent) { $cmdArgs += " --operating-system" } $upgradePlan = Invoke-KvaCtl -arguments $cmdArgs -activity $activity $upgradePlan = $upgradePlan -join "`n" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $upgradePlan Write-Status -moduleName $moduleName -Verbose -msg "`n$upgradePlan`n" if ($upgradePlan -like "*no upgrade found*") { return } if (-not $PSCmdlet.ShouldProcess($Name, "Update the managed Kubernetes cluster")) { return } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_provisioning_image_gallery) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_on_kubernetes_version, $cluster.KubernetesVersion)) # We may need to re-add the same image, to pick up just OS update Add-GalleryImage -imageType Linux -k8sVersion $cluster.KubernetesVersion -activity $activity if ($cluster.WindowsNodeCount -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $cluster.KubernetesVersion -activity $activity } if ($nextVersion) { Add-GalleryImage -imageType Linux -k8sVersion $nextVersion -activity $activity if ($cluster.WindowsNodeCount -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $nextVersion -activity $activity } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_cluster_upgrade) $cmdArgs = "cluster upgrade --clustername $($cluster.Name) --kubeconfig ""$kubeconfig""" if (($null -ne $nextVersion) -and ($nextVersion -ne "")) { $cmdArgs += " --kubernetes-version $nextVersion" } if ($operatingSystem.IsPresent) { $cmdArgs += " --operating-system" } Invoke-KvaCtl -arguments $cmdArgs -showOutput -activity $activity try { $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $cluster.Name -outputLocation $targetClusterKubeconfig -activity $activity if ($nextVersion) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_waiting_for_upgrade_to_complete) Wait-ForClusterUpgrade -expectedVersion $nextVersion -kubeconfigFile $targetClusterKubeconfig } } finally { Remove-Item -Path $targetClusterKubeconfig -Force -ErrorAction Ignore } } function Set-KvaClusterNodeCount { <# .DESCRIPTION Sets the configuration of a cluster. Currently used to scale the cluster control plane or to scale the worker nodes. .PARAMETER Name Name of the cluster .PARAMETER controlPlaneNodeCount The number of control plane nodes to scale to .PARAMETER linuxNodeCount The number of Linux worker nodes to scale to .PARAMETER windowsNodeCount The number of Windows worker nodes to scale to .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='controlplane')] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $linuxNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $windowsNodeCount, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $cluster = Get-KvaCluster -Name $Name -activity $activity $controlPlane = $cluster.ControlPlaneNodeCount $linuxWorker = $cluster.LinuxNodeCount $windowsWorker = $cluster.WindowsNodeCount $kubeconfig = Get-KvaCredential -activity $activity if ($PSCmdlet.ParameterSetName -ieq "controlplane") { if ($controlPlaneNodeCount -lt 1) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_min_control_plane_node, $cluster.Name)) } if (($cluster.ControlPlaneNodeCount -gt 1) -and ($controlPlaneNodeCount -lt 3)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_ha_min_node, $cluster.Name)) } $controlPlane = $controlPlaneNodeCount $cmdControlPlaneArgs = "cluster scale --clustername=$Name --controlplane=$controlPlane --kubeconfig ""$kubeconfig""" Invoke-KvaCtl -arguments $cmdControlPlaneArgs -showOutput -activity $activity } elseif ($PSCmdlet.ParameterSetName -ieq "worker") { $defaultLinuxNodepoolName = GetDefaultLinuxNodepoolName -clusterName $Name $defaultWindowsNodepoolName = GetDefaultWindowsNodepoolName -clusterName $Name $linuxNodePool = Get-KvaClusterNodePool -ClusterName $Name -Name $defaultLinuxNodepoolName $windowsNodePool = Get-KvaClusterNodePool -ClusterName $Name -Name $defaultWindowsNodepoolName if ($null -ne $windowsNodePool) { if ($windowsNodeCount -gt 0) { Add-GalleryImage -imageType Windows -k8sVersion $cluster.KubernetesVersion -activity $activity } $windowsWorker = $windowsNodeCount $cmdWindowsWorkerArgs = "cluster nodepool scale --clustername=$Name --nodepoolname=$defaultWindowsNodepoolName --replicas=$windowsWorker --kubeconfig ""$kubeconfig""" Invoke-KvaCtl -arguments $cmdWindowsWorkerArgs -showOutput -activity $activity } if ($null -ne $linuxNodePool) { $linuxWorker = $linuxNodeCount $cmdLinuxWorkerArgs = "cluster nodepool scale --clustername=$Name --nodepoolname=$defaultLinuxNodepoolName --replicas=$linuxWorker --kubeconfig ""$kubeconfig""" Invoke-KvaCtl -arguments $cmdLinuxWorkerArgs -showOutput -activity $activity } } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function New-KvaClusterNodePool { <# .SYNOPSIS Creates a new nodepool in a cluster. .DESCRIPTION Creates a new nodepool in a cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count The number of worker nodes in the nodepool .PARAMETER OSType OS type of the node pool. Defaults to Linux .PARAMETER VMSize The VM size to use for the worker nodes. Defaults to Standard_K8S3_v1 .PARAMETER MaxPodCount The maximum number of pods that can run on a worker node .PARAMETER Taints A list of taints to put on each worker node .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [int] $Count = $global:defaultWorkerNodeCount, [Parameter()] [VmSize] $VMSize = $global:defaultWorkerVmSize, [Parameter()] [OsType] $OSType = $global:defaultWorkerNodeOS, [Parameter()] [int] $MaxPodCount = 0, [Parameter()] [String[]] $Taints, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_getting_cluster_details) $cluster = Get-KvaCluster -Name $ClusterName -activity $activity $kubernetesVersion = $cluster.KubernetesVersion Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_check_if_image_in_gallery) Add-GalleryImage -imageType $OSType -k8sVersion $kubernetesVersion -activity $activity $publicKey = Get-Content -Path (Get-SshPublicKey) $publicKey = $publicKey.Split(" ") $publicKey = $($publicKey[0]+" "+$publicKey[1]) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_creating_np_yaml) $yaml = $nodePoolYamlTemplate -f $Name, $Count, $VMSize, $OSType, $publicKey if ($MaxPodCount -gt 0) { $yaml += "`r`nmaxpodcount: $MaxPodCount" } if ($Taints.Count -gt 0) { $taintsYaml = "taints:" foreach ($t in $Taints) { $taintsYaml += "`r`n - ""$t""" } $yaml += "`r`n$taintsYaml" } $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$ClusterName-$Name.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } $msg = "Creating node pool $Name in cluster $ClusterName..." Write-StatusWithProgress -activity $activity -status $msg -moduleName $moduleName $cmdArgs = "cluster nodepool create --clustername $ClusterName --nodepoolconfig ""$yamlFile"" --kubeconfig ""$kubeconfig""" Invoke-KvaCtl -arguments $cmdArgs -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Get-KvaClusterNodePool { <# .SYNOPSIS Returns the nodepool(s) of a cluster. .DESCRIPTION Returns the nodepool(s) of a cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter()] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_gathering_cluster_information) $cluster = Get-KvaCluster -Name $ClusterName -activity $activity $k8sVersion = $cluster[0].PackageVersion $msg="" $cmdArgs="" if (![string]::IsNullOrEmpty($name)) { $msg = "Retrieving nodepool $Name from cluster $ClusterName..." $cmdArgs = "cluster nodepool get --clustername=$ClusterName --nodepoolname=$Name --kubeconfig=""$kubeconfig""" } else { $msg = "Retrieving cluster $ClusterName's nodepools..." $cmdArgs = "cluster nodepool list --clustername=$ClusterName --kubeconfig=""$kubeconfig""" } Write-StatusWithProgress -activity $activity -status $msg -moduleName $moduleName $nodepools = Invoke-KvaCtl -arguments $cmdArgs -activity $activity | ConvertFrom-Json $result = @() foreach($np in $nodepools) { $vmSize = [VmSize]::Default $vmSize = [Enum]::Parse([VmSize], $($np.azurestackhcinodepool.hardwareprofile.vmsize), $true) $statusProps = [ordered]@{ 'Phase' = $($np.status.phase); 'Details' = $($np.status.details); } if ($null -ne $np.status.lasterror) { $statusProps.Insert($o.Count, 'Error', $np.status.lasterror) if ($null -ne $np.status.reason) { $statusProps.Insert($o.Count, 'Reason', $np.status.reason) } } $props = [ordered]@{ 'Status' = $($statusProps); 'ClusterName' = $($ClusterName); 'NodePoolName' = $($np.name); 'Version' = $($k8sVersion); 'OsType' = $($np.azurestackhcinodepool.osprofile.ostype); 'NodeCount' = $(if ($np | Get-Member replicas) { $np.replicas } else { 0 }); 'VmSize' = $($vmSize); 'Phase' = $($np.status.phase); } $n = New-Object -TypeName PsObject -Property $props if ($np | Get-Member maxPodCount) { $n | Add-Member -NotePropertyName 'MaxPodCount' -NotePropertyValue $np.maxPodCount } if ($np | Get-Member taints) { $n | add-member -notepropertyname 'Taints' -notepropertyvalue $np.taints } $result += $n } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) return $result } function Set-KvaClusterNodePool { <# .SYNOPSIS Scale a cluster's nodepool. .DESCRIPTION Scale a cluster's nodepool. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count Node count to scale to .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true)] [int] $Count, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_gathering_cluster_information) $cluster = Get-KvaCluster -Name $ClusterName -activity $activity $k8sVersion = $cluster.KubernetesVersion Write-StatusWithProgress -activity $activity -status $($KvaLocMessage.kva_getting_nodepool_details) -moduleName $moduleName $nodepool = Get-KvaClusterNodePool -ClusterName $ClusterName -Name $Name Write-StatusWithProgress -activity $activity -status $($KvaLocMessage.kva_check_if_image_in_gallery) -moduleName $moduleName Add-GalleryImage -imageType $nodepool.OsType -k8sVersion $k8sVersion -activity $activity $msg = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_scaling_nodepool, $Name, $ClusterName, $Count)) Write-StatusWithProgress -activity $activity -status $msg -moduleName $moduleName $cmdArgs = "cluster nodepool scale --clustername=$ClusterName --nodepoolname=$Name --replicas=$Count --kubeconfig=""$kubeconfig""" Invoke-KvaCtl -arguments $cmdArgs -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Remove-KvaClusterNodePool { <# .SYNOPSIS Delete a nodepool in a managed Kubernetes cluster. .DESCRIPTION Delete a nodepool in a managed Kubernetes cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } if ($PSCmdlet.ShouldProcess($Name, "Delete the node pool in the managed Kubernetes cluster")) { Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity $msg = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_deleting_nodepool, $Name, $ClusterName)) Write-StatusWithProgress -activity $activity -status $msg -moduleName $moduleName $cmdArgs = "cluster nodepool delete --clustername=$ClusterName --nodepoolname=$Name --kubeconfig=""$kubeconfig""" Invoke-KvaCtl -arguments $cmdArgs -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) return $result } } function Remove-KvaCluster { <# .DESCRIPTION Removes a cluster from the deployment. .PARAMETER Name Name of the cluster .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.kva_removing_cluster) Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_cluster_removal_in_progress) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster delete --clustername $Name --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Update-Kva { <# .DESCRIPTION Update a the kva .PARAMETER version Optional version to udate to .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$version, [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $curState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($null -ne $curState) { switch ($curState) { ([InstallState]::Updating) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_in_progress, $moduleName)) return } } } $currentVersion = Get-KvaVersion $currentKvaK8sVersion = $global:config[$moduleName]["kvaK8sVersion"] Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_current_version, $currentVersion)) # If no version is specified, try to move to the latest if (!$version) { # If no version is specified, use the latest $release = Get-LatestRelease -moduleName $moduleName $version = $release.Version Set-KvaConfigValue -name "version" -value $version $release = Get-ProductRelease -Version $version -module $moduleName } else { if ($version -eq $currentVersion) { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_already_in_expected_version, $version)) return } $release = Get-ProductRelease -Version $version -module $moduleName Set-KvaConfigValue -name "version" -value $version } $workingDir = $global:config[$moduleName]["workingDir"] Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_updating_to_version , $version)) try { # Set the new K8s version Set-KvaConfigValue -name "kvaK8sVersion" -value ("v" +$release.CustomData.ManagementNodeImageK8sVersion) Set-KvaConfigValue -name "installationPackageDir" -value $([io.Path]::Combine($workingDir, $version)) Set-KvaConfigValue -name "kubeconfig" -value $([io.Path]::Combine($workingDir, $version, $kubeconfigMgmtFile)) Get-KvaConfigYaml | Out-Null Update-KvaInternal } catch { Set-KvaConfigValue -name "kvaK8sVersion" -value $currentKvaK8sVersion Set-KvaConfigValue -name "version" -value $currentVersion Set-KvaConfigValue -name "installationPackageDir" -value $([io.Path]::Combine($workingDir, $currentVersion)) Set-KvaConfigValue -name "kubeconfig" -value $([io.Path]::Combine($workingDir, $currentVersion, $kubeconfigMgmtFile)) Get-KvaConfigYaml | Out-Null Write-Warning $_.Exception.Message -ErrorAction Continue # Revert Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_reverting_to_version , $currentVersion)) Update-KvaInternal throw } } function Update-KvaInternal { <# .DESCRIPTION Update KVA .PARAMETER activity Activity name to use when updating progress #> param ( [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_updating_to, $(Get-KvaVersion))) # 1. Invalidate all gallery images # Gallery images are downloaded and managed by KVA. So it is okay to reset it completely for now. # If this assumption changes, change the logic here to just remove what KVA brings in Reset-GalleryImage New-Item -ItemType Directory -Force -Path $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:yamlDirectoryName)) | Out-Null # 2. Update the binaries Get-KvaRelease -version $(Get-KvaVersion) -activity $activity if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Install-KvaBinaries -nodeName $_.Name } } else { Install-KvaBinaries -nodeName ($env:computername) } # 3. Upgrade the management appliance $cluster = Get-KvaClusterInternal -Name $global:config[$moduleName]["kvaName"] Write-Status -moduleName $moduleName $($GenericLocMessage.kva_provisioning_image_gallery) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_on_kubernetes_version, $cluster.KubernetesVersion)) $nextVersion = $global:config[$moduleName]["kvaK8sVersion"] Add-GalleryImage -imageType Linux -k8sVersion $nextVersion -activity $activity $kubeconfig = Get-KvaCredential -activity $activity if (-not $global:config[$modulename]["insecure"]) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_updating_identity) $clusterName = $($global:config[$modulename]["kvaName"]) $kvaIdentity = Invoke-MocIdentityRotate -name $clusterName -encode Set-KvaConfigValue -name "identity" -value $kvaIdentity } $updateConfig = Get-KvaUpdateConfigYaml $deploymentManifestLocation = $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:cloudOperatorYaml)) $privateKey = Get-SshPrivateKey # Fix for October upgrade Update-ConfigMapPauseImages Invoke-KvaCtl -arguments "upgrade --configfile ""$updateConfig"" --kubeconfig ""$kubeconfig"" --cloudop ""$deploymentManifestLocation"" --sshprivatekey ""$privateKey"" " -showOutput -activity $activity } function Add-KvaGalleryImage { <# .DESCRIPTION Downloads an image and adds it to the cloud gallery. .PARAMETER kubernetesVersion Optional, override the default version of kubernetes that the image will use. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $kubernetesVersion, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity if (-not $kubernetesVersion) { $kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"] } Add-GalleryImage -imagetype "Linux" -k8sVersion $kubernetesVersion -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Get-KvaVersion { <# .DESCRIPTION Get the current KVA version .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity return $global:config[$modulename]["version"] } function Get-KvaConfig { <# .DESCRIPTION Loads and returns the current KVA configuration. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) Import-KvaConfig -activity $activity Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_getting_configuration, $moduleName)) $global:config[$modulename]["installState"] = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" $global:config[$modulename]["controlplaneVmSize"] = Get-ConfigurationValue -module $moduleName -type ([Type][VmSize]) -name "controlplaneVmSize" return $global:config[$modulename] } function Import-KvaConfig { <# .DESCRIPTION Loads a configuration from persisted storage. If no configuration is present then a default configuration can be optionally generated and persisted. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Switch] $createIfNotPresent, [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $($GenericLocMessage.comm_importing_configuration) if (Test-Configuration -moduleName $moduleName) { Import-Configuration -moduleName $moduleName } else { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_cannot_deploy, $moduleName)) } Write-StatusWithProgress -activity $activity -module $moduleName -status $($GenericLocMessage.comm_importing_configuration_completed) } function Set-KvaRegistration { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. .PARAMETER azureResourceGroup azureResourceGroup is the name of the azure resource group to place arc resources. .PARAMETER azureLocation azureLocation is the name of the azure location where the resource group lives. #> [CmdletBinding()] param ( [String] $azureResourceGroup, [String] $azureLocation ) $kvaRegistration = Get-KvaRegistration if ([string]::IsNullOrWhiteSpace($kvaRegistration.azureResourceGroup)) { Set-KvaConfigValue -name "azureResourceGroup" -value $azureResourceGroup Set-KvaConfigValue -name "azureLocation" -value $azureLocation } } function Get-KvaRegistration { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. #> $obj = New-Object -TypeName psobject $obj | Add-Member -MemberType NoteProperty -Name azureResourceGroup -Value (Get-KvaConfigValue -name "azureResourceGroup") $obj | Add-Member -MemberType NoteProperty -Name azureLocation -Value (Get-KvaConfigValue -name "azureLocation") return $obj } function Set-KvaConfig { <# .DESCRIPTION Configures KVA by persisting the specified parameters to the registry. Any parameter which is not explictly provided by the user will be defaulted. #> [CmdletBinding()] param ( [string] $activity = $MyInvocation.MyCommand.Name, [String] $kvaName = (New-Guid).Guid, [String] $workingDir = $global:defaultWorkingDir, [String] $imageDir, [String] $version, [String] $stagingShare = $global:defaultStagingShare, [String] $cloudLocation = $global:defaultCloudLocation, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet, [VmSize] $controlplaneVmSize = $global:defaultMgmtControlPlaneVmSize, [String] $kvaPodCIDR = $global:defaultPodCidr, [Switch] $kvaSkipWaitForBootstrap, [ProxySettings] $proxySettings = $null, [Switch] $skipUpdates, [Switch] $skipHostLimitChecks, [Switch] $insecure, [String] $macPoolStart, [String] $macPoolEnd, [switch] $useStagingShare, [ContainerRegistry] $containerRegistry = $null, [String] $catalog = $script:catalogName, [String] $ring = $script:ringName, [int] $cloudAgentPort = $global:defaultCloudAgentPort, [int] $cloudAgentAuthorizerPort = $global:defaultCloudAuthorizerPort, [String] $deploymentId = [Guid]::NewGuid().ToString(), [int] $tokenExpiryDays = $script:defaultTokenExpiryDays, [parameter(DontShow)] [int] $operatorTokenValidity = $global:operatorTokenValidity, [parameter(DontShow)] [int] $addonTokenValidity = $global:addonTokenValidity, [parameter(DontShow)] [Switch] $enablePreview ) # Import the existing config, if any try { Import-KvaConfig -activity $activity } catch {} $currentState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($currentState) { switch ($currentState) { ([InstallState]::NotInstalled) { # Fresh install break } Default { Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_current_state, $currentState)) throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_config_state, $moduleName, $currentState)) } } } if ($enablePreview.IsPresent) { $catalog = "aks-hci-stable-catalogs-ext" $ring = "earlyaccesspreview" } Confirm-Configuration -useStagingShare:$useStagingShare.IsPresent -stagingShare $stagingShare Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName Set-ContainerRegistryConfiguration -containerRegistry $containerRegistry # If okay to proceed, overwrite Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_creating_configuration, $moduleName)) Set-KvaConfigValue -name "workingDir" -value $workingDir Set-KvaConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, "$catalog.json")) New-Item -ItemType Directory -Force -Path $workingDir | Out-Null if (!$imageDir) { $imageDir = [io.Path]::Combine($workingDir, $global:imageDirectoryName) New-Item -ItemType Directory -Force -Path $workingDir | Out-Null } Set-KvaConfigValue -name "imageDir" -value $imageDir Set-KvaConfigValue -name "stagingShare" -value $stagingShare Set-KvaConfigValue -name "group" -value "clustergroup" Set-KvaConfigValue -name "cloudLocation" -value $cloudLocation Set-KvaConfigValue -name "moduleVersion" -value $moduleVersion Set-KvaConfigValue -name "catalog" -value $catalog Set-KvaConfigValue -name "ring" -value $ring Set-KvaConfigValue -name "deploymentId" -value $deploymentId Set-KvaConfigValue -name "operatorTokenValidity" -value $operatorTokenValidity Set-KvaConfigValue -name "addonTokenValidity" -value $addonTokenValidity Set-VNetConfiguration -module $moduleName -vnet $vnet Set-KvaConfigValue -name "controlplaneVmSize" -value $controlplaneVmSize Set-KvaConfigValue -name "skipUpdates" -value $skipUpdates.IsPresent Set-KvaConfigValue -name "insecure" -value $insecure.IsPresent Set-KvaConfigValue -name "useStagingShare" -value $useStagingShare.IsPresent Set-KvaConfigValue -name "macPoolStart" -value $macPoolStart Set-KvaConfigValue -name "macPoolEnd" -value $macPoolEnd Set-KvaConfigValue -name "kvaName" -value $kvaName Set-KvaConfigValue -name "kvaPodCidr" -value $kvaPodCIDR Set-KvaConfigValue -name "kvaSkipWaitForBootstrap" -value $kvaSkipWaitForBootstrap.IsPresent Set-KvaConfigValue -name "cloudAgentPort" -value $cloudAgentPort Set-KvaConfigValue -name "cloudAgentAuthorizerPort" -value $cloudAgentAuthorizerPort Set-KvaConfigValue -name "tokenExpiryDays" -value $tokenExpiryDays if (-not $version) { # If no version is specified, use the latest from the product catalog $release = Get-LatestRelease -moduleName $moduleName $version = $release.Version Set-KvaConfigValue -name "version" -value $version } else { Get-LatestCatalog -moduleName $moduleName | Out-Null # This clears the cache $release = Get-ProductRelease -Version $version -module $moduleName Set-KvaConfigValue -name "version" -value $version } $installationPackageDir = ([io.Path]::Combine($workingDir, $version)) Set-KvaConfigValue -name "installationPackageDir" -value $installationPackageDir New-Item -ItemType Directory -Force -Path $installationPackageDir | Out-Null if (-not $release.CustomData.ManagementNodeImageK8sVersion) { throw $($KvaLocMessage.kva_k8s_version_unknown) } Set-KvaConfigValue -name "kvaK8sVersion" -value ("v" +($release.CustomData.ManagementNodeImageK8sVersion)) Set-KvaConfigValue -name "kubeconfig" -value $([io.Path]::Combine($workingDir, $version, $kubeconfigMgmtFile)) # Check if this is rehydration of an already running deployment if ((Test-Path $global:kvaCtlFullPath)) { try { Get-Kva | Out-Null Set-KvaConfigValue -name "installState" -value ([InstallState]::Installed) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_existing_configuration_loaded, $moduleName)) return } catch { Write-Verbose -Message $_ } } Set-KvaConfigValue -name "installState" -value ([InstallState]::NotInstalled) Save-ConfigurationDirectory -moduleName $moduleName -WorkingDir $workingDir Save-Configuration -moduleName $moduleName Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_new_configuration_saved, $moduleName)) } function Confirm-Configuration { <# .DESCRIPTION Validate if the configuration can be used for the deployment .PARAMETER useStagingShare Requests a staging share to be used for downloading binaries and images (for private testing) .PARAMETER stagingShare The staging share endpoint to use when useStagingShare is requested #> param ( [Switch] $useStagingShare, [String] $stagingShare ) if ($useStagingShare.IsPresent -and [string]::IsNullOrWhiteSpace($stagingShare)) { throw $($KvaLocMessage.generic_staging_share_unspecified) } } function Set-KvaConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Object] $value ) Set-ConfigurationValue -name $name -value $value -module $moduleName } function Get-KvaConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value #> param ( [String] $name ) return Get-ConfigurationValue -name $name -module $moduleName } function Get-KvaConfigYaml { <# .DESCRIPTION Sets Configuration for the management appliance. .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER group The name of the group in which the vault resides #> param ( [String]$kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"], [String]$group = $global:config[$modulename]["group"] ) $installationPackageDir = $global:config[$moduleName]["installationPackageDir"] $yamlFile = $($installationPackageDir+"\yaml\appliance.yaml") if (-not (Test-Path $yamlFile)) { New-Item -ItemType File -Force -Path $yamlFile | Out-Null } # ACL the yaml so that it is only readable by administrator Set-SecurePermissionFile -Path $yamlFile $publicKey = Get-Content -Path (Get-SshPublicKey) $publicKey = $publicKey.Split(" ") $publicKey = $($publicKey[0]+" "+$publicKey[1]) $containerRegistryUser = $global:config[$moduleName]["containerRegistryUser"] $containerRegistryPass = $global:config[$moduleName]["containerRegistryPass"] if(-not [String]::IsNullOrEmpty($containerRegistryPass)) { $securePass = $containerRegistryPass | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $containerRegistryUser, $securePass $containerRegistryPass = $credential.GetNetworkCredential().Password } $deploymentManifestLocation = $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:cloudOperatorYaml)) $cloudFqdn = Get-CloudFqdn $clusterName = $($global:config[$modulename]["kvaName"]) $cloudPort = $global:config[$moduleName]["cloudAgentPort"].ToString() $cloudAuthPort = $global:config[$moduleName]["cloudAgentAuthorizerPort"].ToString() $kvaIdentity = $global:config[$moduleName]["identity"] $vnet = Get-VNetConfiguration -module $moduleName $operatorTokenValidity = $global:config[$moduleName]["operatorTokenValidity"] $addonTokenValidity = $global:config[$moduleName]["addonTokenValidity"] # Decide whether to use the macpool or not $useMacPool = (-not [string]::IsNullOrWhiteSpace( $global:config[$modulename]["macPoolStart"]) -and -not [string]::IsNullOrWhiteSpace($global:config[$modulename]["macPoolEnd"])) $macPoolName = "" if ($useMacPool) { $macPoolName = $vnet.MacPoolName } $dnsserver = "" if(-Not [string]::IsNullOrEmpty($vnet.IpAddressPrefix)) { foreach ($dns in $vnet.DnsServers) { if(-Not [string]::IsNullOrEmpty($dns)) { $dnsserver += "`n - " + $dns } } } $kvaRegistration = Get-KvaRegistration if (($kvaRegistration.azureResourceGroup -ne "") -and ($kvaRegistration.azureLocation -ne "")) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } Write-Status -moduleName $moduleName $($KvaLocMessage.kva_creating_configuration) $waitForBootstrap = -not (Get-ConfigurationValue -module $moduleName -Name "kvaSkipWaitForBootstrap" -type ([Type][System.Boolean])) $proxySettings = Get-ProxyConfiguration -moduleName $moduleName $configMaps = Get-KvaConfigMaps $kvaProductInfo = Get-KvaProductInfo $isHyperThreadingEnabled = Get-MocHyperThreadingEnabled $yaml = @" clustername: $clusterName kubernetesversion: $kubernetesVersion sshauthorizedkey: $publicKey lowprivilegekubeconfig: false encryptsecrets: true waitforbootstrapcompletion: $waitForBootstrap containerregistry: name: $($global:config[$moduleName]["containerRegistryServer"]) username: $containerRegistryUser password: $containerRegistryPass networking: controlplanecidr: $($global:mgmtControlPlaneCidr) clustercidr: $($global:mgmtClusterCidr) podcidr: $($global:config[$moduleName]["kvaPodCidr"]) proxy: http: "$($proxySettings.HTTP)" https: "$($proxySettings.HTTPS)" noproxy: "$($proxySettings.NoProxy)" certfilename: "$($proxySettings.CertName)" deploymentmanifest: cloudoperatormanifestpath: $deploymentManifestLocation cni: type: calico applianceagents: onboardingagent: subscriptionid: $subscriptionid tenantid: $tenantid resourcegroup: $resourcegroup location: $location infrastructure: azure_stack_hci billingagent: hyperthreading: $isHyperThreadingEnabled azurestackhciprovider: cloudagent: address: $cloudFqdn port: $cloudPort authenticationport: $cloudAuthPort loginconfig: $kvaIdentity insecure: $($global:config[$moduleName]["insecure"]) appliancevm: imagename: "" vmsize: $(Get-ConfigurationValue -module $moduleName -name "controlplaneVmSize" -type ([Type][VmSize])) loadbalancer: imagename: "" location: $($global:config[$modulename]["cloudLocation"]) group: $group storagecontainer: $($global:cloudStorageContainer) virtualnetwork: name: "$($vnet.Name)" vswitchname: "$($vnet.VswitchName)" type: "Transparent" macpoolname: $macPoolName vlanid: $($vnet.VlanID) ipaddressprefix: $($vnet.IpAddressPrefix) gateway: $($vnet.Gateway) dnsservers: $dnsserver vippoolstart: $($vnet.VipPoolStart) vippoolend: $($vnet.VipPoolEnd) k8snodeippoolstart: $($vnet.K8snodeIPPoolStart) k8snodeippoolend: $($vnet.K8snodeIPPoolEnd) tokenvalidities: addontokenvalidity: $addonTokenValidity operatortokenvalidity: $operatorTokenValidity "@ if ($proxySettings.CertName) { $yaml += @" `ncertificates: - filename: "$($proxySettings.CertName)" contentb64: "$($proxySettings.CertContent)" "@ } if ($configMaps) { $yaml += "`n$configMaps" } if ($kvaProductInfo) { $yaml += "`n$kvaProductInfo" } Set-Content -Path $yamlFile -Value $yaml if ($null -ne $err -and $err.count -gt 0) { throw $err } Set-KvaConfigValue -name "kvaconfig" -value $yamlFile return $yamlFile } function Get-KvaUpdateConfigYaml { <# .DESCRIPTION Returns a configuration file for appliance upgrade. This configuration is merged with the existing config stored within the appliance. It is mainly used to provide updated version information and config maps. #> $installationPackageDir = $global:config[$moduleName]["installationPackageDir"] $yamlFile = $($installationPackageDir+"\yaml\appliance-update.yaml") if (-not (Test-Path $yamlFile)) { New-Item -ItemType File -Force -Path $yamlFile | Out-Null } $kvaIdentity = $global:config[$moduleName]["identity"] $kubernetesVersion = $global:config[$modulename]["kvaK8sVersion"] Write-Status -moduleName $moduleName $($KvaLocMessage.kva_creating_update_configuration) $yaml = @" kubernetesversion: $kubernetesVersion azurestackhciprovider: cloudagent: loginconfig: $kvaIdentity "@ $configMaps = Get-KvaConfigMaps if ($configMaps) { $yaml += "`n$configMaps" } $kvaProductInfo = Get-KvaProductInfo if ($kvaProductInfo) { $yaml += "`n$kvaProductInfo" } Set-Content -Path $yamlFile -Value $yaml if ($null -ne $err -and $err.count -gt 0) { throw $err } return $yamlFile } function Repair-KvaCerts { <# .DESCRIPTION Attempts to repair failed TLS to cloudagent .PARAMETER Name Name of the cluster to fix certs on .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER force Force repair(without checks) .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(DefaultParameterSetName = 'Default')] param ( [Parameter(ParameterSetName='target',Mandatory=$true)] [String] $Name, [Parameter()] [String] $sshPrivateKeyFile, [Parameter()] [Switch] $force, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Write-Status -moduleName $moduleName $($KvaLocMessage.kva_repair) Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity $privateKey = Get-SshPrivateKey if ($PSCmdlet.ParameterSetName -ieq "target") { $group = $("$global:cloudGroupPrefix-$Name") If ($null -eq $Name) { throw $($KvaLocMessage.kva_clustername_missing) } Invoke-KvaCtl -arguments "cluster repair --clustername $Name --kubeconfig ""$kubeconfig"" --sshprivatekey ""$privateKey"" --tags ""Group=$group"" --force=$force" -showOutput -activity $activity return } $group = $global:config[$moduleName]["group"] $installationPackageDir = $global:config[$moduleName]["installationPackageDir"] $repairConfig = $($installationPackageDir+"\yaml\appliance-repair.yaml") if (-not (Test-Path $repairConfig)) { New-Item -ItemType File -Force -Path $repairConfig | Out-Null } if (-not $global:config[$modulename]["insecure"]) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_updating_identity) $clusterName = $($global:config[$modulename]["kvaName"]) $kvaIdentity = Invoke-MocIdentityRotate -name $clusterName -encode Set-KvaConfigValue -name "identity" -value $kvaIdentity } $kvaIdentity = $global:config[$moduleName]["identity"] Write-Status -moduleName $moduleName $($KvaLocMessage.kva_creating_repair_configuration) $yaml = @" azurestackhciprovider: cloudagent: loginconfig: $kvaIdentity "@ Set-Content -Path $repairConfig -Value $yaml if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KvaCtl -arguments "repair --configfile ""$repairConfig"" --kubeconfig ""$kubeconfig"" --sshprivatekey ""$privateKey"" --tags ""Group=$group"" --force=$force" -showOutput -activity $activity } function Get-KvaLogs { <# .DESCRIPTION Collects all the logs from the deployment .PARAMETER Path Path to store the logs .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String]$path, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $logDir = [io.Path]::Combine($path, $moduleName) New-Item -ItemType Directory -Force -Path $logDir | Out-Null Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.comm_collecting_configuration) $global:config[$moduleName] > $logDir"\KvaConfig.txt" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.comm_collecting_module_info) Get-Command -Module Kva | Sort-Object -Property Source > $($logDir+"\moduleinfo.txt") if (Test-Path -Path $script:logFilePath) { Copy-Item -Path $script:logFilePath -Destination $logDir } if (Test-Path -Path $global:config[$moduleName]["kubeconfig"]) { Write-SubStatus -moduleName $moduleName $($GenericLocMessage.comm_collecting_kubernetes_logs) try { $clusters = Invoke-Kubectl -arguments $("get akshciclusters -o json") | ConvertFrom-Json foreach ($cluster in $clusters.items) { $clusterName = $cluster.metadata.name Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_collecting_logs_for_cluster, $clusterName)) if ($clusterName -ine $global:config[$moduleName]["kvaName"]) { $rando = Get-Random $kubeconfigFileLocation = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $clusterName -outputLocation $kubeconfigFileLocation } else { $kubeconfigFileLocation = Get-KvaCredential } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_collecting_cluster_logs, $clusterName)) try{ Invoke-Kubectl -kubeconfig $kubeconfigFileLocation -arguments "get all,pvc,pv,sc -A -o yaml" > "${logDir}\worker_objects_$clusterName.json" $clusterLogDirectory = [io.Path]::Combine($logDir, "clusterlogs_$clusterName") Invoke-Kubectl -kubeconfig $kubeconfigFileLocation -arguments $("cluster-info dump -A --output-directory=`"$clusterLogDirectory`"") | Out-Null }catch{} if ($clusterName -ine $global:config[$moduleName]["kvaName"]) { Remove-Item -Path $kubeconfigFileLocation -Force -ErrorAction Ignore } } } catch [Exception] { Write-Status -moduleName $moduleName -msg $($GenericLocMessage.generic_exception) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() } } else { Write-SubStatus -moduleName $moduleName $($GenericLocMessage.comm_skipping_log_collection) } Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Get-KvaEventLog { <# .DESCRIPTION Gets all the event logs from Kva Module #> Get-WinEvent -ProviderName $moduleName -ErrorAction Ignore } function Repair-KvaCluster { <# .DESCRIPTION Attempts to repair failed TLS on a cluster by reprovisioning control plane certs .PARAMETER Name Name of the node to reprovision certs on .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER fixCertificates If specified, will attempt to reprovision control plane certs to fix TLS errors .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $sshPrivateKeyFile, [Parameter()] [Switch] $fixCertificates, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity if ($fixCertificates.IsPresent) { $kvaConfig = Get-KvaConfigYaml Invoke-KvaCtl -arguments "cluster addons certs repair --cluster-name $Name --sshprivatekey ""$sshPrivateKeyFile"" --configfile ""$kvaConfig"" " -showOutput -activity $activity } } function Set-ContainerRegistryConfiguration { <# .DESCRIPTION Sets the container registry configuration .PARAMETER containerRegistry Container registry settings #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '')] param ( [Parameter()] [ContainerRegistry] $containerRegistry ) $server = "" $user = "" $pass = "" if ($containerRegistry) { $server = $containerRegistry.Server if ($containerRegistry.Credential.Username) { $user = $containerRegistry.credential.UserName } if ($containerRegistry.Credential.Password) { $pass = $containerRegistry.credential.Password | ConvertFrom-SecureString -Key $global:credentialKey } } else { $server = $script:defaultContainerRegistryServer $user = $script:defaultContainerRegistryU $ss = ConvertTo-SecureString $script:defaultContainerRegistryP -AsPlainText -Force $pass = $ss | ConvertFrom-SecureString -Key $global:credentialKey } Set-ConfigurationValue -name "containerRegistryServer" -value $server -module $moduleName Set-ConfigurationValue -name "containerRegistryUser" -value $user -module $moduleName Set-ConfigurationValue -name "containerRegistryPass" -value $pass -module $moduleName } #endregion #region Billing function Sync-KvaBilling { <# .DESCRIPTION Helper function to sync billing. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity try { $syncResult = Invoke-KvaCtl -arguments "cluster addons billing sync --kubeconfig ""$kubeconfig""" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_sync_billing_failed, $_)) } return $syncResult } function Get-KvaBillingRecords { <# .DESCRIPTION Helper function to list billing records. .PARAMETER outputformat Output format of result .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $outputformat = "json", [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $kubeconfig = Get-KvaCredential -activity $activity try { $recordsResult = Invoke-KvaCtl -arguments "cluster addons billing get-records --kubeconfig ""$kubeconfig"" --outputformat=$outputformat" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_get_records_failed, $_)) } return $recordsResult } function Get-KvaBillingStatus { <# .DESCRIPTION Helper function to get billing status. .PARAMETER outputformat Output format of result .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String] $outputformat = "json", [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity $yamlFile = $global:config[$moduleName]["kubeconfig"] try { $statusResult = Invoke-KvaCtl -arguments "cluster addons billing get-status --kubeconfig ""$yamlFile"" --outputformat=$outputformat" -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_billing_failed, $_)) } return $statusResult } #endregion #region Installation and Provisioning functions function Install-KvaInternal { <# .DESCRIPTION The main deployment method for KVA. This function is responsible for provisioning files, deploying the agents. .PARAMETER activity Activity name to use when writing progress #> param ( [String] $activity = $MyInvocation.MyCommand.Name ) Set-KvaConfigValue -name "installState" -value ([InstallState]::Installing) try { Get-KvaRelease -version $(Get-KvaVersion) -activity $activity if (Test-MultiNodeDeployment) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Install-KvaBinaries -nodeName $_.Name } } else { Install-KvaBinaries -nodeName ($env:computername) } if (-not $global:config[$modulename]["insecure"]) { Write-Status -moduleName $moduleName $($KvaLocMessage.kva_creating_identity) $clusterName = $($global:config[$modulename]["kvaName"]) $kvaIdentity = New-MocIdentity -name $clusterName -validityDays $global:config[$moduleName]["tokenExpiryDays"] -fqdn $cloudFqdn -location $global:config[$modulename]["cloudLocation"] -port $cloudPort -authport $cloudAuthPort -encode Set-KvaConfigValue -name "identity" -value $kvaIdentity New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "IdentityContributor" | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "RoleContributor" | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "CertificateReader" | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "VipPoolReader" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "GroupContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "KeyVaultContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "VirtualNetworkContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "LBContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "NetworkInterfaceContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "VMContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null New-MocRoleAssignmentWhenAvailable -identityName $clusterName -roleName "SecretContributor" -location $global:config[$modulename]["cloudLocation"] | Out-Null } $yamlFile = Get-KvaConfigYaml -kubernetesVersion $global:config[$modulename]["kvaK8sVersion"] -group $global:config[$modulename]["group"] $kubeconfig = $global:config[$modulename]["kubeconfig"] Invoke-KvaCtlWithAzureContext -arguments "create --configfile ""$yamlFile"" --outfile ""$kubeconfig"" " -showOutputAsProgress -activity $activity } catch { Set-KvaConfigValue -name "installState" -value ([InstallState]::InstallFailed) throw $_ } Set-SecurePermissionFile -Path $kubeconfig Set-KvaConfigValue -name "installState" -value ([InstallState]::Installed) Write-Status -moduleName $moduleName $($KvaLocMessage.kva_installation_complete) } function Get-KvaRelease { <# .DESCRIPTION Download a KVA release (e.g. package, images, etc) .PARAMETER version Release version .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $($KvaLocMessage.kva_discovering_release) $productRelease = Get-ProductRelease -version $version -moduleName $moduleName if (-not $productRelease.CustomData.ManagementNodeImageK8sVersion) { throw $($KvaLocMessage.kva_k8s_version_unknown) } $k8sVersion = $productRelease.CustomData.ManagementNodeImageK8sVersion Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_required_kubernetes_version, $k8sVersion)) Add-GalleryImage -imageType "Linux" -k8sVersion $k8sVersion -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_management_cluster_provisioned) Get-ReleaseContent -name $script:productName -version $version -activity $activity if (-not ($global:config[$modulename]["useStagingShare"])) { Test-AuthenticodeBinaries -workingDir $global:config[$moduleName]["installationPackageDir"] -binaries $script:kvaBinaries } } function Get-KvaConfigMaps { <# .DESCRIPTION Retrieve KVA config maps #> Write-Status -moduleName $moduleName $($KvaLocMessage.kva_get_configmaps) if ([string]::IsNullOrWhiteSpace($global:config[$modulename]["version"])) { throw $($KvaLocMessage.kva_unable_to_fetch_version) } $productRelease = Get-ProductRelease -version $global:config[$modulename]["version"] -moduleName $moduleName $yaml = @" configmaps: - name: $script:productInfoMapName namespace: $script:productInfoMapNamespace data: offer: "$($productRelease.ProductName)" version: "$($global:config[$modulename]["version"])" catalog: "$($global:config[$modulename]["catalog"])" audience: "$($global:config[$modulename]["ring"])" deploymentid: "$($global:config[$modulename]["deploymentId"])" "@ return $yaml } function Get-KvaProductInfo { <# .DESCRIPTION Retrieve KVA product details #> Write-Status -moduleName $moduleName $($KvaLocMessage.kva_get_product_details) if ([string]::IsNullOrWhiteSpace($global:config[$modulename]["version"])) { throw $($KvaLocMessage.kva_unable_to_fetch_version) } $productRelease = Get-ProductRelease -version $global:config[$modulename]["version"] -moduleName $moduleName $yaml = @" product: name: "$($productRelease.ProductName)" version: "$($global:config[$modulename]["version"])" catalog: "$($global:config[$modulename]["catalog"])" audience: "$($global:config[$modulename]["ring"])" deploymentid: "$($global:config[$modulename]["deploymentId"])" "@ return $yaml } function Get-ReleaseContent { <# .DESCRIPTION Download all required files and packages for the specified release .PARAMETER name Release name .PARAMETER version Release version .PARAMETER destination Destination directory for the content .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [string] $destination = $global:config[$moduleName]["installationPackageDir"], [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $destination = $destination -replace "\/", "\" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_discovering_release_content, $name)) $productRelease = Get-ProductRelease -version $version -moduleName $moduleName # find requested release foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { if ($subProductRelease.ProductName -ieq $name) { $versionManifestPath = [io.Path]::Combine($destination, $("$name-release.json")) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_downloading_release_content_to, $name, $destination)) $downloadParams = Get-ReleaseDownloadParameters -name $subProductRelease.ProductStream -version $subProductRelease.Version -destination $destination -parts 3 -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if (-not ($global:config[$moduleName]["useStagingShare"])) { if ($releaseInfo.Files.Count -ne 1) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_wrong_release_files_count, $name, $releaseInfo.Files.Count)) } $packagename = $releaseInfo.Files[0] -replace "\/", "\" # Temporary until cross-platform signing is available $auth = Get-AuthenticodeSignature -filepath $packagename if (($global:expectedAuthResponse.status -ne $auth.status) -or ($auth.SignatureType -ne $global:expectedAuthResponse.SignatureType)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_authenticode_failed, $global:expectedAuthResponse.status, $global:expectedAuthResponse.SignatureType, $auth.status, $auth.SignatureType)) } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_expanding_package, $name, $packagename, $destination)) $expandoutput = expand.exe -r $packagename $destination -f:* Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_expand_result, $expandoutput)) } $versionJson = $subProductRelease | ConvertTo-Json -depth 100 set-content -path $versionManifestPath -value $versionJson -encoding UTF8 return } } } throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_no_release_content, $name, $version)) } function Initialize-KvaEnvironment { <# .DESCRIPTION Executes steps to prepare the environment for day 0 operations. .PARAMETER createConfigIfNotPresent Whether the call should create a new deployment configuration if one is not already present. .PARAMETER activity Activity name to use when updating progress #> param ( [Switch]$createConfigIfNotPresent, [String]$activity = $MyInvocation.MyCommand.Name ) Get-MocConfig -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -module $moduleName -status $($GenericLocMessage.comm_discovering_configuration) Import-KvaConfig -createIfNotPresent:($createConfigIfNotPresent.IsPresent) -activity $activity Write-StatusWithProgress -activity $activity -module $moduleName -status $($GenericLocMessage.comm_applying_configuration) Initialize-Environment -checkForUpdates:$false -moduleName $script:moduleName } function Install-KvaBinaries { <# .DESCRIPTION Copies KVA binaries to a node .PARAMETER nodeName The node to execute on. #> param ( [Parameter(Mandatory=$true)] [String]$nodeName ) Install-Binaries -module $moduleName -nodeName $nodeName -binariesMap $kvaBinariesMap } function Uninstall-KvaBinaries { <# .DESCRIPTION Copies KVA binaries to a node .PARAMETER nodeName The node to execute on. #> param ( [Parameter(Mandatory=$true)] [String]$nodeName ) Uninstall-Binaries -module $moduleName -nodeName $nodeName -binariesMap $kvaBinariesMap # Cleanup log files associated with kva binary Invoke-Command -ComputerName $nodeName -ScriptBlock { $tmpFile = $args[0] if (Test-Path -Path $tmpFile) { Remove-Item -Path $tmpFile -Force -ErrorAction Ignore } } -ArgumentList $script:logFilePath } #endregion #region Verification Functions function Test-KvaConfiguration { <# .DESCRIPTION A basic sanity test to make sure that a node is ready to deploy kubernetes. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) Test-KvaInstallation -nodeName $nodeName } function Test-KvaInstallation { <# .DESCRIPTION Sanity checks an installation to make sure that all expected binaries are present. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) Write-SubStatus -moduleName $moduleName $($GenericLocMessage.generic_testing_expected_binaries) Test-Binary -nodeName $nodeName -binaryName $global:kvaCtlFullPath Test-Binary -nodeName $nodeName -binaryName $global:kubeCtlFullPath } function Get-KvaLatestVersion { <# .DESCRIPTION Get the current KVA release version .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $($GenericLocMessage.comm_discovering_latest_version) $catalog = Get-LatestCatalog -moduleName $moduleName $productRelease = $catalog.ProductStreamRefs[0].ProductReleases[0] # find latest kva release foreach($subProductStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $subProductStream.ProductReleases) { if ($subProductRelease.ProductName -ieq $script:productName) { return $subProductRelease.Version } } } throw $($KvaLocMessage.kva_unknown_kva_version) } function Reset-GalleryImage { <# .DESCRIPTION Resets all gallery image #> Write-Status -moduleName $moduleName $($GenericLocMessage.comm_resetting_galleryimage) # 1. Invalidate all gallery images # Gallery images are downloaded and managed by KVA. So it is okay to reset it completely for now. # If this assumption changes, change the logic here to just remove what KVA brings in Reset-MocGalleryImage -location $global:config[$modulename]["cloudLocation"] # 2. Cleanup all vhd files $imagepath = $global:config[$moduleName]["imageDir"] Get-ChildItem -Path $imagepath -Filter *.vhdx | ForEach-Object { $tmpFile = $_.FullName Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_removing_image, $tmpFile)) Remove-Item -Path $tmpFile -Force -ErrorAction Ignore } # 3. Cleanup all json manifest cache files Get-ChildItem -Path $imagepath -Filter *.json | ForEach-Object { $tmpFile = $_.FullName Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_removing_image_cache, $tmpFile)) Remove-Item -Path $tmpFile -Force -ErrorAction Ignore } } function Add-GalleryImage { <# .DESCRIPTION Checks the gallery to see if the requested image is present. If it is missing, this function downloads the missing image and adds it to the gallery. .PARAMETER imageName Name of the gallery image to provision .PARAMETER activity Activity name to use when updating progress #> param ( # [Parameter(Mandatory=$true)] # [String] $imageName, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String] $imageType, [Parameter(Mandatory=$true)] [String] $k8sVersion, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.comm_provisioning_galleryimage) $imageName = Get-KubernetesGalleryImageName -imagetype $imageType -k8sVersion $k8sVersion Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_requested_image, $imageName)) Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_checking_existing_gallery_image) $galleryImage = $null $mocLocation = $global:config[$modulename]["cloudLocation"] try { $galleryImage = Get-MocGalleryImage -name $imageName -location $mocLocation } catch {} if ($null -ine $galleryImage) { Write-SubStatus -moduleName $moduleName $($GenericLocMessage.comm_image_already_present_in_gallery) return } Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_obtain_download_information_for_image, $imageName)) $imageRelease = Get-ImageReleaseManifest -imageVersion $(Get-KvaVersion) -operatingSystem $imageType -k8sVersion $k8sVersion Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_downloading_image, $imageName)) $result = Get-ImageRelease -imageRelease $imageRelease -imageDir $global:config[$moduleName]["imageDir"] Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_adding_image_to_cloud_gallery, $imageName)) New-MocGalleryImage -name $imageName -location $mocLocation -imagePath $result -container $global:cloudStorageContainer | Out-Null } function Invoke-KvaCtl { <# .DESCRIPTION Executes a KVACTL command. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER activity Activity name to use when updating progress. #> param ( [string] $arguments, [switch] $ignoreError, [switch] $showOutput, [string] $activity = $MyInvocation.MyCommand.Name ) return Invoke-CommandLine -command $kvaCtlFullPath -arguments $("$arguments") -ignoreError:$ignoreError.IsPresent -showOutputAsProgress:$showOutput.IsPresent -progressActivity $activity -moduleName $moduleName } function Invoke-KvaCtlWithAzureContext { <# .DESCRIPTION Executes a command and optionally ignores errors. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER showOutputAsProgress Optionally, show output from the executing command as progress bar updates. .PARAMETER credential credential is a PSCredential holding a user's Service Principal. .PARAMETER activity The activity name to display when showOutputAsProgress was requested. #> [CmdletBinding()] param ( [Parameter()] [String]$command, [Parameter()] [String]$arguments, [Parameter()] [Switch]$ignoreError, [Parameter()] [Switch]$showOutput, [Parameter()] [Switch]$showOutputAsProgress, [Parameter()] [PSCredential] $credential, [Parameter()] [string] $activity = $MyInvocation.MyCommand.Name ) $obj = New-Object -TypeName psobject if ($null -ne $credential) { $obj | Add-Member -MemberType NoteProperty -Name clientId -Value $credential.GetNetworkCredential().UserName $obj | Add-Member -MemberType NoteProperty -Name clientSecret -Value $credential.GetNetworkCredential().Password } else { $armAccessToken = Get-AzAccessToken $graphAccessToken = Get-GraphAccessToken $obj | Add-Member -MemberType NoteProperty -Name armAccessToken -Value $armAccessToken.Token $obj | Add-Member -MemberType NoteProperty -Name graphAccessToken -Value $graphAccessToken.Token $azContext = Get-AzContext if ($azContext.Account.Type -eq "ServicePrincipal") { $obj | Add-Member -MemberType NoteProperty -Name clientId -Value $azContext.Account.Id $obj | Add-Member -MemberType NoteProperty -Name clientSecret -Value $azContext.Account.ExtendedProperties.ServicePrincipalSecret } } $arguments += " --azure-creds-stdin" try { if ($showOutputAsProgress.IsPresent) { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") | ForEach-Object { $status = $_ -replace "`t"," - "; Write-StatusWithProgress -activity $activity -moduleName $moduleName -Status $status }) 2>&1 } elseif ($showOutput.IsPresent) { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") | Out-Default) 2>&1 } else { $result = ($obj | ConvertTo-Json | & $kvaCtlFullPath $arguments.Split(" ") 2>&1) } } catch { if ($ignoreError.IsPresent) { return } throw } $out = $result | Where-Object {$_.gettype().Name -ine "ErrorRecord"} # On a non-zero exit code, this may contain the error #$outString = ($out | Out-String).ToLowerInvariant() if ($LASTEXITCODE) { if ($ignoreError.IsPresent) { return } $err = $result | Where-Object {$_.gettype().Name -eq "ErrorRecord"} throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_non_zero_params , $command, $arguments, $LASTEXITCODE, $err)) } return $out } function Get-KvaClusterCredential { <# .DESCRIPTION Obtains the credentials for a cluster as a kubeconfig and outputs it to the requested location (by default the location is the current users .kube directory). .PARAMETER Name Name of the cluster to obtain the credential/kubeconfig for. .PARAMETER outputLocation Location to output the credential/kubeconfig file to. .PARAMETER adAuth To get the Active Directory SSO version of the kubeconfig. .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $outputLocation = $($env:USERPROFILE+"\.kube\config"), [Parameter(Mandatory=$false)] [Switch] $adAuth, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity #TODO Fix this from the kvasdk layer if ($Name -eq $global:config[$moduleName]["kvaName"]) { throw $($KvaLocMessage.kva_invalid_cluster) } $kubeconfig = Get-KvaCredential -activity $activity if (-not (Test-Path $kubeconfig)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_missing_kubeconfig, $kubeconfig)) } # TODO move directory check to kva # # The following line will only return true if the passed in location # is a valid directory. If the path doesn't exist or if it points a # file it will return false. # # In the case where the user passes in an invalid path, we will fail at a later point as # its hard to tell whether they are passing valid location to a file to be written or an # path that doesn't exist. if (Test-Path -Path $outputLocation -PathType Container) { $outputLocation = [io.Path]::Combine($outputLocation, "kubeconfig-$Name") } # Ensure that the cluster exists $cluster = Get-KvaCluster -Name $Name -activity $activity if ($adAuth.IsPresent) { $exePath = [io.Path]::Combine($global:installDirectory, "kubectl-adsso.exe") if (-not [System.IO.File]::Exists($exePath)) { Get-ClientCredPluginRelease -version $(Get-KvaVersion) -destinationDir $global:installDirectory -activity $activity } Invoke-KvaCtl -arguments "cluster retrieve --clustername $Name --kubeconfig ""$kubeconfig"" --outfile ""$outputLocation"" --adauth" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_ad_kubeconfig_written_to, $outputLocation)) } else { Invoke-KvaCtl -arguments "cluster retrieve --clustername $Name --kubeconfig ""$kubeconfig"" --outfile ""$outputLocation""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_kubeconfig_will_be_written_to, $outputLocation)) } Set-SecurePermissionFile -Path $outputLocation } function Get-KvaCredential { <# .DESCRIPTION Obtains the credentials for kva as a kubeconfig and outputs it to the default location). .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name)" } $kubeconfigPath = $global:config[$moduleName]["kubeconfig"] if (Test-Path $kubeconfigPath) { return $kubeconfigPath } $yamlFile = Get-KvaConfigYaml Invoke-KvaCtl -arguments "retrieve --configfile ""$yamlFile"" --outfile ""$kubeconfigPath"" " -showOutput -activity $activity Set-SecurePermissionFile -Path $kubeconfigPath return $kubeconfigPath } function Get-ClientCredPluginRelease { <# .DESCRIPTION Download the client cred plugin release content .PARAMETER Version Version .PARAMETER destinationDir Destination directory .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter(Mandatory=$true)] [string] $destinationDir, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Get-ReleaseContent -name "client-cred-plugin" -version $version -activity $activity $srcPath = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "kubectl-adsso.exe") Copy-Item $srcPath -Destination $destinationDir } function Wait-ForKubernetesSecret { <# .DESCRIPTION Polls the vault for a secret to become available and returns the secret value. .PARAMETER name Name of the secret .PARAMETER sleepDuration Duration to sleep for between attempts to retrieve the secret #> param ( [String]$name, [int]$sleepDuration=20 ) Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_polling_cluster, $name)) -moduleName $moduleName while($true) { try { $clusterSecret = Invoke-Kubectl -arguments $("get secrets/$name") } catch {} if ($null -ne $clusterSecret) { break } Sleep $sleepDuration } return $clusterSecret } function Set-KvaGMSAWebhook { <# .DESCRIPTION Internal helper function that installs gMSA webhook for a cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity # ensures cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null try { # Schedules a gmsa webhook deployment onto the cluster master node $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: gmsa-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: gmsa-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-gmsa-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("apply -f ""$yamlFile"" ") $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # wait for gmsa-webhook to be ready $sleepDuration = 5 $namespace = 'kube-system' $selector = 'app=gmsa-webhook' Write-SubStatus $($KvaLocMessage.kva_waiting_for_gmsa_webhook) -moduleName $moduleName while($true) { $result = (Invoke-Kubectl -ignoreError -kubeconfig $targetClusterKubeconfig -arguments $("wait --for=condition=Ready --timeout=5m -n $namespace pod -l $selector")) if ($null -ne $result) { break } Start-Sleep $sleepDuration } #Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_pod_is_ready, $podFriendlyName)) -moduleName $moduleName } finally { Remove-Item -Path $targetClusterKubeconfig } } function Reset-KvaGMSAWebhook { <# .DESCRIPTION Internal helper function that uninstalls gMSA webhook for a cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when writing progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity # ensures cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null if (-not (Invoke-Kubectl -arguments $("get addon gmsa-webhook-$Name") -ignoreError)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_gmsa_addon_not_installed, $Name)) } try { $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: gmsa-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: gmsa-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-gmsa-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("delete -f ""$yamlFile"" ") } finally { Remove-Item $yamlFile } } function Set-KvaGMSACredentialSpec { <# .DESCRIPTION Helper function to install setup for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecFilePath File Path of the JSON cred spec file .PARAMETER credSpecName Name of the k8s credential spec object the user would like to designate .PARAMETER secretName Name of the Kubernetes secret object storing the Active Directory user credentials and gMSA domain .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER serviceAccount Name of the K8s service account assigned to read the k8s gMSA credspec object .PARAMETER clusterRoleName Name of the K8s clusterrole assigned to use the k8s gMSA credspec object .PARAMETER overwrite Overwrites existing gMSA setup .PARAMETER activity Activity name to use when writing progress #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$credSpecFilePath, [Parameter(Mandatory=$true)] [String]$credSpecName, [Parameter(Mandatory=$true)] [String]$secretName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [switch]$overwrite, [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity try { $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # Converting credSpecName, secretName, serviceAccount, and clusterRoleName to lowercase to conform with RFC1123 $credSpecName = $credSpecName.ToLower() $secretName = $secretName.ToLower() $serviceAccount = $serviceAccount.ToLower() $clusterRoleName = $clusterRoleName.ToLower() $roleBindingName = 'allow-' + $serviceAccount + '-svc-account-read-on-' + $credSpecName # check if namespace is created if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get namespace $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get namespace $secretNamespace", $secretNamespace, "namespace check")) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_namespace_not_created, $secretNamespace, $secretNamespace)) } } # check if a gmsa webhook pod is running $webhookPodRunning = (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get pods -n kube-system") -ignoreError) | Select-String -Pattern 'gmsa-webhook.*1/1.*Running' if (-not $webhookPodRunning) { throw $($KvaLocMessage.kva_gmsa_not_installed) } # check if serviceaccount exists if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get serviceaccount $serviceAccount -n $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get serviceaccount $serviceAccount -n $secretNamespace", "$serviceAccount in namespace $secretNamespace", "service account check")) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_namespace_not_found, $serviceAccount, $secretNamespace, $serviceAccount, $secretNamespace)) } } # check if any of the resources already exist # if overwrite, delete them # if not overwrite, throw error if ($overwrite.IsPresent) { if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete GMSACredentialSpec $credSpecName") } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete clusterRole $clusterRoleName") } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete roleBinding $roleBindingName -n $secretNamespace") } } else { if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_namespace_exists, $kva_namespace_not_found)) } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_cluster_role_exists, $clusterRoleName)) } if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_name_exists_in_namespace, $roleBindingName, $secretNamespace)) } } # check if credspec file exists If (-not (Test-Path -Path $credSpecFilePath)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_file_path_exists, $credSpecFilePath)) } # catch errors when get-content or convertfrom-json $credSpecJsonRaw = Get-Content $credSpecFilePath -raw | ConvertFrom-Json $adServiceAccount = $credSpecJsonRaw.DomainJoinConfig.MachineAccountName $domainName = $credSpecJsonRaw.DomainJoinConfig.DNSName $netbios = $credSpecJsonRaw.DomainJoinConfig.NetBiosName $gmsaSid = $credSpecJsonRaw.DomainJoinConfig.Sid $gmsaGuid = $credSpecJsonRaw.DomainJoinConfig.Guid if (-not ($adServiceAccount -and $domainName -and $netbios -and $gmsaSid -and $gmsaGuid)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_file_path_exists, $credSpecFilePath)) } # create credspec yaml $credSpecYaml = @" apiVersion: windows.k8s.io/v1alpha1 kind: GMSACredentialSpec metadata: name: $credSpecName credspec: ActiveDirectoryConfig: GroupManagedServiceAccounts: - Name: $adServiceAccount Scope: $netbios - Name: $adServiceAccount Scope: $domainName HostAccountConfig: PortableCcgVersion: "1" PluginGUID: "{FF4E3124-6831-44A0-8C34-06EC66E148C6}" PluginInput: SecretName=$($secretName);SecretNamespace=$($secretNamespace) CmsPlugins: - ActiveDirectory DomainJoinConfig: DnsName: $domainName DnsTreeName: $domainName Guid: $gmsaGuid MachineAccountName: $adServiceAccount NetBiosName: $netbios Sid: $gmsaSid "@ # apply and remove credspec yaml if ($PSCmdlet.ShouldProcess("$credspecYaml", $credSpecName, "credspec creation")) { $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$credSpecName-$rando.yaml") Set-Content -Path $yamlFile -Value $credSpecYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f ""$yamlFile"" ") Remove-Item $yamlFile } # create cluster role yaml $clusterRoleYaml = @" apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: $clusterRoleName rules: - apiGroups: ["windows.k8s.io"] resources: ["gmsacredentialspecs"] verbs: ["use"] resourceNames: [$($credSpecName)] "@ # apply and remove cluster role yaml if ($PSCmdlet.ShouldProcess("$clusterRoleYaml", $clusterRoleName, "clusterrole creation")) { $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$clusterRoleName-clusterrole-$rando.yaml") Set-Content -Path $yamlFile -Value $clusterRoleYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f ""$yamlFile"" ") Remove-Item $yamlFile } # create rolebinding yaml $roleBindingYaml = @" apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: allow-$($serviceAccount)-svc-account-read-on-$($credSpecName) namespace: $($secretNamespace) subjects: - kind: ServiceAccount name: $($serviceAccount) namespace: $($secretNamespace) roleRef: kind: ClusterRole name: $clusterRoleName apiGroup: rbac.authorization.k8s.io "@ # apply and remove rolebinding yaml if ($PSCmdlet.ShouldProcess("$roleBindingYaml", "allow-$($serviceAccount)-svc-account-read-on-$($credSpecName)", "clusterrolebinding creation")) { # apply yaml file and delete $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\$clusterRoleName-rolebinding-$rando.yaml") Set-Content -Path $yamlFile -Value $roleBindingYaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { Remove-Item $yamlFile throw $err } Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("apply -f ""$yamlFile"" ") Remove-Item $yamlFile } } finally { Remove-Item -Path $targetClusterKubeconfig -ErrorAction Ignore } } function Reset-KvaGMSACredentialSpec { <# .DESCRIPTION Helper function to uninstall setup for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecName Name of the k8s credential spec object the user would like to designate .PARAMETER serviceAccount K8s service account assigned to read the k8s gMSA credspec object .PARAMETER clusterRoleName Name of the K8s clusterrole assigned to use the k8s gMSA credspec object .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER activity Activity name to use when writing progress #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$credSpecName, [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity try { $noDeleteCount = 0 $rando = Get-Random $targetClusterKubeconfig = $($env:USERPROFILE+"\.kube\$Name-kubeconfig-$rando") Get-KvaClusterCredential -Name $Name -outputLocation $targetClusterKubeconfig # check if namespace is created if (-not (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get namespace $secretNamespace") -ignoreError)) { if ($PSCmdlet.ShouldProcess("kubectl get namespace $secretNamespace", $secretNamespace, "namespace check")) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_namespace_not_created, $secretNamespace, $secretNamespace)) } } # delete GMSACredentialSpec object if exists if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get GMSACredentialSpec $credSpecName") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete GMSACredentialSpec $credSpecName", $credSpecName, "credspec deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete GMSACredentialSpec $credSpecName") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_gmsacredentialspec_does_not_exist, $credSpecName)) } # delete clusterrole object if exists if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get clusterRole $clusterRoleName") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete $clusterRoleName", $clusterRoleName, "clusterrole deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete clusterRole $clusterRoleName") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_clusterrole_does_not_exist, $clusterRoleName)) } # delete rolebinding object if exists $roleBindingName = 'allow-' + $serviceAccount + '-svc-account-read-on-' + $credSpecName if (Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("get roleBinding $roleBindingName -n $secretNamespace") -ignoreError) { if ($PSCmdlet.ShouldProcess("kubectl delete $roleBindingName -n $secretNamespace", "allow-$($serviceAccount)-svc-account-read-on-$($credSpecName)", "clusterrolebinding deletion")) { Invoke-Kubectl -kubeconfig $targetClusterKubeconfig -arguments $("delete roleBinding $roleBindingName -n $secretNamespace") } } else { $noDeleteCount += 1 Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_rolebinding_does_not_exist, $roleBindingName, $secretNamespace)) } # give feedback on no objects were deleted if ($noDeleteCount -eq 3) { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_gmsacredentialspec_clusterrole_rolebinding_does_not_exist, $credSpecName, $clusterRole, $rolebindingName)) } } finally { Remove-Item -Path $targetClusterKubeconfig -ErrorAction Ignore } } #endregion #region Image functions function Get-ImageReleaseManifest { <# .DESCRIPTION Discovers the requested image and returns the release manifest for it .PARAMETER imageVersion Image release version to target .PARAMETER operatingSystem Image operating system to target .PARAMETER k8sVersion Kubernetes version to target #> param( [parameter(Mandatory=$true)] [String] $imageVersion, [parameter(Mandatory=$true)] [String] $operatingSystem, [Parameter(Mandatory=$true)] [String] $k8sVersion ) $k8sVersion = $k8sVersion.TrimStart("v") $productRelease = Get-ProductRelease -version $imageVersion -moduleName $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { $vhdInfo = Get-ImageReleaseVhdInfo -release $subProductRelease if (-not $vhdInfo) { continue } if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ine $operatingSystem) { continue } foreach($pkg in $vhdInfo.CustomData.K8SPackages) { if ($pkg.Version -ieq $k8sVersion) { return $subProductRelease } } } } throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_image_not_found, $imageVersion, $operatingSystem, $k8sVersion)) } function Get-ImageReleaseVhdInfo { <# .DESCRIPTION Discovers and returns vhd information for the specified image release .PARAMETER imageRelease Image release manifest #> param( [parameter(Mandatory=$True)] [PSCustomObject] $release ) foreach ($fileRelease in $release.ProductFiles) { if ($fileRelease.CustomData.Type -ieq "vhd") { return $fileRelease } } return $null } function Get-ImageRelease { <# .DESCRIPTION Download the specified image release. .PARAMETER imageRelease Image release manifest .PARAMETER imageDir Directory for local image store. #> param( [parameter(Mandatory=$True)] [PSCustomObject] $imageRelease, [parameter(Mandatory=$True)] [String] $imageDir ) $downloadpath = "$imageDir\$([System.IO.Path]::GetRandomFileName().Split('.')[0])" New-Item -ItemType Directory -Force -Confirm:$false -Path $downloadpath | Out-Null try { $vhdInfo = Get-ImageReleaseVhdInfo -release $imageRelease $k8sVersion = $vhdInfo.CustomData.K8SPackages[0].Version $imageGalleryName = Get-KubernetesGalleryImageName -imagetype $vhdInfo.CustomData.BaseOSImage.OperatingSystem -k8sVersion $k8sVersion $imageVersionCurrent = $imageRelease.Version $imageVersionManifestPath = "$ImageDir\$imageGalleryName.json" $destinationpath = $("$ImageDir\$imageGalleryName.vhdx" -replace "\/", "\") if (Test-Path $imageVersionManifestPath) { $OldImageData = get-content $imageVersionManifestPath | ConvertFrom-Json $OldImageVersion = $OldImageData.Version Write-SubStatus -moduleName $moduleName "Existing image $imageVersionManifestPath has version $OldImageVersion . Requested version is $imageVersionCurrent" if ($imageVersionCurrent -ieq $OldImageVersion) { if (Test-Path -Path $destinationpath) { Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_existing_image_upto_date) return $destinationpath } Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_existing_image_not_present) } } Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_download_image_with_version, $($imageRelease.ProductStream), $imageVersionCurrent, $downloadPath)) $downloadParams = Get-ReleaseDownloadParameters -name $imageRelease.ProductStream -version $imageVersionCurrent -destination $downloadPath -parts 10 -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if ($global:config[$moduleName]["useStagingShare"]) { $imageFile = $releaseInfo.Files[0] -replace "\/", "\" } else { $imageFile = Expand-SfsImage -files $releaseInfo.Files -destination $downloadPath -workingDirectory $downloadPath } $imageJson = $imageRelease | ConvertTo-Json -depth 100 Set-Content -path $imageversionManifestPath -value $imageJson -encoding UTF8 -Confirm:$false if (test-path $destinationpath) { Remove-Item $destinationpath -Force -Confirm:$false } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_moving_image, $imageFile, $destinationpath)) Move-Item -Path $imageFile -Destination $destinationpath -Confirm:$false } finally { Remove-Item -Force $downloadpath -Recurse -Confirm:$false } return $destinationpath } function Expand-SfsImage { <# .DESCRIPTION Expand and verify a SFS image download using authenticode. NOTE: This is temporary until cross-platform signing is available in Download SDK. .PARAMETER files The downloaded image files (expected to be a image file zip and a companion cab). .PARAMETER destination Destination for the expanded image file. .PARAMETER workingDirectory Working directory to use for zip and cab file expansion. #> param( [Parameter(Mandatory=$True)] [String[]] $files, [Parameter(Mandatory=$True)] [String] $destination, [Parameter(Mandatory=$True)] [String] $workingDirectory ) Write-Status -moduleName $moduleName $($GenericLocMessage.comm_verifying_image_companion_file_download) [String[]]$cabfile = $files | Where-Object { $_ -match "\.cab$"} if (($null -eq $cabfile) -or ($cabfile.count -ne 1)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_wrong_cab_file_count, $cabfile.count)) } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_verify_authenticode_signature, $cabFile)) $auth = Get-AuthenticodeSignature -filepath $cabfile if (($global:expectedAuthResponse.status -ne $auth.status) -or ($global:expectedAuthResponse.SignatureType -ne $auth.SignatureType)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_authenticode_failed, "KVA image companion file", $global:expectedAuthResponse.status, $global:expectedAuthResponse.SignatureType, $auth.status, $auth.SignatureType)) } Write-Status -moduleName $moduleName $($GenericLocMessage.comm_expanding_image_companion_file) $expandDir = "$WorkingDirectory\expand_" + [System.IO.Path]::GetRandomFileName().Split('.')[0] New-Item -ItemType Directory -Force -Confirm:$false -Path $expandDir | Out-Null $expandoutput = expand.exe $cabfile $expandDir Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_expand_output, $expandoutput)) $manifest = Get-ChildItem $expandDir | select-object -first 1 $manifestcontents = get-content $manifest.fullname | convertfrom-json $packageAlgo = $manifestcontents.PackageVerification.VerificationDescriptor.Algorithm $packageHash = $manifestcontents.PackageVerification.VerificationDescriptor.FileHash $packageName = $manifestcontents.PackageVerification.VerificationDescriptor.Filename Write-Status -moduleName $moduleName $($GenericLocMessage.comm_verifying_image_file_download) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_verification_request, $packageName, $packageAlgo, $packageHash)) [string[]]$imagezip = $files | Where-Object { $_ -match "$packageName$"} if ($null -eq $imagezip) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_unable_to_locate_image_file, $packageName)) } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_calculating_hash_for_archive, $packageAlgo, $imagezip[0])) $hash = Get-Base64Hash -file $imagezip[0] -algorithm $packageAlgo if ($packageHash -ne $hash) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_unexpected_hash, $packageHash, $($imagezip[0]), $hash)) } Write-Status -moduleName $moduleName $($GenericLocMessage.comm_expanding_image_file_archive) $contentsDir = "$expandDir\contents" New-Item -ItemType Directory -Force -Confirm:$false -Path $contentsdir | Out-Null Expand-Archive -path $imagezip[0] -destinationpath $contentsDir -Confirm:$false | Out-Null Remove-Item $imagezip[0] -Confirm:$false [System.IO.FileInfo[]]$workimage = Get-ChildItem -r $contentsDir | Where-Object { (-not $_.psiscontainer) -and ($_.name -match "\.vhdx$")} if ($workimage.count -ne 1) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_invalid_file_count_expansion, $($workimage.count))) } $content0 = $manifestcontents.ContentVerification[0].VerificationDescriptor Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_calculating_hash_for_file, $packageAlgo, $workimage[0].fullname)) $hash = Get-Base64Hash -file $workimage[0].fullname -algorithm $content0.algorithm if ($content0.FileHash -ne $hash) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_unexpected_hash, $($content0.FileHash), $($workimage[0].fullname), $hash)) } $image = $("$destination\" + $workimage[0].name) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_moving_image_file, $image)) Move-item -Path $workimage[0].fullname -Destination $image -Confirm:$false return $image } function Get-Base64Hash { <# .DESCRIPTION Obtain the base64 byte hash of the specified file. Used to verify SFS binaries .PARAMETER file File to generate the base64 hash for .PARAMETER algorithm Hashing algorithm to use #> param ( [Parameter(Mandatory=$True)] [string] $file, [Parameter(Mandatory=$True)] [string] $algorithm ) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_generating_base64_hash, $file, $algorithm)) $diskhash_hex = Get-FileHash -algo $algorithm -path $file [byte[]] $diskhash_bin = for ([int]$i = 0; $i -lt $diskhash_hex.hash.length; $i += 2) { [byte]::parse($diskhash_hex.hash.substring($i, 2), [System.Globalization.NumberStyles]::HexNumber) } return [convert]::ToBase64String($diskhash_bin) } function Get-GraphAccessToken { <# .DESCRIPTION Gets the Graph Access Token #> $azContext = Get-AzContext $graphTokenItemResource = $global:graphEndpointResourceIdAzureCloud if($azContext.Environment -eq $global:azureChinaCloud) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureChinaCloud } elseif($azContext.Environment -eq $global:azureUSGovernment) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureUSGovernment } elseif($azContext.Environment -eq $global:azureGermanCloud) { $graphTokenItemResource = $global:graphEndpointResourceIdAzureGermancloud } elseif($azContext.Environment -eq $global:azurePPE) { $graphTokenItemResource = $global:graphEndpointResourceIdAzurePPE } return Get-AzAccessToken -ResourceUrl $graphTokenItemResource } function New-KvaArcConnection { <# .DESCRIPTION Helper function to add the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } # Ensure that the cluster exists Get-KvaCluster -Name $Name | Out-Null $kubeconfig = Get-KvaCredential -activity $activity $arguments = "cluster addons arc install" $arguments += " --cluster-name $Name" $arguments += " --kubeconfig ""$kubeconfig""" $arguments += " --subscription-id $subscriptionId" $arguments += " --tenant-id $tenantId" $arguments += " --resource-group $resourceGroup" $arguments += " --location $location" Invoke-KvaCtlWithAzureContext -arguments $arguments -showOutputAsProgress -activity $activity -credential $credential Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_arc_onboarding_agent_installed_to_cluster) Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_watch_progress_for_arc_agents) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName } function Remove-KvaArcConnection { <# .DESCRIPTION Helper function to add the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) Initialize-KvaEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { $azContext = Get-AzContext $subscriptionid = $azContext.Subscription.Id $tenantid = $azContext.Tenant.Id $resourcegroup = $global:config[$moduleName]["azureResourceGroup"] $location = $global:config[$moduleName]["azureLocation"] } # Ensure that the cluster exists Get-KvaCluster -Name $Name | Out-Null $kubeconfig = Get-KvaCredential -activity $activity $arguments = "cluster addons arc uninstall" $arguments += " --cluster-name $Name" $arguments += " --kubeconfig ""$kubeconfig""" $arguments += " --subscription-id $subscriptionId" $arguments += " --tenant-id $tenantId" $arguments += " --resource-group $resourceGroup" $arguments += " --location $location" Invoke-KvaCtlWithAzureContext -arguments $arguments -showOutputAsProgress -activity $activity -credential $credential Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_onboarding_agent_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName } function Set-KvaCsiSmb { <# .DESCRIPTION Installs csi smb plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_installing_csi_smb_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb install --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_smb_installation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Set-KvaCsiNfs { <# .DESCRIPTION Installs csi nfs plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_installing_csi_nfs_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs install --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_nfs_installation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Reset-KvaCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_uninstalling_csi_smb_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb uninstall --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_smb_uninstallation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Reset-KvaCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_uninstalling_csi_nfs_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs uninstall --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_nfs_uninstallation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Set-KvaCsiSmb { <# .DESCRIPTION Installs csi smb plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_installing_csi_smb_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb install --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_smb_installation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Set-KvaCsiNfs { <# .DESCRIPTION Installs csi nfs plugin to an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_installing_csi_nfs_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs install --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_nfs_installation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Reset-KvaCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_uninstalling_csi_smb_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csismb uninstall --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_smb_uninstallation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Reset-KvaCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin from an AKS-HCI cluster. .PARAMETER ClusterName Cluster Name #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $ClusterName -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_uninstalling_csi_nfs_to_cluster) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "cluster addons csinfs uninstall --cluster-name $ClusterName --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_csi_nfs_uninstallation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Set-KvaHciMonitoring { <# .DESCRIPTION Installs monitoring infrastructure on AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER storageSizeGB Amount of storage for Prometheus in GB .PARAMETER retentionTimeHours metrics retention time in hours. (min 2 hours, max 876000 hours(100 years)) .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [int] $storageSizeGB, [Parameter(Mandatory=$true)] [int] $retentionTimeHours, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $hundredYearsInHours = 876000 if (($retentionTimeHours -lt 2) -or ($retentionTimeHours -gt $hundredYearsInHours)) { throw $($KvaLocMessage.kva_invalid_retention_time) } if ($storageSizeGB -lt 1) { throw $($KvaLocMessage.kva_zero_storage_size) } Initialize-KvaEnvironment -activity $activity $kvaCluster=Get-KvaCluster -Name $Name -activity $activity if($kvaCluster.LinuxNodeCount -lt 1) { throw $($KvaLocMessage.kva_linux_node_required) } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_installing_monitoring) $kubeconfig = Get-KvaCredential -activity $activity ## check if akshci-monitoring is already installed. $installedAddons = Invoke-KvaCtl -arguments "cluster addons list --cluster-name $Name --kubeconfig ""$kubeconfig"" --outputformat json" -activity $activity | ConvertFrom-Json foreach ($addon in $installedAddons) { if ($addon.name -eq "akshci-monitoring") { #vishal return "AksHci Monitoring is already installed" } } Invoke-KvaCtl -arguments "cluster addons akshci-monitoring install --cluster-name $Name --kubeconfig ""$kubeconfig"" --storageSizeGB $storageSizeGB --retentionTimeHours $retentionTimeHours" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_monitoring_installation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Reset-KvaHciMonitoring { <# .DESCRIPTION Uninstalls monitoring infrastructure from AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } Initialize-KvaEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_uninstalling_monitoring) $kubeconfig = Get-KvaCredential -activity $activity ## check if akshci-monitoring is already installed. $installedAddons = Invoke-KvaCtl -arguments "cluster addons list --cluster-name $Name --kubeconfig ""$kubeconfig"" --outputformat json" -activity $activity | ConvertFrom-Json $akshciMonitoringInstalled = $false foreach ($addon in $installedAddons) { if ($addon.name -eq "akshci-monitoring") { $akshciMonitoringInstalled = $true break } } if($akshciMonitoringInstalled -eq $false) { throw $($KvaLocMessage.kva_akshci_monitoring_not_installed) } Invoke-KvaCtl -arguments "cluster addons akshci-monitoring uninstall --cluster-name $Name --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_monitoring_uninstallation_complete) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Test-KvaAzureConnection { <# .DESCRIPTION Tests the connection to Azure. #> $azContext = Get-AzContext if([string]::IsNullOrEmpty($azContext.Subscription.Id)) { throw $($KvaLocMessage.kva_azure_connection_failed) } try { Get-AzSubscription -SubscriptionId $azContext.Subscription.Id -ErrorAction Stop | Out-Null } catch [Exception] { throw $($KvaLocMessage.kva_azure_connection_failed) } $aksHciRegistration = Get-AksHciRegistration if ([string]::IsNullOrWhiteSpace($aksHciRegistration.azureResourceGroup)) { throw $($KvaLocMessage.kva_azure_resource_group_not_found) } } #endregion #region Cluster Networking commands function New-KvaClusterNetwork { <# .DESCRIPTION Create network settings to be used for the target clusters. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNetDHCP = New-AksHciClusterNetwork -name e1 -vswitchName External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE $clusterVNetStatic = New-AksHciClusterNetwork -name e1 -vswitchName External -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_validating_network_configuration) $vnet = New-VirtualNetwork -name $name -vswitchName $vswitchName -vlanID $vlanID -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend $dnsserver = "" if(-Not [string]::IsNullOrEmpty($vnet.IpAddressPrefix)) { foreach ($dns in $vnet.DnsServers) { if(-Not [string]::IsNullOrEmpty($dns)) { $dnsserver += "`n - " + $dns } } } $yaml = @" virtualnetwork: name: "$($vnet.Name)" vswitchname: "$($vnet.VswitchName)" type: "Transparent" vlanid: $($vnet.VlanID) ipaddressprefix: $($vnet.IpAddressPrefix) gateway: $($vnet.Gateway) dnsservers: $dnsserver vippoolstart: $($vnet.VipPoolStart) vippoolend: $($vnet.VipPoolEnd) k8snodeippoolstart: $($vnet.K8snodeIPPoolStart) k8snodeippoolend: $($vnet.K8snodeIPPoolEnd) "@ $yamlFile = $($global:config[$modulename]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$name.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_creating_network) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "network create --networkconfig ""$yamlFile"" --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) return $vnet } function Remove-KvaClusterNetwork { <# .DESCRIPTION Remove a virtual network object for a target cluster .PARAMETER name The name of the vnet .NOTES The cmdlet will throw an exception if the network is still being used. The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE Remove-AksHciClusterNetwork -name e1 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($KvaLocMessage.kva_removing_network) Write-SubStatus -moduleName $moduleName $($KvaLocMessage.kva_network_removal_in_progress) $kubeconfig = Get-KvaCredential -activity $activity Invoke-KvaCtl -arguments "network delete --networkname $name --kubeconfig ""$kubeconfig""" -showOutput -activity $activity Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Get-KvaClusterNetwork { <# .DESCRIPTION Gets the VirtualNetwork object for a target cluster given either the vnet name or the cluster name. If no parameter is given, all vnet's are returned. .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $name" } Initialize-KvaEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($KvaLocMessage.kva_retrieving_network) -moduleName $moduleName Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_retrieving_configuration_for_network, $name)) -moduleName $global:KvaModule $retrievedVnet = Get-KvaClusterNetworkInternal -name $name -clusterName $clusterName -activity $activity Write-SubStatus $($KvaLocMessage.kva_successfully_retrieved_network) -moduleName $global:KvaModule return $retrievedVnet } function Get-KvaClusterNetworkInternal { <# .DESCRIPTION This function is similar to Get-KvaNetworkCluster excpet it does not update the powershell status. It is intended to be called by other cmdlets that do update the status .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) $kubeconfig = Get-KvaCredential -activity $activity $queryParam = "" if ($name) { $queryParam = "--networkname $name" } elseif ($clusterName) { $queryParam = "--clustername $clusterName" } $kvaVnets = Invoke-KvaCtl -arguments "network get $queryParam --kubeconfig ""$kubeconfig"" --outputformat json" -activity $activity | ConvertFrom-Json $vnet = @() foreach ($kvaVnet in $kvaVnets) { $vnet += [VirtualNetwork]::new($kvaVnet.virtualnetwork.name, $kvaVnet.virtualnetwork.vswitchname, $kvaVnet.virtualnetwork.ipaddressprefix, $kvaVnet.virtualnetwork.gateway, $kvaVnet.virtualnetwork.dnsservers, $kvaVnet.virtualnetwork.macpoolname, $kvaVnet.virtualnetwork.vlanid, $kvaVnet.virtualnetwork.vippoolstart, $kvaVnet.virtualnetwork.vippoolend, $kvaVnet.virtualnetwork.k8snodeippoolstart, $kvaVnet.virtualnetwork.k8snodeippoolend) } return $vnet } function Enable-KvaPreview { <# .SYNOPSIS Enable AKSHCI catalog and ring configuration to expose early access preview builds. .DESCRIPTION Enable AKSHCI catalog and ring configuration to expose early access preview builds. .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter()] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter()] [String] $ring = "earlyaccesspreview" ) #Set KvaConfig for preview Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_enabling_preview, $moduleName)) Set-KvaConfigValue -name "catalog" -value $catalog Set-KvaConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_configuration_for_module_update, $moduleName)) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Disable-KvaPreview { <# .SYNOPSIS Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .DESCRIPTION Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .PARAMETER activity Activity name to use when updating progress. .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter()] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter()] [String] $ring = "stable" ) #Set KvaConfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_disabling_preview, $moduleName)) Set-KvaConfigValue -name "catalog" -value $catalog Set-KvaConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_configuration_for_module_update, $moduleName)) Write-Status -moduleName $moduleName $($GenericLocMessage.generic_done) } function Update-ConfigMapPauseImages { <# .SYNOPSIS Update-Kva workaround for Github Issues https://github.com/Azure/aks-hci/issues/147 and https://github.com/Azure/aks-hci/issues/148 .DESCRIPTION Patches the KVA ConfigMaps, which hold the pause version metadata, to the correct pause version. This will prevent users from hitting Github Issues https://github.com/Azure/aks-hci/issues/147 and https://github.com/Azure/aks-hci/issues/148 .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [String]$activity = $MyInvocation.MyCommand.Name ) try { $deploymentManifestLocation = $([io.Path]::Combine($global:config[$modulename]["installationPackageDir"], $global:cloudOperatorYaml)) $file = Get-Content -Path $deploymentManifestLocation -Raw $separator = "---" $objects = $file -split $separator, 0, "multiline" $newFile = "" foreach ($object in $objects) { if (($object.Contains("kubernetes: v1.21.2") -or $object.Contains("kubernetes: v1.21.1")) -and $object.Contains('mariner: "2021-10-01"')) { $object = $object -Replace ' pause: "3.2"', ' pause: "3.4.1"' } $newFile += $object + "---" } $newFile | Set-Content -Path $deploymentManifestLocation } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } } #endregion # SIG # Begin signature block # MIInugYJKoZIhvcNAQcCoIInqzCCJ6cCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDNQF+d+lFUMlox # FJ5DxlxsvNhL0TtVNSmYt4ca685OS6CCDYEwggX/MIID56ADAgECAhMzAAACUosz # qviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjEwOTAyMTgzMjU5WhcNMjIwOTAxMTgzMjU5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDQ5M+Ps/X7BNuv5B/0I6uoDwj0NJOo1KrVQqO7ggRXccklyTrWL4xMShjIou2I # sbYnF67wXzVAq5Om4oe+LfzSDOzjcb6ms00gBo0OQaqwQ1BijyJ7NvDf80I1fW9O # L76Kt0Wpc2zrGhzcHdb7upPrvxvSNNUvxK3sgw7YTt31410vpEp8yfBEl/hd8ZzA # v47DCgJ5j1zm295s1RVZHNp6MoiQFVOECm4AwK2l28i+YER1JO4IplTH44uvzX9o # RnJHaMvWzZEpozPy4jNO2DDqbcNs4zh7AWMhE1PWFVA+CHI/En5nASvCvLmuR/t8 # q4bc8XR8QIZJQSp+2U6m2ldNAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUNZJaEUGL2Guwt7ZOAu4efEYXedEw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDY3NTk3MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAFkk3 # uSxkTEBh1NtAl7BivIEsAWdgX1qZ+EdZMYbQKasY6IhSLXRMxF1B3OKdR9K/kccp # kvNcGl8D7YyYS4mhCUMBR+VLrg3f8PUj38A9V5aiY2/Jok7WZFOAmjPRNNGnyeg7 # l0lTiThFqE+2aOs6+heegqAdelGgNJKRHLWRuhGKuLIw5lkgx9Ky+QvZrn/Ddi8u # TIgWKp+MGG8xY6PBvvjgt9jQShlnPrZ3UY8Bvwy6rynhXBaV0V0TTL0gEx7eh/K1 # o8Miaru6s/7FyqOLeUS4vTHh9TgBL5DtxCYurXbSBVtL1Fj44+Od/6cmC9mmvrti # yG709Y3Rd3YdJj2f3GJq7Y7KdWq0QYhatKhBeg4fxjhg0yut2g6aM1mxjNPrE48z # 6HWCNGu9gMK5ZudldRw4a45Z06Aoktof0CqOyTErvq0YjoE4Xpa0+87T/PVUXNqf # 7Y+qSU7+9LtLQuMYR4w3cSPjuNusvLf9gBnch5RqM7kaDtYWDgLyB42EfsxeMqwK # WwA+TVi0HrWRqfSx2olbE56hJcEkMjOSKz3sRuupFCX3UroyYf52L+2iVTrda8XW # esPG62Mnn3T8AuLfzeJFuAbfOSERx7IFZO92UPoXE1uEjL5skl1yTZB3MubgOA4F # 8KoRNhviFAEST+nG8c8uIsbZeb08SeYQMqjVEmkwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIZjzCCGYsCAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAlKLM6r4lfM52wAAAAACUjAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgtgUxhYn3 # ZQrd7PTV32xgrkLF2mMTOKD5b5jiKKBvkX0wQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQA+t8fhBHOjUGRd54+XSa5ep+WMz3j45e6cfH19npkI # PuXCGy+LXudq2VNk5M7Fy8Lo5RnT8ZxNo9Qi7Ei8gF6Vs8rNq3vHlon03a6+ZcXf # 80QOcMbhZMLJAT3PUfSN/tUENHxbeFNqNS3A5v1uiFPfakqtZzPPvYNNFFhdpNl6 # fxlv52n1BElLD19qT2pQlRPL/yy63Aj6Zd8XPp9LBO6Vw0dlAJtOrP6qLIugvKBJ # yi2ldhxxA2aDFiwkEnTRe8dWM7PMNMj1z4Xz7mcJ6vL1GzcUzu4rVaCK5yHUYDLc # u5Rp81Ex2s20+bnf1vSXAnV5qjQfCKhKYQFsmzsrJYK4oYIXGTCCFxUGCisGAQQB # gjcDAwExghcFMIIXAQYJKoZIhvcNAQcCoIIW8jCCFu4CAQMxDzANBglghkgBZQME # AgEFADCCAVkGCyqGSIb3DQEJEAEEoIIBSASCAUQwggFAAgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIDlvOxJ+F9LOfrWkKCYIh19hAfmtuBkizv0JzNYG # eFdoAgZhwkvDeoQYEzIwMjIwMTE5MDA0NTQ0LjQ2MlowBIACAfSggdikgdUwgdIx # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1p # Y3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhh # bGVzIFRTUyBFU046RDA4Mi00QkZELUVFQkExJTAjBgNVBAMTHE1pY3Jvc29mdCBU # aW1lLVN0YW1wIFNlcnZpY2WgghFoMIIHFDCCBPygAwIBAgITMwAAAY/zUajrWnLd # zAABAAABjzANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0Eg # MjAxMDAeFw0yMTEwMjgxOTI3NDZaFw0yMzAxMjYxOTI3NDZaMIHSMQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQg # SXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOkQwODItNEJGRC1FRUJBMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNlMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAmVc+/rXP # Fx6Fk4+CpLrubDrLTa3QuAHRVXuy+zsxXwkogkT0a+XWuBabwHyqj8RRiZQQvdvb # Oq5NRExOeHiaCtkUsQ02ESAe9Cz+loBNtsfCq846u3otWHCJlqkvDrSr7mMBqwcR # Y7cfhAGfLvlpMSojoAnk7Rej+jcJnYxIeN34F3h9JwANY360oGYCIS7pLOosWV+b # xug9uiTZYE/XclyYNF6XdzZ/zD/4U5pxT4MZQmzBGvDs+8cDdA/stZfj/ry+i0XU # YNFPhuqc+UKkwm/XNHB+CDsGQl+ZS0GcbUUun4VPThHJm6mRAwL5y8zptWEIocbT # eRSTmZnUa2iYH2EOBV7eCjx0Sdb6kLc1xdFRckDeQGR4J1yFyybuZsUP8x0dOsEE # oLQuOhuKlDLQEg7D6ZxmZJnS8B03ewk/SpVLqsb66U2qyF4BwDt1uZkjEZ7finIo # UgSz4B7fWLYIeO2OCYxIE0XvwsVop9PvTXTZtGPzzmHU753GarKyuM6oa/qaTzYv # rAfUb7KYhvVQKxGUPkL9+eKiM7G0qenJCFrXzZPwRWoccAR33PhNEuuzzKZFJ4De # aTCLg/8uK0Q4QjFRef5n4H+2KQIEibZ7zIeBX3jgsrICbzzSm0QX3SRVmZH//Aqp # 8YxkwcoI1WCBizv84z9eqwRBdQ4HYcNbQMMCAwEAAaOCATYwggEyMB0GA1UdDgQW # BBTzBuZ0a65JzuKhzoWb25f7NyNxvDAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJl # pxtTNRnpcjBfBgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAx # MCgxKS5jcmwwbAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3Rh # bXAlMjBQQ0ElMjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoG # CCsGAQUFBwMIMA0GCSqGSIb3DQEBCwUAA4ICAQDNf9Oo9zyhC5n1jC8iU7NJY39F # izjhxZwJbJY/Ytwn63plMlTSaBperan566fuRojGJSv3EwZs+RruOU2T/ZRDx4VH # esLHtclE8GmMM1qTMaZPL8I2FrRmf5Oop4GqcxNdNECBClVZmn0KzFdPMqRa5/0R # 6CmgqJh0muvImikgHubvohsavPEyyHQa94HD4/LNKd/YIaCKKPz9SA5fAa4phQ4E # vz2auY9SUluId5MK9H5cjWVwBxCvYAD+1CW9z7GshJlNjqBvWtKO6J0Aemfg6z28 # g7qc7G/tCtrlH4/y27y+stuwWXNvwdsSd1lvB4M63AuMl9Yp6au/XFknGzJPF6n/ # uWR6JhQvzh40ILgeThLmYhf8z+aDb4r2OBLG1P2B6aCTW2YQkt7TpUnzI0cKGr21 # 3CbKtGk/OOIHSsDOxasmeGJ+FiUJCiV15wh3aZT/VT/PkL9E4hDBAwGt49G88gSC # O0x9jfdDZWdWGbELXlSmA3EP4eTYq7RrolY04G8fGtF0pzuZu43A29zaI9lIr5ul # KRz8EoQHU6cu0PxUw0B9H8cAkvQxaMumRZ/4fCbqNb4TcPkPcWOI24QYlvpbtT9p # 31flYElmc5wjGplAky/nkJcT0HZENXenxWtPvt4gcoqppeJPA3S/1D57KL3667ep # Ir0yV290E2otZbAW8DCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUw # DQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5n # dG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9y # YXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhv # cml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9z # b2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw # ggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg # 4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aO # RmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41 # JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5 # LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL # 64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9 # QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj # 0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqE # UUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0 # kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435 # UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB # 3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTE # mr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwG # A1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNV # HSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNV # HQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo # 0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29m # dC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5j # cmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDAN # BgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4 # sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th54 # 2DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRX # ud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBew # VIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0 # DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+Cljd # QDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFr # DZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFh # bHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7n # tdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+ # oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6Fw # ZvKhggLXMIICQAIBATCCAQChgdikgdUwgdIxCzAJBgNVBAYTAlVTMRMwEQYDVQQI # EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv # ZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046RDA4Mi00QkZE # LUVFQkExJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WiIwoB # ATAHBgUrDgMCGgMVAD5NL4IEdudIBwdGoCaV0WBbQZpqoIGDMIGApH4wfDELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9z # b2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJKoZIhvcNAQEFBQACBQDlkbN0MCIY # DzIwMjIwMTE5MDU0NTI0WhgPMjAyMjAxMjAwNTQ1MjRaMHcwPQYKKwYBBAGEWQoE # ATEvMC0wCgIFAOWRs3QCAQAwCgIBAAICHXICAf8wBwIBAAICETAwCgIFAOWTBPQC # AQAwNgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEK # MAgCAQACAwGGoDANBgkqhkiG9w0BAQUFAAOBgQALN+vQNrYbh6RT0XwgpyySbJ1Z # KoeFK6VcoYvYlN6+L25SZulcx+Cbr2gtTDX4o5nYlgT1hEr66p+UPBJKPVPfdSNY # u/CQQYBHW9RhHwJIgCjmpNBlIOwuxWsWsuInCUta3V/wQF/eO+1y0A0ntyAOjSr+ # f6gcJB56l2gDY5ZmaTGCBA0wggQJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwAhMzAAABj/NRqOtact3MAAEAAAGPMA0GCWCGSAFlAwQCAQUAoIIB # SjAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEIGe/ # v7r3AMJw1BiPPYIPeTwilgLed/1ZzbIdusbhfhNLMIH6BgsqhkiG9w0BCRACLzGB # 6jCB5zCB5DCBvQQgl3IFT+LGxguVjiKm22ItmO6dFDWW8nShu6O6g8yFxx8wgZgw # gYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAY/zUajrWnLd # zAABAAABjzAiBCDk5JrPtNz/mEzpflm81KD7+MBS7ag4zW9WZM+s2EeLVzANBgkq # hkiG9w0BAQsFAASCAgCMkmwDA8DxxL5oepl+UIEQ/QMWHEkeuramwQIGseNsQ2Ql # CUra3Suzzodgz5HS2B0K3gqGMhQTWL8Klp314/BvwgWxeRHlj/gaFu0g94Y5bjTj # utjYG/SfrHVgYLIjmKGb6h2Mo9Takfvrl0JOLgV/pxdqx+HjRAJ+YPtyYmr76YaG # B3JkrZXuNftbuc8ZoJhXYnPWg8hAmD8dSmCRCD6F3pHkdRCzMcNLbXSMVtXJU9AD # +DNNJ7sOmLRG+QibYCULGu640nGpu/JdBb7xfpyN9Sx6mWwgfJ1/Jx4i3yC8aOrF # 9y4m/n+utL8mCKQULmO6dS+KTq6BUG6117udWZNtuNzlFoWZ3/X54L9j9Nx/+OsE # i5Gxy6EnRnjLPAZsBt2KOeJRQkC03LZfgaJTlTfbANZV5c/0HMPpiBcAYZNp2TG4 # QP6b4HFzVvSrFBL4K9EUcQvEI4fnOGByf9W4tLXwz5db5cr6n+uIyW/DlAcIk+kZ # CvnKxL1jN+Hx2QQf2DPdByJI7tYUaz5lQOAQbMwhzzB9+W5uC/avgC5Lcn2T3xuh # 1cc/QuDBLTQqOPNZ0F3ElzUv6qdF1hujv7RgKSG/13jWayez5oX+dOsQpiUK71HU # 208ZQBy50x69U3NrC4L7d9OGoPxYUgH+7SlzGw31W1fUAqOlXvW2dlduY543Fg== # SIG # End signature block |