Common.psm1
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # Common/Shared Functions, Types, and Variables # ######################################################################################### #requires -runasadministrator #region Enums Add-Type -TypeDefinition @" using System.Management.Automation; public enum InstallState { NotInstalled, Installing, InstallFailed, Updating, UpdateFailed, Uninstalling, UninstallFailed, Installed }; public enum DeploymentType { None, SingleNode, MultiNode }; public enum LoadBalancerType { unstacked_haproxy, stacked_kube_vip }; public enum VmSize { Default, Standard_A2_v2, Standard_A4_v2, Standard_D2s_v3, Standard_D4s_v3, Standard_D8s_v3, Standard_D16s_v3, Standard_D32s_v3, Standard_DS2_v2, Standard_DS3_v2, Standard_DS4_v2, Standard_DS5_v2, Standard_DS13_v2, Standard_K8S_v1, Standard_K8S2_v1, Standard_K8S3_v1, Standard_NK6, Standard_NV6, Standard_NV12 }; public class VirtualNetwork { public string Name; public string VswitchName; public string IpAddressPrefix; public string Gateway; public string[] DnsServers; public string MacPoolName; public int Vlanid; public string VipPoolStart; public string VipPoolEnd; public string K8snodeIPPoolStart; public string K8snodeIPPoolEnd; public VirtualNetwork ( string Name, string VswitchName, string IpAddressPrefix, string Gateway, string[] DnsServers, string MacPoolName, int Vlanid, string VipPoolStart, string VipPoolEnd, string K8snodeIPPoolStart, string K8snodeIPPoolEnd ) { this.Name = Name; this.VswitchName = VswitchName; this.IpAddressPrefix = IpAddressPrefix; this.Gateway = Gateway; this.DnsServers = DnsServers; this.MacPoolName = MacPoolName; this.Vlanid = Vlanid; this.VipPoolStart = VipPoolStart; this.VipPoolEnd = VipPoolEnd; this.K8snodeIPPoolStart = K8snodeIPPoolStart; this.K8snodeIPPoolEnd = K8snodeIPPoolEnd; } public override string ToString() { return string.Format("{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}\n{7}\n{8}\n{9}", this.Name, this.IpAddressPrefix, this.Gateway, string.Join(",", this.DnsServers), this.MacPoolName, this.Vlanid, this.VipPoolStart, this.VipPoolEnd, this.K8snodeIPPoolStart, this.K8snodeIPPoolEnd); } } public class ProxySettings { public string Name; public string HTTP; public string HTTPS; public string NoProxy; public string CertFile; public PSCredential Credential; public ProxySettings ( PSCredential Credential, string Name = "", string HTTP = "", string HTTPS = "", string NoProxy = "", string CertFile = "" ) { this.Name = Name; this.HTTP = HTTP; this.HTTPS = HTTPS; this.NoProxy = NoProxy; this.CertFile = CertFile; this.Credential = Credential; } public override string ToString() { return string.Format("{0}\n{1}\n{2}\n{3}\n{4}\n{5}", this.Name, this.HTTP, this.HTTPS, this.NoProxy, this.CertFile, this.Credential.ToString()); } } public class ContainerRegistry { public string Server; public PSCredential Credential; public ContainerRegistry ( PSCredential Credential, string Server = "" ) { this.Server = Server; this.Credential = Credential; } public override string ToString() { return string.Format("{0}\n{1}", this.Server, this.Credential.ToString()); } } "@ Add-Type -Language CSharp -ReferencedAssemblies "System.Numerics.dll" @" using System; using System.Numerics; using System.Net; namespace AKSHCI { public class IPRange { public IPAddress start, end; public override string ToString() { return string.Format("{0} - {1}", start, end); } } public static class IPUtilities { public static int CompareIpAddresses(IPAddress ipAddress1, IPAddress ipAddress2) { byte[] ipAddress1Bytes = ipAddress1.GetAddressBytes(); byte[] ipAddress2Bytes = ipAddress2.GetAddressBytes(); Array.Reverse(ipAddress1Bytes); Array.Reverse(ipAddress2Bytes); Array.Resize<byte>(ref ipAddress1Bytes, 5); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned Array.Resize<byte>(ref ipAddress2Bytes, 5); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned BigInteger ipAddress1BigInt = new BigInteger(ipAddress1Bytes); BigInteger ipAddress2BigInt = new BigInteger(ipAddress2Bytes); return BigInteger.Compare(ipAddress1BigInt, ipAddress2BigInt); } public static IPAddress GetLastIpInCidr(IPAddress ipAddressStr, int prefixLength) { BigInteger fullMask = new BigInteger(0xFFFFFFFF); BigInteger mask = ((fullMask >> (prefixLength)) & fullMask); byte[] endAddress = mask.ToByteArray(); Array.Resize<byte>(ref endAddress, 4); Array.Reverse(endAddress); byte[] ipAddress = ipAddressStr.GetAddressBytes(); if(ipAddress.Length != endAddress.Length) { throw new System.InvalidOperationException("Address and prefix length are both expected to be IPv4 (" + ipAddress.Length + " != " + endAddress.Length + ")"); } for(int i = 0; i < ipAddress.Length; i++) { endAddress[i] = (byte) (endAddress[i] | ipAddress[i]); } return new IPAddress(endAddress); } public static IPAddress ToIPAddress(BigInteger bi) { var bytes = bi.ToByteArray(); Array.Resize<byte>(ref bytes, 4); Array.Reverse(bytes); return new IPAddress(bytes); } public static BigInteger ToBigInteger(IPAddress ip) { var ipBytes = ip.GetAddressBytes(); Array.Reverse(ipBytes); Array.Resize<byte>(ref ipBytes, ipBytes.Length + 1); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned return new BigInteger(ipBytes); } public static void ToRange(string CIDR, out IPAddress start, out IPAddress end) { var s = CIDR.Split('/'); start = IPAddress.Parse(s[0]); var prefixLength = int.Parse(s[1]); end = AKSHCI.IPUtilities.GetLastIpInCidr(start, prefixLength); } public static bool ValidateRange(string rangeStart, string rangeEnd) { var start = IPAddress.Parse(rangeStart); var startBI = ToBigInteger(start); var end = IPAddress.Parse(rangeEnd); var endBI = ToBigInteger(end); if (endBI < startBI) { return false; } return true; } public static bool ValidateIPInCIDR(string ip, string CIDR) { var ipaddress = IPAddress.Parse(ip); var ipBI = ToBigInteger(ipaddress); IPAddress cidrStart, cidrEnd; ToRange(CIDR, out cidrStart, out cidrEnd); var cidrStartBI = ToBigInteger(cidrStart); var cidrEndBI = ToBigInteger(cidrEnd); if ((ipBI >= cidrStartBI) && (ipBI <= cidrEndBI)) { return true; } return false; } public static bool ValidateRangeInCIDR(string rangeStart, string rangeEnd, string CIDR) { if (ValidateIPInCIDR(rangeStart, CIDR) && ValidateIPInCIDR(rangeEnd, CIDR)) { return true; } return false; } public static IPRange[] GetVMIPPool(string vippoolStart, string vippoolEnd, string CIDR) { var start = IPAddress.Parse(vippoolStart); var startBI = ToBigInteger(start); var end = IPAddress.Parse(vippoolEnd); var endBI = ToBigInteger(end); IPAddress cidrStart, cidrEnd; ToRange(CIDR, out cidrStart, out cidrEnd); var cidrStartBI = ToBigInteger(cidrStart); var cidrEndBI = ToBigInteger(cidrEnd); if ((startBI == cidrStartBI) && (endBI == cidrEndBI)) { throw new Exception(string.Format("The VIP pool range ({0} - {1}) is too large. There is no space to allocate IP addresses for VM's. Try decreasing the size of the VIP pool.", vippoolStart, vippoolEnd)); } if (startBI == cidrStartBI) { var ippoolstart = ToIPAddress(endBI + 1); var ippoolend = ToIPAddress(cidrEndBI); return new IPRange[] { new IPRange{ start = ippoolstart, end = ippoolend } }; } else if (endBI == cidrEndBI) { var ippoolstart = ToIPAddress(cidrStartBI); var ippoolend = ToIPAddress(startBI - 1); return new IPRange[] { new IPRange { start = ippoolstart, end = ippoolend } }; } else { var ippool1start = ToIPAddress(cidrStartBI); var ippool1end = ToIPAddress(startBI - 1); var ippool2start = ToIPAddress(endBI + 1); var ippool2end = ToIPAddress(cidrEndBI); return new IPRange[] { new IPRange { start = ippool1start, end = ippool1end }, new IPRange { start = ippool2start, end = ippool2end } }; } } } } "@; #endregion #region Module constants $global:AksHciModule = "AksHci" $global:MocModule = "Moc" $global:KvaModule = "Kva" $global:DownloadModule = "DownloadSdk" $global:CommonModule = "Common" $global:configurationKeys = @{ $global:AksHciModule = "HKLM:SOFTWARE\Microsoft\${global:AksHciModule}PS"; $global:MocModule = "HKLM:SOFTWARE\Microsoft\${global:MocModule}PS"; $global:KvaModule = "HKLM:SOFTWARE\Microsoft\${global:KvaModule}PS"; } $global:repositoryName = "PSGallery" $global:repositoryNamePreview = "PSGallery" $global:repositoryUser = "" $global:repositoryPass = "" #endregion #region VM size definitions $global:vmSizeDefinitions = @( # Name, CPU, MemoryGB ([VmSize]::Default, "4", "4"), ([VmSize]::Standard_A2_v2, "2", "4"), ([VmSize]::Standard_A4_v2, "4", "8"), ([VmSize]::Standard_D2s_v3, "2", "8"), ([VmSize]::Standard_D4s_v3, "4", "16"), ([VmSize]::Standard_D8s_v3, "8", "32"), ([VmSize]::Standard_D16s_v3, "16", "64"), ([VmSize]::Standard_D32s_v3, "32", "128"), ([VmSize]::Standard_DS2_v2, "2", "7"), ([VmSize]::Standard_DS3_v2, "2", "14"), ([VmSize]::Standard_DS4_v2, "8", "28"), ([VmSize]::Standard_DS5_v2, "16", "56"), ([VmSize]::Standard_DS13_v2, "8", "56"), ([VmSize]::Standard_K8S_v1, "4", "2"), ([VmSize]::Standard_K8S2_v1, "2", "2"), ([VmSize]::Standard_K8S3_v1, "4", "6") # Dont expose GPU size until its supported #([VmSize]::Standard_NK6, "6", "12"), #([VmSize]::Standard_NV6, "6", "64"), #([VmSize]::Standard_NV12, "12", "128") ) #endregion #region Pod names and selectors $global:managementPods = @( ("Cloud Operator", "cloudop-system", "control-plane=controller-manager"), ("Cluster API core", "capi-system", "cluster.x-k8s.io/provider=cluster-api"), ("Bootstrap kubeadm", "capi-kubeadm-bootstrap-system", "cluster.x-k8s.io/provider=bootstrap-kubeadm"), ("Control Plane kubeadm", "capi-kubeadm-control-plane-system", "cluster.x-k8s.io/provider=control-plane-kubeadm"), ("Cluster API core Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=cluster-api"), ("Bootstrap kubeadm Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=bootstrap-kubeadm"), ("Control Plane kubeadm Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=control-plane-kubeadm"), ("AzureStackHCI Provider", "caph-system", "cluster.x-k8s.io/provider=infrastructure-azurestackhci"), ("AzureStackHCI Provider Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=infrastructure-azurestackhci") ) #endregion #region Classes class NetworkPlugin { [string] $Name static [string] $Default = "flannel" # TODO: we can't default to calico yet because windows doesn't suppor it NetworkPlugin( [string] $name ) { $curatedName = $name.ToLower() if ($curatedName -ne "flannel" -and $curatedName -ne "calico") { throw "Invalid CNI '$curatedName'. The only supported CNIs are 'flannel' and 'calico'" } $this.Name = $curatedName } NetworkPlugin() { $this.Name = [NetworkPlugin]::Default } } #endregion #region Script Constants $global:installDirectoryName = "AksHci" $global:workingDirectoryName = "AksHci" $global:imageDirectoryName = "AksHciImageStore" $global:yamlDirectoryName = "yaml" $global:cloudConfigDirectoryName = "wssdcloudagent" $global:nodeConfigDirectoryName = "wssdagent" $global:installDirectory = $($env:ProgramFiles + "\" + $global:installDirectoryName) $global:defaultworkingDir = $($env:SystemDrive + "\" + $global:workingDirectoryName) $global:defaultStagingShare = "" $global:nodeAgentBinary = "wssdagent.exe" $global:cloudAgentBinary = "wssdcloudagent.exe" $global:nodeCtlBinary = "nodectl.exe" $global:cloudCtlBinary = "mocctl.exe" $global:kubectlBinary = "kubectl.exe" $global:kvactlBinary = "kvactl.exe" $global:cloudOperatorYaml = "cloud-operator.yaml" $global:nodeAgentFullPath = [io.Path]::Combine($global:installDirectory, $global:nodeAgentBinary) $global:cloudAgentFullPath = [io.Path]::Combine($global:installDirectory, $global:cloudAgentBinary) $global:nodeCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:nodeCtlBinary) $global:cloudCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:cloudCtlBinary) $global:kubeCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:kubectlBinary) $global:kvaCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:kvactlBinary) $script:psConfigKeyName = "psconfig" $script:psConfigJson = "psconfig.json" $global:psConfigDirectoryRoot = $($env:USERPROFILE) $global:mocMetadataRoot = $($env:USERPROFILE + "\.wssd") $global:mocMetadataDirectory = [io.Path]::Combine($global:mocMetadataRoot, "mocctl" ) $global:kvaMetadataDirectory = [io.Path]::Combine($global:mocMetadataRoot, "kvactl" ) $global:accessFileLocation = [io.Path]::Combine($global:mocMetadataDirectory, "cloudconfig") $global:defaultCloudConfigLocation = $($env:SystemDrive + "\programdata\" + $global:cloudConfigDirectoryName) $global:defaultNodeConfigLocation = $($env:SystemDrive + "\programdata\" + $global:nodeConfigDirectoryName) $global:defaultTargetK8Version = "v1.19.7" $global:defaultMgmtReplicas = 1 $global:defaultMgmtControlPlaneVmSize = [VmSize]::Standard_A4_v2 $global:defaultControlPlaneVmSize = [VmSize]::Standard_A4_v2 $global:defaultLoadBalancerVmSize = [VmSize]::Standard_A4_v2 $global:defaultWorkerVmSize = [VmSize]::Standard_K8S3_v1 $global:defaultNodeAgentPort = 45000 $global:defaultNodeAuthorizerPort = 45001 $global:defaultCloudAgentPort = 55000 $global:defaultCloudAuthorizerPort = 65000 $global:defaultVipPoolName = "clusterVipPool" $global:defaultMacPoolStart = "" $global:defaultMacPoolEnd = "" $global:defaultVlanID = 0 $global:failoverCluster = $null $global:cloudAgentAppName = "ca" $global:cloudName = "moc-cloud" $global:defaultCloudLocation = "MocLocation" $global:cloudGroupPrefix = "clustergroup" $global:cloudStorageContainer = "MocStorageContainer" $global:cloudMacPool = "MocMacPool" $global:defaultPodCidr = "10.244.0.0/16" $global:mgmtClusterCidr = "10.200.0.0/16" $global:mgmtControlPlaneCidr = "10.240.0.0/24" $global:workloadPodCidr = "10.244.0.0/16" $global:workloadServiceCidr = "10.96.0.0/12" $global:defaultProxyExemptions = "localhost,127.0.0.1,.svc,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" $global:credentialKey = (24,246,163,38,50,244,215,218,223,10,65,98,19,1,149,106,190,141,144,180,157,135,211,143) $global:clusterNameRegex = "^[a-z0-9][a-z0-9-]*[a-z0-9]$" $global:defaultLogLineCount = 500000 $global:operatorTokenValidity = 60 $global:addonTokenValidity = 60 $global:certificateValidityFactor = 1.0 # Temporary until cross-platform signing is available $global:expectedAuthResponse = @{ "Status" = "Valid"; "SignatureType" = "Authenticode"; "StatusMessage" = "" } #endregion #region User Configuration and Defaults #region Configuration Functions function Set-VNetConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet ) Set-ConfigurationValue -name "vnetName" -value $vnet.Name -module $module Set-ConfigurationValue -name "vswitchName" -value $vnet.VswitchName -module $module Set-ConfigurationValue -name "ipaddressprefix" -value $vnet.IpAddressPrefix -module $module Set-ConfigurationValue -name "gateway" -value $vnet.Gateway -module $module Set-ConfigurationValue -name "dnsservers" -value ($vnet.DnsServers -join ",") -module $module Set-ConfigurationValue -name "macpoolname" -value $vnet.MacPoolName -module $module Set-ConfigurationValue -name "vlanid" -value $vnet.Vlanid -module $module Set-ConfigurationValue -name "vnetvippoolstart" -value $vnet.VipPoolStart -module $module Set-ConfigurationValue -name "vnetvippoolend" -value $vnet.VipPoolEnd -module $module Set-ConfigurationValue -name "k8snodeippoolstart" -value $vnet.K8snodeIPPoolStart -module $module Set-ConfigurationValue -name "k8snodeippoolend" -value $vnet.K8snodeIPPoolEnd -module $module } function Get-VNetConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module ) $vnet_name = Get-ConfigurationValue -name "vnetName" -module $module $vnet_vswitchname = Get-ConfigurationValue -name "vswitchName" -module $module $vnet_ipaddressprefix = Get-ConfigurationValue -name "ipaddressprefix" -module $module $vnet_gateway = Get-ConfigurationValue -name "gateway" -module $module $vnet_dnsservers = (Get-ConfigurationValue -name "dnsservers" -module $module) -split "," $vnet_macpoolname = Get-ConfigurationValue -name "macpoolname" -module $module $vnet_vlanid = Get-ConfigurationValue -name "vlanid" -module $module $vnet_vippoolstart = Get-ConfigurationValue -name "vnetvippoolstart" -module $module $vnet_vippoolend = Get-ConfigurationValue -name "vnetvippoolend" -module $module $vnet_k8snodeippoolstart = Get-ConfigurationValue -name "k8snodeippoolstart" -module $module $vnet_k8snodeippoolend = Get-ConfigurationValue -name "k8snodeippoolend" -module $module return [VirtualNetwork]::new($vnet_name, $vnet_vswitchname, $vnet_ipaddressprefix, $vnet_gateway, $vnet_dnsservers, $vnet_macpoolname, $vnet_vlanid, $vnet_vippoolstart, $vnet_vippoolend, $vnet_k8snodeippoolstart, $vnet_k8snodeippoolend) } function New-VirtualNetwork { <# .DESCRIPTION A wrapper around [VirutalNetwork]::new that Validates parameters before returning a VirtualNetwork object .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER MacPoolName The name of the mac pool .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .EXAMPLE New-VirtualNetwork -name External -vswitchname External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-VirtualNetwork -name "defaultswitch" -vswitchname "Default Switch" -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [String] $MacPoolName = $global:cloudMacPool, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend ) Test-ValidNetworkName -Name $name | Out-Null if ($dnsservers) { foreach ($dns in $dnsservers) { try { Test-ValidEndpoint -endpoint $dns } catch { throw ("$dnsservers is not a valid list of ip addresses. Please enter a valid list of ip addresses: E.g. -dnsservers 4.4.4.4, 8.8.8.8") } } } if ($ipaddressprefix -or $gateway -or $dnsservers -or $k8snodeippoolstart -or $k8snodeippoolend) { if (-not $ipaddressprefix -or -not $gateway -or -not $dnsservers -or -not $k8snodeippoolstart -or -not $k8snodeippoolend) { throw "ipaddressprefix, gateway, dnsservers, k8snodeippoolstart, and k8snodeippoolend must all be specified to use a static ip configuration" } } if ($ipaddressprefix) { Test-ValidCIDR -CIDR $ipaddressprefix } Test-ValidPool -PoolStart $vippoolstart -PoolEnd $vippoolend -CIDR $ipaddressprefix if ($k8snodeippoolstart -and $k8snodeippoolend) { Test-ValidPool -PoolStart $k8snodeippoolstart -PoolEnd $k8snodeippoolend -CIDR $ipaddressprefix } return [VirtualNetwork]::new($name, $vswitchname, $ipaddressprefix, $gateway, $dnsservers, $MacPoolName, $vlanID, $vippoolstart, $vippoolend, $k8snodeippoolstart, $k8snodeippoolend) } #region configuration function Save-ConfigurationDirectory { <# .DESCRIPTION Saves the workingDir of configuration in registry. Handles multinode as well. .PARAMETER WorkingDir WorkingDir to be persisted .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName, [Parameter(Mandatory=$true)] [String] $WorkingDir ) $configDir = [io.Path]::Combine($WorkingDir, "." + $moduleName) Write-Status "Saving Configuration Directory [$configDir]" -moduleName $moduleName if (Test-MultiNodeDeployment) { # *. If Multinode, replicate this across all nodes Get-ClusterNode -ErrorAction Stop | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { $regPath = $args[0] $regKey = $args[1] $regValue = $args[2] if (!(Test-Path ($regPath))) { New-Item -Path $regPath | Out-Null } Set-ItemProperty -Path $regPath -Name $regKey -Value $regValue -Force | Out-Null } -ArgumentList @($global:configurationKeys[$moduleName], $script:psConfigKeyName, $configDir) } } else { # *. If Standalone, store it locally if (!(Test-Path ($global:configurationKeys[$moduleName]))) { New-Item -Path $global:configurationKeys[$moduleName] | Out-Null } Set-ItemProperty -Path $global:configurationKeys[$moduleName] -Name $script:psConfigKeyName -Value $configDir -Force | Out-Null } } function Set-SecurePermissionFolder { <# .DESCRIPTION Initialize folder with appropriate permissions .PARAMETER Path path of config folder. #> param ( [parameter(Mandatory=$true)] [string]$Path ) $acl = Get-Acl $Path $acl.SetAccessRuleProtection($true,$false) $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators","FullControl","ContainerInherit,ObjectInherit", "None", "Allow") $acl.SetAccessRule($accessRule) $acl | Set-Acl $Path } function Set-SecurePermissionFile { <# .DESCRIPTION Initialize file with appropriate permissions .PARAMETER Path path of file. #> param ( [parameter(Mandatory=$true)] [string]$Path ) # ACL the yaml so that it is only readable by administrator $acl = Get-Acl $Path $acl.SetAccessRuleProtection($true,$false) $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators","FullControl","Allow") $acl.SetAccessRule($accessRule) $acl | Set-Acl $Path } function Reset-ConfigurationDirectory { <# .DESCRIPTION Cleanup workingDir info in registry that has been saved .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) Write-Status "Resetting Configuration Directory" -moduleName $moduleName if (Test-MultiNodeDeployment) { # *. If Multinode, remove this across all nodes Get-ClusterNode -ErrorAction Stop | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { $regPath = $args[0] Remove-Item -Path $regPath -Force -ErrorAction SilentlyContinue | Out-Null } -ArgumentList @($global:configurationKeys[$moduleName]) } } else { # *. If Standalone, remove it locally Remove-Item -Path $global:configurationKeys[$moduleName] -Force -ErrorAction SilentlyContinue | Out-Null } } function Get-ConfigurationDirectory { <# .DESCRIPTION Gets the Working Directory of configuration from the registry .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # 1. If the psconfig path is available, try to use it. $regVal = Get-ItemPropertyValue -Path $global:configurationKeys[$moduleName] -Name $script:psConfigKeyName -ErrorAction SilentlyContinue if ($regVal) { return $regVal } # 2. If not, use the default path return [io.Path]::Combine($global:psConfigDirectoryRoot, "." + $moduleName) } function Get-ConfigurationFile { <# .DESCRIPTION Get the configuration file to be used for persisting configurations .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) $configFile = [io.Path]::Combine((Get-ConfigurationDirectory -moduleName $moduleName), $script:psConfigJson) # Write-Status "Configuration file for $moduleName => [$configFile]" return $configFile } function Test-Configuration { <# .DESCRIPTION Tests if a configuration exists .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # Write-Status "Testing Configuration for $moduleName" return Test-Path -Path (Get-ConfigurationFile -moduleName $moduleName) } function Reset-Configuration { <# .DESCRIPTION Resets the configuration Resets also the configuration info that persisted in registry. Does a double cleanup so the one in working dir as well as user directory is removed .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) Write-Status "Resetting Configuration" -moduleName $moduleName # 1. Remove the shared configuration if (Test-Configuration -moduleName $moduleName) { Remove-Item -Path (Get-ConfigurationDirectory -moduleName $moduleName) -Recurse -Force -ErrorAction SilentlyContinue } Reset-ConfigurationDirectory -moduleName $moduleName # 2. Remove the local configuration, if any if (Test-Configuration -moduleName $moduleName) { Remove-Item -Path (Get-ConfigurationDirectory -moduleName $moduleName) -Recurse -Force -ErrorAction SilentlyContinue } $global:config[$moduleName] = @{} } function Save-Configuration { <# .DESCRIPTION saves a configuration to persisted storage .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # Write-Status "Saving Configuration for $moduleName" $configFile = Get-ConfigurationFile -moduleName $moduleName $configDir = [IO.Path]::GetDirectoryName($configFile) if (!(Test-Path $configDir)) { New-Item -ItemType Directory -Force -Path $configDir | Out-Null } ConvertTo-Json -InputObject $global:config[$moduleName] | Out-File -FilePath $configFile } function Import-Configuration { <# .DESCRIPTION Loads a configuration from persisted storage .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # Write-Status "Importing Configuration for $moduleName" $tmp = ConvertFrom-Json -InputObject (Get-Content (Get-ConfigurationFile -moduleName $moduleName) -Raw) $global:config[$moduleName] = @{} $tmp.psobject.Properties | ForEach-Object { $global:config[$moduleName][$_.Name] = $_.Value} } function Set-ConfigurationValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Parameter(Mandatory=$true)] [String] $module, [Object] $value ) $global:config[$module][$name] = $value Save-Configuration -moduleName $module } function Get-ConfigurationValue { <# .DESCRIPTION Retrieves a configuration value from the registry .PARAMETER type The expected type of the value being retrieved .PARAMETER value Name of the configuration value #> param ( [Type] $type = [System.String], [Parameter(Mandatory=$true)] [String] $module, [String] $name ) $value = $null if (Test-Configuration -moduleName $module) { Import-Configuration -moduleName $module $value = $global:config[$module][$name] } switch($type.Name) { "Boolean" { if (!$value) {$value = 0} return [System.Convert]::ToBoolean($value) } "UInt32" { if (!$value) {$value = 0} return [System.Convert]::ToUInt32($value) } "VmSize" { if (!$value) {$value = 0} return [Enum]::Parse([VmSize], $value, $true) } "DeploymentType" { if (!$value) {$value = 0} return [Enum]::Parse([DeploymentType], $value, $true) } "InstallState" { if (!$value) {$value = 0} return [Enum]::Parse([InstallState], $value, $true) } "LoadBalancerType" { if (!$value) {$value = 0} return [Enum]::Parse([LoadBalancerType], $value, $true) } Default { if (!$value) {$value = ""} return $value } } } #endregion function Test-IsProductInstalled() { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name to test for installation state .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $moduleName, [Parameter()] [String]$activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Verifying product installation state") $currentState = Get-InstallState -module $moduleName if (-not $currentState) { return $false } switch($currentState) { $([InstallState]::NotInstalled) { return $false } $([InstallState]::InstallFailed) { return $false } } return $true } function Get-InstallState { <# .DESCRIPTION Returns the installation state for a product/module. May return $null. .PARAMETER moduleName The module name to query #> param ( [String] $moduleName ) $state = Get-ConfigurationValue -module $moduleName -name "InstallState" -type ([Type][InstallState]) if (-not $state) { $state = [InstallState]::NotInstalled } Write-SubStatus -moduleName $moduleName $("Installation state is: $state") return $state } function Test-ValidEndpoint { <# .DESCRIPTION Validates that an endpoint is a valid ip address This function exists to validate that a endpoint is a valid ip address. If the enpoint is not a valid ip address, it throws. .PARAMETER endpoint A string representing an IP address #> param ( [String] $endpoint ) if (-not [ipaddress]::TryParse($endpoint,[ref]$null)) { throw ("$endpoint is not a valid ip address. Please enter a valid ip address in the form 1.1.1.1") } } function Test-ValidK8sObjectName { <# .DESCRIPTION Validates the format of a name that will be used as the name of a kubernetes object This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name The name of an k8s object .PARAMETER Type Currently we are only using k8s names for clusters and networks #> param ( [String] $Name, [ValidateSet("cluster","network")] [String] $Type ) if (-not ($Name -cmatch $clusterNameRegex)) { throw $("'$Name' is not a valid $Type name. Names must be lower-case and match the regex pattern: '$clusterNameRegex'") } return $true } function Test-ValidClusterName { <# .DESCRIPTION Validates the format of a cluster name. This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name A cluster name #> param ( [String] $Name ) return Test-ValidK8sObjectName -Name $Name -Type "cluster" } function Test-ValidNetworkName { <# .DESCRIPTION Validates the format of a network name. This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name A network name #> param ( [String] $Name ) return Test-ValidK8sObjectName -Name $Name -Type "network" } function Test-ForUpdates { <# .DESCRIPTION Check if a module is up to date and provide the option to update it. .PARAMETER moduleName The name of the module. .PARAMETER repositoryName Powershell repository name. .PARAMETER repositoryUser Powershell repository username. .PARAMETER repositoryPass Powershell repository password. #> param ( [String] $moduleName, [String] $repositoryName, [String] $repositoryUser, [String] $repositoryPass ) if ($global:config[$moduleName]["skipUpdates"]) { return } Write-Status $("Check module updates") -moduleName $moduleName $proxyParameters = @{} $proxyConfig = Get-ProxyConfiguration -moduleName $moduleName if ($proxyConfig.HTTPS) { $proxyParameters.Add("Proxy", $proxyConfig.HTTPS) } elseif ($proxyConfig.HTTP) { $proxyParameters.Add("Proxy", $proxyConfig.HTTP) } if ($proxyConfig.Credential) { $proxyParameters.Add("ProxyCredential", $proxyConfig.Credential) } $current = Get-InstalledModule -Name "PowershellGet" -ErrorAction SilentlyContinue if (($null -eq $current) -or ($current.version -lt 1.6.0)) { Write-SubStatus "PowershellGet is too old and needs to be updated. Updating now...`n" -moduleName $moduleName $parameters = @{ Name = "PowershellGet" Force = $true Confirm = $false SkipPublisherCheck = $true } $parameters += $proxyParameters Install-Module @parameters Write-SubStatus "PowershellGet was updated. This window *must* now be closed. Please re-run the script to continue." -moduleName $moduleName exit 0 } $patToken = $repositoryPass | ConvertTo-SecureString -AsPlainText -Force $repositoryCreds = New-Object System.Management.Automation.PSCredential($repositoryUser, $patToken) $current = Get-InstalledModule -Name $moduleName -ErrorAction SilentlyContinue # TODO: Skip alpha/preview updates for now if ($current.Repository -ieq $global:repositoryNamePreview) { Write-SubStatus "Your current version is a pre-release. Updates will be skipped.`n" -moduleName $moduleName return } $parameters = @{ Name = $moduleName Repository = $repositoryName Credential = $repositoryCreds ErrorAction = "SilentlyContinue" } $parameters += $proxyParameters $latest = Find-Module @parameters if (($null -eq $current) -or ($null -eq $latest)) { Write-SubStatus "Warning: Unable to check for updates" -moduleName $moduleName return } Write-SubStatus $("Installed module version is "+$current.version) -moduleName $moduleName Write-SubStatus $("Latest module version is "+$latest.version) -moduleName $moduleName if ([System.Version]$current.version -ge [System.Version]$latest.version) { Write-SubStatus "You are already up to date" -moduleName $moduleName return } Write-SubStatus $("A newer version of "+$moduleName+" is available!") -moduleName $moduleName $title = 'Recommended update' $question = "Do you want to update to the latest version of the module/binaries? If you choose 'Yes' then we will perform a full cleanup of your deployment before applying the updates.`n" $choices = '&Yes', '&No' $decision = $Host.UI.PromptForChoice($title, $question, $choices, 0) if ($decision -eq 0) { Write-Status "Installing updates" -moduleName $moduleName $parameters = @{ Name = $moduleName Credential = $repositoryCreds Force = $true } $parameters += $proxyParameters Update-Module @parameters $current = Get-InstalledModule -Name $moduleName -ErrorAction SilentlyContinue Write-SubStatus $("Installed module version is now "+$current.version+"`n") -moduleName $moduleName if ([System.Version]$current.version -ge [System.Version]$latest.version) { Write-SubStatus $("Removing older versions of the module...`n") -moduleName $moduleName Get-InstalledModule -Name $moduleName -AllVersions | Where-Object {$_.Version -ne $current.Version} | Uninstall-Module -Force -Confirm:$false -ErrorAction:SilentlyContinue Write-SubStatus "The update was successful! This window *must* now be closed. Please re-run the script to continue." -moduleName $moduleName exit 0 } throw $("The update operation was not successful. Your installed module version ("+$current.version+") is still older than the latest version ("+$latest.version+").") } } function Initialize-Environment { <# .DESCRIPTION Executes steps to prepare the environment for operations. This includes checking to ensure that the module is up to date and that a configuration is present. .PARAMETER moduleName The name of the module. .PARAMETER repositoryName Powershell repository name. .PARAMETER repositoryUser Powershell repository username. .PARAMETER repositoryPass Powershell repository password. .PARAMETER checkForUpdates Should the script check for updates before proceeding to prepare the environment. .PARAMETER createIfNotPresent Should the script create a new deployment configuration if one is not present. #> param ( [Parameter()] [String] $moduleName, [Parameter()] [String] $repositoryName = $global:repositoryName, [Parameter()] [String] $repositoryUser = $global:repositoryUser, [Parameter()] [String] $repositoryPass = $global:repositoryPass, [Parameter()] [Switch]$checkForUpdates, [Parameter()] [Switch]$createIfNotPresent ) Initialize-ProxyEnvironment -moduleName $moduleName if ($checkForUpdates.IsPresent) { Test-ForUpdates -moduleName $moduleName -repositoryName $repositoryName -repositoryUser $repositoryUser -repositoryPass $repositoryPass } } function Get-FirewallRules { <# .DESCRIPTION Obtains the firewall rules needed for agent communication #> $firewallRules = @( ("wssdagent GRPC server port", "TCP", $global:config[$global:MocModule]["nodeAgentPort"]), ("wssdagent GRPC authentication port", "TCP", $global:config[$global:MocModule]["nodeAgentAuthorizerPort"]), ("wssdcloudagent GRPC server port", "TCP", $global:config[$global:MocModule]["cloudAgentPort"]), ("wssdcloudagent GRPC authentication port", "TCP", $global:config[$global:MocModule]["cloudAgentAuthorizerPort"]) ) return $firewallRules } #endregion #region General helper functions function Write-StatusWithProgress { <# .DESCRIPTION Outputs status to progress and to console .PARAMETER status The status message .PARAMETER activity The progress activity .PARAMETER percentage The progress percentage. 100% will output progress completion .PARAMETER moduleName The module name. Will become a prefix for console output #> [CmdletBinding()] param ( [String] $status = "", [String] $activity = "Status", [Int] $percentage = -1, [Switch] $completed, [Parameter(Mandatory=$true)] [String] $moduleName ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } $message = $status if ($moduleName) { $eventmessage = $("$activity - $message") $message = $("$moduleName - $message") Write-ModuleEventLog -moduleName $moduleName -entryType Information -eventId 1 -message $eventmessage } Write-Progress -Activity $activity -Status $message -PercentComplete $percentage -Completed:$completed.IsPresent Write-Status -msg $status -moduleName $moduleName } function Write-Status { <# .DESCRIPTION Outputs status to the console with a prefix for readability .PARAMETER msg The message/object to output .PARAMETER moduleName The module name. Will be used as a prefix #> [CmdletBinding()] param ( [Object]$msg, [Parameter(Mandatory=$true)] [String]$moduleName ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } if ($msg) { $time = Get-Date -DisplayHint Time Write-Verbose "[$time] [$moduleName] $msg`n" } } function Write-SubStatus { <# .DESCRIPTION Outputs sub-status to the console with a indent for readability .PARAMETER msg The message/object to output .PARAMETER moduleName The module name. Will be used as a prefix .PARAMETER indentChar Char to use as the indent/bulletpoint of the status message #> [CmdletBinding()] param ( [Object]$msg, [Parameter(Mandatory=$true)] [String]$moduleName, [String]$indentChar = "`t`t" ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } Write-Status -msg $($indentChar + $msg) -moduleName $moduleName } function Test-Process { <# .DESCRIPTION Test if a process is running. .PARAMETER processName The process name to test. .PARAMETER nodeName The node to test on. #> param ( [String]$processName, [String]$nodeName ) Invoke-Command -ComputerName $nodeName -ScriptBlock { $processName = $args[0] $process = Get-Process -Name $processName -ErrorAction SilentlyContinue if(!$process) { throw $("$processName is not running on " + ($env:computername)) } } -ArgumentList $processName } function Test-LocalFilePath { <# .DESCRIPTION Returns true if the path appears to be local. False otherwise. .PARAMETER path Path to be tested. #> param( [String]$path ) $path = [System.Environment]::ExpandEnvironmentVariables($path) $clusterStorage = Join-Path -Path $([System.Environment]::ExpandEnvironmentVariables("%systemdrive%")) -ChildPath 'clusterstorage' if($path.StartsWith('\\') -or $path.ToLower().StartsWith($clusterStorage.ToLower())) { return $false } return $true } function Stop-AllProcesses { <# .DESCRIPTION Kill all instance of the given process name. #> param ( [string]$processName ) Get-Process -Name $processName -ErrorAction SilentlyContinue | ForEach-Object { $_ | Stop-Process -Force } } function Copy-FileLocal { <# .DESCRIPTION Copies a file locally. .PARAMETER source File source. .PARAMETER destination File destination. #> param ( [String]$source, [String]$destination ) Write-SubStatus "Copying $source to $destination " -moduleName $global:CommonModule Copy-Item -Path $source -Destination $destination } function Copy-FileToRemoteNode { <# .DESCRIPTION Copies a file to a remote node. .PARAMETER source File source. .PARAMETER destination File destination. .PARAMETER remoteNode The remote node to copy to. #> param ( [String]$source, [String]$destination, [String]$remoteNode ) $remotePath = "\\$remoteNode\" + ($destination).Replace(":", "$") Write-SubStatus "Copying $source to $remotePath " -moduleName $global:CommonModule $remoteDir = [IO.Path]::GetDirectoryName($remotePath) if (!(Test-Path $remoteDir)) { New-Item -ItemType Directory -Force -Path $remoteDir | Out-Null } Copy-Item -Path $source -Destination $remotePath } function Test-ForWindowsFeatures { <# .DESCRIPTION Installs any missing required OS features. .PARAMETER features The features to check for. .PARAMETER nodeName The node to execute on. #> param ( [String[]]$features, [String]$nodeName ) Write-Status "Check for required OS features on $nodeName" -moduleName $global:MocModule $nodeEditionName = Invoke-Command -ComputerName $nodeName -ScriptBlock { return (get-itemproperty "hklm:\software\microsoft\windows nt\currentversion").editionid } if (-not ($nodeEditionName -match 'server')) { throw "This product is only supported on server editions" } $remoteRebootRequired = Invoke-Command -ComputerName $nodeName -ScriptBlock { $rebootRequired = $false foreach($feature in $args[0]) { write-verbose $(" - Checking the status of feature '$feature'") $wf = Get-WindowsFeature -Name "$feature" if ($null -eq $wf) { throw $("Windows feature '$feature' does not seem to be present in this OS version and therefore cannot be enabled.") } if ($wf.InstallState -ine "Installed") { write-verbose $(" - Installing missing feature '$feature' ...") $result = Install-WindowsFeature -Name "$feature" -WarningAction SilentlyContinue if ($result.RestartNeeded) { $rebootRequired = $true } } } return $rebootRequired } -ArgumentList (, $features) if ($remoteRebootRequired) { Write-Status $("OS features were installed and a reboot is required to complete the installation") -moduleName $global:MocModule Read-Host $("Press enter when you are ready to reboot $nodeName ...") Restart-Computer -ComputerName $nodeName -Force } } function Enable-Remoting { <# .DESCRIPTION Enables powershell remoting on the local machine. #> Write-Status "Enabling powershell remoting..." -moduleName $global:MocModule Enable-PSRemoting -Force -Confirm:$false winrm quickconfig -q -force } function Get-HostRoutingInfo { <# .DESCRIPTION Obtains the host routing information. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) return Invoke-Command -ComputerName $nodeName -ScriptBlock { $oldProgressPreference = $global:progressPreference $global:progressPreference = 'silentlyContinue' $computerName = "www.msftconnecttest.com" $routingInfo = Test-NetConnection -DiagnoseRouting -ComputerName $computerName -ErrorAction SilentlyContinue $global:progressPreference = $oldProgressPreference if (!$routingInfo -or !$routingInfo.RouteDiagnosticsSucceeded) { throw $("Unable to obtain host routing. Connectivity test to $computerName failed.") } return $routingInfo } } function Get-HostAdapterName { <# .DESCRIPTION Obtains the name of the best host network adapter. Uses routing to determine the host interface name to return. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) $routingInfo = Get-HostRoutingInfo -nodeName $nodeName return $routingInfo.OutgoingInterfaceAlias } function Get-HostIp { <# .DESCRIPTION Obtains the hosts IP address. Uses routing to determine the best host IPv4 address to use. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) $routingInfo = Get-HostRoutingInfo -nodeName $nodeName return $routingInfo.SelectedSourceAddress.IPv4Address } function Test-Binary { <# .DESCRIPTION A basic sanity test to make sure that this system is ready to deploy kubernetes. .PARAMETER nodeName The node to execute on. .PARAMETER binaryName Binary to check. #> param ( [String]$nodeName, [String]$binaryName ) Invoke-Command -ComputerName $nodeName -ScriptBlock { if ( !(Get-Command $args -ErrorAction SilentlyContinue )) { throw $("Expected binary $args is missing.") } } -ArgumentList $binaryName } function Get-CloudFqdn { <# .DESCRIPTION Determines the right FQDN to use for cloudagent based on the type of deployment and script args #> return $global:config[$global:MocModule]["cloudFqdn"] } function Get-SshPublicKey { <# .DESCRIPTION Get the SSH Public Key that is configured to be used by the deployment #> return $global:config[$global:MocModule]["sshPublicKey"] } function Get-SshPrivateKey { <# .DESCRIPTION Get the SSH Private Key that is configured to be used by the deployment #> return $global:config[$global:MocModule]["sshPrivateKey"] } #region invoke methods function Invoke-CommandLine { <# .DESCRIPTION Executes a command and optionally ignores errors. .PARAMETER command Comamnd to execute. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER showOutputAsProgress Optionally, show output from the executing command as progress bar updates. .PARAMETER progressActivity The activity name to display when showOutputAsProgress was requested. .PARAMETER moduleName The calling module name to show in output logging. #> param ( [String]$command, [String]$arguments, [Switch]$ignoreError, [Switch]$showOutput, [Switch]$showOutputAsProgress, [String]$progressActivity, [Parameter(Mandatory=$true)] [String]$moduleName ) try { if ($showOutputAsProgress.IsPresent) { $result = (& $command $arguments.Split(" ") | ForEach-Object { $status = $_ -replace "`t"," - "; Write-StatusWithProgress -activity $progressActivity -moduleName $moduleName -Status $status }) 2>&1 } elseif ($showOutput.IsPresent) { $result = (& $command $arguments.Split(" ") | Out-Default) 2>&1 } else { $result = (& $command $arguments.Split(" ") 2>&1) } } catch { if ($ignoreError.IsPresent) { return } throw } $out = $result | Where-Object {$_.gettype().Name -ine "ErrorRecord"} # On a non-zero exit code, this may contain the error #$outString = ($out | Out-String).ToLowerInvariant() if ($LASTEXITCODE) { $err = $result | Where-Object {$_.gettype().Name -eq "ErrorRecord"} $errMessage = "$command $arguments returned a non zero exit code $LASTEXITCODE [$err]" if ($ignoreError.IsPresent) { $ignoreMessage = "[IGNORED ERROR] $errMessage" Write-Status -msg $ignoreMessage -moduleName $moduleName Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message $errMessage return } throw $errMessage } return $out } function Invoke-Kubectl { <# .DESCRIPTION Executes a kubectl command. .PARAMETER kubeconfig The kubeconfig file to use. Defaults to the management kubeconfig. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. #> param ( [string] $kubeconfig = $global:config[$global:KvaModule]["kubeconfig"], [string] $arguments, [switch] $ignoreError, [switch] $showOutput ) return Invoke-CommandLine -command $global:kubeCtlFullPath -arguments $("--kubeconfig=$kubeconfig $arguments") -showOutput:$showOutput.IsPresent -ignoreError:$ignoreError.IsPresent -moduleName $global:KvaModule } function Invoke-MocShowCommand { <# .DESCRIPTION Executes a cloudagent command either against a local cloudagent (single node deployment) or a cluster generic service (multi-node/cluster deployments). .PARAMETER arguments Arguments to pass to cloud ctl. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). #> param ( [String]$arguments, [Switch]$ignoreError, [ValidateSet("tsv", "csv", "yaml", "json")] [string]$output = "json" ) $arguments += " --output $output" $out = Invoke-MocCommand -arguments $arguments -ignoreError:$ignoreError.IsPresent if ([string]::IsNullOrWhiteSpace($out)) { return } return $out | ConvertFrom-Json } function Invoke-MocListCommand { <# .DESCRIPTION Executes a cloudagent command either against a local cloudagent (single node deployment) or a cluster generic service (multi-node/cluster deployments). .PARAMETER arguments Arguments to pass to cloud ctl. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). #> param ( [String]$arguments, [Switch]$ignoreError, [ValidateSet("tsv", "csv", "yaml", "json")] [string]$output = "json", [string]$filter ) $arguments += " --output $output" if ($filter) { $arguments += " --query ""$filter""" } $out = Invoke-MocCommand -arguments $arguments -ignoreError:$ignoreError.IsPresent if ([string]::IsNullOrWhiteSpace($out) -or $out -like "No *") { return } Write-Verbose "$out" return $out | ConvertFrom-Json } function Invoke-MocCommand { <# .DESCRIPTION Executes a cloudagent command either against a local cloudagent (single node deployment) or a cluster generic service (multi-node/cluster deployments). .PARAMETER arguments Arguments to pass to cloud ctl. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). #> param ( [String]$arguments, [Switch]$ignoreError ) if (-not (Get-Command $global:cloudCtlFullPath -ErrorAction SilentlyContinue)) { throw $("Unable to find command "+$global:cloudCtlFullPath) } $cloudFqdn = Get-CloudFqdn $cmdArgs = "--cloudFqdn $cloudFqdn $arguments" if ($global:config[$global:MocModule]["insecure"]) { $cmdArgs += " --debug" } $response = Invoke-CommandLine -Command $global:cloudCtlFullPath -Arguments $cmdArgs -ignoreError:$ignoreError -moduleName $global:MocModule Write-Status $response -moduleName $global:MocModule return $response } function Invoke-NodeCommand { <# .DESCRIPTION Executes a nodeagent command. .PARAMETER arguments Arguments to pass to node ctl. #> param ( [String]$arguments ) if (-not (Get-Command $global:nodeCtlFullPath -ErrorAction SilentlyContinue)) { throw $("Unable to find command "+$global:nodeCtlFullPath) } $cmdArgs = "$arguments" if ($global:config[$global:MocModule]["insecure"]) { $cmdArgs += " --debug" } Invoke-CommandLine -Command $global:nodeCtlFullPath -Arguments $cmdArgs -moduleName $global:MocModule } function Invoke-MocLogin { <# .DESCRIPTION Provisions the Script to have access to node ctl .PARAMETER nodeName The node to execute on. #> param ( [Parameter(Mandatory=$true)] [String]$loginYaml ) Invoke-MocCommand $(" security login --loginpath $loginYaml --identity") } #end region function Compress-Directory { <# .DESCRIPTION Util for zipping folders .PARAMETER ZipFilename output zip file name .PARAMETER SourceDir directory to compress #> param ( [Parameter(Mandatory=$true)] [String]$ZipFilename, [Parameter(Mandatory=$true)] [String]$SourceDir ) if (Test-Path $ZipFilename) { $title = 'ZipFile already exists' $question = "Do you want to overwrite it?`n" $choices = '&Yes', '&No' $decision = $Host.UI.PromptForChoice($title, $question, $choices, 0) if ($decision -eq 0) { Remove-Item -Path $ZipFilename -Force } else { throw [System.IO.IOException] "file already exists" } } Add-Type -Assembly System.IO.Compression.FileSystem $compressionLevel = [System.IO.Compression.CompressionLevel]::Optimal [System.IO.Compression.ZipFile]::CreateFromDirectory($SourceDir,$ZipFilename, $compressionLevel, $false) } #endregion #region Resource limit helper functions function Convert-ParametersToString { <# .DESCRIPTION Takes a dictionary of parameters and converts them to a string representation. .PARAMETER argDictionary Dictionary of arguments (e.g. obtained from $PSBoundParameters). .PARAMETER stripAsJob Optionally remove 'AsJob' if it's included in the argDictionary. #> param ( [System.Collections.IDictionary] $argDictionary, [Switch] $stripAsJob ) $strArgs = "" foreach ($key in $argDictionary.Keys) { if (($key -ieq "AsJob") -and ($stripAsJob.IsPresent)) { continue } $seperator = " " $val = $argDictionary[$key] if (($val -eq $true) -or ($val -eq $false)) { $seperator = ":$" } $strArgs += " -$key$seperator$val" } return $strArgs } #endregion #region prechecks function Test-HCIRegistration { <# .DESCRIPTION Check the SKU of node and if HCI substrate then check the registration status of the node #> $osResult = Get-CimInstance -Namespace root/CimV2 -ClassName Win32_OperatingSystem -Property OperatingSystemSKU # PRODUCT_AZURESTACKHCI_SERVER_CORE = 406 # Check if the substrate is HCI if ($osResult.OperatingSystemSKU -eq 406) { #Check if the HCI node is registered else throw error $regStatus = (Get-AzureStackHCI).RegistrationStatus if ($regStatus -ine "Registered") { throw "HCI cluster node ($env:computername) is not registered, registrationStatus is $regStatus" } } } function Test-ClusterHealth { <# .DESCRIPTION Check if cluster node and cluster network are up #> #Check the ClusterNode Get-ClusterNode -ErrorAction Stop | ForEach-Object { if ($_.State -ine "Up") { throw "Cluster node $_ is not Up. Its current state is $($_.State)." } } Get-ClusterNetwork -ErrorAction Stop | ForEach-Object { if ($_.State -ine "Up") { throw "Cluster network $_ is not Up.Its current state is $($_.State)." } } } #endregion #region Asynchronous job helpers function Get-BackgroundJob { <# .DESCRIPTION Returns a background job by name (supports wildcards and may return multiple jobs) .PARAMETER name Name of the job(s). #> param ( [Parameter(Mandatory=$true)] [String] $name ) return Get-Job -Name $name -ErrorAction SilentlyContinue } function New-BackgroundJob { <# .DESCRIPTION Creates a new background job. .PARAMETER name Name of the job. .PARAMETER cmdletName Cmdlet to execute as a job .PARAMETER argDictionary Argument dictionary to pass to the job cmdlet. .PARAMETER scheduledJob Optionally, use a scheduled job instead of a regular job. This allows the job to execute fully in the background and be accessible cross-session. Note that less progress information is available while this type of job is running. .PARAMETER allowDuplicateJobs Optionally, allow a new job to be created even if a job with the same name already exists and is still executing. #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $cmdletName, [Parameter(Mandatory=$true)] [System.Collections.IDictionary] $argDictionary, [Parameter()] [Switch] $scheduledJob, [Parameter()] [Switch] $allowDuplicateJobs ) if (-not $allowDuplicateJobs.IsPresent) { $jobs = Get-BackgroundJob -name $name foreach ($job in $jobs) { if (($job.State -ieq "Completed") -or ($job.State -ieq "Failed")) { continue } throw $("A job with the name '$name' already exists and has not yet completed or failed. Please wait or remove it using Remove-Job.") } } $strArgs = Convert-ParametersToString($argDictionary) -stripAsJob if ($scheduledJob.IsPresent) { if (-not $strArgs) { # Always pass at least one arg $strArgs = '-Verbose:$false' } $options = New-ScheduledJobOption -RunElevated -HideInTaskScheduler return Register-ScheduledJob -Name $name -ScheduledJobOption $options -RunNow -ScriptBlock { param($p1,$p2) #$VerbosePreference = "continue" Invoke-Expression $("$p1 $p2") } -ArgumentList $cmdletName,$strArgs } else { return Start-Job -Name $name -ScriptBlock { Invoke-Expression $("$using:cmdletName $using:strArgs") } } } #endregion #region Wait for resource functions function Wait-ForCloudAgentEndpoint { <# .DESCRIPTION Waits for the cloudagent generic service VIP/FQDN to be functional (i.e. wait for DNS to propogate). .PARAMETER sleepDuration Duration to sleep for between attempts .PARAMETER timeout Duration until timeout of waiting .PARAMETER activity Activity name to use when updating progress #> param ( [int]$sleepDuration=20, [int]$timeout=3600, #seconds in a hour [String]$activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -status $("Waiting for cloudagent API endpoint to be accessible...") -moduleName $global:MocModule Write-SubStatus $("Warning: this depends on DNS propogation and can take between 10-30 minutes in some environments...`n") -moduleName $global:MocModule $endpoint = Get-CloudFqdn ## Start the timer $timer = [Diagnostics.Stopwatch]::StartNew() while(($timer.Elapsed.TotalSeconds -lt $timeout)) { Write-SubStatus $("Testing cloudagent endpoint: $endpoint") -moduleName $global:MocModule $location = $null try { Clear-DnsClientCache if (-Not ($global:config[$global:MocModule]["insecure"])) { try { Invoke-MocLogin -loginYaml $($global:config[$global:MocModule]["mocLoginYAML"]) | Out-Null } catch { Write-Verbose -Message $_ } } $location = Get-MocLocation } catch { Write-Verbose -Message $_ } if ($null -ne $location) { Write-SubStatus $("Cloudagent VIP is working.") -moduleName $global:MocModule return $true } Sleep $sleepDuration } return $false } #endregion function Get-FailoverCluster { <# .DESCRIPTION Safe wrapper around checking for the presence of a failover cluster. #> # Check if failover cluster powershell module was installed # and only run Get-Cluster in that case if (Get-Command "Get-Cluster" -errorAction SilentlyContinue) { return Get-Cluster -ErrorAction SilentlyContinue } return $null } function Get-KubernetesGalleryImageName { <# .DESCRIPTION Returns the appropriate gallery image name based on kubernetes versions .PARAMETER k8sVersion Kubernetes version .PARAMETER imageType Image Type #> param ( [Parameter(Mandatory=$true)] [String]$k8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String]$imageType ) $tmpVersion = ($k8sVersion.TrimStart("v").Replace('.', '-')) switch ($imageType) { "Windows" { return "Windows_k8s" } "Linux" { return "Linux_k8s_" + $tmpVersion } } } function Test-MultiNodeDeployment { <# .DESCRIPTION Returns true if the script believes this is a multi-node deployment. False otherwise. #> $failoverCluster = Get-FailoverCluster return ($nil -ne $failoverCluster) } function Get-Ipv4MaskFromPrefix { <# .DESCRIPTION Transforms an IP prefix length to an IPv4 net mask. .PARAMETER PrefixLength Length of the prefix #> param ( [Parameter(Mandatory=$true)] [int] $PrefixLength ) Add-Type -Language CSharp -ReferencedAssemblies "System.Numerics.dll" @" using System; using System.Numerics; namespace AKSHCI { public static class Ipv4MaskCompute { public static byte[] GetIpv4MaskFromPrefix(int prefixLength) { BigInteger fullMask = new BigInteger(0xFFFFFFFF); BigInteger mask = ((fullMask << (32 - prefixLength)) & fullMask); return mask.ToByteArray(); } } } "@; if ($PrefixLength -lt 0) { throw "Invalid prefix length $PrefixLength" } if ($PrefixLength -eq 0) { return "0.0.0.0" } if ($PrefixLength -gt 32) { throw "Invalid prefix length $PrefixLength" } $maskArray = [AKSHCI.Ipv4MaskCompute]::GetIpv4MaskFromPrefix($PrefixLength) return "$($maskArray[3]).$($maskArray[2]).$($maskArray[1]).$($maskArray[0])" } function Get-ClusterNetworkPrefixForIp { <# .DESCRIPTION Returns the cluster network prefix length associated with the given IP, or $null if not found. .PARAMETER IpAddress Find the cluster network associated with this IP address #> param ( [Parameter(Mandatory=$true)] [System.Net.IPAddress] $IpAddress ) $v4Networks = (Get-ClusterNetwork -ErrorAction SilentlyContinue | Where-Object { $_.Ipv4Addresses.Count -gt 0 }) foreach($v4Network in $v4Networks) { for($i = 0; $i -lt $v4Network.Ipv4Addresses.Count; $i++) { [System.Net.IPAddress]$ipv4 = $null $clusIpv4 = $v4Network.Ipv4Addresses[$i] if (-Not [System.Net.IPAddress]::TryParse($clusIpv4, [ref] $ipv4)) { Write-Warning "Ignoring failover cluster network IPAddress '$clusIpv4' as it couldn't be parsed as an IP address." continue } $lastIp = [AKSHCI.IPUtilities]::GetLastIpInCidr($ipv4, $v4Network.Ipv4PrefixLengths[$i]) if([AKSHCI.IPUtilities]::CompareIpAddresses($ipAddress, $ipv4) -ge 0 -AND [AKSHCI.IPUtilities]::CompareIpAddresses($ipAddress, $lastIp) -le 0) { return $v4Network.Ipv4PrefixLengths[$i] } } } throw "Could not create the failover cluster generic role. No cluster network could be found for IP '$IpAddress'" } function Add-FailoverClusterGenericRole { <# .DESCRIPTION Creates a generic service role in failover cluster (similar to Add-ClusterGenericServiceRole), but allows for fine tuning network configuration. .PARAMETER staticIpCidr Static IP and network prefix, using the CIDR format (Example: 192.168.1.2/16) .PARAMETER serviceDisplayName Display name of the service (Example: "WSSD cloud agent service") .PARAMETER clusterGroupName Name of the cluster group (Example: ca-2f87825b-a4af-473f-8a33-8e3bdd5f9b61) .PARAMETER serviceName Name of the service binary (Example: wssdcloudagent) .PARAMETER serviceParameters Service start parameters (Example: --fqdn ca-2f87825b-a4af-473f-8a33-8e3bdd5f9b61.contoso.com) #> param ( [string] $staticIpCidr, [string] $serviceDisplayName, [string] $clusterGroupName, [string] $serviceName, [string] $serviceParameters ) Add-ClusterGroup -Name $clusterGroupName -GroupType GenericService -ErrorAction Stop | Out-Null $dnsName = Add-ClusterResource -Name "$clusterGroupName" -ResourceType "Network Name" -Group $clusterGroupName -ErrorAction Stop $dnsName | Set-ClusterParameter -Multiple @{"Name"="$clusterGroupName";"DnsName"="$clusterGroupName"} -ErrorAction Stop if ([string]::IsNullOrWhiteSpace($staticIpCidr)) { $networkList = Get-ClusterNetwork -ErrorAction SilentlyContinue | Where-Object { $_.Role -eq "ClusterAndClient" } foreach ($network in $networkList) { $IPResourceName = "IPv4 Address on $($network.Address)" $IPAddress = Add-ClusterResource -Name $IPResourceName -ResourceType "IP Address" -Group $clusterGroupName -ErrorAction Stop $IPAddress | Set-ClusterParameter -Multiple @{"Network"=$network;"EnableDhcp"=1} -ErrorAction Stop Add-ClusterResourceDependency -Resource "$clusterGroupName" -Provider $IPResourceName -ErrorAction Stop | Out-Null } } else { $staticIpCidrArray = $staticIpCidr.Split("/") #Split a string of format x.x.x.x/pp to an array of x.x.x.x and pp if ($staticIpCidrArray.Length -eq 1) { $prefixLength = Get-ClusterNetworkPrefixForIp -IpAddress $staticIpCidrArray[0] } else { $prefixLength = $staticIpCidrArray[1] } $subnetMask = Get-Ipv4MaskFromPrefix -PrefixLength $prefixLength $IPResourceName = "IPv4 Address $($staticIpCidrArray[0])" $IPAddress = Add-ClusterResource -Name $IPResourceName -ResourceType "IP Address" -Group $clusterGroupName -ErrorAction Stop $IPAddress | Set-ClusterParameter -Multiple @{"Address"=$staticIpCidrArray[0];"SubnetMask"=$subnetMask;"EnableDhcp"=0} -ErrorAction Stop Add-ClusterResourceDependency -Resource "$clusterGroupName" -Provider $IPResourceName -ErrorAction Stop | Out-Null } $ServiceConfig = Add-ClusterResource -Name $serviceDisplayName -ResourceType "Generic Service" -Group $clusterGroupName -ErrorAction Stop Add-ClusterResourceDependency -Resource $serviceDisplayName -Provider "$clusterGroupName" -ErrorAction Stop | Out-Null $ServiceConfig | Set-ClusterParameter -Multiple @{"ServiceName"=$serviceName;"StartupParameters"=$serviceParameters;"UseNetworkName"=1} -ErrorAction Stop # Start the cluster group and wait at most 5 minutes for it to come online Start-ClusterGroup -Name $clusterGroupName -Wait 300 -ErrorAction Stop # Make sure the cluster group is not in the pending state. Wait for up to 5 minutes for the transition to happen $caGroupState = (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop).State $waitStart = [DateTime]::get_Now() while ($caGroupState -eq [Microsoft.FailoverClusters.PowerShell.ClusterGroupState]::Pending) { if (([DateTime]::get_Now() - $waitStart) -ge [Timespan]::FromMinutes(5)) { $failedResources = [String]::Join(", ", (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop | Get-ClusterResource -ErrorAction Stop | Where-Object { $_.State -ne [Microsoft.FailoverClusters.PowerShell.ClusterResourceState]::Online }).Name) Remove-ClusterGroup -Name $clusterGroupName -RemoveResources -Force -ErrorAction Stop throw "Timed out while trying to start the cloud agent generic service in failover cluster. The cluster resource group is in the '$caGroupState' state. Resources in 'failed' or 'pending' states: '$failedResources'" } Start-Sleep 1 $caGroupState = (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop).State } # Perform a final validation $caGroupState = (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop).State if ($caGroupState -ne [Microsoft.FailoverClusters.PowerShell.ClusterGroupState]::Online) { $failedResources = [String]::Join(", ", (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop | Get-ClusterResource -ErrorAction Stop | Where-Object { $_.State -ne [Microsoft.FailoverClusters.PowerShell.ClusterResourceState]::Online }).Name) Remove-ClusterGroup -Name $clusterGroupName -RemoveResources -Force -ErrorAction Stop throw "Failed to start the cloud agent generic service in failover cluster. The cluster resource group is in the '$caGroupState' state. Resources in 'failed' or 'pending' states: '$failedResources'" } } function Get-TargetCapiClusters { <# .DESCRIPTION Returns all CapiCluster resources #> $tmpClusters = @() Get-CapiClusters | ForEach-Object { if ($null -eq $_.spec.management) { $tmpClusters += ($_) } } return $tmpClusters } function Get-CapiClusters { <# .DESCRIPTION Returns all AksHciCluster resources #> Write-Status $("Retrieving clusters") -moduleName $global:KvaModule $capiClusters = Invoke-Kubectl -arguments $("get akshciclusters -o json") | ConvertFrom-Json if ($null -eq $capiClusters) { throw $("No cluster information was found.") } $clusters = @() foreach($capiCluster in $capiClusters.items) { $clusters += Get-CapiCluster -Name $capiCluster.metadata.name } return $clusters } function Get-CapiCluster { <# .DESCRIPTION Returns the requested CapiCluster resource .PARAMETER Name Name of the cluster #> param ( [Parameter()] [String]$Name ) Write-Status $("Retrieving configuration for workload cluster '$Name'") -moduleName $global:KvaModule $capiCluster = Invoke-Kubectl -ignoreError -arguments $("get akshciclusters/$Name -o json") | ConvertFrom-Json if ($null -eq $capiCluster) { throw $("A workload cluster with the name '$Name' was not found.") } $linuxNodepool = Invoke-Kubectl -ignoreError -arguments $("get akshcinodepools/$Name-default-linux-nodepool -o json") | ConvertFrom-Json if ($null -eq $linuxNodepool) { Write-SubStatus "A workload nodepool with the name '$Name-default-linux-nodepool' was not found." -moduleName $global:KvaModule } $windowsNodepool = Invoke-Kubectl -ignoreError -arguments $("get akshcinodepools/$Name-default-windows-nodepool -o json") | ConvertFrom-Json if ($null -eq $windowsNodepool) { Write-SubStatus "A workload cluster with the name '$Name-default-windows-nodepool' was not found." -moduleName $global:KvaModule } Write-SubStatus "Successfully retrieved cluster information." -moduleName $global:KvaModule $cniConfig = $capiCluster.spec.clusterConfiguration.cniConfiguration.configuration $primaryNetworkPlugin = [NetworkPlugin]::Default if (-Not [string]::IsNullOrWhiteSpace($cniConfig)) { if ($cniConfig.Contains(':')) { $primaryNetworkPlugin = $cniConfig.Substring(0, $cniConfig.IndexOf(':')) # only keep the cni name, remove the version } else { $primaryNetworkPlugin = $cniConfig } } # For ease, calculate linux and windows replica counts + VmSizes and append them as a member of the returned object $linuxWorkerReplicas = 0 $linuxWorkerVmSize = [VmSize]::Default if ($null -ne $linuxNodepool) { $linuxWorkerReplicas = $linuxNodepool.spec.replicas $linuxWorkerVmSize = [Enum]::Parse([VmSize], $($linuxNodepool.spec.infrastructureProfile.hardwareProfile.vmsize), $true) } $windowsWorkerReplicas = 0 $windowsWorkerVmSize = [VmSize]::Default if ($null -ne $windowsNodepool) { $windowsWorkerReplicas = $windowsNodepool.spec.replicas $windowsWorkerVmSize = [Enum]::Parse([VmSize], $($windowsNodepool.spec.infrastructureProfile.hardwareProfile.vmsize), $true) } $capiCluster | Add-Member -NotePropertyName linuxWorkerReplicas -NotePropertyValue $linuxWorkerReplicas $capiCluster | Add-Member -NotePropertyName linuxWorkerVmSize -NotePropertyValue $linuxWorkerVmSize $capiCluster | Add-Member -NotePropertyName windowsWorkerReplicas -NotePropertyValue $windowsWorkerReplicas $capiCluster | Add-Member -NotePropertyName windowsWorkerVmSize -NotePropertyValue $windowsWorkerVmSize $capiCluster | Add-Member -NotePropertyName primaryNetworkPlugin -NotePropertyValue $primaryNetworkPlugin return $capiCluster } #region Logging and Monitoring #region Cleanup Functions function Install-Binaries { <# .DESCRIPTION Copies AksHci binaries to a node .PARAMETER nodeName The node to execute on. .PARAMETER module The module #> param ( [Parameter(Mandatory=$true)] [String]$nodeName, [Parameter(Mandatory=$true)] [String]$module, [Parameter(Mandatory=$true)] [Hashtable] $binariesMap ) Write-Status "Installing $module binaries on Node $nodeName" -moduleName $module Invoke-Command -ComputerName $nodeName -ScriptBlock { $path = $args[0] New-Item -ItemType Directory -Force -Path $path $envPath = [Environment]::GetEnvironmentVariable("PATH") if($envPath -notlike $("*$path*")) { [Environment]::SetEnvironmentVariable("PATH", "$envPath;$path") [Environment]::SetEnvironmentVariable("PATH", "$envPath;$path", "Machine") } } -ArgumentList $global:installDirectory | out-null $binariesMap.Keys | foreach-object { Copy-FileToRemoteNode -source $([io.Path]::Combine($global:config[$module]["installationPackageDir"], $_)) -remoteNode $nodeName -destination $binariesMap[$_] } } function Uninstall-Binaries { <# .DESCRIPTION Copies AksHci binaries to a node .PARAMETER nodeName The node to execute on. .PARAMETER module The module #> param ( [Parameter(Mandatory=$true)] [String]$nodeName, [Parameter(Mandatory=$true)] [String]$module, [Parameter(Mandatory=$true)] [Hashtable] $binariesMap ) Write-Status "Uninstalling $module binaries on Node $nodeName" -moduleName $module Invoke-Command -ComputerName $nodeName -ScriptBlock { $binaries = $args[0] $binaries.Keys | Foreach-Object { Remove-Item -Path $binaries[$_] -force -ErrorAction SilentlyContinue } } -ArgumentList $binariesMap | out-null } #endregion #region Catalog helpers function Get-Catalog { <# .DESCRIPTION Get the Catalog for AksHci. This would include a set of product release versions .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String]$moduleName ) $cacheFile = $global:config[$moduleName]["manifestCache"] if ((Test-Path $cacheFile)) { return (Get-Content $cacheFile | ConvertFrom-Json) } $provider = Get-DownloadProvider -module $moduleName $downloadParams = @{ Name = $global:config[$moduleName]["catalog"] Audience = $global:config[$moduleName]["ring"] Provider = $provider } if ($global:config[$moduleName]["useStagingShare"]) { $downloadParams.Add("Endpoint", $global:config[$moduleName]["stagingShare"]) } $catalog = Get-DownloadSdkCatalog @downloadParams $cacheFile = $global:config[$moduleName]["manifestCache"] $catalogJson = $catalog | ConvertTo-Json -depth 100 Add-Content -path $cacheFile -value $catalogJson -encoding UTF8 return $catalog } function Get-LatestCatalog { <# .DESCRIPTION Get the latest catalog for AksHci by clearing the cache and redownloading the latest .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) # Clean the catalog cache, so we download the latest Clear-CatalogCache -module $moduleName return Get-Catalog -module $moduleName } function Clear-CatalogCache { <# .DESCRIPTION Removes any cached copy of the catalog .PARAMETER moduleName Module name #> $cacheFile = $global:config[$moduleName]["manifestCache"] if ((Test-Path $cacheFile)) { Remove-Item $cacheFile -Force } # Sometimes this path wouldnt exist. Try to initialize it here $dirPath = [io.Path]::GetDirectoryName($cacheFile) if (!(Test-Path $dirPath)) { New-Item -ItemType Directory -force -Path $dirPath } } function Get-ProductRelease { <# .DESCRIPTION Gets the Product Release manifest for the specified Version .PARAMETER version The requested release version .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter(Mandatory=$true)] [String] $moduleName ) $catalog = Get-Catalog -moduleName $moduleName foreach($productRelease in $catalog.ProductStreamRefs[0].ProductReleases) { if ($productRelease.Version -ieq $version) { return $productRelease } } throw "A release with version $version was NOT FOUND" } function Get-ProductReleasesUptoVersion { <# .DESCRIPTION Get all of the Product Release Manifests up to the specified Version .PARAMETER version Requested version #> param ( [String]$version, [Parameter(Mandatory=$true)] [String]$moduleName ) # Assumption here is that the ordering of values in catalog release stream is latest at top. $releaseList = @() $catalog = Get-Catalog -moduleName $moduleName foreach($productRelease in $catalog.ProductStreamRefs[0].ProductReleases) { $releaseList += $productRelease if ($productRelease.Version -ieq $version) { break } } if ($releaseList.Count -eq 0) { throw "$version is NOT FOUND" } return $releaseList } function Get-LatestRelease { <# .DESCRIPTION Get the latest release of AksHci by refreshing the catalog and returning the latest release from the updated catalog .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $catalog = Get-LatestCatalog -moduleName $moduleName if (-not $catalog) { throw $("The latest release catalog could not be retrieved at this time. Please retry later") } return $catalog.ProductStreamRefs[0].ProductReleases[0] } function Get-ReleaseDownloadParameters { <# .DESCRIPTION .PARAMETER name Release name to be downloaded .PARAMETER version Release version to be downloaded .PARAMETER destination The destination for the download .PARAMETER parts How many download parts to use (concurrency) .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [String] $destination, [Parameter()] [Int] $parts = 1, [Parameter(Mandatory=$true)] [String] $moduleName ) $provider = Get-DownloadProvider -module $moduleName $downloadParams = @{ Provider = $provider Name = $name Version = $version Destination = $destination Parts = $parts } if ($global:config[$moduleName]["useStagingShare"]) { $downloadParams.Add("Endpoint", $global:config[$moduleName]["stagingShare"]) $downloadParams.Add("CatalogName", $global:config[$moduleName]["catalog"]) $downloadParams.Add("Audience", $global:config[$moduleName]["ring"]) } return $downloadParams } function Get-DownloadProvider { <# .DESCRIPTION Returns an appropriate download provider based on the current module configuration .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String]$moduleName ) if ($global:config[$moduleName]["useStagingShare"]) { $endpoint = $($global:config[$moduleName]["stagingShare"]) if ($endpoint.StartsWith("http")) { return "http" } elseif ($endpoint.StartsWith("//") -or $endpoint.StartsWith("\\") -or $endpoint.Contains(":")) { return "local" } else { throw "Unsupported staging share endpoint: $endpoint" } } return "sfs" } function Get-TargetClusterKubernetesVersions { <# .DESCRIPTION Get the Kubernetes Versions used for the target clusters .PARAMETER Version Version #> try { $tmp = Get-TargetClusterKubernetesReferences return $tmp.Keys } catch { # Workaround return @() } } function Get-TargetClusterKubernetesReferences { <# .DESCRIPTION Get the Kubernetes Versions used for the target clusters #> $k8sversionsInUse = @{} Get-TargetCapiClusters | ForEach-Object { $tmp = $_ $k8sversion = $tmp.spec.clusterConfiguration.kubernetesVersion if (!$k8sversionsInUse.ContainsKey($k8sversion)) { $k8sversionsInUse += @{$k8sversion = @();} } # Add references to the target cluster name $k8sversionsInUse[$k8sversion] += ($tmp.metadata.Name) } return $k8sversionsInUse } #endregion #region File download helpers function Test-AuthenticodeBinaries { <# .DESCRIPTION Validates binary integrity via authenticode .PARAMETER workingDir Location of the binaries to be tested .PARAMETER binaries The list of binaries to be tested #> param ( [Parameter(Mandatory=$true)] [string] $workingDir, [Parameter(Mandatory=$true)] [string[]] $binaries ) Write-Status "Verifying Authenticode binaries" -moduleName $global:DownloadModule $workingDir = $workingDir -replace "\/", "\" foreach ($binary in $binaries) { $name = $("$workingDir/$binary") -replace "\/", "\" Write-SubStatus $("Checking signature for binary: $name") -moduleName $global:DownloadModule $auth = Get-AuthenticodeSignature -FilePath $name if (($global:expectedAuthResponse.status -ne $auth.status) -or ($global:expectedAuthResponse.SignatureType -ne $auth.SignatureType)) { throw $("Binary $name failed authenticode verification. Expected status=$($global:expectedAuthResponse.status) and type=$($global:expectedAuthResponse.SignatureType) but received status=$($auth.status) and type=$($auth.SignatureType)") } Write-SubStatus "Verified Signature for $name" -moduleName $global:DownloadModule } } #endregion #region EventLog function New-ModuleEventLog { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) New-EventLog -LogName "AKSHCI" -Source $moduleName -ErrorAction Ignore } function Write-ModuleEventLog { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name to test for installation state .PARAMETER message The message that has to be logged .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $moduleName, [Parameter(Mandatory=$true)] [String] $message, [Parameter(Mandatory=$true)] [System.Diagnostics.EventLogEntryType] $entryType, [Parameter(Mandatory=$true)] [int] $eventId ) Write-EventLog -LogName "AKSHCI" -Source $moduleName -EventID $eventId -EntryType $entryType -Message $message } #endregion #region Validation functions function Test-ValidCIDR { <# .DESCRIPTION This function validates that CIDR is valid by checking: 1. That the CIDR notation is correct (IP/prefix) 2. That the IP is valid. 3. That the prefix length is between 1 and 30 .PARAMETER CIDR The CIDR in the form IP/prefixlength. E.g. 10.0.0.0/24 #> param ( [Parameter(Mandatory=$true)] [string] $CIDR ) $x = $CIDR.Split('/') if ($x.Length -ne 2) { throw "Invalid CIDR ($CIDR). CIDR should be of the form 10.0.0.0/24." } $ip = $x[0] $prefix = [int]::Parse($x[1]) Test-ValidEndpoint -endpoint $ip # The minimum prefix length is /30 (which leaves 2 usable ip addresses. 1 for mgmt cluster VM, 1 for mgmt k8s VIP) if (($prefix -lt 1) -or ($prefix -gt 30)) { throw "Invalid prefix length ($prefix). The prefix must be between 1 and 30." } } function Test-ValidPool { <# .DESCRIPTION This function validates that the pool start/end are valid by checking: 1. That the pool start/end are valid ip addresses 2. That the pool end comes after or is equal (1 IP) to the pool start. 3. If CIDR is also given, it validates the range is within the CIDR. .PARAMETER PoolStart The starting ip address of the pool .PARAMETER PoolEnd The ending ip address of the pool .PARAMETER CIDR The CIDR from where the pool should come from. Note that if CIDR is given, it is expected to have already been validated. #> param ( [Parameter(Mandatory=$true)] [string] $PoolStart, [Parameter(Mandatory=$true)] [string] $PoolEnd, [string] $CIDR ) Test-ValidEndpoint -endpoint $PoolStart Test-ValidEndpoint -endpoint $PoolEnd $valid = [AKSHCI.IPUtilities]::ValidateRange($PoolStart, $PoolEnd) if (-not $valid) { throw "Invalid range $PoolStart - $PoolEnd" } if ($CIDR) { $valid = [AKSHCI.IPUtilities]::ValidateRangeInCIDR($PoolStart, $PoolEnd, $CIDR) if (-not $valid) { throw "Range $PoolStart - $PoolEnd is not in $CIDR" } } } #endregion #region Proxy server functions function Set-ProxyConfiguration { <# .DESCRIPTION Sets the proxy server configuration for a module .PARAMETER proxySettings Proxy server settings .PARAMETER moduleName The module name #> param ( [Parameter()] [ProxySettings] $proxySettings, [Parameter(Mandatory=$true)] [String] $moduleName ) $http = "" $https = "" $noProxy = "" $certFile = "" $user = "" $pass = "" if ($proxySettings) { $http = $proxySettings.HTTP $https = $proxySettings.HTTPS $noProxy = $proxySettings.NoProxy $certFile = $proxySettings.CertFile Test-ProxyConfiguration -http $http -https $https -certFile $certFile if ($proxySettings.Credential.Username) { $user = $proxySettings.Credential.UserName } if ($proxySettings.Credential.Password) { $pass = $proxySettings.Credential.Password | ConvertFrom-SecureString -Key $global:credentialKey } } Set-ConfigurationValue -name "proxyServerUsername" -value $user -module $moduleName Set-ConfigurationValue -name "proxyServerPassword" -value $pass -module $moduleName Set-ConfigurationValue -name "proxyServerHTTP" -value $http -module $moduleName Set-ConfigurationValue -name "proxyServerHTTPS" -value $https -module $moduleName Set-ConfigurationValue -name "proxyServerNoProxy" -value $noProxy -module $moduleName Set-ConfigurationValue -name "proxyServerCertFile" -value $certFile -module $moduleName Initialize-ProxyEnvironment -moduleName $moduleName } function Initialize-ProxyEnvironment { <# .DESCRIPTION Applies proxy settings to the current process environment .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $proxySettings = Get-ProxyConfiguration -moduleName $moduleName if ($proxySettings.HTTP -or $proxySettings.HTTPS) { Set-DownloadSdkProxy -Http "$($proxySettings.HTTP)" -Https "$($proxySettings.HTTPS)" -NoProxy "$($proxySettings.NoProxy)" } } function Get-ProxyConfiguration { <# .DESCRIPTION Returns a custom PSObject containing the complete HTTP, HTTPS, NoProxy, and CertFile setting strings .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $proxyHTTP = "" $proxyHTTPS = "" $proxyCertName = "" $proxyCertContentB64 = "" if ($global:config[$moduleName]["proxyServerHTTP"]) { $proxyHTTP = Get-ProxyWithCredentials -proxyServer $($global:config[$moduleName]["proxyServerHTTP"]) -proxyUsername $($global:config[$moduleName]["proxyServerUsername"]) -proxyPass $($global:config[$moduleName]["ProxyServerPassword"]) } if ($global:config[$moduleName]["proxyServerHTTPS"]) { $proxyHTTPS = Get-ProxyWithCredentials -proxyServer $($global:config[$moduleName]["proxyServerHTTPS"]) -proxyUsername $($global:config[$moduleName]["proxyServerUsername"]) -proxyPass $($global:config[$moduleName]["ProxyServerPassword"]) } if (($global:config[$moduleName]["proxyServerCertFile"]) -and (Test-Path $global:config[$moduleName]["proxyServerCertFile"])) { $content = Get-Content -Encoding Byte -Path $global:config[$moduleName]["proxyServerCertFile"] if ($content) { $proxyCertName = "proxy-cert.crt" $proxyCertContentB64 = [Convert]::ToBase64String($content) } } $proxyConfig = [ordered]@{ 'HTTP' = $proxyHTTP; 'HTTPS' = $proxyHTTPS; 'NoProxy' = $global:config[$moduleName]["proxyServerNoProxy"]; 'CertPath' = $global:config[$moduleName]["proxyServerCertFile"]; 'CertName' = $proxyCertName; 'CertContent' = $proxyCertContentB64 } if ($($global:config[$moduleName]["proxyServerUsername"]) -and $($global:config[$moduleName]["ProxyServerPassword"])) { $securePass = $($global:config[$moduleName]["ProxyServerPassword"]) | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $($global:config[$moduleName]["proxyServerUsername"]), $securePass $proxyConfig.Add("Credential", $credential) } $result = @() $result += New-Object -TypeName PsObject -Property $proxyConfig return $result } function Get-ProxyWithCredentials { <# .DESCRIPTION Returns a complete proxy string with credentials in the URI format (e.g. http://user:pass@server.com:8080) .PARAMETER proxyServer Proxy server string URI .PARAMETER proxyUsername Proxy server username .PARAMETER proxyPass Proxy server password (this is a secure string representation, not plaintext) #> param ( [Parameter(Mandatory=$true)] [String] $proxyServer, [Parameter()] [String] $proxyUsername, [Parameter()] [String] $proxyPass ) $uri = Test-ValidProxyServer -proxyServer $proxyServer $proxyString = $($uri.Scheme + "://") if ($proxyUsername -and $proxyPass) { $securePass = $proxyPass | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $proxyUsername, $securePass $proxyString += $($credential.UserName + ":" + $credential.GetNetworkCredential().Password + "@") } $proxyString += $uri.Authority return $proxyString } function Test-ProxyConfiguration { <# .DESCRIPTION Validates the provided proxy server configuration. On failure, would throw. .PARAMETER http HTTP proxy server configuration .PARAMETER https HTTPS proxy server configuration .PARAMETER certFile Path to a CA certificate file used to establish trust with a HTTPS proxy server #> param ( [Parameter()] [String] $http, [Parameter()] [String] $https, [Parameter()] [String] $certFile ) if ($http) { Test-ValidProxyServer -proxyServer $http | Out-Null } if ($https) { Test-ValidProxyServer -proxyServer $https | Out-Null } if ($certFile) { if (-not (Test-Path $certFile)) { throw $("The proxy server certificate file '$certFile' was not found") } } } function Test-ValidProxyServer { <# .DESCRIPTION Validates the provided proxy server string .PARAMETER proxyServer Proxy server string in absolute URI format (e.g. http://proxy.com:3128) #> param ( [String] $proxyServer ) $uri = $null $result = [System.URI]::TryCreate($proxyServer, [System.UriKind]::Absolute, [ref]$uri) if (-not $result) { throw $("The proxy server string '" + $proxyServer + "' is not a valid absolute URI (e.g. http://server.com:8080)") } switch($uri.Scheme) { $([System.URI]::UriSchemeHttp) { break } $([System.URI]::UriSchemeHttps) { break } Default { throw $("The proxy server string '" + $proxyServer + "' does not use a support URI scheme (e.g. http or https)") } } return $uri } #endregion # SIG # Begin signature block # MIIjhQYJKoZIhvcNAQcCoIIjdjCCI3ICAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDZ54gWDdkRQ/Fq # az6MORjBJcBIj+ZMt9+IJgVlWokVLaCCDYEwggX/MIID56ADAgECAhMzAAAB32vw # LpKnSrTQAAAAAAHfMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjAxMjE1MjEzMTQ1WhcNMjExMjAyMjEzMTQ1WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC2uxlZEACjqfHkuFyoCwfL25ofI9DZWKt4wEj3JBQ48GPt1UsDv834CcoUUPMn # s/6CtPoaQ4Thy/kbOOg/zJAnrJeiMQqRe2Lsdb/NSI2gXXX9lad1/yPUDOXo4GNw # PjXq1JZi+HZV91bUr6ZjzePj1g+bepsqd/HC1XScj0fT3aAxLRykJSzExEBmU9eS # yuOwUuq+CriudQtWGMdJU650v/KmzfM46Y6lo/MCnnpvz3zEL7PMdUdwqj/nYhGG # 3UVILxX7tAdMbz7LN+6WOIpT1A41rwaoOVnv+8Ua94HwhjZmu1S73yeV7RZZNxoh # EegJi9YYssXa7UZUUkCCA+KnAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUOPbML8IdkNGtCfMmVPtvI6VZ8+Mw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDYzMDA5MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAnnqH # tDyYUFaVAkvAK0eqq6nhoL95SZQu3RnpZ7tdQ89QR3++7A+4hrr7V4xxmkB5BObS # 0YK+MALE02atjwWgPdpYQ68WdLGroJZHkbZdgERG+7tETFl3aKF4KpoSaGOskZXp # TPnCaMo2PXoAMVMGpsQEQswimZq3IQ3nRQfBlJ0PoMMcN/+Pks8ZTL1BoPYsJpok # t6cql59q6CypZYIwgyJ892HpttybHKg1ZtQLUlSXccRMlugPgEcNZJagPEgPYni4 # b11snjRAgf0dyQ0zI9aLXqTxWUU5pCIFiPT0b2wsxzRqCtyGqpkGM8P9GazO8eao # mVItCYBcJSByBx/pS0cSYwBBHAZxJODUqxSXoSGDvmTfqUJXntnWkL4okok1FiCD # Z4jpyXOQunb6egIXvkgQ7jb2uO26Ow0m8RwleDvhOMrnHsupiOPbozKroSa6paFt # VSh89abUSooR8QdZciemmoFhcWkEwFg4spzvYNP4nIs193261WyTaRMZoceGun7G # CT2Rl653uUj+F+g94c63AhzSq4khdL4HlFIP2ePv29smfUnHtGq6yYFDLnT0q/Y+ # Di3jwloF8EWkkHRtSuXlFUbTmwr/lDDgbpZiKhLS7CBTDj32I0L5i532+uHczw82 # oZDmYmYmIUSMbZOgS65h797rj5JJ6OkeEUJoAVwwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVWjCCFVYCAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAd9r8C6Sp0q00AAAAAAB3zAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgESEGeGo7 # rOo3cI2m9B/ogMNre9XHPomjVEMdhkWX0xUwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQBKJjiSm47GcVw1ZrGFuiBS26s1QVixpDAnwYtgyJwT # eBzI9ZXw0JZH9N/y/FjIE1Va3ZzCIkct2lTI7AhFOFk6f1rzYBrdokLl0vZTNBXA # iIuBi9Sf3jWQPIuIzdYbVQFi3jcOMrl+s2pMI7rn2dixd+sIRlt9w7YsEFAX/tdD # sUb6ffhwln+MLh//Ssu24IIlMDtKjJTcxdbi4gVuai+Ljl+gTMwbWPqql/QX9q/X # aXTEhjREeYyF4JonhW/vB0xxuWXOSTorvPd8C4AFMf7u9AzxlFS59fgs1bs39y6w # YqbDGuvCKvtLCcJ0nU71MlWg2yCpZozR4wIisVbOjWCpoYIS5DCCEuAGCisGAQQB # gjcDAwExghLQMIISzAYJKoZIhvcNAQcCoIISvTCCErkCAQMxDzANBglghkgBZQME # AgEFADCCAVEGCyqGSIb3DQEJEAEEoIIBQASCATwwggE4AgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIJzi89J5VXcn/1ltWmFT1ZwhX5eG1iZMFW903DCm # nNaZAgZgichXoXIYEzIwMjEwNTIxMjAzMDQ0LjY5N1owBIACAfSggdCkgc0wgcox # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1p # Y3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOkU1QTYtRTI3Qy01OTJFMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNloIIOOzCCBPEwggPZoAMCAQICEzMAAAFHnY/x5t4xg1kAAAAAAUcw # DQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcN # MjAxMTEyMTgyNTU1WhcNMjIwMjExMTgyNTU1WjCByjELMAkGA1UEBhMCVVMxEzAR # BgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1p # Y3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2Eg # T3BlcmF0aW9uczEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046RTVBNi1FMjdDLTU5 # MkUxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtBQNM6X32KFk/BJ8YaprfzEt6Lj34 # G+VLjzgfEgOGSVd1Mu7nCphK0K4oyPrzItgNRjB4gUiKq6GzgxdDHgZPgTEvm57z # sascyGrybWkf3VVr8bqf2PIgGvwKDNEgVcygsEbuWwXz9Li6M7AOoD4TB8fl4ATm # +L7b4+lYDUMJYMLzpiJzM745a0XHiriUaOpYWfkwO9Hz6uf+k2Hq7yGyguH8naPL # MnYfmYIt2PXAwWVvG4MD4YbjXBVZ14ueh7YlqZTMua3n9kT1CZDsHvz+o58nsoam # XRwRFOb7LDjVV++cZIZLO29usiI0H79tb3fSvh9tU7QC7CirNCBYagNJAgMBAAGj # ggEbMIIBFzAdBgNVHQ4EFgQUtPjcb95koYZXGy9DPxN49dSCsLowHwYDVR0jBBgw # FoAU1WM6XIoxkPNDe3xGG8UzaFqFbVUwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDov # L2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljVGltU3RhUENB # XzIwMTAtMDctMDEuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0 # cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNUaW1TdGFQQ0FfMjAx # MC0wNy0wMS5jcnQwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDCDAN # BgkqhkiG9w0BAQsFAAOCAQEAUMQOyjV+ea2kEtXqD0cOfD2Z2PFUIy5kLkGU53RD # GcfhlzIR9QlTgZLqTEhgLLuCSy6jcma+nPg7e5Xg1oqCZcZJRwtRPzS1F6/M6YR3 # 5H3brN0maVnPrmrQ91kkfsNqDTtuWDiAIBfkNEgCpQZCb4OV3HMu5L8eZzg5dUaJ # 7XE+LBuphJSLFJtabxYt4fkCQxnTD2z50Y32ZuXiNmFFia7qVq+3Yc3mmW02+/KW # H8P1HPiobJG8crGYgSEkxtkUXGdoutwGWW88KR9RRcM/4GKLqt2OQ8AWEQb7shgM # 8pxNvu30TxejRApa4WAfOAejTG4+KzBm67XjVZ2IlXAPkjCCBnEwggRZoAMCAQIC # CmEJgSoAAAAAAAIwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRp # ZmljYXRlIEF1dGhvcml0eSAyMDEwMB4XDTEwMDcwMTIxMzY1NVoXDTI1MDcwMTIx # NDY1NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNV # BAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQG # A1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggEiMA0GCSqGSIb3 # DQEBAQUAA4IBDwAwggEKAoIBAQCpHQ28dxGKOiDs/BOX9fp/aZRrdFQQ1aUKAIKF # ++18aEssX8XD5WHCdrc+Zitb8BVTJwQxH0EbGpUdzgkTjnxhMFmxMEQP8WCIhFRD # DNdNuDgIs0Ldk6zWczBXJoKjRQ3Q6vVHgc2/JGAyWGBG8lhHhjKEHnRhZ5FfgVSx # z5NMksHEpl3RYRNuKMYa+YaAu99h/EbBJx0kZxJyGiGKr0tkiVBisV39dx898Fd1 # rL2KQk1AUdEPnAY+Z3/1ZsADlkR+79BL/W7lmsqxqPJ6Kgox8NpOBpG2iAg16Hgc # sOmZzTznL0S6p/TcZL2kAcEgCZN4zfy8wMlEXV4WnAEFTyJNAgMBAAGjggHmMIIB # 4jAQBgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQU1WM6XIoxkPNDe3xGG8UzaFqF # bVUwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1Ud # EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186aGMQwVgYD # VR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwv # cHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEB # BE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9j # ZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwgaAGA1UdIAEB/wSBlTCB # kjCBjwYJKwYBBAGCNy4DMIGBMD0GCCsGAQUFBwIBFjFodHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vUEtJL2RvY3MvQ1BTL2RlZmF1bHQuaHRtMEAGCCsGAQUFBwICMDQe # MiAdAEwAZQBnAGEAbABfAFAAbwBsAGkAYwB5AF8AUwB0AGEAdABlAG0AZQBuAHQA # LiAdMA0GCSqGSIb3DQEBCwUAA4ICAQAH5ohRDeLG4Jg/gXEDPZ2joSFvs+umzPUx # vs8F4qn++ldtGTCzwsVmyWrf9efweL3HqJ4l4/m87WtUVwgrUYJEEvu5U4zM9GAS # inbMQEBBm9xcF/9c+V4XNZgkVkt070IQyK+/f8Z/8jd9Wj8c8pl5SpFSAK84Dxf1 # L3mBZdmptWvkx872ynoAb0swRCQiPM/tA6WWj1kpvLb9BOFwnzJKJ/1Vry/+tuWO # M7tiX5rbV0Dp8c6ZZpCM/2pif93FSguRJuI57BlKcWOdeyFtw5yjojz6f32WapB4 # pm3S4Zz5Hfw42JT0xqUKloakvZ4argRCg7i1gJsiOCC1JeVk7Pf0v35jWSUPei45 # V3aicaoGig+JFrphpxHLmtgOR5qAxdDNp9DvfYPw4TtxCd9ddJgiCGHasFAeb73x # 4QDf5zEHpJM692VHeOj4qEir995yfmFrb3epgcunCaw5u+zGy9iCtHLNHfS4hQEe # gPsbiSpUObJb2sgNVZl6h3M7COaYLeqN4DMuEin1wC9UJyH3yKxO2ii4sanblrKn # QqLJzxlBTeCG+SqaoxFmMNO7dDJL32N79ZmKLxvHIa9Zta7cRDyXUHHXodLFVeNp # 3lfB0d4wwP3M5k37Db9dT+mdHhk4L7zPWAUu7w2gUDXa7wknHNWzfjUeCLraNtvT # X4/edIhJEqGCAs0wggI2AgEBMIH4oYHQpIHNMIHKMQswCQYDVQQGEwJVUzETMBEG # A1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWlj # cm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBP # cGVyYXRpb25zMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpFNUE2LUUyN0MtNTky # RTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcG # BSsOAwIaAxUAq6fBtEENocNASMqL03zGJS0wZd2ggYMwgYCkfjB8MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQg # VGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIFAORR78UwIhgPMjAy # MTA1MjExNjM3MjVaGA8yMDIxMDUyMjE2MzcyNVowdjA8BgorBgEEAYRZCgQBMS4w # LDAKAgUA5FHvxQIBADAIAgEAAgMAgtIwCAIBAAIDAVwQMAoCBQDkU0FFAgEAMDYG # CisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEA # AgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAolbZnYJjGVQjjEkJdiZ9Mp4/sNjVA/FY # Xl6wI9fQP+CHdJaTwG/SV7T6wEDt369Z/omZOuX3tQPp8t+prGOW4VhUqwRWFbjn # YHPy0Bkd3Q956Z3YDN2thJV5I4AyzyxbPV56bNrZ317osIFB3xP7ztL3qzarL3mj # GQaPTMOxMLoxggMNMIIDCQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0Eg # MjAxMAITMwAAAUedj/Hm3jGDWQAAAAABRzANBglghkgBZQMEAgEFAKCCAUowGgYJ # KoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCBtu+buyGgv # POTDEyauyv6gd7pwp0uO0QyKCtsE2OHDNzCB+gYLKoZIhvcNAQkQAi8xgeowgecw # geQwgb0EIHvbPBIDlM+6BsiJk7/YfWGuKwBUi3DMOxxvRaqKGOmFMIGYMIGApH4w # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAFHnY/x5t4xg1kAAAAA # AUcwIgQgp/uc1xBrIue3RFiedzfEHhNxybK46o64VHo1r241HG0wDQYJKoZIhvcN # AQELBQAEggEAFgpdsaz3X2BgZgTtUCl0nX0zJDioGGxU8M9xCeaFzoKu3xPDIyEP # VnXslahYe+/NJGE9pPSlrrT44yvT4UUmLpo2GW1fPAySjAk62bbOqnJoYGeVodRw # lwuouEZxqTp/AULstfgwHaS6BPigoVCuGzrqbTr5Dnl3zZfR+20zo4C/xS2enA4r # 8GVuOaDeNX5ynpaqiAzVoC1+Y30UvbbebNnTU0s8UBC9W27cMOFq2pfvPMBpr36L # XspeWe6eh0kgOm7/Qrz353yq6MoSfYLT8eya3hj1YtPX0+g9rFbMK24IEvAxyLjP # Q2AFm1Y+eFVQES7Ka7epwGtgE1qHgfIxSA== # SIG # End signature block |