AzStackHci.EnvironmentChecker

1.2100.2455.236

Obs/bin/ObsDep/content/Powershell/ObservabilityConfig.psm1

<###################################################
# #
# Copyright (c) Microsoft. All rights reserved. #
# #
##################################################>

class ObservabilityConfig
{
    static RegisterObservabilityEventSource([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        foreach ($eventSource in $eventSources)
        {
            $eventSourceRootName = $eventSource.EventSourceRootName
            $eventSourceLogName = $eventSource.EventSourceLogName
            $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes
            $eventSourcePublisher = $eventSource.EventSourcePublisher

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog"
            try
            {
                # Register eventsource on hosts
                Invoke-Command `
                    -ComputerName $hostIps `
                    -Credential $localAdminCredential `
                    -Authentication Credssp `
                    -ScriptBlock {
                        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                        $manifestDllPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.dll"
                        $manifestPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.man"

                        # Grant read only access to manifest files
                        Trace-Execution "Granting read only access to $manifestPath"
                        icacls $manifestPath /grant Everyone:R
                        Trace-Execution "Granting read only access to $manifestDllPath"
                        icacls $manifestDllPath /grant Everyone:R


                        $publisherList = wevtutil ep
                        if ($publisherList.Contains($Using:eventSourcePublisher))
                        {
                            Trace-Execution "Publisher $Using:eventSourcePublisher already exists. Uninstalling."
                            wevtutil uninstall-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"
                            Trace-Execution "Successfully uninstalled publisher $Using:eventSourcePublisher."
                        }

                        # Register the EventSource with Windows
                        Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath"
                        wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"

                        Trace-Execution "wevtutil setting log size to $Using:eventSourceLogSizeInBytes"
                        wevtutil set-log $Using:eventSourceLogName /MaxSize:"$Using:eventSourceLogSizeInBytes"
                    }
            }
            catch
            {
                Trace-Execution "Registering Observability EventSource failed with error: $_"
                throw $_
            }

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded."
        }
    }

    static CreateObservabilityVolume([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityVolumeLabel
        $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveSizeInBytes

        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        $PhysicalDriveLetter = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.PhysicalDriveLetter
        $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName
        $volumeLocation = "${PhysicalDriveLetter}:"
        $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName

        Write-ObservabilityVolumeCreationStartTelemetry `
            -ComputerNames ($hostIps -join " ") `
            -VolumeFilePath $volumePath `
            -VolumeAccessPath $driveAccessPath `
            -VolumeLabel $volumeLabel `
            -VolumeSize $volumeSize

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Add Observability Volume to each Host
        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $VolumePath,

                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $VolumeLocation,

                [string]
                [Parameter(Mandatory=$true)]
                $VolumeLabel,

                [string]
                [Parameter(Mandatory=$true)]
                $VolumeSize
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
            Add-ObservabilityVolumeWithRetry -Path $VolumePath -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $VolumeSize -StaticSize
            Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $VolumePath
            Add-MountVolumeScheduledTask -Path $VolumePath
        }

        $argList = @($volumePath, $driveAccessPath, $volumeLocation, $volumeLabel, $volumeSize)
        Invoke-Command `
            -ComputerName $hostIps `
            -ScriptBlock $scriptBlock `
            -ArgumentList $argList `
            -Credential $localAdminCredential `
            -Authentication Credssp
        Write-ObservabilityLog "Observability Volume Setup Succeeded"
    }

    static SetUptimeScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        Write-InfoLog "Obtaining local admin credential..."
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        Write-InfoLog "Starting Uptime scheduled task Setup"
        Write-InfoLog "Hosts: $hostNames"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1"
            $hostName = $env:COMPUTERNAME
            Write-InfoLog "Starting uptime scheduled task on host $hostName."
            try
            {
                Set-UptimeTaskWithRetry
                Write-InfoLog "Uptime scheduled task on host $hostName succeeded."
            }
            catch
            {
                Write-ErrorLog "Uptime scheduled task on host $hostName failed with exception $_ ."
                throw $_
            }
        }

        Invoke-Command -ComputerName $hostIps -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp | Out-Null
        Write-InfoLog "Uptime scheduled task completed."
    }

    static SetObservabilityEventTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $eventName, $eventScriptBlock)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        Write-InfoLog "Starting $eventName events task Setup"
        Write-InfoLog "Hosts: $hostNames"
        $exceptionMessage = ""
        $scriptBlock = $eventScriptBlock

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        foreach ($hostIp in $hostIps)
        {
            Write-InfoLog "Starting $eventName events task on host $hostIp."
            $result = Invoke-Command -ComputerName $hostIp -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp

            if($result -eq $true)
            {
                Write-InfoLog "$eventName events task on host $hostIp succeeded."
            }
            else
            {
                Write-ErrorLog "$eventName events task on host $hostIp failed with exception $result."
                $exceptionMessage += "${hostIp}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }
    }

    static SetCensusEventScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
            Set-CensusTaskWithRetry
        }

        [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "CensusEvent", $scriptBlock)

        Write-InfoLog "Census events scheduled task completed."
    }

    static SetRegistrationEventOneTimeTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
            Set-RegistrationEventsTaskWithRetry
        }

        [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "RegistrationEvents", $scriptBlock)

        Write-InfoLog "Registration events one-time task completed."
    }

    static CreateVolumeFoldersAndPruner([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent
        $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent
        $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes
        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilitySubFolderConfigFileName
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        Write-ObservabilityLog "Starting Observability Volume Folder and Pruner Setup"
        Write-ObservabilityLog "Hosts: $hostNames"
        Write-ObservabilityLog "Volume folder mount access path: $driveAccessPath"
        Write-ObservabilityLog "Observability subfolder folder cleanup threshold: $folderCleanupThresholdPercent"
        Write-ObservabilityLog "Observability subfolder folder free space threshold: $folderFreeSpaceThresholdPercent"
        Write-ObservabilityLog "Observability volume purge frequency in minutes: $purgeFolderFrequencyInMinutes"

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Add Observability Volume to each Host
        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [Parameter(Mandatory=$true)]
                $FolderCleanupThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $FolderFreeSpaceThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $PurgeFolderFrequencyInMinutes,

                [string]
                [Parameter(Mandatory=$true)]
                $SubFolderConfigFileName
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
            New-VolumeFoldersAndPrunerWithRetry `
                -AccessPath $AccessPath `
                -CleanupThresholdPercent $FolderCleanupThresholdPercent `
                -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent `
                -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes `
                -SubFolderConfigFileName $SubFolderConfigFileName
        }

        $argList = @(
            $driveAccessPath,
            $folderCleanupThresholdPercent,
            $folderFreeSpaceThresholdPercent,
            $purgeFolderFrequencyInMinutes,
            $subFolderConfigFileName
        )
        Invoke-Command `
            -ComputerName $hostIps `
            -ScriptBlock $scriptBlock `
            -ArgumentList $argList `
            -Credential $localAdminCredential `
            -Authentication Credssp
        Write-ObservabilityLog "Observability subfolder and pruner Setup Succeeded"
    }

    # Set registry key HKLM\Software\Microsoft\AzureStack DeviceType to AzureEdge
    static SetDeviceTypeRegistryKey([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        Write-ObservabilityLog "SetDeviceTypeRegistryKey start."

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                    Write-Verbose "Setting HKLM\Software\Microsoft\AzureStack DeviceType registry key to AzureEdge."
                    $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\'
                    if (!(Test-Path -Path $registryPath))
                    {
                        New-Item -Path $registryPath -Force
                    }
                    New-ItemProperty -Path $registryPath -Name 'DeviceType' -PropertyType 'String' -Value "AzureEdge" -Force
                    Write-Verbose "Finished DeviceType Registry key setup on $($env:COMPUTERNAME)"
                }
            Write-ObservabilityLog "Succeeded in setting AzureStack DeviceType Registry key."
        }
        catch
        {
            Write-ObservabilityErrorLog "[Error] DeviceType registry key set up failed with an error : $_ "
            throw $_
        }
    }

    # Set up AzureStack environment to enable Autologger and UTC Telemetry via MA
    # Reg key set up needs to happend upfornt during host configuration
    static SetUpUtcExporterFeature([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA."

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                    $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                    $utcExporterPath = "$($env:SystemRoot)\System32\UtcExporters"
                    # Create UtcExporters folder in system32
                    if (-not (Test-Path -Path $utcExporterPath))
                    {
                        New-Item -Path "$($env:SystemRoot)\System32" -ItemType Directory -Name 'UtcExporters' -Force
                    }

                    Write-Verbose "Set up the required regkey to enable utc exporter Feature"
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack' -Name 'AllowExporters' -PropertyType 'DWORD' -Value 1 -Force
                    New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters' -Force
                    New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'DllPath' -PropertyType 'String' -Value "$($utcExporterPath)\UtcGenevaExporter.dll" -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'GenevaNamespace' -PropertyType 'String' -Value "AEOppeTely" -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\TestHooks' -Name 'SkipSignatureMitigation' -PropertyType 'DWORD' -Value 1 -Force

                    # Update UTC Service binary
                    Write-Verbose "Updating UTC service dll"
                    Stop-Service "diagtrack" -Force
                    Start-Sleep -Seconds 10
                    Write-Verbose "Copy UTC Exporter dll to $($utcExporterPath)"
                    Copy-Item -Path "$observabilityNugetPath\lib\net472\UtcGenevaExporter.dll" -Destination $utcExporterPath -Force
                    Start-Service "diagtrack"

                    Write-Verbose "Finished UTC Exporter setup on $($env:COMPUTERNAME)"
                }
            Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA succeeded."
        }
        catch
        {
            Write-ObservabilityErrorLog "[Error] UTC Exporter set up failed with an error : $_ "
            throw $_
        }
    }

    # Install VC Runtime on all hosts
    static InstallVcRuntime([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }
        Write-ObservabilityLog "Starting installation of X64 VC Redistributable."

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                Write-ObservabilityLog "[Starting] installation of VC Redistributable on host $($env:COMPUTERNAME)"
                $vcRedistFilePath = "$observabilityNugetPath\content\VS17\VC_redist.x64.exe"
                if(Test-Path -Path $vcRedistFilePath) {
                    Start-Process -File $vcRedistFilePath -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow
                } else {
                    $errMsg = "[ERROR] VC Redistributable [$vcRedistilePath] not found on host $($env:COMPUTERNAME)"
                    Write-ObservabilityErrorLog $errMsg
                    throw $errMsg
                }
                Write-ObservabilityLog "[Finished] installation of VC Redistributable on host $($env:COMPUTERNAME)"
            }
        }
        catch
        {
            Write-ObservabilityErrorLog "[Failed] installation of VC Redistributable: $_ "
            throw $_
        }
        Write-ObservabilityLog "Finished installation of X64 VC Redistributable."
    }

    static CreateObservabilityVolumeOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Starting observability volume setup on NC VMs"
        $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name
        $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName
        $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityVolumeLabel
        $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveSizeInBytes
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent
        $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent
        $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes
        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $VolumePath,

                [string]
                [parameter(Mandatory=$true)]
                $HostComputerName,

                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $FolderCleanupThresholdPercent,

                [string]
                [parameter(Mandatory=$true)]
                $FolderFreeSpaceThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $PurgeFolderFrequencyInMinutes,

                [string]
                [parameter(Mandatory=$true)]
                $SubFolderConfigFileName,

                [PSCredential]
                [parameter(Mandatory=$true)]
                $Credential
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            $remoteVolumePath = "\\$HostComputerName\$VolumePath"
            $remoteVolumePath = $remoteVolumePath -replace ":", "$"
            Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $remoteVolumePath
            Add-MountVolumeScheduledTask -Path $remoteVolumePath -Credential $Credential
            New-VolumeFoldersAndPrunerWithRetry `
                -AccessPath $AccessPath `
                -CleanupThresholdPercent $FolderCleanupThresholdPercent `
                -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent `
                -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes `
                -SubFolderConfigFileName $SubFolderConfigFileName
            Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName
        }

        if(-not($ncVMs -is [System.Array])) # Only 1 VM exists
        {
            $ncVMs = @($ncVMs)
        }
        foreach($ncVM in $ncVMs)
        {
            $hostComputerName = $env:COMPUTERNAME
            Write-ObservabilityLog "Starting observability volume onboarding for $ncVM."
            try
            {
                Get-VM $ncVM | Out-Null # Test that VM is accessbile
            }
            catch
            {
                Write-ObservabilityLog "$ncVM not found on $hostComputerName. No op."
                continue
            }
            $vmHardDrivePath = (Get-VMHardDiskDrive $ncVM).Path
            if($vmHardDrivePath -is [System.Array])
            {
                $vmHardDrivePath = $vmHardDrivePath[0]
            }
            $volumeLocation = Split-Path $vmHardDrivePath -Parent
            $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName
            Write-ObservabilityLog "Adding observability volume at volume path $volumePath"
            Add-ObservabilityVolumeWithRetry -Path $volumePath -AccessPath $driveAccessPath -VolumeLabel $volumeLabel -Size $volumeSize -StaticSize
            if((Get-DiskImage $volumePath).Attached)
            {
                Dismount-VHD $volumePath
            }
            $argList = @(
                $volumePath,
                $hostComputerName,
                $driveAccessPath,
                $folderCleanupThresholdPercent,
                $folderFreeSpaceThresholdPercent,
                $purgeFolderFrequencyInMinutes,
                $subFolderConfigFileName,
                $domainUserCredential
            )
            Invoke-Command `
                -ComputerName $ncVM `
                -ScriptBlock $scriptBlock `
                -Credential $domainUserCredential `
                -Authentication Credssp `
                -ArgumentList $argList
            Write-ObservabilityLog "Finished observability volume onboarding for $ncVM."
        }
    }

    static RegisterObservabilityEventSourceOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource
        $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name

        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        foreach ($eventSource in $eventSources)
        {
            $eventSourceRootName = $eventSource.EventSourceRootName
            $eventSourceLogName = $eventSource.EventSourceLogName
            $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog"
            $scriptBlock = {
                param (
                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceRootName,

                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceLogName,

                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceLogSizeInBytes
                )
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                $manifestDllPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.dll"
                $manifestPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.man"

                # Grant read only access to manifest files
                Trace-Execution "Granting read only access to $manifestPath"
                icacls $manifestPath /grant Everyone:R
                Trace-Execution "Granting read only access to $manifestDllPath"
                icacls $manifestDllPath /grant Everyone:R

                # Register the EventSource with Windows
                Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath"
                wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"

                Trace-Execution "wevtutil setting log size to $EventSourceLogSizeInBytes"
                wevtutil set-log $EventSourceLogName /MaxSize:"$EventSourceLogSizeInBytes"
            }

            try
            {
                # Register eventsource on hosts
                Invoke-Command `
                    -ComputerName $ncVMs `
                    -ScriptBlock $scriptBlock `
                    -Credential $domainUserCredential `
                    -Authentication Credssp `
                    -ArgumentList @($eventSourceRootName, $eventSourceLogName, $eventSourceLogSizeInBytes)
            }
            catch
            {
                Trace-Execution "Registering Observability EventSource failed with error: $_"
                throw $_
            }

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded."
        }
    }

    static SetFolderQuotas ([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        Write-ObservabilityLog "Starting observability volume fsrm quota setup on $hostNames"
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $SubFolderConfigFileName
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName
        }
        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock $scriptBlock `
                -ArgumentList @($driveAccessPath, $subFolderConfigFileName)
        }
        catch
        {
            Write-ObservabilityErrorLog "[Failed] setup of FSRM Quotas: $_ "
            throw $_
        }
    }

    static [PSCredential] GetLocalCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Obtaining local admin credential..."
        $localAdminCredential = $null
        try
        {
            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
            $securityInfo = $cloudRole.PublicInfo.SecurityInfo
            $localAdmin = $securityInfo.LocalUsers.User | Where-Object Role -eq $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID
            $localAdminCredential = $Parameters.GetCredential($localAdmin.Credential)
        }
        catch
        {
            Write-ObservabilityErrorLog "Failed to obtain local admin credentials: $_ "
            throw $_
        }
        Write-ObservabilityLog "Local admin credential obtained."

        if ($localAdminCredential.UserName -eq "removed")
        {
            Write-ObservabilityLog "The local admin credential obtained has been removed. Using domain credential instead."
            return [ObservabilityConfig]::GetDomainCredential($Parameters)
        }
        return $localAdminCredential
    }

    static [PSCredential] GetDomainCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Obtaining domain user credentials..."
        $domainUserCredential = $null
        try
        {
            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
            $securityInfo = $cloudRole.PublicInfo.SecurityInfo
            $physicalMachinesRole = $Parameters.Configuration.Role
            $domainName = $Parameters.Roles.Domain.PublicConfiguration.PublicInfo.DomainConfiguration.Fqdn

            $domainUser = $securityInfo.DomainUsers.User | Where-Object Role -EQ $physicalMachinesRole.PrivateInfo.Accounts.DomainUserAccountID
            $domainUserCredential = $Parameters.GetCredential($domainUser.Credential)
            # Making sure that the credential is domain qualified, PowerShell Direct require credential to be domain qualified
            if($($domainUserCredential.GetNetworkCredential().Domain) -eq "")
            {
                $domainUserCredential = New-Object -TypeName PSCredential -ArgumentList "$domainName\$($domainUserCredential.UserName)", $($domainUserCredential.Password)
            }
            Write-ObservabilityLog "Domain user credentials obtained."
        }
        catch
        {
            Write-ObservabilityErrorLog "Failed to obtain domain user credentials: $_ "
            throw $_
        }
        return $domainUserCredential
    }

    static InstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $exceptionMessage = ""
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        try
        {
            Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation start."

            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration

            $cloudId = $cloudRole.PublicInfo.CloudId
            $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare

            $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters)

            $scriptBlock = {
                param (
                    [string]
                    [parameter(Mandatory=$true)]
                    $AgentMsiPath,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $AccessToken,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $SubscriptionId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $TenantId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $AccountId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $ResourceGroupName,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $CloudId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $EnvironmentName,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $Region,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $StampId
                )

                $errorMessage = ""
                $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath
                Start-Transcript -Append -Path $setHostNICIPAddressLogFile
                try
                {
                    $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                    Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                    $ResourceName = Get-ArcResourceName
                    Write-ObservabilityLog "$env:COMPUTERNAME Going to install Arc-for-server agent..."
                    Install-ArcForServerAgent `
                        -AgentMsiPath $AgentMsiPath `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -TenantId $TenantId `
                        -ResourceGroupName $ResourceGroupName `
                        -EnvironmentName $EnvironmentName `
                        -Region $Region `
                        -ResourceName $ResourceName | Out-Null

                    Write-ObservabilityLog "$env:COMPUTERNAME Going to download Observability Extension..."
                    Install-ArcForServerExtensions `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -TenantId $TenantId `
                        -AccountId $AccountId `
                        -ResourceGroupName $ResourceGroupName `
                        -ResourceName $ResourceName `
                        -EnvironmentName $EnvironmentName `
                        -Region $Region | Out-Null

                    $hostNameHash = Get-Sha256Hash -ClearString (hostname)
                    $nodeId = "$($StampId)-$hostNameHash"
                    Write-BootstrapNodeIdAndHardwareIdHashTelemetry -BootstrapNodeId $nodeId
                }
                catch{
                    $errorMessage = $PSItem.ToString()
                    Write-ObservabilityErrorLog $errorMessage
                    throw $PSItem.Exception.Message
                }
                finally
                {
                    Write-ArcForServerInstallationStopTelemetry `
                        -ComputerName $env:COMPUTERNAME `
                        -Message "Bootstrap Observability installation end" `
                        -ExceptionDetails $errorMessage
                }
            }

            $argList = @(
                $arcForServerMsiFilePath,
                $registrationParams.ArmAccessToken,
                $registrationParams.SubscriptionId,
                $registrationParams.TenantId,
                $registrationParams.AccountId,
                $registrationParams.ResourceGroupName,
                $cloudId,
                $registrationParams.EnvironmentName,
                $registrationParams.Region,
                $registrationParams.ResourceName
            )

            Invoke-Command `
                -ComputerName $hostIps `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $localAdminCredential `
                -Authentication Credssp
        }
        catch
        {
            $errMsg = "Bootstrap Observability installation failed with $_"
            $exceptionMessage += $errMsg
            Write-ObservabilityErrorLog $errMsg
            throw
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation end."
    }

    static UninstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation start."
        
        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }
        
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)
        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration

        $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare
        Write-ObservabilityLog "$env:COMPUTERNAME Arc For Server MSI Path: $arcForServerMsiFilePath"

        $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters)


        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AgentMsiPath,

                [string]
                [Parameter(Mandatory=$true)]
                $AccessToken,

                [string]
                [Parameter(Mandatory=$true)]
                $SubscriptionId,

                [string]
                [Parameter(Mandatory=$true)]
                $AccountId,

                [string]
                [Parameter(Mandatory=$true)]
                $ResourceGroupName,

                [string]
                [Parameter(Mandatory=$true)]
                $CloudId
            )

            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            # Import MiscConstants to remove GMAScenario reg key
            $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent"
            $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content"
            Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking
            $gmaScenarioRegKeyPath = $MiscConstants.GMAScenarioRegKey.Path
            $gmaScenarioRegKeyName = $MiscConstants.GMAScenarioRegKey.Name
            if (Test-RegKeyExists -Path $gmaScenarioRegKeyPath -Name $gmaScenarioRegKeyName)
            {
                Remove-ItemProperty `
                    -Path $gmaScenarioRegKeyPath `
                    -Name $gmaScenarioRegKeyName `
                    -Force
                Write-ObservabilityLog "Removed registry key $gmaScenarioRegKeyName at path $gmaScenarioRegKeyPath"
            }

            Uninstall-ArcForServerExtensions `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -AccountId $AccountId `
                        -ResourceGroupName $ResourceGroupName `
                        -CloudId $CloudId


            $result = Uninstall-ArcForServerAgent `
                        -AgentMsiPath $AgentMsiPath `
                        -AccessToken $AccessToken
 
            return $result
        }

        # Install Arc for server agent on each Host
        $exceptionMessage = ""
        foreach($hostIp in $hostIps)
        {
            $argList = @($arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.CloudId)
            $result = Invoke-Command `
                -ComputerName $hostIp `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $domainUserCredential `
                -Authentication Credssp
            if ($result -eq $true)
            {
                Write-ObservabilityLog "Arc for server agent setup on host $hostIp succeeded."
            }
            else
            {
                Write-ObservabilityLog "Arc for server agent setup on host $hostIp failed with exception $result."
                $exceptionMessage += "${hostIp}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation end."
    }

        <#
        .SYNOPSIS
        Evaluation function to identify what type of GMA configuration action to use.
        Following are the cases involved during evaluation :
            case 1 : If the interface is invoked in case of Deployment then run configuration on all nodes.
            case 2 : If the interface is invoked in case of AddNode or RepairNode then run configuration on only that node.
             
        .EXAMPLE
        [GMATenantJson]::EvaluateGMAConfigurationType($Parameters)
 
        .PARAMETER Parameters
        The ECE role parameters for this interface.
    #>
    static [CloudEngine.Actions.ConditionalActionDescription] EvaluateListenerModeConfigurationType([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $ErrorActionPreference = "Stop"

        ## For classes in PS, we need to use the call stack to get the current executing function.
        $functionName = $(Get-PSCallStack)[0].FunctionName

        try {
            Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

            $cloudRolePath = "ObservabilityConfig"
            $listenerModeOnAllNodes = "SetListenerModeOnAllNodes"
            $listenerModeOnOneNode = "SetListenerModeOnOneNode"
            $onAllNodesAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnAllNodes)
            $onOneNodeAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnOneNode)

            $nodeName = Get-ExecutionContextNodeName -Parameters $Parameters

            if($null -ne $nodeName)
            {
                # In add node or repair node case, ExecutionContextNodeName should have exactly one value.
                Trace-Execution "Execution context node was determined. Listener Mode will be set on $nodeName only."
                return $onOneNodeAction
            }

            # In regular deployment case, ExecutionContextNodeName will be null.
            Trace-Execution "No exceution context node was determined. Listener Mode will set on all the nodes."
            return $onAllNodesAction
        }
        catch {
            Trace-Error "$functionName :Failed to evaluate ListenerMode configuration type on node $($env:COMPUTERNAME) due to following error : $_"
            throw
        }

    }

    static StopArcExtensionObservabilityAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                $ObsAgentServiceName = "AzureStack Observability Agent"
                $ObsAgentServiceDisplayName = "AzureStack Arc Extension Observability Agent"
                $stopDelayInSeconds = 10

                $service = Get-Service | Where-Object {$_.Name -eq $ObsAgentServiceName -and $_.DisplayName -eq $ObsAgentServiceDisplayName }
                if($service)
                {
                    if ($service.Status -ne "Stopped")
                    {
                        Write-ObservabilityLog "Stopping service $ObsAgentServiceName with display name $ObsAgentServiceDisplayName on host $($env:COMPUTERNAME)"
                        $service | Stop-Service
                        Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $ObsAgentServiceName before unregistering."
                        Start-Sleep -Seconds $stopDelayInSeconds
                    }
                    Write-ObservabilityLog "Unregistering service $ObsAgentServiceName."
                    sc.exe delete $ObsAgentServiceName
                    Write-ObservabilityLog "Successfully unregistered service $ObsAgentServiceDisplayName"
                }
                else
                {
                    Write-ObservabilityLog "Service $ObsAgentServiceName not found. Noop."
                }
            }
    }

    static StopRemSupAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                # Remote Support Service names - Arc Extension, Env Checker
                $agentServiceNames = @("Observability Remote Support Agent", "RemoteSupportAgent")
                $stopDelayInSeconds = 10
                $maxRetries = 3

                foreach ($agentServiceName in $agentServiceNames)
                {
                    $attempt = 1
                    $service = Get-Service -Name $agentServiceName -ErrorAction "SilentlyContinue"
                    if($service)
                    {
                        $serviceStatus = $service.Status
                        while ($attempt -le $maxRetries -and -not [string]::IsNullOrWhiteSpace($serviceStatus))
                        {
                            try
                            {
                                if ($serviceStatus -ne "Stopped")
                                {
                                    Write-ObservabilityLog "Stopping service $agentServiceName Status: $serviceStatus, Attempt: $attempt."
                                    $service | Stop-Service
                                    Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $agentServiceName before unregistering, Attempt: $attempt."
                                    Start-Sleep -Seconds $stopDelayInSeconds
                                }
                                Write-ObservabilityLog "Unregistering service $agentServiceName , Attempt: $attempt."
                                sc.exe delete $agentServiceName
                                Write-ObservabilityLog "Successfully unregistered service $agentServiceName , Attempt: $attempt."
                            }
                            catch{
                                Write-ObservabilityLog "Exception caught during stopping/removing service $agentServiceName . $($_.Exception)"
                                Write-ObservabilityLog "Retrying ..."
                            }

                            $attempt = $attempt + 1
                            $service = Get-Service -Name $agentServiceName -ErrorAction "SilentlyContinue"
                            $serviceStatus = $service.Status
                        }

                        if($attempt -eq $maxRetries)
                        {
                            Write-ObservabilityErrorLog "All retries $maxRetries exhausted. $agentServiceName is not unregistered."
                            throw
                        }
                    }
                    else
                    {
                        Write-ObservabilityLog "Service $agentServiceName not found. NoOp."
                    }
                }
            }
    }

    static SetWatchdogToListenerMode([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
                Write-ObservabilityLog "Setting HKLM\Software\Microsoft\AzureStack\Observability WatchdogListenerMode registry key to 1 on $($env:COMPUTERNAME)."
                $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability'
                if (!(Test-Path -Path $registryPath))
                {
                    New-Item -Path $registryPath -Force
                }
                New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 1 -Force
                Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 1 on $($env:COMPUTERNAME)"

                $timeLimitInSeconds = 600
                $retryPeriod = 10
                $elapsed = 0

                $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue
                $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue
                while($maHostProcess -or $maCoreProcess)
                {
                    if($maHostProcess)
                    {
                        $pids = $maHostProcess.Id | Out-String
                        Write-ObservabilityLog "MonAgentHost Active Process Ids: `n $pids"
                    }
                    if($maCoreProcess)
                    {
                        $pids = $maCoreProcess.Id | Out-String
                        Write-ObservabilityLog "MonAgentCore Active Process Ids: `n $pids"
                    }

                    if($elapsed -gt $timeLimitInSeconds)
                    {
                        $errMsg = "WatchdogListenerMode has failed to stop Monitoring agent processes after $timeLimitInSeconds."
                        Write-ObservabilityErrorLog $errMsg
                        throw $errMsg
                    }
                    Write-ObservabilityLog "Monitoring Agent processes have not yet stopped. Retrying in $retryPeriod seconds."
                    Start-Sleep -Seconds $retryPeriod
                    $elapsed += $retryPeriod
                    $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue
                    $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue
                }
                Write-ObservabilityLog "MonitoringAgent has been successfully stopped after Watchdog has been set to listener mode."
            }
    }

    static SetGMATenantJsonRegistryKeys([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        Write-ObservabilityLog "SetGMATenantJsonRegistryKeys start"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent"
        $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content"
        Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking

        ## Get PhysicalDriveLetter to place GMA cache folder
        $PhysicalDriveLetter = $Parameters.Roles["GMATenantJson"].PublicConfiguration.PublicInfo.PhysicalDriveLetter
        $volumeName = "${PhysicalDriveLetter}:"

        ## GMA cache folder path
        $gmaCacheFolderName = $Parameters.Roles["GMATenantJson"].PublicConfiguration.PublicInfo.GMACacheFolderName
        $gmaCacheFolderPath = Join-Path -Path $volumeName -ChildPath $gmaCacheFolderName

        ## Determine GcsEnvironment
        $gcsEnvironment = "Prod" ## default environment

        ## Check if the reg key created for CI exists or not, if yes then change the GCSEnvironment to point to PPE.
        if ((Test-RegKeyExists -Path "HKLM:\Software\Microsoft\SQMClient\" -Name "IsCIEnv") -eq $true) {
            $gcsEnvironment = "Ppe"
        }

        $envInfoFilePath = "$GmaPackageContentPath\EnvironmentInfo.json"
        $tenantInfoContent = Get-Content $envInfoFilePath -Raw | ConvertFrom-Json
        $envInfo = $tenantInfoContent.$GcsEnvironment

        # Settings from EnvironmentInfo.json
        $gcsEndpoint = $envInfo.EndPoint
        $gcsAccount = $envInfo.Account
        $genevaConfigVersion = $envInfo.ConfigVersion

        # Settings from ECE
        $assemblyVersion = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version
        $registrationSubscriptionId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationSubscriptionId
        $registrationResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceGroupName
        $stampId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceName
        $gcsRegionName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationRegion
        $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node.Name
        $cloudId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.CloudId
        $deviceArmResourceUri = "/Subscriptions/$registrationSubscriptionId/resourceGroups/$registrationResourceGroupName/providers/Microsoft.AzureStackHCI/clusters/$stampId"
        $osBuild = Get-OSBuildVersion
        $registrationArcResourceGroupName = $registrationResourceGroupName
        $arcResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationArcServerResourceGroupName

        if (![String]::IsNullOrEmpty($arcResourceGroupName)) {
            Trace-Execution "Using RegistrationArcServerResourceGroupName $arcResourceGroupName for ArcServer resource in Tenant Json"
            $registrationArcResourceGroupName = $arcResourceGroupName
        }

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)
        Invoke-Command `
            -ComputerName $hostIps `
            -Credential $localAdminCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
                $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $Using:gmaNugetName) -ChildPath "content"
                Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking

               # Will unset GMA scenario registry path at bootstrap uninstall
                Set-GMAScenarioRegistryKeyToBootstrap | Out-Null

                # For Bootstrap Scenario, nodeid is stampid-sha256(hostname)
                $hostNameHash = Get-Sha256Hash -ClearString (hostname)
                $nodeId = "$($Using:stampId)-$hostNameHash"
                $arcAgentResourceId = Get-ArcResourceId -SubscriptionId $Using:registrationSubscriptionId -ResourceGroupName $Using:registrationArcResourceGroupName -CloudId $Using:cloudId

                $configTypes = @("Telemetry", "Diagnostics", "Health", "Metrics")
                foreach($configType in $configTypes)
                {
                    $envInfo = $Using:envInfo
                    $gcsNameSpace = $envInfo.Namespaces.$configType

                    if ($configType -eq "Metrics")
                    {
                        $gcsNameSpace = Get-MetricsNamespaceRegionMapping -Region $using:gcsRegionName -MetricsNameSpace $envInfo.Namespaces.$configType
                    }

                    # cacheLocalPath will be overwritten by Extension Install script because cache location for bootstrap is not known until install time
                    $cacheLocalPath = Join-Path -Path $Using:gmaCacheFolderPath -ChildPath $($configType + "Cache")
                    Set-TenantConfigRegistryKeys `
                        -ConfigType $configType `
                        -Version "1.0" `
                        -GcsAuthIdType "AuthMSIToken" `
                        -GcsEnvironment $Using:gcsEndpoint `
                        -GcsGenevaAccount $Using:gcsAccount `
                        -GcsNamespace $gcsNameSpace `
                        -GcsRegion $Using:gcsRegionName `
                        -GenevaConfigVersion $Using:genevaConfigVersion `
                        -LocalPath $cacheLocalPath `
                        -DisableUpdate "true" `
                        -DisableCustomImds "true" `
                        -MONITORING_AEO_REGION $Using:gcsRegionName `
                        -MONITORING_AEO_DEVICE_ARM_RESOURCE_URI $Using:deviceArmResourceUri `
                        -MONITORING_AEO_STAMPID $Using:stampId `
                        -MONITORING_AEO_CLUSTER_NAME $Using:clusterName `
                        -MONITORING_AEO_OSBUILD $Using:osBuild `
                        -MONITORING_AEO_ASSEMBLYBUILD $Using:assemblyVersion `
                        -MONITORING_AEO_NODEID $nodeId `
                        -MONITORING_AEO_NODE_ARC_RESOURCE_URI $arcAgentResourceId `
                        -MONITORING_AEO_CLUSTER_NODE_NAME "%COMPUTERNAME%"
                }
                $hostname = hostname
                Write-ObservabilityLog "SetGMATenantJsonRegistryKeys on $hostname succeeded."
            }
    }

    static GenerateTelemetryFromCachedFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files start."

        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration
        $domainCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)
        $cloudId = $cloudRole.PublicInfo.CloudId

        # Get host names using execution context
        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $CorrelationId

            )

            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            $logOrchestratorNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.LogOrchestrator"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            $result = Invoke-CachedTelemetryFilesParsing `
                        -LogOrchestratorNugetPath $logOrchestratorNugetPath `
                        -CorrelationId $CorrelationId

            return $result
        }

        # Install Arc for server agent on each Host
        $exceptionMessage = ""
        foreach($hostName in $hostNames)
        {
            $argList = @($cloudId)
            $result = Invoke-Command `
                -ComputerName $hostName `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $domainCredential `
                -Authentication Credssp
            if ($result -eq $true)
            {
                Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName succeeded."
            }
            else
            {
                Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName failed with exception $result."
                $exceptionMessage += "${hostName}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files end."
    }

    static SyncDiagnosticLevel([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $computerName = Get-Cluster | Select-Object -expand Name
        $resourceId = Get-AzureStackHCI | Select-Object -expand AzureResourceUri
        $diagnosticLevel = if ($Parameters.Roles["GMATenantJson"].PublicConfiguration.PublicInfo.StreamingDataClient) {"Basic"} Else {"Off"}
        $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters)

        $retryAttempt = 0
        $retrySleepTimeInSeconds = 10
        $Retries = 5
        $success = $false
        while(-not($success) -and ($retryAttempt -lt $Retries))
        {
            $retryAttempt = $retryAttempt + 1
            try
            {
                Trace-Execution "Setting Diagnostic Level to $diagnosticLevel. Attempt $retryAttempt of $Retries"
                Set-AzStackHCI -ComputerName $computerName `
                                -ResourceId $resourceId `
                                -DiagnosticLevel $diagnosticLevel `
                                -ArmAccessToken $registrationParams.ArmAccessToken `
                                -AccountId $registrationParams.AccountId `
                                -Confirm:$false `
                                -Verbose

                Trace-Execution "Invoking Sync-AzureStackHCI"
                Sync-AzureStackHCI -Verbose
                if ( $(Get-AzureStackHCI | Select-Object -expand DiagnosticLevel) -eq $diagnosticLevel) { $success = $true }
                else { throw "The Diagnostic Level does not match" }
            }
            catch
            {
                if ($retryAttempt -lt $Retries)
                {
                    $exceptionMessage = $_.Exception.Message
                    Trace-Execution "Failure during syncing diagnostic level: '$exceptionMessage'. Retrying."
                }
                else
                {
                    Trace-Execution "Syncing Diagnostic Level failed with error: $_"
                    throw $_
                }
                Start-Sleep -Seconds $retrySleepTimeInSeconds
            }
        }

        Trace-Execution "Syncing Diagnostic Level successful"
    }

    static [Hashtable] GetRegistrationParameters([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        $cloudDeploymentNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.CloudDeployment"
        Import-Module Az.Accounts -Force

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }


        Trace-Execution "Getting registration parameters from ECE role parameters"
        $nodeNames      = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration
        $securityInfo   = $cloudRole.PublicInfo.SecurityInfo
        $registrationParams = @{
            EnvironmentName         = $cloudRole.PublicInfo.RegistrationCloudName
            SubscriptionId          = $cloudRole.PublicInfo.RegistrationSubscriptionId
            TenantId                = $cloudRole.PublicInfo.RegistrationTenantId
            Region                  = $cloudRole.PublicInfo.RegistrationRegion
            ResourceGroupName       = $cloudRole.PublicInfo.RegistrationResourceGroupName
            ResourceName            = $cloudRole.PublicInfo.RegistrationResourceName
            CloudId                 = $cloudRole.PublicInfo.CloudId
            EnableAzureArcServer    = [System.Convert]::ToBoolean($cloudRole.PublicInfo.EnableAzureArcServer)
            ComputerName            = $nodeNames | Select-Object -First 1
        }

        if (![String]::IsNullOrEmpty($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName)) {
            Trace-Execution "Using RegistrationArcServerResourceGroupName $($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName) for ArcServer registration"
            $registrationParams.ResourceGroupName = $cloudRole.PublicInfo.RegistrationArcServerResourceGroupName
        }

        Trace-Execution "Getting AccessTokenfor ArcForServer using $($registrationParams | ConvertTo-Json -Depth 2)"
        $registrationParameterSet = $cloudRole.PublicInfo.RegistrationParameterSet
        if ($registrationParameterSet -eq "DefaultSet") {
            Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using user token cache"
            $registrationTokenCacheUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationTokenCacheID
            $registrationTokenCacheCred = $Parameters.GetCredential($registrationTokenCacheUser.Credential)
            Trace-Execution "RegistrationTokenCacheCred $($registrationTokenCacheCred.UserName)"
            $clientId = $cloudRole.PublicInfo.RegistrationClientId
            Trace-Execution "Using clientId $clientId to get access token"
            Trace-Execution "Going to use RegistrationHelpers $cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1"
            Import-Module "$cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1"
            $armAccessToken     = Get-AccessToken -AzureEnvironment $registrationParams.EnvironmentName -TenantId $registrationParams.TenantId -TokenCacheCred $registrationTokenCacheCred -ClientId $clientId
            Trace-Execution "Access tokens using token cache is $($armAccessToken.AccessToken.Length)"

            $registrationParams += @{
                AccountId           = $registrationTokenCacheCred.UserName
                ArmAccessToken      = $armAccessToken.AccessToken
            }
        } else {
            Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using service principal"
            $registrationSPUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationSPID
            $registrationSPCred = $Parameters.GetCredential($registrationSPUser.Credential)
            Trace-Execution "RegistrationSPCred AppId $($registrationSPCred.UserName)"
            Login-AzAccount -Environment $registrationParams.EnvironmentName -Credential $registrationSPCred -Tenant $registrationParams.TenantId -ServicePrincipal
            $armAccessToken = Get-AzAccessToken -Verbose
            Trace-Execution "Access tokens using service principal is $($armAccessToken.Token.Length)"

            $registrationParams += @{
                AccountId           = $registrationSPCred.UserName
                ArmAccessToken      = $armAccessToken.Token
            }
        }

        if($registrationParams.ArmAccessToken.Length -eq 0)
        {
            throw "GetRegistrationParameters failed to retrieve AccessToken"
        }

        return $registrationParams
    }
}
# SIG # Begin signature block
# MIInvwYJKoZIhvcNAQcCoIInsDCCJ6wCAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCClWfHFRu/98+Mr
# v4MFF+KyRc4qyXZXl1h1lz/PRUWqcaCCDXYwggX0MIID3KADAgECAhMzAAADTrU8
# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU
# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1
# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm
# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa
# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq
# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk
# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31
# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2
# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d
# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM
# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh
# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX
# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir
# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8
# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A
# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H
# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGZ8wghmbAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIJbJqRBd/g/Rx7BGT2kDajU7
# H8Ty069mjaFNUp9VOQ0kMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEADnTvKiCvGLMt2W+j1jEUPwWgR/Kc2XAE3Kx38jJnqphUYpIz27YwNYMD
# WRKkUwDRSVfgt4rzxsJqcdGcu6LwQgOl932lX8bXzx899QzyzLXjRkUkoOq8AYkQ
# OgH8QjE1CnVSAyU5rSpeIZKTyyPly36TCpV+AWYVdPcgDUnLZqsfXe5qXcw+oufQ
# qeUtUnjsKffA3oz8z7o5+KqsYq/Pp6sJOcWylRrKDUjP0H4ZC7bJNvdm64X1TLoi
# cKJnaybjPcDc7Evc6jQ7GXP4cGTUEPY+Ze7YvkOsp7CDUJNwSgioa1VGPnLvj00f
# yhciWJZrGz/OnO/RwaI6DvoKHc3jVqGCFykwghclBgorBgEEAYI3AwMBMYIXFTCC
# FxEGCSqGSIb3DQEHAqCCFwIwghb+AgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq
# hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCAjMtcGJYO9HM0Iu7DGiYZpp2nB0ZdWf6p5kogno0oqWwIGZQtY1EX9
# GBMyMDIzMDkyMjA4MzIwMi43MzZaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl
# bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO
# Ojg2REYtNEJCQy05MzM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT
# ZXJ2aWNloIIReDCCBycwggUPoAMCAQICEzMAAAG3IScaB6IqhkYAAQAAAbcwDQYJ
# KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjIw
# OTIwMjAyMjE0WhcNMjMxMjE0MjAyMjE0WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV
# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv
# c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl
# cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4NkRGLTRC
# QkMtOTMzNTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC
# AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMf9z1dQNBNkTBq3HJclypjQ
# cJIlDAgpvsw4vHJe06n532RKGkcn0V7p65OeA1wOoO+8NsopnjPpVZ8+4s/RhdMC
# MNPQJXoWdkWOp/3puIEs1fzPBgTJrdmzdyUYzrAloICYx722gmdpbNf3P0y5Z2gR
# O48sWIYyYeNJYch+ZfJzXqqvuvq7G8Nm8IMQi8Zayvx+5dSGBM5VYHBxCEjXF9EN
# 6Qw7A60SaXjKjojSpUmpaM4FmVec985PNdSh8hOeP2tL781SBan92DT19tfNHv9H
# 0FAmE2HGRwizHkJ//mAZdS0s6bi/UwPMksAia5bpnIDBOoaYdWkV0lVG5rN0+ltR
# z9zjlaH9uhdGTJ+WiNKOr7mRnlzYQA53ftSSJBqsEpTzCv7c673fdvltx3y48Per
# 6vc6UR5e4kSZsH141IhxhmRR2SmEabuYKOTdO7Q/vlvAfQxuEnJ93NL4LYV1IWw8
# O+xNO6gljrBpCOfOOTQgWJF+M6/IPyuYrcv79Lu7lc67S+U9MEu2dog0MuJIoYCM
# iuVaXS5+FmOJiyfiCZm0VJsJ570y9k/tEQe6aQR9MxDW1p2F3HWebolXj9su7zrr
# ElNlHAEvpFhcgoMniylNTiTZzLwUj7TH83gnugw1FCEVVh5U9lwNMPL1IGuz/3U+
# RT9wZCBJYIrFJPd6k8UtAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQUs/I5Pgw0JAVh
# DdYB2yPII8l4tOwwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD
# VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j
# cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG
# CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw
# MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD
# CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAA2dZMybhVxSXTbJ
# zFgvNiMCV5/Ayn5UuzJU495YDtcefold0ehR9QBGBhHmAMt10WYCHz2WQUyM3mQD
# 4IsHfEL1JEwgG9tGq71ucn9dknLBHD30JvbQRhIKcvFSnvRCCpVpilM8F/YaWXC9
# VibSef/PU2GWA+1zs64VFxJqHeuy8KqrQyfF20SCnd8zRZl4YYBcjh9G0GjhJHUP
# AYEx0r8jSWjyi2o2WAHD6CppBtkwnZSf7A68DL4OwwBpmFB3+vubjgNwaICS+fkG
# VvRnP2ZgmlfnaAas8Mx7igJqciqq0Q6An+0rHj1kxisNdIiTzFlu5Gw2ehXpLrl5
# 9kvsmONVAJHhndpx3n/0r76TH+3WNS9UT9jbxQkE+t2thif6MK5krFMnkBICCR/D
# VcV1qw9sg6sMEo0wWSXlQYXvcQWA65eVzSkosylhIlIZZLL3GHZD1LQtAjp2A5F7
# C3Iw4Nt7C7aDCfpFxom3ZulRnFJollPHb3unj9hA9xvRiKnWMAMpS4MZAoiV4O29
# zWKZdUzygp7gD4WjKK115KCJ0ovEcf92AnwMAXMnNs1o0LCszg+uDmiQZs5eR7jz
# dKzVfF1z7bfDYNPAJvm5pSQdby3wIOsN/stYjM+EkaPtUzr8OyMwrG+jpFMbsB4c
# fN6tvIeGtrtklMJFtnF68CcZZ5IAMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ
# mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT
# Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m
# dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh
# dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1
# WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH
# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD
# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB
# BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK
# NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg
# fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp
# rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d
# vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9
# 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR
# Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu
# qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO
# ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb
# oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6
# bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t
# AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW
# BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb
# UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz
# aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku
# aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA
# QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2
# VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu
# bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw
# LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93
# d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt
# MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q
# XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6
# U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt
# I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis
# 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp
# kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0
# sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e
# W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ
# sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7
# Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0
# dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ
# tB1VM1izoXBm8qGCAtQwggI9AgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx
# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT
# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh
# bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4
# NkRGLTRCQkMtOTMzNTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy
# dmljZaIjCgEBMAcGBSsOAwIaAxUAyGdBGMObODlsGBZmSUX2oWgfqcaggYMwgYCk
# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH
# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD
# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF
# AOi3KKQwIhgPMjAyMzA5MjIwNDQwMDRaGA8yMDIzMDkyMzA0NDAwNFowdDA6Bgor
# BgEEAYRZCgQBMSwwKjAKAgUA6LcopAIBADAHAgEAAgIKmjAHAgEAAgISEjAKAgUA
# 6Lh6JAIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAID
# B6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAECO5laKqKLCnIOukl5E
# k5GLTffJnr51w1yN1F4035G4UhhNu7PicVibckByJmzSegWE2Fvk4ZOLQAqENZGp
# pjnkVGuCzaeJtiC4ciBO5JPXipDSquW4Jv0fKOYTZ6xoK6HN7qItPdRegNmHfsx3
# JOlWNbtWxuDNT7alEAy+jfkDMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMx
# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT
# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUt
# U3RhbXAgUENBIDIwMTACEzMAAAG3IScaB6IqhkYAAQAAAbcwDQYJYIZIAWUDBAIB
# BQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQx
# IgQg8rxw84UzcKyb4IPwomC1oZMEg8ty8/9rDNPr/o2IVnYwgfoGCyqGSIb3DQEJ
# EAIvMYHqMIHnMIHkMIG9BCBsJ3jTsh7aL8hNeiYGL5/8IBn8zUfr7/Q7rkM8ic1w
# QTCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw
# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x
# JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAABtyEn
# GgeiKoZGAAEAAAG3MCIEINkdmaQbxevriOmGg4cmW1GUc1+gqpOinWVkpG+TekR+
# MA0GCSqGSIb3DQEBCwUABIICAJvVO1NSyI4MixHb2jhFccaN3OmFom6drUVpsoWO
# 5TvkAqrUTsnGU8/wOOMDOcsBrFJClFb3ybdAI2tnujzt2HF6for/l6rV3ew1VITY
# 7otSxE/V2JjlqBGh2AQ3FmmtamyjHowtYRklLt1QT9Xazal5EHt3M82UcuplW+pR
# l3r1LxkGV1m/hc/8zxXK3xgcQYqOjypH0aTzCC0AbSfDRv+ffhoZDvye3FScFlYJ
# 8NlK/VYNy2/gmpnnfQcpq0W8EhsS327+PiZrq3Cx5A9kwXCVIgdIdPvbUBm7/10C
# yrMEWDE725wp0d78MVq6YLWG1OpVALiaBz7VnTZyaVqXv4CjR33dfvzmOIjZLChF
# 1LAwG9ut7sxk9Qh92XWjnVtcGP0hkPmKg21amyjnSxst9GgEcozpqO47ovp2x04V
# kaRqB6JoddQ4x287bepjH4F3XIYnMl3CpQzz2As3ClbTVl6BmsLmDBGa2gJSRKGV
# 20hkXOsiQe0Q2yfYhLVPonKw3N1ntQMvXk/r7TM0LvzVGqguDI8Zqcy04cJSGe8/
# ShwfFRhzVuX+9IePl86gCw+iOWfNtp5kopumqJKT4vCtotDIgkh5n82GIAJptTqS
# sdZwa/+lAhO57Z3k3+XK7wr5a9ykLXQd+E+eLp8LyZ2wU8rqbfffpS5/Glgvn6yt
# m6hE
# SIG # End signature block