Obs/bin/ObsAgent/lib/Scripts/LogCollectionHelper.psm1
<##############################################################
# # # Copyright (C) Microsoft Corporation. All rights reserved. # # # ##############################################################> Import-Module $PSScriptRoot\GenericHelper.psm1 -Force -Verbose:$false function Invoke-ScriptBlockWithRetries { param ( [Parameter(Mandatory = $true)] [ScriptBlock] $ScriptBlock, [Parameter(Mandatory = $false)] [Object] $Argument, [Parameter(Mandatory = $true)] [int] $MaxTries, [Parameter(Mandatory = $false)] [int] $IntervalInSeconds = 30 ) $functionName = "$($MyInvocation.MyCommand.Name)" Trace-Progress "$functionName : Retrying max $MaxTries interval [$IntervalInSeconds] ScriptBlock = [$ScriptBlock]" $attempt = 1 $success = $false do { Trace-Progress "$functionName : Starting attempt $attempt" try { $result = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $Argument -ErrorAction Stop $success = $true } catch { $message = "Exception occurred while trying to execute scriptblock command:" + $_.Exception.ToString() Trace-Progress "$functionName : $message" if ($attempt -ge $MaxTries) { throw } Start-Sleep -Seconds $IntervalInSeconds } finally { Trace-Progress "$functionName : Completed attempt $attempt; Status = $success" $attempt++ } } while (!$success) return $result } function Get-WindowsEventLog { Param ( [parameter(Mandatory=$true)] [string[]] $ComputerNames, [parameter(Mandatory=$false)] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [parameter(Mandatory=$true)] [string[]] $LogPattern, [parameter(Mandatory=$false)] [DateTime] $EventsFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $EventsToDate = (Get-Date), [Parameter(Mandatory=$false)] [REF] $ExcludedEndpoints, [parameter(Mandatory=$true)] [PSObject] $Roles, [parameter(Mandatory=$true)] [string] $CurrentRole, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [parameter(Mandatory=$true)] [bool] $LocalMode ) $functionName = "$($MyInvocation.MyCommand.Name)_$CurrentRole" # Get the time span in milliseconds function Get-TimeSpan($Date) { $timeSpan = New-TimeSpan -Start $Date -End (Get-Date) return [Math]::Round($timeSpan.TotalMilliseconds) } # Calculate number of milliseconds and prepare the WEvtUtil parameter to filter based on date/time $toSpan = Get-TimeSpan -Date $EventsToDate $fromSpan = Get-TimeSpan -Date $EventsFromDate $exportLogJobs = @() # Copy logs from remote machine to local machine foreach($computerName in $ComputerNames) { $session = $null $machineName = $computerName.Split('.')[0] Trace-Progress "$functionName : computername = [$computerName] machinename = [$machineName]" if (!$LocalMode) { if ($ComputerPSSessions) { Trace-Progress "$functionName :Checking if the session for $computerName session is valid in ComputerPSSessions array" if ($ComputerPSSessions.ContainsKey($computerName)) { $session = $ComputerPSSessions[$computerName] if (($null -ne $session) -and ( ($session.State -ne "Opened") -or ($session.Availability -ne "Available") ) ) { if($session) { #if we had opened the session previously, close it before overwriting this variable with new session. Remove-PSSession -Session $session -ErrorAction SilentlyContinue } Trace-Progress "$functionName :The session for $computerName went into state = [$($session.state)] , availabilty = [$($session.Availability)], ! Reinitializing!" $session = Initialize-PSSession -ComputerPSSessions $ComputerPSSessions -ComputerFqdn $computerName -ExcludedEndpoints ([REF]$ExcludedEndpoints.Value) if ($null -ne $session) { $ComputerPSSessions[$computerName] = $session } } } else { Trace-Progress "$functionName :$computerName session not found in ComputerPSSessions[] array, unable to collect event logs " } } else { Trace-Progress "$functionName :Creating a PSSession to [$computerName] as ComputerPSSessions[] array is null" $session = New-PSSession -ComputerName $computerName -ErrorAction SilentlyContinue # $ComputerPsSessions are not provided and we are opening a new session for each computername, we need to close these before we leave. } } if ($LocalMode -or (($null -ne $session) -and ($session.State -eq "Opened") -and ($session.Availability -eq "Available"))) { if ($LocalMode) { $logPath = "$($env:TEMP)WinEvents$CurrentRole\" } else { $logPath = Invoke-Command -Session $session {$tmp = "$($env:TEMP)WinEvents$using:CurrentRole\" ; $tmp = $tmp.ToLower().Replace("c:","\\$($env:ComputerName)\c$"); return $tmp} } Trace-Progress "$functionName :Log path computer = $logPath -- for $computerName " $initblock = [ScriptBlock]::Create("Import-Module -Name '$PSScriptRoot\LogCollectionHelper.psm1' -Force; Import-Module -Name '$PSScriptRoot\GenericHelper.psm1' -Force") # Collect logs on remote machine if ($LocalMode) { $exportLogJobs += Start-Job -ScriptBlock { Collect-WindowsEventLogs -LogFolder $using:logPath -FromSpan $using:fromSpan -ToSpan $using:ToSpan -LogPattern $using:LogPattern } -InitializationScript $initblock -ErrorAction Continue } else { Invoke-Command -Session $session $initBlock $exportLogJobs += Invoke-Command -AsJob -Session $session { Collect-WindowsEventLogs -LogFolder $using:logPath -FromSpan $using:fromSpan -ToSpan $using:ToSpan -LogPattern $using:LogPattern } -ErrorAction Continue } } else { if($null -eq $session) { Trace-Progress "$functionName :Could not establish a PS session with the computer [$computerName]." -Warning } else { Trace-Progress "$functionName :Session with the computer [$computerName] is stale - Session state = [$($session.State)], Session Availability =[$($session.Availability)]" -Warning } } } Trace-Progress "$functionName :Kicked off $($exportLogJobs.count) jobs to collect windows events" try { $ProgressPreference = "SilentlyContinue" $exportLogJobOutput = $exportLogJobs | Wait-job | Receive-Job Trace-Progress "$functionName :Finished waiting for jobs count = [$($exportLogJobs.Count)]" Write-Output $exportLogJobs # dont change to trace-progress Trace-Progress "$functionName :DestPathWithRoleName = [$DestPathWithRoleName]" foreach ($o in $exportLogJobOutput) { Trace-Progress "$functionName :Job retruned logpath = [$($o.logPath)] from computer = $($o.ComputerName)" if (-not [string]::IsNullOrEmpty($o.logPath)) { Trace-Progress "$functionName :Copying from Source: $($o.logPath) to Destination: $DestPathWithRoleName" try { $windowsEventFiles = Get-ChildItem -Path $o.logPath -File -Recurse } catch { Trace-Progress -Message "$functionName :Failed to get files from $($o.logPath) on $($o.computerName). Error: $_" -Warning } if (($null -ne $windowsEventFiles ) -or ($windowsEventFiles.Count -gt 0)) { $destPath = Join-Path -Path $DestPathWithRoleName -ChildPath $o.ComputerName try { Trace-Progress "$functionName :Creating new directory $destPath" New-ASPath -Path $destPath -Type Directory Copy-Item -Path $o.logPath -Destination $destPath -Force -Recurse } catch { Trace-Progress "$functionName :Failed to copy logs from $($o.logPath). Error: $_" -Warning } Remove-Item $o.logPath -Force -Recurse -ErrorAction SilentlyContinue } } else { Trace-Progress "$functionName :No logs copied as path on remote machine was empty. $($o.logPath)" } } Trace-Progress "$functionName :all evtx logs from all role vm's complete." $allEvtxCollectionSuccess = $true } finally { Trace-Progress "$functionName :In Finally block" if(!$allEvtxCollectionSuccess) { Trace-Progress "$functionName :Finally block- Unclean Exit detected, stopping all export jobs, if in progress" $exportLogJobs | Stop-Job $exportLogJobs | Receive-Job } Trace-Progress "$functionName :In Finally block, removing all job" $exportLogJobs | remove-job -ErrorAction SilentlyContinue } } function Collect-WindowsEventLogs { Param ( [parameter(Mandatory=$true)] [string] $LogFolder, [parameter(Mandatory=$true)] [double] $FromSpan, [parameter(Mandatory=$true)] [double] $ToSpan, [parameter(Mandatory=$true)] [string[]] $LogPattern ) if (-not (Test-Path $logFolder)) { $null = New-Item -ItemType Directory -Path $logFolder } $timestamps = @{} $qParameter = "*[System[TimeCreated[timediff(@SystemTime) <= $fromSpan] and TimeCreated[timediff(@SystemTime) >=$toSpan]]]" foreach ($lp in $logPattern) { $eventLogs = Get-WinEvent -ListLog $lp -Force -ErrorAction SilentlyContinue if (!$eventLogs.count) { $timestamps.$lp = @{} } else { $eventLogs | Foreach-Object { $fileSuffix = "Event_"+$_.LogName.Replace("/","-")+".EVTX" $logFile = $logFolder + $fileSuffix $locale = (Get-Culture).Name # Export log file using the WEvtUtil command-line tool # For Analytical and Debug log: disable => export => enable, as export cannot be performed over an enabled direct channel. $directChannel = $false $allLatestTimeCreated = $null if ($_.LogType -in @('Analytical','Debug')) { if ($_.IsEnabled) { $directChannel = $true # Disable Logs WEvtUtil.exe sl /e:false $_.LogName } } else { # We cant collect latest time in O(1) for Analytical and Debug Log, so leave it as null # Here are are collecting the latest time for Regular Logs only $allLatestTimeCreated = Get-WinEvent -logname $_.LogName -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" } # Export logs based on query to file with overwrite WEvtUtil.exe epl $_.LogName $logFile /q:$qParameter /ow:true $allOldestTimeCreated = Get-WinEvent -logname $_.LogName -oldest -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" if ($directChannel -eq $true) { # Enable Logs echo y | WEvtUtil.exe sl /e:true $_.LogName | out-null } # Archive logs (saves all locale specific information to allow reading of events without publisher) # WEvtUtil.exe al $logFile /l:$locale $copiedLatestTimeCreated = Get-WinEvent -path $logFile -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" $copiedOldestTimeCreated = Get-WinEvent -path $logFile -oldest -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" if ($allOldestTimeCreated -or $copiedOldestTimeCreated) { $timestamps.($_.LogName) = @{} } if ($allOldestTimeCreated) { $timestamps.($_.LogName).all = @{ oldestTimeCreated = $allOldestTimeCreated; latestTimeCreated = $allLatestTimeCreated } } if ($copiedOldestTimeCreated) { $timestamps.($_.LogName).copied = @{ oldestTimeCreated = $copiedOldestTimeCreated; latestTimeCreated = $copiedLatestTimeCreated } } } } } # Return the computerName and the logFolder @{ ComputerName = $env:ComputerName VMName = $null logPath = $logFolder timestamps = $timestamps } } # # For security reasons we strictly restrict the files to be included to certain extentions # function Get-FileLog { Param ( [parameter(Mandatory=$true, ParameterSetName='File')] [string[]] $ComputerNames, [parameter(Mandatory=$false, ParameterSetName='File')] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [parameter(Mandatory=$true)] [string[]] $SourceLogFilePaths, [parameter(Mandatory=$false)] [DateTime] $FilesFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $FilesToDate = (Get-Date), [parameter(Mandatory=$true, ParameterSetName='CSV')] [string] $CSVLogsFolderName, [parameter(Mandatory=$true)] [string] $Role, [Parameter(Mandatory=$false, ParameterSetName='File')] [REF]$ExcludedEndpoints, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [parameter(Mandatory=$true)] [bool] $LocalMode, [parameter(Mandatory=$false)] [bool] $IsArcA = $false, [parameter(Mandatory=$true)] [bool] $ToSMBShare ) Trace-EnteringMethod $functionName = "$($MyInvocation.MyCommand.Name)_$Role" $CSVLogsCopied = @() $ProgressPreference = "SilentlyContinue" foreach($logPath in $SourceLogFilePaths) { if ($logPath.Contains('$')) { # The path might contain environment variables, hence expanding it to actual path. # Example for valid environment variables: $env:WinDir, $env:SystemDrive, $env:ProgramData. # Avoid using environment variables that are different per user ex. $env:temp. $logPath = $ExecutionContext.InvokeCommand.ExpandString($logPath) } # Copy-Item -FromSession has a bug where it does not respect wild card over remote, as well as failing to copy some logs due to file locks. # Manually copying the files by mapping the drive. $logPathLeaf = Split-Path -Path $logPath -Leaf $logPathParent = Split-Path -Path $logPath -Parent if ($PsCmdlet.ParameterSetName -eq "CSV") { try { if ($logPathParent -notin $CSVLogsCopied) { Trace-Progress "$functionName :Copying from $logPath" $CSVLogDestRelativePath = $CSVLogsFolderName $items = $null if (Test-Path $logPath -ErrorAction SilentlyContinue) { Trace-Progress "$functionName : Copying csv logs from Source: $logPathParent to Destination: $CSVLogDestRelativePath" $items = Get-FilteredChildItem -Path $logPath -FromDate $FilesFromDate -ToDate $FilesToDate -IsArcA $IsArcA if (($null -ne $items.filteredItems) -and ($items.filteredItems.Count -gt 0)) { Copy-FilteredChildItem -Items $items.filteredItems -Source $logPathParent -ChildFolder $CSVLogDestRelativePath -DestPathWithRoleName $DestPathWithRoleName $cabFiles = $items.filteredItems.Name | Where-Object { $_.EndsWith(".cab") } # If we are sending the logs to an SMB Share, then we want all files compressed if(($cabFiles -ne $null) -and (-not $toSMBShare)) { Extract-CabFiles -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $CSVLogDestRelativePath } } else { Trace-Progress "$functionName : Skipping Copy-FilteredChildItem and checking for cab files, as items.FilteredItems is null." } <# if ($items.filesToSkipCompression.Count -gt 0) { Trace-Progress "$functionName : total files to skip compression = $($items.filesToSkipCompression.Count)" Copy-FilteredChildItem -Items $items.filesToSkipCompression -Source $logPathParent -ChildFolder $CSVLogDestRelativePath -ZipPipeline $UncompressedPipeline } #> $CSVLogsCopied += $logPathParent } else { Trace-Progress "$functionName :Folder $logPath does not exist. Logs from '$logPath' were not collected." -Warning } } } catch { Trace-Progress "$functionName : Failed to copy CSV logs at log path : $logpath. Error : $_" -Error } } elseif ($PsCmdlet.ParameterSetName -eq "File") { # Copy logs from remote machine to local machine foreach($computerName in $ComputerNames) { try { $session = $null $machineName = $computerName.Split('.')[0] $destRelativePath = $machineName if (!$LocalMode) { if ($ComputerPSSessions) { if ($ComputerPSSessions.ContainsKey($computerName)) { $session = $ComputerPSSessions[$computerName] if (($null -ne $session) -and ($session.State -ne "Opened")) { Trace-Progress "$functionName :The session for $computerName went into $($session.state) state! Reinitializing!" $session = Initialize-PSSession -ComputerPSSessions $ComputerPSSessions -ComputerFqdn $computerName -ExcludedEndpoints ([REF]$ExcludedEndpoints.Value) if ($null -ne $session) { $ComputerPSSessions[$computerName] = $session } } } } else { Trace-Progress "$functionName :Creating a PSSession to $computerName" $session = New-PSSession -ComputerName $computerName -ErrorAction SilentlyContinue } } if (!$LocalMode -and (($null -eq $session) -or ($session.State -ne "Opened"))) { Trace-progress -Message "$functionName :Could not establish a PS session with the computer. Logs were not copied from this computer." -Warning } else { Trace-Progress "$functionName : Copying from $logPath" if (!$LocalMode) { $logPathRoot = ("\\$computerName\$($logPathParent -replace ':', '$')").TrimEnd('\') $mappedDriveName = "Remote" + $machineName $mappedDrive = Get-PSDrive $mappedDriveName -ErrorAction SilentlyContinue if ((-not $mappedDrive)) { Trace-Progress "$functionName : Creating mapped drive : $mappedDriveName" $mappedDrive = New-PSDrive -Name $mappedDriveName -PSProvider FileSystem -Root $logPathRoot -ErrorVariable DriveError -ErrorAction SilentlyContinue if ($DriveError.count -gt 0) { $err = $DriveError[0] $errorMessage = $err.Exception.Message Trace-Progress "$functionName : Error creating mapped drive : $errorMessage" -Warning } } else { Trace-Progress "$functionName : Mapped drive for $mappedDriveName exists" } } if ($LocalMode -or $mappedDrive) { if ($LocalMode) { $logPathRoot = $logPathParent.TrimEnd('\') $newLogPath = $logPath } else { $newLogPath = $mappedDriveName + ':' + $logPathLeaf } Trace-Progress "$functionName : newLogPath = [$newLogPath]" $items = $null if (Test-Path $newLogPath -ErrorAction Continue) { if ($LocalMode) { Trace-Progress "$functionName :Copying file logs from Source: $newLogPath to Destination: $DestPathWithRoleName $destRelativePath" } else { Trace-Progress "$functionName :Copying file logs from Source: $newLogPath (Remote is mapped drive for $logPathRoot) to Destination: $DestPathWithRoleName $destRelativePath" } $items = Get-FilteredChildItem -Path $newLogPath -FromDate $FilesFromDate -ToDate $FilesToDate -IsArcA $IsArcA Trace-Progress "$functionName :Obtained files = $($items.count)" if (($null -ne $items.filteredItems) -and ($items.filteredItems.Count -gt 0)) { Copy-FilteredChildItem -Items $items.filteredItems -Source $logPathRoot -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $destRelativePath -ComputerName $machineName $cabFiles = $items.filteredItems.Name | Where-Object { $_.EndsWith(".cab") } # If we are sending the logs to an SMB Share, then we want all files compressed if (($cabFiles -ne $null) -and (-not $toSMBShare)) { Extract-CabFiles -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $destRelativePath } Trace-Progress "$functionName :Completed Copy-FilteredChildItems" } else { Trace-Progress "$functionName : Skipping Copy-FilteredChildItem and checking for cab files, as items.FilteredItems is null." } } else { Trace-Progress "$functionName :Folder $newLogPath does not exist on $computerName. Logs from '$logPath' were not collected." -Warning } if (!$LocalMode) { Trace-Progress "$functionName :Removing PS Drive $mappedDrive" Remove-PSDrive $mappedDrive -Verbose } } } } catch { Trace-Progress "$functionName : Failure to collect logs for log path : $LogPath on computer : $ComputerName. Error: $_" -Error if ($mappedDrive) { Remove-PSDrive $mappedDrive } } } if (-not $ComputerPSSessions -and ($null -ne $session)) { Remove-PSSession -Session $session -ErrorAction SilentlyContinue } } else{ # We should never see this, if we see this means error in role xml. Trace-Progress "$functionName :Folder [$logPath] - [$logPathParent] is neither file log or CSV log check..." -Warning } } } # # Gets files according to filtered extensions and date range. # returns the child items based on the filtered criteria # function Get-FilteredChildItem { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $Path, [Parameter(Mandatory=$true)] [DateTime] $FromDate, [Parameter(Mandatory=$true)] [DateTime] $ToDate, [parameter(Mandatory=$false)] [switch] $IncludeDumpFile, [parameter(Mandatory=$false)] [bool] $IsArcA = $false ) Trace-EnteringMethod $functionName = $($MyInvocation.MyCommand.Name) $allowedFileExtensions = '*.txt','*.log','*.etl','*.out','*.xml','*.htm','*.html','*.mta','*.evtx','*.tsf','*.json','*.zip','*.csv','*.err','*.cab' # Note following file extensions are omitted - '*.blg', ,'*.trace', '*.bin' if ($isArcA) { $allowedFileExtensions = $allowedFileExtensions + "*.dtr", "*.bin" } if ($IncludeDumpFile) { $allowedFileExtensions = $allowedFileExtensions + "*.dmp" } $dateFilterExt = @('*.bin') $excludedFiles = @('*unattend.xml') #$skipCompressionFileExtensions = @('*.bin', '*.zip', '*.cab') #$reservedFiles = 'MpSupportFiles.cab','IPInformation.txt','Cluster.log','ClusterHealth.log','gMSAInformation.txt','IISSiteInformation.txt','SLBStateInformation.txt','AzureStackAlerts.json' #$reservedFolders = @('ServiceFabricLogs', 'OEMLogs','StorageDiagnosticInfo','dcdiag', 'SDN', 'NetworkControllerState') #$reservedPattern = @('MonAgentHost', 'AzureStack_Validation') $filesToSkipCompression = @() Trace-Progress "$functionName :Path : $Path" try { if (Test-Path -Path $Path -PathType leaf) { Trace-Progress "$functionName testpath success - path is a leaf = $Path" $unfilteredItems = Get-ChildItem -Path $Path -Force -ErrorAction stop } else { Trace-Progress "$functionName testpath is not a leaf = $Path" $unfilteredItems = Get-ChildItem -Path $Path -Recurse -Force -ErrorAction stop } Trace-Progress "$functionName : Found $($unfilteredItems.Count) unfiltered items in $Path." # Apply special filtering for bin files based on date range as logs get added to these files incrementally, so we cannot depend on creation/modification date. if ($allowedFileExtensions | Where-Object {$dateFilterExt -Contains $_}) { Trace-Progress "$functionName allowedFileExtentions $allowedFileExtensions - dateFilterExt = $dateFilterExt" $items1 = @() try { $childItems = Get-ChildItem -Path $Path -Include $dateFilterExt -Recurse -Force -ErrorAction stop Trace-Progress "$functionName childItems = $($childItems.count) , childItems = $($childItems -join ',')" } catch { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, Powershell Exception: $_" -Error } $directories = $childItems | Group-Object Directory Trace-Progress "$functionName : Obtained $($directories.count) directories" foreach ($directory in $directories) { Trace-Progress "$functionName : Processing direcotry = $directory " $files = @($directory.Group | Sort-Object CreationTime,Name) if ($files.Count -le 2) { $items1 += $files } elseif (($FromDate -le $ToDate) -and ($ToDate -ge $files[0].CreationTime)) { # Start from the first file modified after FromDate $filesModifiedAfterFromDate = $files | Where-Object {$_.LastWriteTime -ge $FromDate} # If there is less then 3 files which were modified after from date, just get the last 3 files to help investigation if ($filesModifiedAfterFromDate.Count -gt 2) { $fromFile = $filesModifiedAfterFromDate[0] } else { # Get last 3 log files if there is no log written in specific time range $fromFile = $files[0 - [math]::min($files.Count, 3)] } # End at the first file modifed after the ToDate. $filesModifiedAfterToDate = $files | Where-Object {$_.LastWriteTime -ge $ToDate} if ($filesModifiedAfterToDate.Count -gt 0) { $toFile = $filesModifiedAfterToDate[0] } else { $toFile = $files[-1] } $fromIndex = [array]::IndexOf($files, $fromFile) $toIndex = [array]::IndexOf($files, $toFile) if($fromIndex -ne '-1' -and $toIndex -ne '-1' -and $fromIndex -le $toIndex) { $items1 += $files[$fromIndex..$toIndex] } } } <# # by disabling this, all the files will be in item1 [System.Array]$tmp = @($items1 | ForEach-Object {$r=@()} {$t=$_; $skipCompressionFileExtensions | ForEach-Object {if ($t -like $_){$r+=$t}}} {$r}) $filesToSkipCompression = $tmp Trace-Progress "$functionName : Zipping skipped for files with dateFilterExt are : $tmp" $items1 = $items1 | Where-Object { $_ -NotIn $filesToSkipCompression } #> } }catch { Trace-Progress "$functionName : Failed while parsing for bin files $_" -Error Trace-Progress -Message "$functionName : StackTrace : $($PSItem.ScriptStackTrace)" -Error } # Rest of files, ex.("*.etl","*.txt","*.log", ..etc) are filtered based on creation/modification date range, except reserved folders/files. # Adding try catch block because powershell throws .net terminating exception which is not ignored by powershell with “ErrorAction SilentlyContinue” try { $ext = $allowedFileExtensions | Where-Object { $_ -notin $dateFilterExt} $items2 = @() # Handles possible arrays of files/folders $pathItemResult = Get-Item $Path foreach ($pathItem in $pathItemResult) { if($pathItem -is [System.IO.DirectoryInfo]) { # Get items recursively for folders $items2 += Get-ChildItem -Path $pathItem -Include $ext -Exclude $excludedFiles -Recurse -Force -ErrorVariable Item2Errors -ErrorAction Continue } elseif ($pathItem -is [System.IO.FileInfo]) { # Get items non recursively for files $items2 += Get-ChildItem -Path $pathItem -Include $ext -Exclude $excludedFiles -Force -ErrorVariable Item2Errors -ErrorAction Continue } else { Trace-Progress "$functionName : Failed to handle '$pathItem' item type: $($pathItem.GetType().FullName)" } } } catch [UnauthorizedAccessException] { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, .Net Exception: $_" -Error # This is a temporary workaround to handle the issue in Bug 4780610, where accessing (by Get-ChildItem above) some of the .blg files copied to our SF clusters' # diagnostic shares result in an AccessDenied error. Since we already have the unfiltered list of items, as a fallback, we will perform the filtering directly # against that list instead of relying on Get-ChildItem. $items2 = Get-ItemsByExtension -UnfilteredItems $unfilteredItems -Include $ext -Exclude $excludedFiles -ErrorAction SilentlyContinue } catch { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, Powershell Exception: $_" -Error } Trace-Progress -Message "$functionName : item2 count = $($items2.count)" $items2 = $items2 | Where-Object {((($_.CreationTime -ge $FromDate) -or ($_.LastWriteTime -ge $FromDate)) -and $_.CreationTime -le $ToDate)} Trace-Progress -Message "FromDate $($FromDate.ToString()) ToDate = $($ToDate.ToString()) " # since we use -ErrorAction continue, most errors will not hit the catch block. See if there were any errors in getting $items2 # Note: Found a bug where Get-ChildItem did not finish getting items if a file is not found (likely because it was pruned or zipped). Solution is to use # -ErrorAction Continue instead of -ErrorAction Stop. foreach ($err in $Item2Errors) { $errorMessage = $err.Exception.Message if ($errorMessage -like "Could not find item *") { $fileNotFound = $errorMessage.Split()[-1].Trim('.') # if file not found is .etl or .blg, check if it was zipped if (($fileNotFound.endswith(".etl")) -or ($fileNotFound.endswith(".blg"))) { $extension = $fileNotFound.Substring($fileNotFound.length - 3) $zippedFileName = $fileNotFound + ".zip" $srcFile = $null try { # This is the new zip file that needs to be copied in lieu of original etl or blg $srcFile = Get-Item -Path $zippedFileName -ErrorAction Stop } catch { Trace-Progress -Message "$functionName : Failed to Fetch the zip file in the abscence of $($extension) [$zippedFileName]" -Error } if ($srcFile -ne $null) { $items2 += $srcFile Trace-Progress -Message "$functionName : Successfully added [$zippedFileName] to list of items to copy" } } else { if (($fileNotFound).EndsWith('.zip')) { # assume the file was pruned, so make it a warning Trace-Progress -Message "$functionName : $errorMessage" -warning } else { Trace-Progress -Message "$functionName : $errorMessage" -Error } } } else { Trace-Progress -Message "$functionName : $errorMessage" -Error } } <# # Contents of reserved folders are always copied. # NOTE: # Each $path will be of the format Remote:SDN # i.e. only the leaf folder name with a prefix of 'Remote:' will be part of the path that needs to be compared with set of reserved folders defined above. # using $path.contains looks for substring which can be faulty, To match the full folder name should use EndsWith # e.g. of faulty comparision is when SDN matches folders 'SDN' and 'SDNDiagnostics' cause all files from both folders to be picked up. $items3 = @() if (($reservedFolders | ForEach-Object {$Path.EndsWith($_)}) -contains $true) { $items3 = Get-ChildItem -Path $Path -Recurse -Force -ErrorAction SilentlyContinue | Where-Object { !$_.PSIsContainer } } # Reserved files are always copied. $items4 = Get-ChildItem -Path $Path -Force -ErrorAction SilentlyContinue | Where-Object {$_.Name -in $reservedFiles} $items5 = @() if (($reservedPattern | ForEach-Object {$Path.Contains($_)}) -contains $true) { $items5 = Get-ChildItem -Path $Path -Force -ErrorAction SilentlyContinue } #> # [System.Array]$tmp1 = @(@($items1) + @($items2) | ForEach-Object {$r=@()} {$t=$_; $skipCompressionFileExtensions | ForEach-Object {if ($t -like $_){$r+=$t}}} {$r}) # $filesToSkipCompression += $tmp1 Trace-Progress "$functionName : adding items1.count = $($items1.count) and items2.count = $($items2.count) after applying time filter" $filteredItems = @($items1) + @($items2) | Sort-Object -Property FullName -Unique Trace-Progress "$functionName : Returning unfilteredItems ($($unfilteredItems.count)), filteredItems ($($filteredItems.count)), filesToSkipCompression ($($filesToSkipCompression.count))" return @{ unfilteredItems = @($unfilteredItems) filteredItems = @($filteredItems) filesToSkipCompression = @($filesToSkipCompression) } } function Get-TimestampsHelper { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $Items ) if ($Items.count -eq 0) { return @{} } $oldestCreationTime = $Items[0].CreationTimeUtc $latestCreationTime = $Items[0].CreationTimeUtc $oldestLastWriteTime = $Items[0].LastWriteTimeUtc $latestLastWriteTime = $Items[0].LastWriteTimeUtc foreach ($item in $Items) { if ($null -ne $item.CreationTimeUtc) { if ($null -eq $oldestCreationTime -or $oldestCreationTime -gt $item.CreationTimeUtc) { $oldestCreationTime = $item.CreationTimeUtc } if ($null -eq $latestCreationTime -or $latestCreationTime -lt $item.CreationTimeUtc) { $latestCreationTime = $item.CreationTimeUtc } } if ($null -ne $item.LastWriteTimeUtc) { if ($null -eq $oldestLastWriteTime -or $oldestLastWriteTime -gt $item.LastWriteTimeUtc) { $oldestLastWriteTime = $item.LastWriteTimeUtc } if ($null -eq $latestLastWriteTime -or $latestLastWriteTime -lt $item.LastWriteTimeUtc) { $latestLastWriteTime = $item.LastWriteTimeUtc } } } return @{ oldestCreationTime = $oldestCreationTime latestCreationTime = $latestCreationTime oldestLastWriteTime = $oldestLastWriteTime latestLastWriteTime = $latestLastWriteTime } } function Get-Timestamps { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $all, [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $copied ) $newDetails = @{} if ($all.count) { $allTimestamps = Get-TimestampsHelper $all $newDetails.all = $allTimestamps $newDetails.all.count = $all.count } if ($copied.count) { $copiedTimestamps = Get-TimestampsHelper $copied $newDetails.copied = $copiedTimestamps $newDetails.copied.count = $copied.count } return $newDetails } # # Copy filtered files recursively by re-creating the folder structure at the destination to match the source. # function Copy-FilteredChildItem { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [Object[]] $Items, [Parameter(Mandatory=$true)] [string] $Source, [Parameter(Mandatory=$true)] [string] $ChildFolder, [Parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [Parameter(Mandatory=$false)] [string] $ComputerName ) $functionName = $($MyInvocation.MyCommand.Name) # Handle paths with wildcard(s); set Source to deepest non-wildcard parent that resolves to a full directory. if ($Source -ne $env:SystemDrive) { $sourceItem = Get-Item $Source while ($sourceItem -isnot [System.IO.DirectoryInfo]) { $Source = Split-Path $Source -Parent $sourceItem = Get-Item $Source } } else { # handle exception when path is $env:systemdrive, in that case get-item $source returns current path not c:/ $Source += "\" $sourceItem = Get-Item $Source } # Catches cases where string path may not match the resolved file path, e.g. # 'C:\Users\ADMINI~1\AppData' (string) vs. C:\Users\Administrator (${item}.FullName) # Also resolves wildcard paths into valid expanded paths. $Source = $sourceItem.FullName.TrimEnd('\') Trace-Progress "$functionName DestPathWithRoleName = $DestPathWithRoleName" foreach ($item in $Items) { $Destination = Join-Path -Path $DestPathWithRoleName -ChildPath $ChildFolder $itemDir = $item.DirectoryName $itemName = $item.FullName if (($null -eq $itemDir) -or ($null -eq $itemName)) { Trace-Progress "$functionName : Null directory or fullname found. Item $item, Directory $itemDir, ItemName $itemName" -Warning # Skip processing this item continue } $dir = $itemDir.Replace($Source, $Destination) $target = $itemName.Replace($Source, $Destination) if (!(Test-Path $dir -ErrorAction Continue)) { Trace-Progress "$functionName Creating new directory: $dir" $null = New-Item $dir -Type Directory } if ($ComputerName) { if ($item.Extension -in @('.bin','.etl')) { $parent = Split-Path $target -Parent $leaf = Split-Path $target -Leaf $target = "$($parent)\$($computerName)_$($leaf)" } } if (!(Test-Path $target -ErrorAction Continue)) { try { Trace-Progress -Message "$functionName : Copying item $($item.FullName) to [$target]" Copy-Item -Path $item.FullName -Destination $target -Force -ErrorAction Stop } catch [System.Management.Automation.ItemNotFoundException], [System.IO.FileNotFoundException] { $errorHResult = "0x$('{0:x8}' -f $_.Exception.HResult)" # Prepare the error message but dont trace immediately $actualErrorMessage = "$functionName : Failed to copy [$($item.FullName)] to $target. HResult : $errorHResult. Error: $_" # Does the file that failed to copy end with .etl or .blg? if yes, maybe it just got converted to .zip, so attempt to copy zip instead. # If copy of that fails too, then trace the original error - $actualErrorMessage if(($item.FullName).EndsWith('.etl') -or ($item.FullName).EndsWith('.blg')) { # This is best case attempt when etl or blg file just got converted to zip file. $extension = ($item | select Extension).Extension $zippedFileName = $item.FullName + ".zip" $target = $target + ".zip" $srcFile = $null try { # This is the new zip file that needs to be copied in lieu of original etl or blg $srcFile = Get-Item -Path $zippedFileName -ErrorAction Stop } catch { Trace-Progress -Message $actualErrorMessage -Error Trace-Progress -Message "Failed to Fetch the zip file in the abscence of $($extension) [$zippedFileName]" } if($srcFile) { try { Copy-Item -Path $zippedFileName -Destination $target -Force -ErrorAction Ignore Trace-Progress -Message "$functionName : attempting to copy ZIP file instead of $($extension) file succeeded [$zippedFileName] to [$target]" } catch { # this is not the original error, we found matching zip file and copying of that failed # this is a best case effort, if this fails trace original error. Trace-Progress -Message "$functionName : copying ZIP file instead of $($extension) file failed as well [$zippedFileName] to [$target]" # also add the original error into the error list. Trace-Progress -Message $actualErrorMessage -Error } } } else { # the file that failed to copy is not an etl or blg, so we dont have an alternative to that. if (($item.FullName).EndsWith('.zip')) { # assume the file was pruned, so make it a warning Trace-Progress -Message $actualErrorMessage -warning } else { Trace-Progress -Message $actualErrorMessage -Error } } } catch { $errorHResult = "0x$('{0:x8}' -f $_.Exception.HResult)" Trace-Progress -Message "$functionName : Failed to copy $($item.FullName) to $target. HResult : $errorHResult. Error: $_" -Error # On failure, display the size of the directories in system drive if($target.StartsWith($env:systemdrive[0])) { $sysDrive = Get-PSDrive $env:systemdrive[0] Trace-Progress -Message "SystemDrive = $($sysDrive.Name), UsedSpace = $($($sysDrive.Used)/1GB), FreeSpace = $($($sysDrive.Free)/1GB) " } #if we are copying to a user specified destination folder, HRESULT will have error incase of diskfull, no need to print folder sizes } } } Trace-Progress "$functionName Complete.." } # # Search for cab files in the input directory and extract the content in to <filename>_cab directory and delete the cab files. # function Extract-CabFiles { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [Parameter(Mandatory=$true)] [string] $ChildFolder ) $functionName = $($MyInvocation.MyCommand.Name) Trace-Progress "$functionName DestPathWithRoleName = $DestPathWithRoleName ChildFolder = $ChildFolder" $searchFolder = Join-Path -Path $DestPathWithRoleName -ChildPath $ChildFolder Trace-Progress "$functionName Going to search CAB files under searchFolder = $searchFolder" $cabFiles = Get-ChildItem -Path $searchFolder -Filter "*.cab" -File -Recurse -ErrorAction Ignore Trace-Progress "$functionName CAB files count $($cabFiles.Count) under searchFolder = $searchFolder" foreach ($cabFile in $cabFiles) { try { Trace-Progress "$functionName Processing CAB file $($cabFile.FullName)" Add-Type -Path "$PSScriptRoot\..\Microsoft.Deployment.Compression.Cab.dll" -ErrorAction Ignore -Verbose:$false | Out-Null $cabObject = New-Object -TypeName "Microsoft.Deployment.Compression.Cab.CabInfo" -ArgumentList $cabFile.FullName $cabDirectoryPath = Join-Path -Path $cabFile.Directory -ChildPath ($cabFile.BaseName+"_CAB") Trace-Progress "$functionName Going to create extract folder $cabDirectoryPath for CAB file $($cabFile.FullName)" $temp = New-Item -Path $cabDirectoryPath -ItemType Directory $cabObject.Unpack($cabDirectoryPath) $internalFileCount = $cabObject.GetFiles().Count $extractedFiles = Get-ChildItem -Path $cabDirectoryPath -Filter "*.*" -File -Recurse Trace-Progress "$functionName CAB file $($cabFile.FullName) Internal File count $internalFileCount extracted file count $($extractedFiles.Count)" $cabObject.Delete() } catch { Trace-Progress "$functionName CAB file $($cabFile.FullName) exception during processing: $($_.Exception.ToString())" Trace-Progress "$functionName CAB file $($cabFile.FullName) exception during processing: $($_.Exception.Message)" -Error } } Trace-Progress "$functionName Complete.." } # # Creates a PowerShell Session if needed. # function Initialize-PSSession { [CmdletBinding()] param( [parameter(Mandatory=$false)] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerFqdn, [Parameter(Mandatory=$false)] [REF]$ExcludedEndpoints ) $functionName = $($MyInvocation.MyCommand.Name) if ($ComputerPSSessions) { if ($ComputerPSSessions.ContainsKey($ComputerFqdn)) { $session = $ComputerPSSessions[$ComputerFqdn] if (($null -ne $session) -and ($session.State -ne "Opened")) { Trace-Progress "$functionName : The session for $ComputerFqdn went into $($session.state) state! Reinitializing!" } } } if ($null -eq $session -or $session.State -ne "Opened") { # Client call for new PS session can hang forever when server side WSMan layer is not responding.To unblock log collection, # we are testing the PS session creation in different thread using start-job if the monitoring job doesn’t return the PS session object in 2 min we declare the server to be in a bad state. $scriptBlock = [ScriptBlock]::Create(${function:Test-PSSession}) $psSessionObject = Invoke-ScriptBlockCommand -ScriptBlock $scriptBlock -ArgumentList $ComputerFqdn -TimeOutInSec 120 # Validate if the server is connectable if (($null -eq $psSessionObject -or $psSessionObject.State -ne 'Opened') -or (!(Test-Connection -ComputerName $ComputerFqdn -Quiet))) { Trace-Progress -Message "$functionName : Computer $ComputerFqdn is unreachable, Could not establish a PS session earlier. Will not retry" -Error $ExcludedEndpoints.Value += $ComputerFqdn return $null } <# New-PSSessionOption paramter: .IdleTimeout : Determines how long the session stays open if the computer does not receive any communication. This includes the heartbeat signal {It means if no operation is happening, session will be open as long as session connection is established and it will help us from create PS session timeout} .OperationTimeout - Determines the maximum time that any operation in the session can run. {This prevent very large file like +25GB copy operation and help us from diskspace issue} .MaxConnectionRetryCount :Specifies the number of times that PowerShell attempts to make a connection to a target machine if the current attempt fails due to network issues. #> $sessionOptions = New-PSSessionOption -OperationTimeout ([timespan]"00:10:00").TotalMilliseconds -MaxConnectionRetryCount 1 -IdleTimeout 600000 $session = New-PSSession -ComputerName $ComputerFqdn -SessionOption $sessionOptions -ErrorAction Continue if ($null -eq $session) { $ExcludedEndpoints.Value += $ComputerFqdn Trace-Progress -Message "$functionName : Could not establish a PS session with the computer $ComputerFqdn." -error } } return $session } <# .SYNOPSIS This is the generic function to execute the command or function as script block in separate powershell thread using start-job and return the job output object. #> function Invoke-ScriptBlockCommand { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [ScriptBlock]$ScriptBlock, [Parameter(Mandatory=$false)] $ArgumentList, [Parameter(Mandatory=$false)] [int]$TimeOutInSec = 120 ) $functionName = $($MyInvocation.MyCommand.Name) $jobOutput = $null # Start and get the monitoring job result try { $monitoringJob = Start-Job -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Verbose $jobOutput = $monitoringJob | Wait-Job -Timeout $TimeOutInSec | Receive-Job $monitoringJob | Stop-Job $monitoringJob | Remove-Job } catch { Trace-Progress "$functionName : ScriptBlock - $ScriptBlock, failed with an error: $_ " -Error } return $jobOutput } function Test-PSSession { [CmdletBinding()] param( [Parameter(Mandatory=$false)] [string] $ComputerFqdn, [Parameter(Mandatory=$false)] [PSCredential] $LocalAdminCredential ) if($LocalAdminCredential) { $session = New-PSSession -ComputerName $ComputerFqdn -Credential $LocalAdminCredential -ErrorAction Continue } else { $session = New-PSSession -ComputerName $ComputerFqdn -ErrorAction Continue } return $session } <# .SYNOPSIS Return items that match the provided filter conditions for file extensions to include/exclude. #> function Get-ItemsByExtension { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [Object[]] $UnfilteredItems, [Parameter(Mandatory=$true)] [string[]] $Include, [Parameter(Mandatory=$true)] [string[]] $Exclude ) $filteredItems = New-Object System.Collections.Generic.List[System.Object] foreach ($unfilteredItem in $UnfilteredItems) { $excludedItem = $false foreach ($extensionToExclude in $Exclude) { if ($unfilteredItem -like $extensionToExclude) { $excludedItem = $true } } if (-not $excludedItem) { foreach ($extensionToInclude in $Include) { if ($unfilteredItem -like $extensionToInclude) { $filteredItems.Add($unfilteredItem) } } } } return $filteredItems } function Get-ContainerStateLog { param ( [parameter(Mandatory=$false)] [DateTime] $FilesFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $FilesToDate = (Get-Date), [parameter(Mandatory=$true)] [string] $Role, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName ) $containerStateLogDirPath = Join-Path -Path $DestPathWithRoleName -ChildPath "ContainerStateLogs" $containerStateErrorLogPath = Join-Path -Path $containerStateLogDirPath -ChildPath "ContainerStateCollectionErrors.txt" Trace-Progress -Message "Start container state log collection of $Role to $containerStateLogDirPath." New-Item $containerStateLogDirPath -ItemType Directory -Force | Out-Null # Collect HCS state $hcsStateLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HcsState.txt" Invoke-ExpressionWithTracing -Expression "hcsdiag list" -TraceFilePath $hcsStateLogFilePath # Collect HNS state $hnsNetworksLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_Networks.txt" $hnsEndpointsLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_Endpoints.txt" $hnsPolicyListLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_PolicyList.txt" $hnsNetworksCommands = @( "Get-HnsNetwork | select Name, Type, ActivityId, ID, @{Name='Subnets'; Expression={ `$_.Subnets | select AddressPrefix, GatewayAddress, ID }} | Out-String", "Get-HnsNetwork | ForEach-Object { Get-HnsNetwork -Id `$_.ID -Detailed } | ConvertTo-Json -Depth 20" ) $hnsEndpointsCommands = @( "Get-HnsEndpoint | select ActivityId, ID, IpAddress, MacAddress, State | Format-Table | Out-String", "Get-HnsEndpoint | ConvertTo-Json -Depth 20" ) foreach ($hnsNetworksCommand in $hnsNetworksCommands) { Invoke-ExpressionWithTracing -Expression $hnsNetworksCommand -TraceFilePath $hnsNetworksLogFilePath } foreach ($hnsEndpointsCommand in $hnsEndpointsCommands) { Invoke-ExpressionWithTracing -Expression $hnsEndpointsCommand -TraceFilePath $hnsEndpointsLogFilePath } Invoke-ExpressionWithTracing -Expression "Get-HnsPolicyList | ConvertTo-Json -Depth 20" -TraceFilePath $hnsPolicyListLogFilePath # Collect Docker engine state $dockerStateEngineLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "DockerState-Engine.txt" $dockerEngineStateCommands = @( "docker version", "docker info", "docker ps -sa", "docker images", "docker volume ls", "docker system df -v", "docker network ls" ) foreach ($dockerEngineStateCommand in $dockerEngineStateCommands) { Invoke-ExpressionWithTracing -Expression $dockerEngineStateCommand -TraceFilePath $dockerStateEngineLogFilePath } $networkIds = docker network ls -q foreach ($networkId in $networkIds) { Invoke-ExpressionWithTracing -Expression "docker inspect $networkId" -TraceFilePath $dockerStateEngineLogFilePath } # Collect container specific diagnostics $allContainerIds = docker ps -aq [System.Collections.Generic.HashSet[string]]$runningContainerIds = docker ps -q # List of SF environment variables to include in the output. Other SF environment variable names starting with "Fabric" will be redacted. $sfEnvironmentVariablesToInclude = [System.Collections.Generic.HashSet[string]]@( "Fabric_ApplicationHostId", "Fabric_ApplicationHostType", "Fabric_ApplicationId", "Fabric_ApplicationName", "Fabric_CodePackageName", "Fabric_Endpoint_InstanceEndpoint", "Fabric_Endpoint_IPOrFQDN_InstanceEndpoint", "Fabric_Folder_App_Log", "Fabric_Folder_App_Temp", "Fabric_Folder_App_Work", "Fabric_Folder_Application", "Fabric_Folder_Application_OnHost", "Fabric_IsContainerHost", "Fabric_NodeId", "Fabric_NodeIPOrFQDN" "Fabric_NodeName" "Fabric_PartitionId", "Fabric_ServiceName", "Fabric_ServicePackageActivationId", "Fabric_ServicePackageName", "Fabric_ServicePackageVersionInstance", "Fabric_ContainerName", "FabricCodePath", "FabricLogRoot" ) foreach ($containerId in $allContainerIds) { try { $dockerInspectOutput = docker inspect $containerId | ConvertFrom-Json for ($i = 0; $i -lt $dockerInspectOutput.Config.Env.Count; $i++) { $envVariablePair = $dockerInspectOutput.Config.Env[$i] -split '=', 2 if ($envVariablePair.Length -eq 2) { $envVariableName = $envVariablePair[0] if ($envVariableName -ieq "AZS_DEPLOYMENT_APPLICATION_NAME") { $applicationName = $envVariablePair[1] -replace "/", "+" } elseif ($envVariableName -ieq "AZS_DEPLOYMENT_SERVICE_NAME") { $serviceName = $envVariablePair[1] } if ($envVariableName.StartsWith("Fabric") -and (-not $sfEnvironmentVariablesToInclude.Contains($envVariableName))) { $redactedEnvVariable = "$envVariableName=[redacted]" $dockerInspectOutput.Config.Env[$i] = $redactedEnvVariable } } else { # Unable to parse environment variable string, so will redact it completely to be safe (i.e., by avoiding leaking sensitive information). $dockerInspectOutput.Config.Env[$i] = "[redacted]" } } $containerStateLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "DockerState-${applicationName}_${serviceName}_${containerId}.txt" Add-Content $containerStateLogFilePath "docker inspect $containerId" Add-Content $containerStateLogFilePath $($dockerInspectOutput | ConvertTo-Json -Depth 10) } catch { Add-Content $containerStateErrorLogPath "Error while collecting docker inspect output of $containerId. ExceptionMessage: $($_.Exception.Message), ExceptionType: $($_.Exception.GetType().Name)" } # Collect running container specific diagnostics. if ($runningContainerIds.Contains($containerId)) { $containerStateCommands = @( "docker top $containerId", "docker stats $containerId --no-stream" ) foreach ($containerStateCommand in $containerStateCommands) { Invoke-ExpressionWithTracing -Expression $containerStateCommand -TraceFilePath $containerStateLogFilePath } } } Trace-Progress -Message "Finished container state log collection." } function Invoke-ExpressionWithTracing { param ( [parameter(Mandatory=$true)] [string] $Expression, [parameter(Mandatory=$true)] [string] $TraceFilePath ) try { Add-Content $TraceFilePath $Expression Invoke-Expression $Expression *>&1 | Add-Content -Path $TraceFilePath Add-Content $TraceFilePath "`n" } catch { Add-Content $containerStateErrorLogPath "Error executing '$Expression'. ExceptionMessage: $($_.Exception.Message), ExceptionType: $($_.Exception.GetType().Name)" } } function Get-ServiceFabricLog { [CmdletBinding()] param( [parameter(Mandatory=$false)] [string] $LogPath = "$env:SystemDrive\ServiceFabricLogs", [parameter(Mandatory=$false)] [DateTime] $FromDate = (Get-Date).AddHours(-4), [parameter(Mandatory=$false)] [DateTime] $ToDate = (Get-Date) ) $packagePath = Join-Path $env:SystemDrive -ChildPath "ServiceFabric\Tools\Microsoft.Azure.ServiceFabric.WindowsServer.SupportPackage.zip" # Expand SF Support Package. $toolsDir = Join-Path $env:SystemDrive -ChildPath "ServiceFabric\Tools" $collectorPath = Join-Path $toolsDir "StandaloneLogCollector.exe" if (-not (Test-Path($collectorPath))) { Trace-Progress "$functionName : Unzipping tool to:$collectorPath" Expand-Archive $packagePath -DestinationPath $toolsDir -Force } $timeStamp = $((Get-Date).ToString('yyyyMMddHHmmss')) $outputPath = "$env:SystemDrive\MASLogs\StandaloneLogCollector_StdOut_$timeStamp.txt" # Perform the log directory cleanup only in default path case to prevent security vulnerability if (Test-Path $LogPath) { Trace-Progress "$functionName : Removing existing ServiceFabric logs in:$LogPath" $null = Remove-Item -Path $LogPath -Recurse -Force } . $collectorPath -Output $LogPath -Mode Collect -StartUtcTime $FromDate.ToUniversalTime() -EndUtcTime $ToDate.ToUniversalTime() > $outputPath 2>&1 Copy-Item -Path $outputPath -Destination $LogPath\ -Force -ErrorAction Continue } Function Get-RoleLogs { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [PSCustomObject] $argumentsObject, [Parameter(Mandatory=$true)] [String] $role ) $OutputPath = $argumentsObject.OutputPath $FromDateG = $argumentsObject.FromDateG $ToDateG = $argumentsObject.ToDateG $FromDate = $argumentsObject.FromDate $ToDate = $argumentsObject.ToDate $roles = $argumentsObject.roles $domain = $argumentsObject.domain $destPath = $argumentsObject.destPath $roleNames = $argumentsObject.roleNames $nodeNames = $argumentsObject.nodeNames $filterByNode = $argumentsObject.FilterByNode $vmRoleNames = $argumentsObject.vmRoleNames $FilterByLogType = $argumentsObject.FilterByLogType $allClusterInfo = $argumentsObject.allClusterInfo $localMode = $argumentsObject.localMode $isArcA = $argumentsObject.isArcAEnv $toSMBShare = $argumentsObject.toSMBShare $functionName = "$($MyInvocation.MyCommand.Name)_$role" $perfRoleStartDate = Get-Date $roleLogDetails = @{"role" = $role; "StartDate" = $perfRoleStartDate} try { Write-Output "`r" Trace-Progress "$functionName : Collecting logs for role: $role" #Trace-InvokingProcessStats -Role ($role+"_Start") if (!$localMode) { $endpointPSSessions = @{} $ExcludedEndpoints = @() } # TODOTODO Override the nodes with node names passed #$nodes = $roles[$role].Nodes $nodes = @() $currentRoleData = $roles[$role] if ("PhysicalMachines" -in $currentRoleData.Nodes) { $nodes += $nodeNames } if ("AllVms" -in $currentRoleData.Nodes -and $vmRoleNames["AllVms"].count -gt 0) { $nodes += $vmRoleNames["AllVms"] } elseif ($role -in $currentRoleData.Nodes -and $vmRoleNames[$role].count -gt 0) { $nodes += $vmRoleNames[$role] } Trace-Progress "$functionName : Nodes to collect logs from for role [$role] = [$($nodes -join ', ')]. Note that this is before applying node filter." <# Thave above elseif should resolve to following code, if there are more specialized roles get added and their rolename is not same as the defined in Get-InfraVMNames() we need to remove the above elseif and update below cases. elseif ("NC" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["NC"] } elseif ("SLB" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["SLB"] } elseif ("GWY" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["GWY"] }#> $logsTobeCollected = (($currentRoleData.FileLog.count -gt 0) -or ($currentRoleData.CSVLog.count ) -or ($currentRoleData.WindowsEventLog.count ) ) # $rolePublicInfoLogs -- This is the xml node will <Logs></Logs> if ($logsTobeCollected) { $roleLogDetails.logsAvailable = $true Trace-Progress "$functionName : Destination path : $OutputPath" $destinationFolderPath = Join-Path -Path $OutputPath -ChildPath $role # Iterate over each end-point and collect logs if ($localMode) { $node = $env:ComputerName $endpoint = if ($null -eq $domain) { $node } else { "$node.$domain" } $endpoints = @($endpoint) } else { $endpoints = @() } if ($filterByNode) { Trace-Progress "$functionName : Node filter list = $($filterByNode -join ', ')" $nodes = $nodes | Where-Object { $_ -in $filterByNode} Trace-Progress "$functionName : Node list after applying node filter = $($nodes -join ', ')" } else { Trace-Progress "$functionName : No node filter specified" } if (!$localMode) { foreach ($node in $nodes) { $session = $null $endpoint = $node + ".$domain" Trace-Progress "$functionName : Creating a PSSession to $endpoint" if ($ExcludedEndpoints -contains $endpoint) { Trace-Progress -Message "$functionName : Could not establish a PS session earlier with the computer $endpoint. Will not retry." -Error } else { $session = Initialize-PSSession -ComputerPSSessions $endpointPSSessions -ComputerFqdn $endpoint -ExcludedEndpoints ([REF]$ExcludedEndpoints) if ($null -ne $session) { $endpointPSSessions[$endpoint] = $session $endpoints += $endpoint } } } Trace-Progress "$functionName : nodescount = [$($nodes.count)] endpointPSSessions count = [ $($endpointPSSessions.Count)], endpoints Count = [$($endpoints.Count)]" } if ($role -in "ServiceFabric") { if ($isArcA -and $localMode) { # the sf log collector only supports max path length of 35, so we collect logs in temp short path dir and then move it. $SFLogTempPath = "$env:SystemDrive\MASLogs\ServiceFabricLogs" Trace-Progress -Message "$functionName : ServiceFabric logs will be temporarily output to:$SFLogTempPath" Get-ServiceFabricLog -LogPath $SFLogTempPath -FromDate $FromDate -ToDate $ToDate Trace-Progress -Message "$functionName : Move ServiceFabric logs to:$OutputPath" Move-Item $SFLogTempPath $OutputPath } } # $endpoints is an empty array if there are no endpoints, it is not null. if($endpoints -gt 0) { # Collecting Windows event logs if ($FilterByLogType -contains 'WindowsEvent') { if ($currentRoleData.WindowsEventLog) { $logPattern = $currentRoleData.WindowsEventLog Trace-Progress -Message "$functionName : Collecting windows event logs with log patterns: $($logPattern -join ', '), with date range: from $FromDateG until $ToDateG, from machines $($endpoints -join ', ')" try { if ($localMode) { Get-WindowsEventLog -ComputerNames $endpoints -LogPattern $logPattern -EventsFromDate $FromDate -EventsToDate $ToDate -Roles $roles -CurrentRole $role ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode } else { Get-WindowsEventLog -ComputerNames $endpoints -ComputerPSSessions $endpointPSSessions -LogPattern $logPattern -EventsFromDate $FromDate -EventsToDate $ToDate ` -ExcludedEndpoints ([REF]$ExcludedEndpoints) -Roles $roles -CurrentRole $role -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode } Trace-Progress "$functionName : Successfully dumped and copied all the windows event log from individual machines to $destinationFolderPath" } catch { Trace-Progress "$functionName : Failed during windows event log collection $($_.Exception.Message)" -Error } } } else { Trace-Progress -Message "$functionName : Skipping WindowsEventLog collection." } # Collecting log files. if ($FilterByLogType -contains 'File') { if ($currentRoleData.FileLog) { $sourceLogPaths = foreach($entry in $currentRoleData.FileLog) { $entry Trace-Progress -Message "$functionName : Collecting files from '$($entry)'." } try { if ($localMode) { Get-FileLog -ComputerNames $endpoints -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } else { Get-FileLog -ComputerNames $endpoints -ComputerPSSessions $endpointPSSessions -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate ` -Role $role -ExcludedEndpoints ([REF]$ExcludedEndpoints) -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } } catch { Trace-Progress "$functionName : Failed during File log collection $($_.Exception.Message)" -Error } } } else { Trace-Progress -Message "$functionName : Skipping FileLog collection." } # Collecting container state. if ($FilterByLogType -contains 'ContainerState') { try { if ($isArcA -and $role -eq "MASLogs") { Get-ContainerStateLog -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role -DestPathWithRoleName $destinationFolderPath } else { Trace-Progress -Message "$functionName : Skipping ContainerState collection for non-ArcA MASLogs." } } catch { Trace-Progress "$functionName : Failed during ContainerState collection $($_.Exception.Message)" -Error } } else { Trace-Progress -Message "$functionName : Skipping ContainerState collection." } if (!$localMode) { #Remove PSSessions Trace-Progress -Message "$functionName : Role : $role, Removing PS Sessions." foreach ($psSession in $endpointPSSessions.Values) { if ($null -ne $psSession) { Remove-PSSession -Session $psSession -ErrorAction SilentlyContinue } } } } if ($FilterByLogType -contains 'CSV') { if ($currentRoleData.CSVLog) { if ($localMode) { $isPrimaryNode = $false # if LocalMode, each node is doing it's own log collection in parallel. Only want the primary node to collect CSV logs. Trace-Progress -Message "$functionName : In Local Mode. Determining primary node, so that only primary node collects CSV logs" try { $cluster = get-Cluster $nodes = get-clusternode -cluster $cluster | where-object {$_.State -eq "Up" } | Sort-Object -Property Name $primaryNode = $nodes[0].Name.ToLower() $isPrimaryNode = $primaryNode -eq ($env:COMPUTERNAME).ToLower() if ($isPrimaryNode) { Trace-Progress -message "$functionName : This is the primary node. This node will collect CSV logs." } else { Trace-Progress -message "$functionName : This is not the primary node. This node will not collect CSV logs." } } catch { # If we can't get primary node, it is likely deployment failed before cluster creation. In this case, # there would be no CSV Logs in cluster storage, as cluster storage is not available. # Even if there were race conditions in copying over CSV Logs, it would not cause log collection to fail. Trace-Progress -message "$functionName : Error getting primary node : $_ Will collect CSV logs on all nodes." $isPrimaryNode = $true } } if (!$localMode -or $isPrimaryNode) { $sourceLogPaths = foreach($entry in $currentRoleData.CSVLog) { $entry Trace-Progress -Message "$functionName : Collecting CSV files from '$entry'." } try { Get-FileLog -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role -CSVLogsFolderName "CSVLogs" ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } catch { Trace-Progress "$functionName : Failed during CSV log collection $($_.Exception.Message)" -Error } } } } else { Trace-Progress -Message "$functionName : Skipping CSV Log collection." } # TODOTODO: When framework support run powershell, we will move this Get-MocLogs to configuration json; after that, we can remove this block. if ($role -eq "MOC_ARB") { # Call Get-MocLogs to get the MOC logs. Get-MocLogs contains the logic to connect to each node and get data. # So, the cmdlet just need to run on primary node. $isPrimaryNode = $false if ($localMode) { # If runs LocalMode, each node is doing it's own log collection in parallel. Only want the primary node to collect Get-Moclogs. try { $cluster = get-Cluster $nodes = get-clusternode -cluster $cluster | Where-Object { $_.State -eq "Up" } | Sort-Object -Property Name $primaryNode = $nodes[0].Name.ToLower() $isPrimaryNode = $primaryNode -eq ($env:COMPUTERNAME).ToLower() } catch { # If we can't get primary node, set current node as PrimaryNode, so that we will call get-MocLogs in this node. Trace-Progress -message "$functionName : Error getting primary node : $_ Will call Get-Moclogs to collect Moc logs on this node." $isPrimaryNode = $true } } # Call Get-MocLogs to get the MOC logs. Get-MocLogs contains the logic to connect to each node and get data. # So, the cmdlet just need to run on primary node. if (!$localMode -or $isPrimaryNode) { $mocLogFolder = Join-Path $destinationFolderPath "MOC" # Call Get-MocLogs to get the MOC logs. We need specify the parameters to just get the MocStore, NodeVirtualizationLogs, and MOC agent logs. # We skipped failover cluster logs. # Currently, the output of Get-MocLogs has some gaps # 1. No time filter. We accept this in this version, and need fix this in future. # 2. The MoC agent logs will be uploaded as json directly. Trace-Progress -Message "$functionName : Calling Get-MocLogs to collect MocLog and save to $mocLogFolder." Get-MocLogs -MocStore -NodeVirtualizationLogs -AgentLogs -path $mocLogFolder # So, we need do some data cleaningup # a. remove the unused EventFile, which already collect by others. # b. rename the files with Extension (text), so that the textFileParser could handle them and upload. Trace-Progress -Message "$functionName : Do the data cleaningup for output of Get-MocLogs." $allFiles = Get-ChildItem -File -Recurse -Path $mocLogFolder foreach($file in $allFiles) { if ($file.Extension -eq ".evtx" ) { Remove-Item -Path $file.FullName } ElseIf (([string]::IsNullOrEmpty($file.Extension)) -or ($file.Extension -eq ".yaml")) { # the file extension is empty or yaml, we want to use text log parser to process it, so add .txt as extension. $newName = "$($file.Name).txt" Rename-item -Path $file.FullName -Newname $newName } } Trace-Progress -Message "$functionName : Successfully copied all MocLogs" } else { Trace-Progress -Message "$functionName : Skipping Get-MocLogs as current node is not primary node" } } } else { $roleLogDetails.logsAvailable = $false Trace-Progress -Message "$functionName : No logs collected for this role as none is specified in input configuration file." } $normalTermination = $true } catch { Trace-Progress -Message "$functionName : Collecting logs failed with error: $_" -Error Trace-Progress -Message "$functionName : StackTrace : $($PSItem.ScriptStackTrace)" -Error $normalTermination = $true } finally { if (!$localMode) { Trace-Progress -Message "$functionName : Role: $role cleaningup endpointPSSessions, current opened sessions count = [$($endpointPSSessions.Values.Count)] " foreach ($psSession in $endpointPSSessions.Values) { if ($null -ne $psSession) { Trace-Progress -Message "$functionName : Removing session = [$psSession] " Remove-PSSession -Session $psSession -ErrorAction SilentlyContinue } } } if ($normalTermination -ne $true) { Trace-Progress -Message "$functionName : $role : unclean exit detected " -Error Trace-Progress -Message "$functionName : $role : Wait 30 seconds for child jobs to complete" Start-Sleep 30 # incase of an unclean exit, give time for sub jobs to complete before exiting parent job #[environment]::Exit(0) #$ZippingJobs.Values | Remove-Job -force } Write-ErrorsIfExist -Role $role #Trace-InvokingProcessStats -Role ($role+"_End") $roleLogCollectionTime = ((Get-Date) - $perfRoleStartDate).TotalMinutes.ToString("0.0##") Trace-Progress -Message "$functionName : Time taken to collect role $role is [$roleLogCollectionTime] Minutes" } } # This method prints the $global:errorList in the calling process (each role and resource provider collection runs as a separate Process) # Ensure this is called almost at the end of the job/role collection function Write-ErrorsIfExist { Param ( [parameter(Mandatory=$true)] [string] $Role ) $functionName = $($MyInvocation.MyCommand.Name) + "_$Role" # this variable is created when any trace-progress with -error is invoked. # Each role runs in its own process, so we can clear the $error automatic variable as well. if (((Test-Path variable:global:errorList) -and $Global:errorList -ne "") -or $Error.Count -gt 0) { Trace-Progress -Message "$functionName : Total entries in Global error list = $($Global:errorList.count)" $errorMessage = "ErrorList: `n" + $Global:errorList + ($Error | Get-Unique | Out-String) Write-Host $errorMessage -ForegroundColor "Red" #Dont change this to trace-progress Trace-Progress -Message $errorMessage $Error.Clear() } else { Trace-Progress -Message "$functionName : No Errors during role $Role" } } function Get-FreeSpace { Param ( [parameter(Mandatory=$true)] [string]$RelativePath ) $destinationFolder = Get-Item -Path (Split-Path $RelativePath -Parent) $fsobuild = new-Object -comobject Scripting.FileSystemObject $destinationFolderObj = $fsobuild.GetFolder($destinationFolder) $freeSpaceBytes = $destinationFolderObj.Drive.FreeSpace $freeSpaceKb = $freeSpaceBytes / 1024 return $freeSpaceKb } Export-ModuleMember -Function Get-FreeSpace Export-ModuleMember -Function Invoke-ScriptBlockWithRetries Export-ModuleMember -Function Write-ErrorsIfExist Export-ModuleMember -Function Get-RoleLogs Export-ModuleMember -Function Get-WindowsEventLog Export-ModuleMember -Function Collect-WindowsEventLogs Export-ModuleMember -Function Get-FileLog Export-ModuleMember -Function Get-FilteredChildItem Export-ModuleMember -Function Copy-FilteredChildItem Export-ModuleMember -Function Initialize-PSSession Export-ModuleMember -Function Test-PSSession Export-ModuleMember -Function Invoke-ScriptBlockCommand # SIG # Begin signature block # MIInwgYJKoZIhvcNAQcCoIInszCCJ68CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCSp2T1iO0gZaSu # f9LULXsRahYd1xUoyEfGESmIO1nQgaCCDXYwggX0MIID3KADAgECAhMzAAADTrU8 # esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU # p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1 # 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm # WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa # +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq # jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk # mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31 # TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2 # kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d # hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM # pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh # JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX # UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir # IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8 # 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A # Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H # tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIF6TePlLioa2ry/8YY99+OyF # wSLvy065tx3N6q+8YibZMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAm0C5mhux2WMQ07NrS3m3eQOjybuLLkvLzbENd4qeHe5udHykOyK1L3qW # LmvL8n3CNGNL1tDeMs98L0pyhLEWH6ZDqZnLlGBtdvPvaEbEtLD6z9TFjp8OcfpF # BaDMTSYpEX+TwP7nA/pgkZ0pbpqhgvRLpfw6ds2u9Om9Y5iVDUvOv0tfCOJjy40F # c+P4o6rEcFEhGX9MjVhKXbzo029KvNZvzuzP6Kr/YkbEvinnG3ApI58OTctQgCFT # Puqh0Nzp20wRNHhAwvxB0VjGAcR8nIWA+9b7FtBS1evFKY7NGeLZfAJkRtBZEaCk # jzse0wL9cUZYG/5hadr9nfuoKCDWYqGCFywwghcoBgorBgEEAYI3AwMBMYIXGDCC # FxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq # hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCDZoR6AdB1ueP/xoQigyvp+spMElHUmfUf3HlJY6/9FPwIGZQrjflAI # GBMyMDIzMDkyMjA4MzEyMy4zMzVaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO # OjJBRDQtNEI5Mi1GQTAxMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAGxypBD7gvwA6sAAQAAAbEwDQYJ # KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjIw # OTIwMjAyMTU5WhcNMjMxMjE0MjAyMTU5WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl # cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjoyQUQ0LTRC # OTItRkEwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIaiqz7V7BvH7IOMPEeDM2Uw # CpM8LxAUPeJ7Uvu9q0RiDBdBgshC/SDre3/YJBqGpn27a7XWOMviiBUfMNff51Nx # KFoSX62Gpq36YLRZk2hN1wigrCO656z5pVTjJp3Q8jdYAJX3ruJea3ccfTgxAgT3 # Uv/sP4w0+yZAYa2JZalV3MBgIFi3VwKFA4ClQcr+V4SpGzqz8faqabmYypuJ35Zn # 8G/201pAN2jDEOu7QaDC0rGyDdwSTVmXcHM46EFV6N2F69nwfj2DZh74gnA1DB7N # FcZn+4v1kqQWn7AzBJ+lmOxvKrURlV/u19Mw1YP+zVQyzKn5/4r/vuYSRj/thZr+ # FmZAUtTAacLzouBENuaSBuOY1k330eMp8nndSNUsUjj/nn7gcdFqzdQNudJb+Xxm # Rwi9LwjA0/8PlOsKTZ8Xw6EEWPVLfNojSuWpZMTaMzz/wzSPp5J02kpYmkdl50lw # yGRLO5X7iWINKmoXySdQmRdiGMTkvRStXKxIoEm/EJxCaI+k4S3+BWKWC07EV5T3 # UG7wbFb4LfvgbbaKM58HytAyjDnO9fEi0vrp8JFTtGhdtwhEEkraMtGVt+CvnG0Z # lH4mvpPRPuJbqE509e6CqmHwzTuUZPFMFWvJn4fPv0d32Ws9jv2YYmE/0WR1fULs # +TxxpWgn1z0PAOsxSZRPAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU9Jtnke8NrYSK # 9fFnoVE0pr0OOZMwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBANjnN5JqpeVShIrQ # IaAQnNVOv1cDEmCkD6oQufX9NGOX28Jw/gdkGtMJyagA0lVbumwQla5LPhBm5LjI # UW/5aYhzSlZ7lxeDykw57wp2AqoMAJm7bXcXtJt/HyaRlN35hAhBV+DmGnBIRcE5 # C2bSFFY3asD50KUSCPmKl/0NFadPeoNqbj5ZUna8VAfMSDsdxeyxjs8r/9Vpqy8l # gIVBqRrXtFt6n1+GFpJ+2AjPspfPO7Y+Y/ozv5dTEYum5eDLDdD1thQmHkW8s0BB # DbIOT3d+dWdPETkf50fM/nALkMEdvYo2gyiJrOSG0a9Z2S/6mbJBUrgrkgPp2HjL # kycR4Nhwl67ehAhWxJGKD2gRk88T2KKXLiRHAoYTZVpHbgkYLspBLJs9C77ZkuxX # uvIOGaId7EJCBOVRMJygtx8FXpoSu3jWEdau0WBMXxhVAzEHTu7UKW3Dw+KGgW7R # Rlhrt589SK8lrPSvPM6PPnqEFf6PUsTVO0bOkzKnC3TOgui4JhlWliigtEtg1SlP # MxcdMuc9uYdWSe1/2YWmr9ZrV1RuvpSSKvJLSYDlOf6aJrpnX7YKLMRoyKdzTkcv # Xw1JZfikJeGJjfRs2cT2JIbiNEGK4i5srQbVCvgCvdYVEVZXVW1Iz/LJLK9XbIkM # MjmECJEsa07oadKcO4ed9vY6YYBGMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh # bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjoy # QUQ0LTRCOTItRkEwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUA7WSxvqQDbA7vyy69Tn0wP5BGxyuggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AOi2szAwIhgPMjAyMzA5MjEyMDE4NTZaGA8yMDIzMDkyMjIwMTg1NlowdzA9Bgor # BgEEAYRZCgQBMS8wLTAKAgUA6LazMAIBADAKAgEAAgIg4gIB/zAHAgEAAgIR9jAK # AgUA6LgEsAIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB # AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAAduu1j9J/tgbUkz # kLxj7ODM9OtgbQ8piql7to7euorbjou4fksrv4qe2tWOzAFtw5UjvPA0P5VP5bmA # 4PxX8t6LWdkYDSo8LHW9FCpumN/iLSitxmQUOMjePnXRXpQPMG5MsuPR+Nl0kxJt # x7ugel7Cdr8X7yPd6UwVMmf+oJhuMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTACEzMAAAGxypBD7gvwA6sAAQAAAbEwDQYJYIZIAWUD # BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B # CQQxIgQgaEzxf2SMfpRijSZwoT6+pze5+01Qre3+gBDQMS7ifp8wgfoGCyqGSIb3 # DQEJEAIvMYHqMIHnMIHkMIG9BCCD7Q2LFFvfqeDoy9gpu35t6dYerrDO0cMTlOIo # mzTPbDCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB # scqQQ+4L8AOrAAEAAAGxMCIEIORiYXHs9mbD/bRiX2DjxNTbd/qyiRl/l5vc9ou3 # Xm7eMA0GCSqGSIb3DQEBCwUABIICAEIBEqjDnPT1apiooIFL9U4OOBFe8csRRsZ4 # XPtrAVS2tAavkeQX9pSXnCfOvISVpfM73zdk3A7wGGPSVQTpwJdOW7xycaA1faXv # fY9kOPqu33/3NYlITiZp10gRFWfhZKz6RhWEyRgL1L1QMEMk1z7GMykpHbCBPK78 # DkTe2xQPSukFWYuK0wi8J90vLVqK0gRDGeFt7KpW6D3KNzINh28vw6Jnzk2JsvOv # YXw7o/DgyP+zVLLhvizWQIC7JHBfqz171Oh7DkAaEoUtEDPiHl6N+XUML2iuFO77 # B6/XJn2/MWFAJJI7Ll95n8KvEhSv6M3jVt6fxFeU967f0yYG5x7+k/EpR/n6seOQ # CUn/96wIXn2VUcMJ7SkS3noO5XMbPjc01UAiRC4FueRoRIqorqiOIBnKtr1McRSk # ImvXbduLBIDhKdJeuUkDH0Jzaw012GFnaH+3E2tLE5GLEyAar1AyKMZ//wFnbe1O # Eh38FG1uAg+bGE1HoGZl2+V1zscOQpTGmyJkaxsmuLdQwGyJEZgBzWqurfLlSDdN # zF7qPhdBMmJE4grEtWOgzcPM6ls5YZ2V2FrjZ2UN2vDzGcYb32AsfIHWAYyfX3Xu # /X7hkAE7DXuOJj7sXK3vxUrZn25G/sIxVHptiRaf0fou+CWFtVnRqRM312zVJCHY # hEQXy1tJ # SIG # End signature block |