AzStackHciStandaloneObservability/package/bin/ObsAgent/lib/Scripts/LogCollectionHelper.psm1
<##############################################################
# # # Copyright (C) Microsoft Corporation. All rights reserved. # # # ##############################################################> Import-Module $PSScriptRoot\GenericHelper.psm1 -Force -Verbose:$false function Invoke-ScriptBlockWithRetries { param ( [Parameter(Mandatory = $true)] [ScriptBlock] $ScriptBlock, [Parameter(Mandatory = $false)] [Object] $Argument, [Parameter(Mandatory = $true)] [int] $MaxTries, [Parameter(Mandatory = $false)] [int] $IntervalInSeconds = 30 ) $functionName = "$($MyInvocation.MyCommand.Name)" Trace-Progress "$functionName : Retrying max $MaxTries interval [$IntervalInSeconds] ScriptBlock = [$ScriptBlock]" $attempt = 1 $success = $false do { Trace-Progress "$functionName : Starting attempt $attempt" try { $result = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $Argument -ErrorAction Stop $success = $true } catch { $message = "Exception occurred while trying to execute scriptblock command:" + $_.Exception.ToString() Trace-Progress "$functionName : $message" if ($attempt -ge $MaxTries) { throw } Start-Sleep -Seconds $IntervalInSeconds } finally { Trace-Progress "$functionName : Completed attempt $attempt; Status = $success" $attempt++ } } while (!$success) return $result } function Get-WindowsEventLog { Param ( [parameter(Mandatory=$true)] [string[]] $ComputerNames, [parameter(Mandatory=$false)] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [parameter(Mandatory=$true)] [string[]] $LogPattern, [parameter(Mandatory=$false)] [DateTime] $EventsFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $EventsToDate = (Get-Date), [Parameter(Mandatory=$false)] [REF] $ExcludedEndpoints, [parameter(Mandatory=$true)] [PSObject] $Roles, [parameter(Mandatory=$true)] [string] $CurrentRole, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [parameter(Mandatory=$true)] [bool] $LocalMode ) $functionName = "$($MyInvocation.MyCommand.Name)_$CurrentRole" # Get the time span in milliseconds function Get-TimeSpan($Date) { $timeSpan = New-TimeSpan -Start $Date -End (Get-Date) return [Math]::Round($timeSpan.TotalMilliseconds) } # Calculate number of milliseconds and prepare the WEvtUtil parameter to filter based on date/time $toSpan = Get-TimeSpan -Date $EventsToDate $fromSpan = Get-TimeSpan -Date $EventsFromDate $exportLogJobs = @() # Copy logs from remote machine to local machine foreach($computerName in $ComputerNames) { $session = $null $machineName = $computerName.Split('.')[0] Trace-Progress "$functionName : computername = [$computerName] machinename = [$machineName]" if (!$LocalMode) { if ($ComputerPSSessions) { Trace-Progress "$functionName :Checking if the session for $computerName session is valid in ComputerPSSessions array" if ($ComputerPSSessions.ContainsKey($computerName)) { $session = $ComputerPSSessions[$computerName] if (($null -ne $session) -and ( ($session.State -ne "Opened") -or ($session.Availability -ne "Available") ) ) { if($session) { #if we had opened the session previously, close it before overwriting this variable with new session. Remove-PSSession -Session $session -ErrorAction SilentlyContinue } Trace-Progress "$functionName :The session for $computerName went into state = [$($session.state)] , availabilty = [$($session.Availability)], ! Reinitializing!" $session = Initialize-PSSession -ComputerPSSessions $ComputerPSSessions -ComputerFqdn $computerName -ExcludedEndpoints ([REF]$ExcludedEndpoints.Value) if ($null -ne $session) { $ComputerPSSessions[$computerName] = $session } } } else { Trace-Progress "$functionName :$computerName session not found in ComputerPSSessions[] array, unable to collect event logs " } } else { Trace-Progress "$functionName :Creating a PSSession to [$computerName] as ComputerPSSessions[] array is null" $session = New-PSSession -ComputerName $computerName -ErrorAction SilentlyContinue # $ComputerPsSessions are not provided and we are opening a new session for each computername, we need to close these before we leave. } } if ($LocalMode -or (($null -ne $session) -and ($session.State -eq "Opened") -and ($session.Availability -eq "Available"))) { if ($LocalMode) { $logPath = "$($env:TEMP)WinEvents$CurrentRole\" } else { $logPath = Invoke-Command -Session $session {$tmp = "$($env:TEMP)WinEvents$using:CurrentRole\" ; $tmp = $tmp.ToLower().Replace("c:","\\$($env:ComputerName)\c$"); return $tmp} } Trace-Progress "$functionName :Log path computer = $logPath -- for $computerName " $initblock = [ScriptBlock]::Create("Import-Module -Name '$PSScriptRoot\LogCollectionHelper.psm1' -Force; Import-Module -Name '$PSScriptRoot\GenericHelper.psm1' -Force") # Collect logs on remote machine if ($LocalMode) { $exportLogJobs += Start-Job -ScriptBlock { Collect-WindowsEventLogs -LogFolder $using:logPath -FromSpan $using:fromSpan -ToSpan $using:ToSpan -LogPattern $using:LogPattern } -InitializationScript $initblock -ErrorAction Continue } else { Invoke-Command -Session $session $initBlock $exportLogJobs += Invoke-Command -AsJob -Session $session { Collect-WindowsEventLogs -LogFolder $using:logPath -FromSpan $using:fromSpan -ToSpan $using:ToSpan -LogPattern $using:LogPattern } -ErrorAction Continue } } else { if($null -eq $session) { Trace-Progress "$functionName :Could not establish a PS session with the computer [$computerName]." -Warning } else { Trace-Progress "$functionName :Session with the computer [$computerName] is stale - Session state = [$($session.State)], Session Availability =[$($session.Availability)]" -Warning } } } Trace-Progress "$functionName :Kicked off $($exportLogJobs.count) jobs to collect windows events" try { $ProgressPreference = "SilentlyContinue" $exportLogJobOutput = $exportLogJobs | Wait-job | Receive-Job Trace-Progress "$functionName :Finished waiting for jobs count = [$($exportLogJobs.Count)]" Write-Output $exportLogJobs # dont change to trace-progress Trace-Progress "$functionName :DestPathWithRoleName = [$DestPathWithRoleName]" foreach ($o in $exportLogJobOutput) { Trace-Progress "$functionName :Job retruned logpath = [$($o.logPath)] from computer = $($o.ComputerName)" if (-not [string]::IsNullOrEmpty($o.logPath)) { Trace-Progress "$functionName :Copying from Source: $($o.logPath) to Destination: $DestPathWithRoleName" try { $windowsEventFiles = Get-ChildItem -Path $o.logPath -File -Recurse } catch { Trace-Progress -Message "$functionName :Failed to get files from $($o.logPath) on $($o.computerName). Error: $_" -Warning } if (($null -ne $windowsEventFiles ) -or ($windowsEventFiles.Count -gt 0)) { $destPath = Join-Path -Path $DestPathWithRoleName -ChildPath $o.ComputerName try { Trace-Progress "$functionName :Creating new directory $destPath" New-ASPath -Path $destPath -Type Directory Copy-Item -Path $o.logPath -Destination $destPath -Force -Recurse } catch { Trace-Progress "$functionName :Failed to copy logs from $($o.logPath). Error: $_" -Warning } Remove-Item $o.logPath -Force -Recurse -ErrorAction SilentlyContinue } } else { Trace-Progress "$functionName :No logs copied as path on remote machine was empty. $($o.logPath)" } } Trace-Progress "$functionName :all evtx logs from all role vm's complete." $allEvtxCollectionSuccess = $true } finally { Trace-Progress "$functionName :In Finally block" if(!$allEvtxCollectionSuccess) { Trace-Progress "$functionName :Finally block- Unclean Exit detected, stopping all export jobs, if in progress" $exportLogJobs | Stop-Job $exportLogJobs | Receive-Job } Trace-Progress "$functionName :In Finally block, removing all job" $exportLogJobs | remove-job -ErrorAction SilentlyContinue } } function Collect-WindowsEventLogs { Param ( [parameter(Mandatory=$true)] [string] $LogFolder, [parameter(Mandatory=$true)] [double] $FromSpan, [parameter(Mandatory=$true)] [double] $ToSpan, [parameter(Mandatory=$true)] [string[]] $LogPattern ) if (-not (Test-Path $logFolder)) { $null = New-Item -ItemType Directory -Path $logFolder } $timestamps = @{} $qParameter = "*[System[TimeCreated[timediff(@SystemTime) <= $fromSpan] and TimeCreated[timediff(@SystemTime) >=$toSpan]]]" foreach ($lp in $logPattern) { $eventLogs = Get-WinEvent -ListLog $lp -Force -ErrorAction SilentlyContinue if (!$eventLogs.count) { $timestamps.$lp = @{} } else { $eventLogs | Foreach-Object { $fileSuffix = "Event_"+$_.LogName.Replace("/","-")+".EVTX" $logFile = $logFolder + $fileSuffix $locale = (Get-Culture).Name # Export log file using the WEvtUtil command-line tool # For Analytical and Debug log: disable => export => enable, as export cannot be performed over an enabled direct channel. $directChannel = $false $allLatestTimeCreated = $null if ($_.LogType -in @('Analytical','Debug')) { if ($_.IsEnabled) { $directChannel = $true # Disable Logs WEvtUtil.exe sl /e:false $_.LogName } } else { # We cant collect latest time in O(1) for Analytical and Debug Log, so leave it as null # Here are are collecting the latest time for Regular Logs only $allLatestTimeCreated = Get-WinEvent -logname $_.LogName -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" } # Export logs based on query to file with overwrite WEvtUtil.exe epl $_.LogName $logFile /q:$qParameter /ow:true $allOldestTimeCreated = Get-WinEvent -logname $_.LogName -oldest -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" if ($directChannel -eq $true) { # Enable Logs echo y | WEvtUtil.exe sl /e:true $_.LogName | out-null } # Archive logs (saves all locale specific information to allow reading of events without publisher) # WEvtUtil.exe al $logFile /l:$locale $copiedLatestTimeCreated = Get-WinEvent -path $logFile -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" $copiedOldestTimeCreated = Get-WinEvent -path $logFile -oldest -MaxEvents 1 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty "TimeCreated" if ($allOldestTimeCreated -or $copiedOldestTimeCreated) { $timestamps.($_.LogName) = @{} } if ($allOldestTimeCreated) { $timestamps.($_.LogName).all = @{ oldestTimeCreated = $allOldestTimeCreated; latestTimeCreated = $allLatestTimeCreated } } if ($copiedOldestTimeCreated) { $timestamps.($_.LogName).copied = @{ oldestTimeCreated = $copiedOldestTimeCreated; latestTimeCreated = $copiedLatestTimeCreated } } } } } # Return the computerName and the logFolder @{ ComputerName = $env:ComputerName VMName = $null logPath = $logFolder timestamps = $timestamps } } # # For security reasons we strictly restrict the files to be included to certain extentions # function Get-FileLog { Param ( [parameter(Mandatory=$true, ParameterSetName='File')] [string[]] $ComputerNames, [parameter(Mandatory=$false, ParameterSetName='File')] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [parameter(Mandatory=$true)] [string[]] $SourceLogFilePaths, [parameter(Mandatory=$false)] [DateTime] $FilesFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $FilesToDate = (Get-Date), [parameter(Mandatory=$true, ParameterSetName='CSV')] [string] $CSVLogsFolderName, [parameter(Mandatory=$true)] [string] $Role, [Parameter(Mandatory=$false, ParameterSetName='File')] [REF]$ExcludedEndpoints, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [parameter(Mandatory=$true)] [bool] $LocalMode, [parameter(Mandatory=$false)] [bool] $IsArcA = $false, [parameter(Mandatory=$true)] [bool] $ToSMBShare ) Trace-EnteringMethod $functionName = "$($MyInvocation.MyCommand.Name)_$Role" $CSVLogsCopied = @() $ProgressPreference = "SilentlyContinue" foreach($logPath in $SourceLogFilePaths) { if ($logPath.Contains('$')) { # The path might contain environment variables, hence expanding it to actual path. # Example for valid environment variables: $env:WinDir, $env:SystemDrive, $env:ProgramData. # Avoid using environment variables that are different per user ex. $env:temp. $logPath = $ExecutionContext.InvokeCommand.ExpandString($logPath) } # Copy-Item -FromSession has a bug where it does not respect wild card over remote, as well as failing to copy some logs due to file locks. # Manually copying the files by mapping the drive. $logPathLeaf = Split-Path -Path $logPath -Leaf $logPathParent = Split-Path -Path $logPath -Parent if ($PsCmdlet.ParameterSetName -eq "CSV") { try { if ($logPathParent -notin $CSVLogsCopied) { Trace-Progress "$functionName :Copying from $logPath" $CSVLogDestRelativePath = $CSVLogsFolderName $items = $null if (Test-Path $logPath -ErrorAction SilentlyContinue) { Trace-Progress "$functionName : Copying csv logs from Source: $logPathParent to Destination: $CSVLogDestRelativePath" $items = Get-FilteredChildItem -Path $logPath -FromDate $FilesFromDate -ToDate $FilesToDate -IsArcA $IsArcA if (($null -ne $items.filteredItems) -and ($items.filteredItems.Count -gt 0)) { Copy-FilteredChildItem -Items $items.filteredItems -Source $logPathParent -ChildFolder $CSVLogDestRelativePath -DestPathWithRoleName $DestPathWithRoleName $cabFiles = $items.filteredItems.Name | Where-Object { $_.EndsWith(".cab") } # If we are sending the logs to an SMB Share, then we want all files compressed if(($cabFiles -ne $null) -and (-not $toSMBShare)) { Extract-CabFiles -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $CSVLogDestRelativePath } } else { Trace-Progress "$functionName : Skipping Copy-FilteredChildItem and checking for cab files, as items.FilteredItems is null." } <# if ($items.filesToSkipCompression.Count -gt 0) { Trace-Progress "$functionName : total files to skip compression = $($items.filesToSkipCompression.Count)" Copy-FilteredChildItem -Items $items.filesToSkipCompression -Source $logPathParent -ChildFolder $CSVLogDestRelativePath -ZipPipeline $UncompressedPipeline } #> $CSVLogsCopied += $logPathParent } else { Trace-Progress "$functionName :Folder $logPath does not exist. Logs from '$logPath' were not collected." -Warning } } } catch { Trace-Progress "$functionName : Failed to copy CSV logs at log path : $logpath. Error : $_" -Error } } elseif ($PsCmdlet.ParameterSetName -eq "File") { # Copy logs from remote machine to local machine foreach($computerName in $ComputerNames) { try { $session = $null $machineName = $computerName.Split('.')[0] $destRelativePath = $machineName if (!$LocalMode) { if ($ComputerPSSessions) { if ($ComputerPSSessions.ContainsKey($computerName)) { $session = $ComputerPSSessions[$computerName] if (($null -ne $session) -and ($session.State -ne "Opened")) { Trace-Progress "$functionName :The session for $computerName went into $($session.state) state! Reinitializing!" $session = Initialize-PSSession -ComputerPSSessions $ComputerPSSessions -ComputerFqdn $computerName -ExcludedEndpoints ([REF]$ExcludedEndpoints.Value) if ($null -ne $session) { $ComputerPSSessions[$computerName] = $session } } } } else { Trace-Progress "$functionName :Creating a PSSession to $computerName" $session = New-PSSession -ComputerName $computerName -ErrorAction SilentlyContinue } } if (!$LocalMode -and (($null -eq $session) -or ($session.State -ne "Opened"))) { Trace-progress -Message "$functionName :Could not establish a PS session with the computer. Logs were not copied from this computer." -Warning } else { Trace-Progress "$functionName : Copying from $logPath" if (!$LocalMode) { $logPathRoot = ("\\$computerName\$($logPathParent -replace ':', '$')").TrimEnd('\') $mappedDriveName = "Remote" + $machineName $mappedDrive = Get-PSDrive $mappedDriveName -ErrorAction SilentlyContinue if ((-not $mappedDrive)) { Trace-Progress "$functionName : Creating mapped drive : $mappedDriveName" $mappedDrive = New-PSDrive -Name $mappedDriveName -PSProvider FileSystem -Root $logPathRoot -ErrorVariable DriveError -ErrorAction SilentlyContinue if ($DriveError.count -gt 0) { $err = $DriveError[0] $errorMessage = $err.Exception.Message Trace-Progress "$functionName : Error creating mapped drive : $errorMessage" -Warning } } else { Trace-Progress "$functionName : Mapped drive for $mappedDriveName exists" } } if ($LocalMode -or $mappedDrive) { if ($LocalMode) { $logPathRoot = $logPathParent.TrimEnd('\') $newLogPath = $logPath } else { $newLogPath = $mappedDriveName + ':' + $logPathLeaf } Trace-Progress "$functionName : newLogPath = [$newLogPath]" $items = $null if (Test-Path $newLogPath -ErrorAction Continue) { if ($LocalMode) { Trace-Progress "$functionName :Copying file logs from Source: $newLogPath to Destination: $DestPathWithRoleName $destRelativePath" } else { Trace-Progress "$functionName :Copying file logs from Source: $newLogPath (Remote is mapped drive for $logPathRoot) to Destination: $DestPathWithRoleName $destRelativePath" } $items = Get-FilteredChildItem -Path $newLogPath -FromDate $FilesFromDate -ToDate $FilesToDate -IsArcA $IsArcA Trace-Progress "$functionName :Obtained files = $($items.count)" if (($null -ne $items.filteredItems) -and ($items.filteredItems.Count -gt 0)) { Copy-FilteredChildItem -Items $items.filteredItems -Source $logPathRoot -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $destRelativePath -ComputerName $machineName $cabFiles = $items.filteredItems.Name | Where-Object { $_.EndsWith(".cab") } # If we are sending the logs to an SMB Share, then we want all files compressed if (($cabFiles -ne $null) -and (-not $toSMBShare)) { Extract-CabFiles -DestPathWithRoleName $DestPathWithRoleName -ChildFolder $destRelativePath } Trace-Progress "$functionName :Completed Copy-FilteredChildItems" } else { Trace-Progress "$functionName : Skipping Copy-FilteredChildItem and checking for cab files, as items.FilteredItems is null." } } else { Trace-Progress "$functionName :Folder $newLogPath does not exist on $computerName. Logs from '$logPath' were not collected." -Warning } if (!$LocalMode) { Trace-Progress "$functionName :Removing PS Drive $mappedDrive" Remove-PSDrive $mappedDrive -Verbose } } } } catch { Trace-Progress "$functionName : Failure to collect logs for log path : $LogPath on computer : $ComputerName. Error: $_" -Error if ($mappedDrive) { Remove-PSDrive $mappedDrive } } } if (-not $ComputerPSSessions -and ($null -ne $session)) { Remove-PSSession -Session $session -ErrorAction SilentlyContinue } } else{ # We should never see this, if we see this means error in role xml. Trace-Progress "$functionName :Folder [$logPath] - [$logPathParent] is neither file log or CSV log check..." -Warning } } } # # Gets files according to filtered extensions and date range. # returns the child items based on the filtered criteria # function Get-FilteredChildItem { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $Path, [Parameter(Mandatory=$true)] [DateTime] $FromDate, [Parameter(Mandatory=$true)] [DateTime] $ToDate, [parameter(Mandatory=$false)] [switch] $IncludeDumpFile, [parameter(Mandatory=$false)] [bool] $IsArcA = $false ) Trace-EnteringMethod $functionName = $($MyInvocation.MyCommand.Name) $allowedFileExtensions = '*.txt','*.log','*.etl','*.out','*.xml','*.htm','*.html','*.mta','*.evtx','*.tsf','*.json','*.zip','*.csv','*.err','*.cab' # Note following file extensions are omitted - '*.blg', ,'*.trace', '*.bin' if ($isArcA) { $allowedFileExtensions = $allowedFileExtensions + "*.dtr", "*.bin" } if ($IncludeDumpFile) { $allowedFileExtensions = $allowedFileExtensions + "*.dmp" } $dateFilterExt = @('*.bin') $excludedFiles = @('*unattend.xml') #$skipCompressionFileExtensions = @('*.bin', '*.zip', '*.cab') #$reservedFiles = 'MpSupportFiles.cab','IPInformation.txt','Cluster.log','ClusterHealth.log','gMSAInformation.txt','IISSiteInformation.txt','SLBStateInformation.txt','AzureStackAlerts.json' #$reservedFolders = @('ServiceFabricLogs', 'OEMLogs','StorageDiagnosticInfo','dcdiag', 'SDN', 'NetworkControllerState') #$reservedPattern = @('MonAgentHost', 'AzureStack_Validation') $filesToSkipCompression = @() Trace-Progress "$functionName :Path : $Path" try { if (Test-Path -Path $Path -PathType leaf) { Trace-Progress "$functionName testpath success - path is a leaf = $Path" $unfilteredItems = Get-ChildItem -Path $Path -Force -ErrorAction stop } else { Trace-Progress "$functionName testpath is not a leaf = $Path" $unfilteredItems = Get-ChildItem -Path $Path -Recurse -Force -ErrorAction stop } Trace-Progress "$functionName : Found $($unfilteredItems.Count) unfiltered items in $Path." # Apply special filtering for bin files based on date range as logs get added to these files incrementally, so we cannot depend on creation/modification date. if ($allowedFileExtensions | Where-Object {$dateFilterExt -Contains $_}) { Trace-Progress "$functionName allowedFileExtentions $allowedFileExtensions - dateFilterExt = $dateFilterExt" $items1 = @() try { $childItems = Get-ChildItem -Path $Path -Include $dateFilterExt -Recurse -Force -ErrorAction stop Trace-Progress "$functionName childItems = $($childItems.count) , childItems = $($childItems -join ',')" } catch { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, Powershell Exception: $_" -Error } $directories = $childItems | Group-Object Directory Trace-Progress "$functionName : Obtained $($directories.count) directories" foreach ($directory in $directories) { Trace-Progress "$functionName : Processing direcotry = $directory " $files = @($directory.Group | Sort-Object CreationTime,Name) if ($files.Count -le 2) { $items1 += $files } elseif (($FromDate -le $ToDate) -and ($ToDate -ge $files[0].CreationTime)) { # Start from the first file modified after FromDate $filesModifiedAfterFromDate = $files | Where-Object {$_.LastWriteTime -ge $FromDate} # If there is less then 3 files which were modified after from date, just get the last 3 files to help investigation if ($filesModifiedAfterFromDate.Count -gt 2) { $fromFile = $filesModifiedAfterFromDate[0] } else { # Get last 3 log files if there is no log written in specific time range $fromFile = $files[0 - [math]::min($files.Count, 3)] } # End at the first file modifed after the ToDate. $filesModifiedAfterToDate = $files | Where-Object {$_.LastWriteTime -ge $ToDate} if ($filesModifiedAfterToDate.Count -gt 0) { $toFile = $filesModifiedAfterToDate[0] } else { $toFile = $files[-1] } $fromIndex = [array]::IndexOf($files, $fromFile) $toIndex = [array]::IndexOf($files, $toFile) if($fromIndex -ne '-1' -and $toIndex -ne '-1' -and $fromIndex -le $toIndex) { $items1 += $files[$fromIndex..$toIndex] } } } <# # by disabling this, all the files will be in item1 [System.Array]$tmp = @($items1 | ForEach-Object {$r=@()} {$t=$_; $skipCompressionFileExtensions | ForEach-Object {if ($t -like $_){$r+=$t}}} {$r}) $filesToSkipCompression = $tmp Trace-Progress "$functionName : Zipping skipped for files with dateFilterExt are : $tmp" $items1 = $items1 | Where-Object { $_ -NotIn $filesToSkipCompression } #> } }catch { Trace-Progress "$functionName : Failed while parsing for bin files $_" -Error Trace-Progress -Message "$functionName : StackTrace : $($PSItem.ScriptStackTrace)" -Error } # Rest of files, ex.("*.etl","*.txt","*.log", ..etc) are filtered based on creation/modification date range, except reserved folders/files. # Adding try catch block because powershell throws .net terminating exception which is not ignored by powershell with “ErrorAction SilentlyContinue” try { $ext = $allowedFileExtensions | Where-Object { $_ -notin $dateFilterExt} $items2 = @() # Handles possible arrays of files/folders $pathItemResult = Get-Item $Path foreach ($pathItem in $pathItemResult) { if($pathItem -is [System.IO.DirectoryInfo]) { # Get items recursively for folders $items2 += Get-ChildItem -Path $pathItem -Include $ext -Exclude $excludedFiles -Recurse -Force -ErrorVariable Item2Errors -ErrorAction Continue } elseif ($pathItem -is [System.IO.FileInfo]) { # Get items non recursively for files $items2 += Get-ChildItem -Path $pathItem -Include $ext -Exclude $excludedFiles -Force -ErrorVariable Item2Errors -ErrorAction Continue } else { Trace-Progress "$functionName : Failed to handle '$pathItem' item type: $($pathItem.GetType().FullName)" } } } catch [UnauthorizedAccessException] { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, .Net Exception: $_" -Error # This is a temporary workaround to handle the issue in Bug 4780610, where accessing (by Get-ChildItem above) some of the .blg files copied to our SF clusters' # diagnostic shares result in an AccessDenied error. Since we already have the unfiltered list of items, as a fallback, we will perform the filtering directly # against that list instead of relying on Get-ChildItem. $items2 = Get-ItemsByExtension -UnfilteredItems $unfilteredItems -Include $ext -Exclude $excludedFiles -ErrorAction SilentlyContinue } catch { Trace-Progress "$functionName : Failed to Get-ChildItem for Path : $Path, Powershell Exception: $_" -Error } Trace-Progress -Message "$functionName : item2 count = $($items2.count)" $items2 = $items2 | Where-Object {((($_.CreationTime -ge $FromDate) -or ($_.LastWriteTime -ge $FromDate)) -and $_.CreationTime -le $ToDate)} Trace-Progress -Message "FromDate $($FromDate.ToString()) ToDate = $($ToDate.ToString()) " # since we use -ErrorAction continue, most errors will not hit the catch block. See if there were any errors in getting $items2 # Note: Found a bug where Get-ChildItem did not finish getting items if a file is not found (likely because it was pruned or zipped). Solution is to use # -ErrorAction Continue instead of -ErrorAction Stop. foreach ($err in $Item2Errors) { $errorMessage = $err.Exception.Message if ($errorMessage -like "Could not find item *") { $fileNotFound = $errorMessage.Split()[-1].Trim('.') # if file not found is .etl or .blg, check if it was zipped if (($fileNotFound.endswith(".etl")) -or ($fileNotFound.endswith(".blg"))) { $extension = $fileNotFound.Substring($fileNotFound.length - 3) $zippedFileName = $fileNotFound + ".zip" $srcFile = $null try { # This is the new zip file that needs to be copied in lieu of original etl or blg $srcFile = Get-Item -Path $zippedFileName -ErrorAction Stop } catch { Trace-Progress -Message "$functionName : Failed to Fetch the zip file in the abscence of $($extension) [$zippedFileName]" -Error } if ($srcFile -ne $null) { $items2 += $srcFile Trace-Progress -Message "$functionName : Successfully added [$zippedFileName] to list of items to copy" } } else { if (($fileNotFound).EndsWith('.zip')) { # assume the file was pruned, so make it a warning Trace-Progress -Message "$functionName : $errorMessage" -warning } else { Trace-Progress -Message "$functionName : $errorMessage" -Error } } } else { Trace-Progress -Message "$functionName : $errorMessage" -Error } } <# # Contents of reserved folders are always copied. # NOTE: # Each $path will be of the format Remote:SDN # i.e. only the leaf folder name with a prefix of 'Remote:' will be part of the path that needs to be compared with set of reserved folders defined above. # using $path.contains looks for substring which can be faulty, To match the full folder name should use EndsWith # e.g. of faulty comparision is when SDN matches folders 'SDN' and 'SDNDiagnostics' cause all files from both folders to be picked up. $items3 = @() if (($reservedFolders | ForEach-Object {$Path.EndsWith($_)}) -contains $true) { $items3 = Get-ChildItem -Path $Path -Recurse -Force -ErrorAction SilentlyContinue | Where-Object { !$_.PSIsContainer } } # Reserved files are always copied. $items4 = Get-ChildItem -Path $Path -Force -ErrorAction SilentlyContinue | Where-Object {$_.Name -in $reservedFiles} $items5 = @() if (($reservedPattern | ForEach-Object {$Path.Contains($_)}) -contains $true) { $items5 = Get-ChildItem -Path $Path -Force -ErrorAction SilentlyContinue } #> # [System.Array]$tmp1 = @(@($items1) + @($items2) | ForEach-Object {$r=@()} {$t=$_; $skipCompressionFileExtensions | ForEach-Object {if ($t -like $_){$r+=$t}}} {$r}) # $filesToSkipCompression += $tmp1 Trace-Progress "$functionName : adding items1.count = $($items1.count) and items2.count = $($items2.count) after applying time filter" $filteredItems = @($items1) + @($items2) | Sort-Object -Property FullName -Unique Trace-Progress "$functionName : Returning unfilteredItems ($($unfilteredItems.count)), filteredItems ($($filteredItems.count)), filesToSkipCompression ($($filesToSkipCompression.count))" return @{ unfilteredItems = @($unfilteredItems) filteredItems = @($filteredItems) filesToSkipCompression = @($filesToSkipCompression) } } function Get-TimestampsHelper { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $Items ) if ($Items.count -eq 0) { return @{} } $oldestCreationTime = $Items[0].CreationTimeUtc $latestCreationTime = $Items[0].CreationTimeUtc $oldestLastWriteTime = $Items[0].LastWriteTimeUtc $latestLastWriteTime = $Items[0].LastWriteTimeUtc foreach ($item in $Items) { if ($null -ne $item.CreationTimeUtc) { if ($null -eq $oldestCreationTime -or $oldestCreationTime -gt $item.CreationTimeUtc) { $oldestCreationTime = $item.CreationTimeUtc } if ($null -eq $latestCreationTime -or $latestCreationTime -lt $item.CreationTimeUtc) { $latestCreationTime = $item.CreationTimeUtc } } if ($null -ne $item.LastWriteTimeUtc) { if ($null -eq $oldestLastWriteTime -or $oldestLastWriteTime -gt $item.LastWriteTimeUtc) { $oldestLastWriteTime = $item.LastWriteTimeUtc } if ($null -eq $latestLastWriteTime -or $latestLastWriteTime -lt $item.LastWriteTimeUtc) { $latestLastWriteTime = $item.LastWriteTimeUtc } } } return @{ oldestCreationTime = $oldestCreationTime latestCreationTime = $latestCreationTime oldestLastWriteTime = $oldestLastWriteTime latestLastWriteTime = $latestLastWriteTime } } function Get-Timestamps { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $all, [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [System.IO.FileSystemInfo[]] $copied ) $newDetails = @{} if ($all.count) { $allTimestamps = Get-TimestampsHelper $all $newDetails.all = $allTimestamps $newDetails.all.count = $all.count } if ($copied.count) { $copiedTimestamps = Get-TimestampsHelper $copied $newDetails.copied = $copiedTimestamps $newDetails.copied.count = $copied.count } return $newDetails } # # Copy filtered files recursively by re-creating the folder structure at the destination to match the source. # function Copy-FilteredChildItem { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [AllowNull()] [AllowEmptyCollection()] [Object[]] $Items, [Parameter(Mandatory=$true)] [string] $Source, [Parameter(Mandatory=$true)] [string] $ChildFolder, [Parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [Parameter(Mandatory=$false)] [string] $ComputerName ) $functionName = $($MyInvocation.MyCommand.Name) # Handle paths with wildcard(s); set Source to deepest non-wildcard parent that resolves to a full directory. if ($Source -ne $env:SystemDrive) { $sourceItem = Get-Item $Source while ($sourceItem -isnot [System.IO.DirectoryInfo]) { $Source = Split-Path $Source -Parent $sourceItem = Get-Item $Source } } else { # handle exception when path is $env:systemdrive, in that case get-item $source returns current path not c:/ $Source += "\" $sourceItem = Get-Item $Source } # Catches cases where string path may not match the resolved file path, e.g. # 'C:\Users\ADMINI~1\AppData' (string) vs. C:\Users\Administrator (${item}.FullName) # Also resolves wildcard paths into valid expanded paths. $Source = $sourceItem.FullName.TrimEnd('\') Trace-Progress "$functionName DestPathWithRoleName = $DestPathWithRoleName" foreach ($item in $Items) { $Destination = Join-Path -Path $DestPathWithRoleName -ChildPath $ChildFolder $itemDir = $item.DirectoryName $itemName = $item.FullName if (($null -eq $itemDir) -or ($null -eq $itemName)) { Trace-Progress "$functionName : Null directory or fullname found. Item $item, Directory $itemDir, ItemName $itemName" -Warning # Skip processing this item continue } $dir = $itemDir.Replace($Source, $Destination) $target = $itemName.Replace($Source, $Destination) if (!(Test-Path $dir -ErrorAction Continue)) { Trace-Progress "$functionName Creating new directory: $dir" $null = New-Item $dir -Type Directory } if ($ComputerName) { if ($item.Extension -in @('.bin','.etl')) { $parent = Split-Path $target -Parent $leaf = Split-Path $target -Leaf $target = "$($parent)\$($computerName)_$($leaf)" } } if (!(Test-Path $target -ErrorAction Continue)) { try { Trace-Progress -Message "$functionName : Copying item $($item.FullName) to [$target]" Copy-Item -Path $item.FullName -Destination $target -Force -ErrorAction Stop } catch [System.Management.Automation.ItemNotFoundException], [System.IO.FileNotFoundException] { $errorHResult = "0x$('{0:x8}' -f $_.Exception.HResult)" # Prepare the error message but dont trace immediately $actualErrorMessage = "$functionName : Failed to copy [$($item.FullName)] to $target. HResult : $errorHResult. Error: $_" # Does the file that failed to copy end with .etl or .blg? if yes, maybe it just got converted to .zip, so attempt to copy zip instead. # If copy of that fails too, then trace the original error - $actualErrorMessage if(($item.FullName).EndsWith('.etl') -or ($item.FullName).EndsWith('.blg')) { # This is best case attempt when etl or blg file just got converted to zip file. $extension = ($item | select Extension).Extension $zippedFileName = $item.FullName + ".zip" $target = $target + ".zip" $srcFile = $null try { # This is the new zip file that needs to be copied in lieu of original etl or blg $srcFile = Get-Item -Path $zippedFileName -ErrorAction Stop } catch { Trace-Progress -Message $actualErrorMessage -Error Trace-Progress -Message "Failed to Fetch the zip file in the abscence of $($extension) [$zippedFileName]" } if($srcFile) { try { Copy-Item -Path $zippedFileName -Destination $target -Force -ErrorAction Ignore Trace-Progress -Message "$functionName : attempting to copy ZIP file instead of $($extension) file succeeded [$zippedFileName] to [$target]" } catch { # this is not the original error, we found matching zip file and copying of that failed # this is a best case effort, if this fails trace original error. Trace-Progress -Message "$functionName : copying ZIP file instead of $($extension) file failed as well [$zippedFileName] to [$target]" # also add the original error into the error list. Trace-Progress -Message $actualErrorMessage -Error } } } else { # the file that failed to copy is not an etl or blg, so we dont have an alternative to that. if (($item.FullName).EndsWith('.zip')) { # assume the file was pruned, so make it a warning Trace-Progress -Message $actualErrorMessage -warning } else { Trace-Progress -Message $actualErrorMessage -Error } } } catch { $errorHResult = "0x$('{0:x8}' -f $_.Exception.HResult)" Trace-Progress -Message "$functionName : Failed to copy $($item.FullName) to $target. HResult : $errorHResult. Error: $_" -Error # On failure, display the size of the directories in system drive if($target.StartsWith($env:systemdrive[0])) { $sysDrive = Get-PSDrive $env:systemdrive[0] Trace-Progress -Message "SystemDrive = $($sysDrive.Name), UsedSpace = $($($sysDrive.Used)/1GB), FreeSpace = $($($sysDrive.Free)/1GB) " } #if we are copying to a user specified destination folder, HRESULT will have error incase of diskfull, no need to print folder sizes } } } Trace-Progress "$functionName Complete.." } # # Search for cab files in the input directory and extract the content in to <filename>_cab directory and delete the cab files. # function Extract-CabFiles { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $DestPathWithRoleName, [Parameter(Mandatory=$true)] [string] $ChildFolder ) $functionName = $($MyInvocation.MyCommand.Name) Trace-Progress "$functionName DestPathWithRoleName = $DestPathWithRoleName ChildFolder = $ChildFolder" $searchFolder = Join-Path -Path $DestPathWithRoleName -ChildPath $ChildFolder Trace-Progress "$functionName Going to search CAB files under searchFolder = $searchFolder" $cabFiles = Get-ChildItem -Path $searchFolder -Filter "*.cab" -File -Recurse -ErrorAction Ignore Trace-Progress "$functionName CAB files count $($cabFiles.Count) under searchFolder = $searchFolder" foreach ($cabFile in $cabFiles) { try { Trace-Progress "$functionName Processing CAB file $($cabFile.FullName)" Add-Type -Path "$PSScriptRoot\..\Microsoft.Deployment.Compression.Cab.dll" -ErrorAction Ignore -Verbose:$false | Out-Null $cabObject = New-Object -TypeName "Microsoft.Deployment.Compression.Cab.CabInfo" -ArgumentList $cabFile.FullName $cabDirectoryPath = Join-Path -Path $cabFile.Directory -ChildPath ($cabFile.BaseName+"_CAB") Trace-Progress "$functionName Going to create extract folder $cabDirectoryPath for CAB file $($cabFile.FullName)" $temp = New-Item -Path $cabDirectoryPath -ItemType Directory $cabObject.Unpack($cabDirectoryPath) $internalFileCount = $cabObject.GetFiles().Count $extractedFiles = Get-ChildItem -Path $cabDirectoryPath -Filter "*.*" -File -Recurse Trace-Progress "$functionName CAB file $($cabFile.FullName) Internal File count $internalFileCount extracted file count $($extractedFiles.Count)" $cabObject.Delete() } catch { Trace-Progress "$functionName CAB file $($cabFile.FullName) exception during processing: $($_.Exception.ToString())" Trace-Progress "$functionName CAB file $($cabFile.FullName) exception during processing: $($_.Exception.Message)" -Error } } Trace-Progress "$functionName Complete.." } # # Creates a PowerShell Session if needed. # function Initialize-PSSession { [CmdletBinding()] param( [parameter(Mandatory=$false)] [HashTable] [ValidateNotNull()] $ComputerPSSessions, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerFqdn, [Parameter(Mandatory=$false)] [REF]$ExcludedEndpoints ) $functionName = $($MyInvocation.MyCommand.Name) if ($ComputerPSSessions) { if ($ComputerPSSessions.ContainsKey($ComputerFqdn)) { $session = $ComputerPSSessions[$ComputerFqdn] if (($null -ne $session) -and ($session.State -ne "Opened")) { Trace-Progress "$functionName : The session for $ComputerFqdn went into $($session.state) state! Reinitializing!" } } } if ($null -eq $session -or $session.State -ne "Opened") { # Client call for new PS session can hang forever when server side WSMan layer is not responding.To unblock log collection, # we are testing the PS session creation in different thread using start-job if the monitoring job doesn’t return the PS session object in 2 min we declare the server to be in a bad state. $scriptBlock = [ScriptBlock]::Create(${function:Test-PSSession}) $psSessionObject = Invoke-ScriptBlockCommand -ScriptBlock $scriptBlock -ArgumentList $ComputerFqdn -TimeOutInSec 120 # Validate if the server is connectable if (($null -eq $psSessionObject -or $psSessionObject.State -ne 'Opened') -or (!(Test-Connection -ComputerName $ComputerFqdn -Quiet))) { Trace-Progress -Message "$functionName : Computer $ComputerFqdn is unreachable, Could not establish a PS session earlier. Will not retry" -Error $ExcludedEndpoints.Value += $ComputerFqdn return $null } <# New-PSSessionOption paramter: .IdleTimeout : Determines how long the session stays open if the computer does not receive any communication. This includes the heartbeat signal {It means if no operation is happening, session will be open as long as session connection is established and it will help us from create PS session timeout} .OperationTimeout - Determines the maximum time that any operation in the session can run. {This prevent very large file like +25GB copy operation and help us from diskspace issue} .MaxConnectionRetryCount :Specifies the number of times that PowerShell attempts to make a connection to a target machine if the current attempt fails due to network issues. #> $sessionOptions = New-PSSessionOption -OperationTimeout ([timespan]"00:10:00").TotalMilliseconds -MaxConnectionRetryCount 1 -IdleTimeout 600000 $session = New-PSSession -ComputerName $ComputerFqdn -SessionOption $sessionOptions -ErrorAction Continue if ($null -eq $session) { $ExcludedEndpoints.Value += $ComputerFqdn Trace-Progress -Message "$functionName : Could not establish a PS session with the computer $ComputerFqdn." -error } } return $session } <# .SYNOPSIS This is the generic function to execute the command or function as script block in separate powershell thread using start-job and return the job output object. #> function Invoke-ScriptBlockCommand { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [ScriptBlock]$ScriptBlock, [Parameter(Mandatory=$false)] $ArgumentList, [Parameter(Mandatory=$false)] [int]$TimeOutInSec = 120 ) $functionName = $($MyInvocation.MyCommand.Name) $jobOutput = $null # Start and get the monitoring job result try { $monitoringJob = Start-Job -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Verbose $jobOutput = $monitoringJob | Wait-Job -Timeout $TimeOutInSec | Receive-Job $monitoringJob | Stop-Job $monitoringJob | Remove-Job } catch { Trace-Progress "$functionName : ScriptBlock - $ScriptBlock, failed with an error: $_ " -Error } return $jobOutput } function Test-PSSession { [CmdletBinding()] param( [Parameter(Mandatory=$false)] [string] $ComputerFqdn, [Parameter(Mandatory=$false)] [PSCredential] $LocalAdminCredential ) if($LocalAdminCredential) { $session = New-PSSession -ComputerName $ComputerFqdn -Credential $LocalAdminCredential -ErrorAction Continue } else { $session = New-PSSession -ComputerName $ComputerFqdn -ErrorAction Continue } return $session } <# .SYNOPSIS Return items that match the provided filter conditions for file extensions to include/exclude. #> function Get-ItemsByExtension { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [Object[]] $UnfilteredItems, [Parameter(Mandatory=$true)] [string[]] $Include, [Parameter(Mandatory=$true)] [string[]] $Exclude ) $filteredItems = New-Object System.Collections.Generic.List[System.Object] foreach ($unfilteredItem in $UnfilteredItems) { $excludedItem = $false foreach ($extensionToExclude in $Exclude) { if ($unfilteredItem -like $extensionToExclude) { $excludedItem = $true } } if (-not $excludedItem) { foreach ($extensionToInclude in $Include) { if ($unfilteredItem -like $extensionToInclude) { $filteredItems.Add($unfilteredItem) } } } } return $filteredItems } function Get-ContainerStateLog { param ( [parameter(Mandatory=$false)] [DateTime] $FilesFromDate = (Get-Date).AddHours(-1), [parameter(Mandatory=$false)] [DateTime] $FilesToDate = (Get-Date), [parameter(Mandatory=$true)] [string] $Role, [parameter(Mandatory=$true)] [string] $DestPathWithRoleName ) $containerStateLogDirPath = Join-Path -Path $DestPathWithRoleName -ChildPath "ContainerStateLogs" $containerStateErrorLogPath = Join-Path -Path $containerStateLogDirPath -ChildPath "ContainerStateCollectionErrors.txt" Trace-Progress -Message "Start container state log collection of $Role to $containerStateLogDirPath." New-Item $containerStateLogDirPath -ItemType Directory -Force | Out-Null # Collect HCS state $hcsStateLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HcsState.txt" Invoke-ExpressionWithTracing -Expression "hcsdiag list" -TraceFilePath $hcsStateLogFilePath # Collect HNS state $hnsNetworksLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_Networks.txt" $hnsEndpointsLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_Endpoints.txt" $hnsPolicyListLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "HnsState_PolicyList.txt" $hnsNetworksCommands = @( "Get-HnsNetwork | select Name, Type, ActivityId, ID, @{Name='Subnets'; Expression={ `$_.Subnets | select AddressPrefix, GatewayAddress, ID }} | Out-String", "Get-HnsNetwork | ForEach-Object { Get-HnsNetwork -Id `$_.ID -Detailed } | ConvertTo-Json -Depth 20" ) $hnsEndpointsCommands = @( "Get-HnsEndpoint | select ActivityId, ID, IpAddress, MacAddress, State | Format-Table | Out-String", "Get-HnsEndpoint | ConvertTo-Json -Depth 20" ) foreach ($hnsNetworksCommand in $hnsNetworksCommands) { Invoke-ExpressionWithTracing -Expression $hnsNetworksCommand -TraceFilePath $hnsNetworksLogFilePath } foreach ($hnsEndpointsCommand in $hnsEndpointsCommands) { Invoke-ExpressionWithTracing -Expression $hnsEndpointsCommand -TraceFilePath $hnsEndpointsLogFilePath } Invoke-ExpressionWithTracing -Expression "Get-HnsPolicyList | ConvertTo-Json -Depth 20" -TraceFilePath $hnsPolicyListLogFilePath # Collect Docker engine state $dockerStateEngineLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "DockerState-Engine.txt" $dockerEngineStateCommands = @( "docker version", "docker info", "docker ps -sa", "docker images", "docker volume ls", "docker system df -v", "docker network ls" ) foreach ($dockerEngineStateCommand in $dockerEngineStateCommands) { Invoke-ExpressionWithTracing -Expression $dockerEngineStateCommand -TraceFilePath $dockerStateEngineLogFilePath } $networkIds = docker network ls -q foreach ($networkId in $networkIds) { Invoke-ExpressionWithTracing -Expression "docker inspect $networkId" -TraceFilePath $dockerStateEngineLogFilePath } # Collect container specific diagnostics $allContainerIds = docker ps -aq [System.Collections.Generic.HashSet[string]]$runningContainerIds = docker ps -q # List of SF environment variables to include in the output. Other SF environment variable names starting with "Fabric" will be redacted. $sfEnvironmentVariablesToInclude = [System.Collections.Generic.HashSet[string]]@( "Fabric_ApplicationHostId", "Fabric_ApplicationHostType", "Fabric_ApplicationId", "Fabric_ApplicationName", "Fabric_CodePackageName", "Fabric_Endpoint_InstanceEndpoint", "Fabric_Endpoint_IPOrFQDN_InstanceEndpoint", "Fabric_Folder_App_Log", "Fabric_Folder_App_Temp", "Fabric_Folder_App_Work", "Fabric_Folder_Application", "Fabric_Folder_Application_OnHost", "Fabric_IsContainerHost", "Fabric_NodeId", "Fabric_NodeIPOrFQDN" "Fabric_NodeName" "Fabric_PartitionId", "Fabric_ServiceName", "Fabric_ServicePackageActivationId", "Fabric_ServicePackageName", "Fabric_ServicePackageVersionInstance", "Fabric_ContainerName", "FabricCodePath", "FabricLogRoot" ) foreach ($containerId in $allContainerIds) { try { $dockerInspectOutput = docker inspect $containerId | ConvertFrom-Json for ($i = 0; $i -lt $dockerInspectOutput.Config.Env.Count; $i++) { $envVariablePair = $dockerInspectOutput.Config.Env[$i] -split '=', 2 if ($envVariablePair.Length -eq 2) { $envVariableName = $envVariablePair[0] if ($envVariableName -ieq "AZS_DEPLOYMENT_APPLICATION_NAME") { $applicationName = $envVariablePair[1] -replace "/", "+" } elseif ($envVariableName -ieq "AZS_DEPLOYMENT_SERVICE_NAME") { $serviceName = $envVariablePair[1] } if ($envVariableName.StartsWith("Fabric") -and (-not $sfEnvironmentVariablesToInclude.Contains($envVariableName))) { $redactedEnvVariable = "$envVariableName=[redacted]" $dockerInspectOutput.Config.Env[$i] = $redactedEnvVariable } } else { # Unable to parse environment variable string, so will redact it completely to be safe (i.e., by avoiding leaking sensitive information). $dockerInspectOutput.Config.Env[$i] = "[redacted]" } } $containerStateLogFilePath = Join-Path -Path $containerStateLogDirPath -ChildPath "DockerState-${applicationName}_${serviceName}_${containerId}.txt" Add-Content $containerStateLogFilePath "docker inspect $containerId" Add-Content $containerStateLogFilePath $($dockerInspectOutput | ConvertTo-Json -Depth 10) } catch { Add-Content $containerStateErrorLogPath "Error while collecting docker inspect output of $containerId. ExceptionMessage: $($_.Exception.Message), ExceptionType: $($_.Exception.GetType().Name)" } # Collect running container specific diagnostics. if ($runningContainerIds.Contains($containerId)) { $containerStateCommands = @( "docker top $containerId", "docker stats $containerId --no-stream" ) foreach ($containerStateCommand in $containerStateCommands) { Invoke-ExpressionWithTracing -Expression $containerStateCommand -TraceFilePath $containerStateLogFilePath } } } Trace-Progress -Message "Finished container state log collection." } function Invoke-ExpressionWithTracing { param ( [parameter(Mandatory=$true)] [string] $Expression, [parameter(Mandatory=$true)] [string] $TraceFilePath ) try { Add-Content $TraceFilePath $Expression Invoke-Expression $Expression *>&1 | Add-Content -Path $TraceFilePath Add-Content $TraceFilePath "`n" } catch { Add-Content $containerStateErrorLogPath "Error executing '$Expression'. ExceptionMessage: $($_.Exception.Message), ExceptionType: $($_.Exception.GetType().Name)" } } Function Get-RoleLogs { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [PSCustomObject] $argumentsObject, [Parameter(Mandatory=$true)] [String] $role ) $OutputPath = $argumentsObject.OutputPath $FromDateG = $argumentsObject.FromDateG $ToDateG = $argumentsObject.ToDateG $FromDate = $argumentsObject.FromDate $ToDate = $argumentsObject.ToDate $roles = $argumentsObject.roles $domain = $argumentsObject.domain $destPath = $argumentsObject.destPath $roleNames = $argumentsObject.roleNames $nodeNames = $argumentsObject.nodeNames $filterByNode = $argumentsObject.FilterByNode $vmRoleNames = $argumentsObject.vmRoleNames $FilterByLogType = $argumentsObject.FilterByLogType $allClusterInfo = $argumentsObject.allClusterInfo $localMode = $argumentsObject.localMode $isArcA = $argumentsObject.isArcAEnv $toSMBShare = $argumentsObject.toSMBShare $functionName = "$($MyInvocation.MyCommand.Name)_$role" $perfRoleStartDate = Get-Date $roleLogDetails = @{"role" = $role; "StartDate" = $perfRoleStartDate} try { Write-Output "`r" Trace-Progress "$functionName : Collecting logs for role: $role" #Trace-InvokingProcessStats -Role ($role+"_Start") if (!$localMode) { $endpointPSSessions = @{} $ExcludedEndpoints = @() } # TODOTODO Override the nodes with node names passed #$nodes = $roles[$role].Nodes $nodes = @() $currentRoleData = $roles[$role] if ("PhysicalMachines" -in $currentRoleData.Nodes) { $nodes += $nodeNames } if ("AllVms" -in $currentRoleData.Nodes -and $vmRoleNames["AllVms"].count -gt 0) { $nodes += $vmRoleNames["AllVms"] } elseif ($role -in $currentRoleData.Nodes -and $vmRoleNames[$role].count -gt 0) { $nodes += $vmRoleNames[$role] } Trace-Progress "$functionName : Nodes to collect logs from for role [$role] = [$($nodes -join ', ')]. Note that this is before applying node filter." <# Thave above elseif should resolve to following code, if there are more specialized roles get added and their rolename is not same as the defined in Get-InfraVMNames() we need to remove the above elseif and update below cases. elseif ("NC" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["NC"] } elseif ("SLB" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["SLB"] } elseif ("GWY" -in $currentRoleData.Nodes) { $nodes += $vmRoleNames["GWY"] }#> $logsTobeCollected = (($currentRoleData.FileLog.count -gt 0) -or ($currentRoleData.CSVLog.count ) -or ($currentRoleData.WindowsEventLog.count ) ) # $rolePublicInfoLogs -- This is the xml node will <Logs></Logs> if ($logsTobeCollected) { $roleLogDetails.logsAvailable = $true Trace-Progress "$functionName : Destination path : $OutputPath" $destinationFolderPath = Join-Path -Path $OutputPath -ChildPath $role # Iterate over each end-point and collect logs if ($localMode) { $node = $env:ComputerName $endpoint = if ($null -eq $domain) { $node } else { "$node.$domain" } $endpoints = @($endpoint) } else { $endpoints = @() } if ($filterByNode) { Trace-Progress "$functionName : Node filter list = $($filterByNode -join ', ')" $nodes = $nodes | Where-Object { $_ -in $filterByNode} Trace-Progress "$functionName : Node list after applying node filter = $($nodes -join ', ')" } else { Trace-Progress "$functionName : No node filter specified" } if (!$localMode) { foreach ($node in $nodes) { $session = $null $endpoint = $node + ".$domain" Trace-Progress "$functionName : Creating a PSSession to $endpoint" if ($ExcludedEndpoints -contains $endpoint) { Trace-Progress -Message "$functionName : Could not establish a PS session earlier with the computer $endpoint. Will not retry." -Error } else { $session = Initialize-PSSession -ComputerPSSessions $endpointPSSessions -ComputerFqdn $endpoint -ExcludedEndpoints ([REF]$ExcludedEndpoints) if ($null -ne $session) { $endpointPSSessions[$endpoint] = $session $endpoints += $endpoint } } } Trace-Progress "$functionName : nodescount = [$($nodes.count)] endpointPSSessions count = [ $($endpointPSSessions.Count)], endpoints Count = [$($endpoints.Count)]" } # $endpoints is an empty array if there are no endpoints, it is not null. if($endpoints -gt 0) { # Collecting Windows event logs if ($FilterByLogType -contains 'WindowsEvent') { if ($currentRoleData.WindowsEventLog) { $logPattern = $currentRoleData.WindowsEventLog Trace-Progress -Message "$functionName : Collecting windows event logs with log patterns: $($logPattern -join ', '), with date range: from $FromDateG until $ToDateG, from machines $($endpoints -join ', ')" try { if ($localMode) { Get-WindowsEventLog -ComputerNames $endpoints -LogPattern $logPattern -EventsFromDate $FromDate -EventsToDate $ToDate -Roles $roles -CurrentRole $role ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode } else { Get-WindowsEventLog -ComputerNames $endpoints -ComputerPSSessions $endpointPSSessions -LogPattern $logPattern -EventsFromDate $FromDate -EventsToDate $ToDate ` -ExcludedEndpoints ([REF]$ExcludedEndpoints) -Roles $roles -CurrentRole $role -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode } Trace-Progress "$functionName : Successfully dumped and copied all the windows event log from individual machines to $destinationFolderPath" } catch { Trace-Progress "$functionName : Failed during windows event log collection $($_.Exception.Message)" -Error } } } else { Trace-Progress -Message "$functionName : Skipping WindowsEventLog collection." } # Collecting log files. if ($FilterByLogType -contains 'File') { if ($currentRoleData.FileLog) { $sourceLogPaths = foreach($entry in $currentRoleData.FileLog) { $entry Trace-Progress -Message "$functionName : Collecting files from '$($entry)'." } try { if ($localMode) { Get-FileLog -ComputerNames $endpoints -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } else { Get-FileLog -ComputerNames $endpoints -ComputerPSSessions $endpointPSSessions -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate ` -Role $role -ExcludedEndpoints ([REF]$ExcludedEndpoints) -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } } catch { Trace-Progress "$functionName : Failed during File log collection $($_.Exception.Message)" -Error } } } else { Trace-Progress -Message "$functionName : Skipping FileLog collection." } # Collecting container state. if ($FilterByLogType -contains 'ContainerState') { try { if ($isArcA -and $role -eq "MASLogs") { Get-ContainerStateLog -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role -DestPathWithRoleName $destinationFolderPath } else { Trace-Progress -Message "$functionName : Skipping ContainerState collection for non-ArcA MASLogs." } } catch { Trace-Progress "$functionName : Failed during ContainerState collection $($_.Exception.Message)" -Error } } else { Trace-Progress -Message "$functionName : Skipping ContainerState collection." } if (!$localMode) { #Remove PSSessions Trace-Progress -Message "$functionName : Role : $role, Removing PS Sessions." foreach ($psSession in $endpointPSSessions.Values) { if ($null -ne $psSession) { Remove-PSSession -Session $psSession -ErrorAction SilentlyContinue } } } } if ($FilterByLogType -contains 'CSV') { if ($currentRoleData.CSVLog) { if ($localMode) { $isPrimaryNode = $false # if LocalMode, each node is doing it's own log collection in parallel. Only want the primary node to collect CSV logs. Trace-Progress -Message "$functionName : In Local Mode. Determining primary node, so that only primary node collects CSV logs" try { $cluster = get-Cluster $nodes = get-clusternode -cluster $cluster | where-object {$_.State -eq "Up" } | Sort-Object -Property Name $primaryNode = $nodes[0].Name.ToLower() $isPrimaryNode = $primaryNode -eq ($env:COMPUTERNAME).ToLower() if ($isPrimaryNode) { Trace-Progress -message "$functionName : This is the primary node. This node will collect CSV logs." } else { Trace-Progress -message "$functionName : This is not the primary node. This node will not collect CSV logs." } } catch { # If we can't get primary node, it is likely deployment failed before cluster creation. In this case, # there would be no CSV Logs in cluster storage, as cluster storage is not available. # Even if there were race conditions in copying over CSV Logs, it would not cause log collection to fail. Trace-Progress -message "$functionName : Error getting primary node : $_ Will collect CSV logs on all nodes." $isPrimaryNode = $true } } if (!$localMode -or $isPrimaryNode) { $sourceLogPaths = foreach($entry in $currentRoleData.CSVLog) { $entry Trace-Progress -Message "$functionName : Collecting CSV files from '$entry'." } try { Get-FileLog -SourceLogFilePaths $sourceLogPaths -FilesFromDate $FromDate -FilesToDate $ToDate -Role $role -CSVLogsFolderName "CSVLogs" ` -DestPathWithRoleName $destinationFolderPath -LocalMode $localMode -IsArcA $isArcA -ToSMBShare $toSMBShare } catch { Trace-Progress "$functionName : Failed during CSV log collection $($_.Exception.Message)" -Error } } } } else { Trace-Progress -Message "$functionName : Skipping CSV Log collection." } # TODOTODO: When framework support run powershell, we will move this Get-MocLogs to configuration json; after that, we can remove this block. if ($role -eq "MOC_ARB") { # Call Get-MocLogs to get the MOC logs. Get-MocLogs contains the logic to connect to each node and get data. # So, the cmdlet just need to run on primary node. $isPrimaryNode = $false if ($localMode) { # If runs LocalMode, each node is doing it's own log collection in parallel. Only want the primary node to collect Get-Moclogs. try { $cluster = get-Cluster $nodes = get-clusternode -cluster $cluster | Where-Object { $_.State -eq "Up" } | Sort-Object -Property Name $primaryNode = $nodes[0].Name.ToLower() $isPrimaryNode = $primaryNode -eq ($env:COMPUTERNAME).ToLower() } catch { # If we can't get primary node, set current node as PrimaryNode, so that we will call get-MocLogs in this node. Trace-Progress -message "$functionName : Error getting primary node : $_ Will call Get-Moclogs to collect Moc logs on this node." $isPrimaryNode = $true } } # Call Get-MocLogs to get the MOC logs. Get-MocLogs contains the logic to connect to each node and get data. # So, the cmdlet just need to run on primary node. if (!$localMode -or $isPrimaryNode) { $mocLogFolder = Join-Path $destinationFolderPath "MOC" # Call Get-MocLogs to get the MOC logs. We need specify the parameters to just get the MocStore, NodeVirtualizationLogs, and MOC agent logs. # We skipped failover cluster logs. # Currently, the output of Get-MocLogs has some gaps # 1. No time filter. We accept this in this version, and need fix this in future. # 2. The MoC agent logs will be uploaded as json directly. Trace-Progress -Message "$functionName : Calling Get-MocLogs to collect MocLog and save to $mocLogFolder." Get-MocLogs -MocStore -NodeVirtualizationLogs -AgentLogs -path $mocLogFolder # So, we need do some data cleaningup # a. remove the unused EventFile, which already collect by others. # b. rename the files with Extension (text), so that the textFileParser could handle them and upload. Trace-Progress -Message "$functionName : Do the data cleaningup for output of Get-MocLogs." $allFiles = Get-ChildItem -File -Recurse -Path $mocLogFolder foreach($file in $allFiles) { if ($file.Extension -eq ".evtx" ) { Remove-Item -Path $file.FullName } ElseIf (([string]::IsNullOrEmpty($file.Extension)) -or ($file.Extension -eq ".yaml")) { # the file extension is empty or yaml, we want to use text log parser to process it, so add .txt as extension. $newName = "$($file.Name).txt" Rename-item -Path $file.FullName -Newname $newName } } Trace-Progress -Message "$functionName : Successfully copied all MocLogs" } else { Trace-Progress -Message "$functionName : Skipping Get-MocLogs as current node is not primary node" } } } else { $roleLogDetails.logsAvailable = $false Trace-Progress -Message "$functionName : No logs collected for this role as none is specified in input configuration file." } $normalTermination = $true } catch { Trace-Progress -Message "$functionName : Collecting logs failed with error: $_" -Error Trace-Progress -Message "$functionName : StackTrace : $($PSItem.ScriptStackTrace)" -Error $normalTermination = $true } finally { if (!$localMode) { Trace-Progress -Message "$functionName : Role: $role cleaningup endpointPSSessions, current opened sessions count = [$($endpointPSSessions.Values.Count)] " foreach ($psSession in $endpointPSSessions.Values) { if ($null -ne $psSession) { Trace-Progress -Message "$functionName : Removing session = [$psSession] " Remove-PSSession -Session $psSession -ErrorAction SilentlyContinue } } } if ($normalTermination -ne $true) { Trace-Progress -Message "$functionName : $role : unclean exit detected " -Error Trace-Progress -Message "$functionName : $role : Wait 30 seconds for child jobs to complete" Start-Sleep 30 # incase of an unclean exit, give time for sub jobs to complete before exiting parent job #[environment]::Exit(0) #$ZippingJobs.Values | Remove-Job -force } Write-ErrorsIfExist -Role $role #Trace-InvokingProcessStats -Role ($role+"_End") $roleLogCollectionTime = ((Get-Date) - $perfRoleStartDate).TotalMinutes.ToString("0.0##") Trace-Progress -Message "$functionName : Time taken to collect role $role is [$roleLogCollectionTime] Minutes" } } # This method prints the $global:errorList in the calling process (each role and resource provider collection runs as a separate Process) # Ensure this is called almost at the end of the job/role collection function Write-ErrorsIfExist { Param ( [parameter(Mandatory=$true)] [string] $Role ) $functionName = $($MyInvocation.MyCommand.Name) + "_$Role" # this variable is created when any trace-progress with -error is invoked. # Each role runs in its own process, so we can clear the $error automatic variable as well. if (((Test-Path variable:global:errorList) -and $Global:errorList -ne "") -or $Error.Count -gt 0) { Trace-Progress -Message "$functionName : Total entries in Global error list = $($Global:errorList.count)" $errorMessage = "ErrorList: `n" + $Global:errorList + ($Error | Get-Unique | Out-String) Write-Host $errorMessage -ForegroundColor "Red" #Dont change this to trace-progress Trace-Progress -Message $errorMessage $Error.Clear() } else { Trace-Progress -Message "$functionName : No Errors during role $Role" } } function Get-FreeSpace { Param ( [parameter(Mandatory=$true)] [string]$RelativePath ) $destinationFolder = Get-Item -Path (Split-Path $RelativePath -Parent) $fsobuild = new-Object -comobject Scripting.FileSystemObject $destinationFolderObj = $fsobuild.GetFolder($destinationFolder) $freeSpaceBytes = $destinationFolderObj.Drive.FreeSpace $freeSpaceKb = $freeSpaceBytes / 1024 return $freeSpaceKb } Export-ModuleMember -Function Get-FreeSpace Export-ModuleMember -Function Invoke-ScriptBlockWithRetries Export-ModuleMember -Function Write-ErrorsIfExist Export-ModuleMember -Function Get-RoleLogs Export-ModuleMember -Function Get-WindowsEventLog Export-ModuleMember -Function Collect-WindowsEventLogs Export-ModuleMember -Function Get-FileLog Export-ModuleMember -Function Get-FilteredChildItem Export-ModuleMember -Function Copy-FilteredChildItem Export-ModuleMember -Function Initialize-PSSession Export-ModuleMember -Function Test-PSSession Export-ModuleMember -Function Invoke-ScriptBlockCommand # SIG # Begin signature block # MIInzgYJKoZIhvcNAQcCoIInvzCCJ7sCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCsBd82iVgeLIWZ # 1JiOUQEXFgsI/fRuQFprrTN/pv3Hv6CCDYUwggYDMIID66ADAgECAhMzAAADTU6R # phoosHiPAAAAAANNMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI4WhcNMjQwMzE0MTg0MzI4WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDUKPcKGVa6cboGQU03ONbUKyl4WpH6Q2Xo9cP3RhXTOa6C6THltd2RfnjlUQG+ # Mwoy93iGmGKEMF/jyO2XdiwMP427j90C/PMY/d5vY31sx+udtbif7GCJ7jJ1vLzd # j28zV4r0FGG6yEv+tUNelTIsFmmSb0FUiJtU4r5sfCThvg8dI/F9Hh6xMZoVti+k # bVla+hlG8bf4s00VTw4uAZhjGTFCYFRytKJ3/mteg2qnwvHDOgV7QSdV5dWdd0+x # zcuG0qgd3oCCAjH8ZmjmowkHUe4dUmbcZfXsgWlOfc6DG7JS+DeJak1DvabamYqH # g1AUeZ0+skpkwrKwXTFwBRltAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUId2Img2Sp05U6XI04jli2KohL+8w # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMDUxNzAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # ACMET8WuzLrDwexuTUZe9v2xrW8WGUPRQVmyJ1b/BzKYBZ5aU4Qvh5LzZe9jOExD # YUlKb/Y73lqIIfUcEO/6W3b+7t1P9m9M1xPrZv5cfnSCguooPDq4rQe/iCdNDwHT # 6XYW6yetxTJMOo4tUDbSS0YiZr7Mab2wkjgNFa0jRFheS9daTS1oJ/z5bNlGinxq # 2v8azSP/GcH/t8eTrHQfcax3WbPELoGHIbryrSUaOCphsnCNUqUN5FbEMlat5MuY # 94rGMJnq1IEd6S8ngK6C8E9SWpGEO3NDa0NlAViorpGfI0NYIbdynyOB846aWAjN # fgThIcdzdWFvAl/6ktWXLETn8u/lYQyWGmul3yz+w06puIPD9p4KPiWBkCesKDHv # XLrT3BbLZ8dKqSOV8DtzLFAfc9qAsNiG8EoathluJBsbyFbpebadKlErFidAX8KE # usk8htHqiSkNxydamL/tKfx3V/vDAoQE59ysv4r3pE+zdyfMairvkFNNw7cPn1kH # Gcww9dFSY2QwAxhMzmoM0G+M+YvBnBu5wjfxNrMRilRbxM6Cj9hKFh0YTwba6M7z # ntHHpX3d+nabjFm/TnMRROOgIXJzYbzKKaO2g1kWeyG2QtvIR147zlrbQD4X10Ab # rRg9CpwW7xYxywezj+iNAc+QmFzR94dzJkEPUSCJPsTFMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGZ8wghmbAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAANNTpGmGiiweI8AAAAA # A00wDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIHnz # FKsvLB5ozK9y51qUO3/NTLdHXHuz7v6n6idstauwMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAv8QwDoscUUtp6Oss5u8xR3s8iJZGFDafpFEf # IKM98Jb7K9yOnO1UJx0O7L75zoLRw+04XNZU9m5UegA6E6RhrbzuTEVR57mKfTwg # 46giwPENDVyxmlgevIit4RENSpGewhPzlhVoA/h6rovrccDICFSoK1SXixV4nQSw # cn0usjgEbdUEgpqqLOgaxdzheIVEJ58DgQQNaLEJzk4sQaYEhhFIE04ZWpIQ3TAH # FE4fF+wbHDO+nV4kJgZwBqwg/8xhsTJWCYNhUlxx2pzHtLxCQ7+bsIBVW0oiqpyz # 6dhMTgQfLyF/EkMHB6ZUwHw3inr94igl+9DBlrURQ9vDaLU2OKGCFykwghclBgor # BgEEAYI3AwMBMYIXFTCCFxEGCSqGSIb3DQEHAqCCFwIwghb+AgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFZBgsqhkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCApUpfCDRPTAyQ4S0vSe478Arg1hwwkg9gS # ZUCBW4D3XwIGZJL88g0NGBMyMDIzMDcxMzE5Mjg1My43NzZaMASAAgH0oIHYpIHV # MIHSMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsT # HVRoYWxlcyBUU1MgRVNOOjhENDEtNEJGNy1CM0I3MSUwIwYDVQQDExxNaWNyb3Nv # ZnQgVGltZS1TdGFtcCBTZXJ2aWNloIIReDCCBycwggUPoAMCAQICEzMAAAGz/iXO # KRsbihwAAQAAAbMwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAg # UENBIDIwMTAwHhcNMjIwOTIwMjAyMjAzWhcNMjMxMjE0MjAyMjAzWjCB0jELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9z # b2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMg # VFNTIEVTTjo4RDQxLTRCRjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt # U3RhbXAgU2VydmljZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALR8 # D7rmGICuLLBggrK9je3hJSpc9CTwbra/4Kb2eu5DZR6oCgFtCbigMuMcY31QlHr/ # 3kuWhHJ05n4+t377PHondDDbz/dU+q/NfXSKr1pwU2OLylY0sw531VZ1sWAdyD2E # QCEzTdLD4KJbC6wmAConiJBAqvhDyXxJ0Nuvlk74rdVEvribsDZxzClWEa4v62EN # j/HyiCUX3MZGnY/AhDyazfpchDWoP6cJgNCSXmHV9XsJgXJ4l+AYAgaqAvN8N+Ep # N+0TErCgFOfwZV21cg7vgenOV48gmG/EMf0LvRAeirxPUu+jNB3JSFbW1WU8Z5xs # LEoNle35icdET+G3wDNmcSXlQYs4t94IWR541+PsUTkq0kmdP4/1O4GD54ZsJ5eU # nLaawXOxxT1fgbWb9VRg1Z4aspWpuL5gFwHa8UNMRxsKffor6qrXVVQ1OdJOS1Jl # evhpZlssSCVDodMc30I3fWezny6tNOofpfaPrtwJ0ukXcLD1yT+89u4uQB/rqUK6 # J7HpkNu0fR5M5xGtOch9nyncO9alorxDfiEdb6zeqtCfcbo46u+/rfsslcGSuJFz # lwENnU+vQ+JJ6jJRUrB+mr51zWUMiWTLDVmhLd66//Da/YBjA0Bi0hcYuO/WctfW # k/3x87ALbtqHAbk6i1cJ8a2coieuj+9BASSjuXkBAgMBAAGjggFJMIIBRTAdBgNV # HQ4EFgQU0BpdwlFnUgwYizhIIf9eBdyfw40wHwYDVR0jBBgwFoAUn6cVXQBeYl2D # 9OXSZacbUzUZ6XIwXwYDVR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3Nv # ZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy # MDIwMTAoMSkuY3JsMGwGCCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1l # LVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUB # Af8EDDAKBggrBgEFBQcDCDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQAD # ggIBAFqGuzfOsAm4wAJfERmJgWW0tNLLPk6VYj53+hBmUICsqGgj9oXNNatgCq+j # Ht03EiTzVhxteKWOLoTMx39cCcUJgDOQIH+GjuyjYVVdOCa9Fx6lI690/OBZFlz2 # DDuLpUBuo//v3e4Kns412mO3A6mDQkndxeJSsdBSbkKqccB7TC/muFOhzg39mfij # GICc1kZziJE/6HdKCF8p9+vs1yGUR5uzkIo+68q/n5kNt33hdaQ234VEh0wPSE+d # CgpKRqfxgYsBT/5tXa3e8TXyJlVoG9jwXBrKnSQb4+k19jHVB3wVUflnuANJRI9a # zWwqYFKDbZWkfQ8tpNoFfKKFRHbWomcodP1bVn7kKWUCTA8YG2RlTBtvrs3CqY3m # ADTJUig4ckN/MG6AIr8Q+ACmKBEm4OFpOcZMX0cxasopdgxM9aSdBusaJfZ3Itl3 # vC5C3RE97uURsVB2pvC+CnjFtt/PkY71l9UTHzUCO++M4hSGSzkfu+yBhXMGeBZq # LXl9cffgYPcnRFjQT97Gb/bg4ssLIFuNJNNAJub+IvxhomRrtWuB4SN935oMfvG5 # cEeZ7eyYpBZ4DbkvN44ZvER0EHRakL2xb1rrsj7c8I+auEqYztUpDnuq6BxpBIUA # lF3UDJ0SMG5xqW/9hLMWnaJCvIerEWTFm64jthAi0BDMwnCwMIIHcTCCBVmgAwIB # AgITMwAAABXF52ueAptJmQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UE # BhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAc # BgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0 # IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1 # WhcNMzAwOTMwMTgzMjI1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O # 1YLT/e6cBwfSqWxOdcjKNVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZn # hUYjDLWNE893MsAQGOhgfWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t # 1w/YJlN8OWECesSq/XJprx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxq # D89d9P6OU8/W7IVWTe/dvI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmP # frVUj9z6BVWYbWg7mka97aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSW # rAFKu75xqRdbZ2De+JKRHh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv # 231fgLrbqn427DZM9ituqBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zb # r17C89XYcz1DTsEzOUyOArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYcten # IPDC+hIK12NvDMk2ZItboKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQc # xWv2XFJRXRLbJbqvUAV6bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17a # j54WcmnGrnu3tz5q4i6tAgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQAB # MCMGCSsGAQQBgjcVAgQWBBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQU # n6cVXQBeYl2D9OXSZacbUzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEw # QTA/BggrBgEFBQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9E # b2NzL1JlcG9zaXRvcnkuaHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/ # MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJ # oEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01p # Y1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYB # BQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9v # Q2VyQXV0XzIwMTAtMDYtMjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3h # LB9nATEkW+Geckv8qW/qXBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x # 5MKP+2zRoZQYIu7pZmc6U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74p # y27YP0h1AdkY3m2CDPVtI1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1A # oL8ZthISEV09J+BAljis9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbC # HcNhcy4sa3tuPywJeBTpkbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB # 9s7GdP32THJvEKt1MMU0sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNt # yo4JvbMBV0lUZNlz138eW0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3 # rsjoiV5PndLQTHa1V1QJsWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcV # v7TOPqUxUYS8vwLBgqJ7Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A24 # 5oyZ1uEi6vAnQj0llOZ0dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lw # Y1NNje6CbaUFEMFxBmoQtB1VM1izoXBm8qGCAtQwggI9AgEBMIIBAKGB2KSB1TCB # 0jELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMk # TWljcm9zb2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1U # aGFsZXMgVFNTIEVTTjo4RDQxLTRCRjctQjNCNzElMCMGA1UEAxMcTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsOAwIaAxUAcYtE6JbdHhKlwkJe # KoCV1JIkDmGggYMwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAN # BgkqhkiG9w0BAQUFAAIFAOhaexYwIhgPMjAyMzA3MTMyMTMxMDJaGA8yMDIzMDcx # NDIxMzEwMlowdDA6BgorBgEEAYRZCgQBMSwwKjAKAgUA6Fp7FgIBADAHAgEAAgIF # IzAHAgEAAgIT1TAKAgUA6FvMlgIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEE # AYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GB # AA6WnuPAJmXUSj5Yp21/2le/lG+jm2M7tD5lqqAznYlbV4GEu0HQZ26AiC06jZnB # kdYxOBsLeQu7vNfTt0Q7M8m/74Q5yiJ6KN20D2NFM/Hgygl6MMxqXcQz9Tj96ScY # EEhwpw1xlrQMteRMjoNDwGKabA3C2M7WRyYimX8sVpyqMYIEDTCCBAkCAQEwgZMw # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAGz/iXOKRsbihwAAQAA # AbMwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRAB # BDAvBgkqhkiG9w0BCQQxIgQgKUP944OYW56tP+yfaL+GKxIjkGqeOXxwp+sAtAF9 # D0cwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9BCCGoTPVKhDSB7ZG0zJQZUM2 # jk/ll1zJGh6KOhn76k+/QjCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQI # EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv # ZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBD # QSAyMDEwAhMzAAABs/4lzikbG4ocAAEAAAGzMCIEIELDJ4uTAIvOOC4GCl91uiSl # XIFlmiKLgYrkmWHKBejVMA0GCSqGSIb3DQEBCwUABIICAE+Y9kgQhCPMGG0muJdI # 4vI+3o87ccRWmcWaN3n2tDVfDiHRXOQmleMzbHhqn+3NWwQhceBFLmSz+suLuZKB # nfnyEFljy59FY1CGAcNoiB488G8jYRfSmjIAYszoxrVk0p5IS4Xa93CkNE9gv96v # jhde8K68TpENde7xniQKTSYSa6QkZK94gyQUAxcUut78F/VpGxB7Ax2MYq+xt9vQ # XYD+52pNBASvMI4UpLmmB6NUWRICQOLkCAodl2P7lj5IOV2IZKbnEmNBF+15X/KR # zdmoIYQYUiIVCoM5kM8W2TgHwoUVj5OjhRmBh8rWB6MUVT4cKUJPAorfOfo1h5Uu # YndOxR1DlHZH5cJnJxShRKy1fCpjPlk/9SpJs7vAb9/Rcq+eWzML49PxvdUmsK7Y # TDcAnhs/0yGW/09WpDoexnP/KYH41hZsqfaAdMwPf9BylLihmJ8CGB7t0M+3SLM1 # LjHA2e6m3C/skR2A09PoNAphE4rMh50Y+mEOSbCnzDuwt5vX9+QnJtjANfQvcdZs # EUAYHf4l85QZIeosHdszqOXN2Hao63Na5S2YirJnmNX/mEqxuEmmubAXPF2Wr8S1 # uxi7GqrF6XbbOzLQfxuPFPRAEjvp8e/T2iJyoPWRaK9Mb5rgKoia3oNFwZo9/VvO # wtA0td7uqhtJ5yfkHBNEo7Lt # SIG # End signature block |