Obs/bin/ObsAgent/lib/Scripts/InvokeLogCollection.psm1
|
<##############################################################
# # # Copyright (C) Microsoft Corporation. All rights reserved. # # # ##############################################################> # # Get logs from various roles of AzureStack. # # Do not change the order of imports Import-Module $PSScriptRoot\LogCollectionHelper.psm1 -Force -Verbose:$false Import-Module $PSScriptRoot\GenericHelper.psm1 -Force -Verbose:$false function GetFileOrFolderSize { Param ( [Parameter(Mandatory=$true)] [string] $RootPath ) $size = [long] (Get-ChildItem -Recurse $RootPaTh -ErrorAction SilentlyContinue -force | Where-Object { -not $_.PSIsContainer } | Measure-Object -Sum Length ).Sum return $size } function Convert-UtctoLocal { Param ( [parameter(Mandatory=$true)] [DateTime] $UtcdateTime ) if ($UtcdateTime.Kind -eq [DateTimeKind]::Utc) { return [System.TimeZoneInfo]::ConvertTimeFromUtc($UtcdateTime, (Get-TimeZone)) } # Override # Using (Get-Date) will get the local time, so assuming user adjusted the time to UTC # need to convert it into string to remove the kind property which is local if we use get-date Trace-Progress "UtcDateTime format before format conversion is $($UtcdateTime.ToString())" $formattedDateTime = $UtcdateTime.ToString("yyyy/MM/dd HH:mm:ss") Trace-Progress "UtcDateTime format after format conversion is $formattedDateTime" return [System.TimeZoneInfo]::ConvertTimeFromUtc($formattedDateTime, (Get-TimeZone)) } function Invoke-GarbageCollector { $functionName = $($MyInvocation.MyCommand.Name) if(((Get-Date) - $Global:LastGarbageCollectedTime).TotalSeconds -gt 60 ) { Trace-Progress -Message "$functionName : Invoking Garbage Collector" [System.GC]::Collect() $Global:LastGarbageCollectedTime = Get-Date } else { Trace-Progress -Message "$functionName : Skipping Garbage Collector invocation" } } function Write-RoleTime { param ( [string]$RoleName, [int]$Empty, [int]$Filled, [float]$Duration ) #no role name is > 32 charaters currently $roleFiller = (32 - $RoleName.Length) $RoleName = ' '*$roleFiller + $RoleName <# This will print double line switch($Filled) { 1 {$line = '║'} default {$line = '╠'+'═'*($Filled-2)+'╣'} } #> switch($Filled) { 1 {$line = '║'} default {$line = '├'+'─'*($Filled-2)+'┤'} } $d ="{0:00.0}" -f $Duration [String]$printLine = "[$RoleName][$d]" + ' '*$Empty + $line Write-Host $printLine } function Show-RoleTime { param( [string]$roleTimesCsvFilePath ) $functionName = $($MyInvocation.MyCommand.Name) try{ $fileContent = Get-Content -Path $roleTimesCsvFilePath $expectedFirstLine = "RoleName, StartTime, EndTime, Minutes" if (($fileContent.GetType().Name -eq "String") -and ($fileContent -eq $expectedFirstLine)) { Trace-Progress "$functionName : There is no role time data. Cannot show role timeline." return } $f = @() if($fileContent[0] -ne $expectedFirstLine) { $f += $expectedFirstLine $f += $fileContent $fileContent = $f } $rawData = $fileContent| ConvertFrom-Csv $SortedData = $rawData | Sort-Object StartTime [System.Globalization.CultureInfo]$provider = [System.Globalization.CultureInfo]::InvariantCulture $format = 'MM/dd/yyyy HH:mm:ss' [ref]$parsedDate = get-date foreach ($obj in $SortedData) { $IsDateStartTime = [DateTime]::TryParseExact($obj.StartTime, $format,$provider,[System.Globalization.DateTimeStyles]::None,$parseddate) $IsDateEndTime = [DateTime]::TryParseExact($obj.EndTime, $format,$provider,[System.Globalization.DateTimeStyles]::None,$parseddate) $RoleNullOrEmpty = [string]::IsNullOrEmpty($obj.RoleName) $IsMinutesFloat = $obj.Minutes -match "^[\d\.]+$" $errorMessage = [System.Text.StringBuilder]::new() if (!$IsDateStartTime) { [void]$errorMessage.Append("Start Time of $($obj.StartTime) is not a valid date. ") } if (!$IsDateEndTime) { [void]$errorMessage.Append("End Time of $($obj.EndTime) is not a valid date. ") } if ($RoleNullOrEmpty) { [void]$errorMessage.Append("Role Name is null or empty. ") } if (!$IsMinutesFloat) { [void]$errorMessage.Append("Minutes of $($obj.Minutes) is not a valid float. ") } if ($errorMessage.Length -ne 0) { $errorMessageStart = "Role Time Information for role $($obj.RoleName) is invalid : " $errorMessageEnd = " Cannot show role timeline." $fullErrorMessage = $errorMessageStart + $errorMessage.ToString() + $errorMessageEnd Trace-Progress "$functionName : $fullErrorMessage" -error return } } $allData = @() $allData += $SortedData | ForEach-Object { $element = @{ RoleName = $_.RoleName StartTime = Get-Date -Date $_.StartTime EndTime = Get-Date -Date $_.EndTime Duration = [float]$_.Minutes } return $element } $data = $allData $minStartime = ($data.StartTime | Measure-Object -Minimum).Minimum $maxEndtime = ($data.EndTime | Measure-Object -Maximum).Maximum $TotalDuration = ($maxEndtime - $minStartime).TotalMinutes #Lets assume screen has 160 + 32 + few characters for printing name and timetime $multiplier = (160/$TotalDuration) Write-Host '**********************Get AzureStackLog Role Timeline**********************' foreach($role in $data) { $empty = [math]::Ceiling((($role.StartTime - $minStartime).TotalMinutes)*$multiplier) $fill = [math]::Ceiling(($role.Duration)*$multiplier) Write-RoleTime -RoleName $role.RoleName -Empty $empty -Filled $fill -Duration $role.Duration } Write-Host $printLine Write-Host '***************************************************************************' } catch { Trace-Progress -Message "$functionName : Failed to graph times for each role exception : $_" } } function TerminateJobAndCopyTempFilesToOutputPath { param ( [Parameter(Mandatory=$true)] [System.Management.Automation.Job]$JobToTerminate, [Parameter(Mandatory=$true)] [string] $TempFolderPath, [Parameter(Mandatory=$true)] [string] $OutputPath, [string] $SharePath, [string] $SasUri ) $functionName = $($MyInvocation.MyCommand.Name) #we do not want to copy all role* folders bcoz, we have few roles with same prefix as # ACS, ACSFabric,ACSWac etc.. # Which could be all running parallel, we need to copy only folders with pattern "$role", "$role-" Trace-Progress -Message "$functionName_$($JobToTerminate.Name) : Forcibly stopping the job " $JobToTerminate | Stop-Job # Force the job to Terminate Trace-Progress -Message "$functionName_$($JobToTerminate.Name) : Stopped, current state = [$($JobToTerminate.State)] ...." Start-Sleep 60 # Give few seconds for child processes to terminate, this can be upto 60 seconds $JobToTerminate | Receive-Job <# #copy files from temp and destination to outputpath for current role only. $RoleFoldersInTemp1 = $TempFolderPath + "\" + $JobToTerminate.Name + "*" # this will capture Storage, Storage-part2, Storage-Uncompressed, Storage-Uncompressed-Part2 $roleFoldersOnly = Get-ChildItem -Path $RoleFoldersInTemp1 -Directory Trace-Progress -message "$functionName_$($JobToTerminate.Name) : [$($roleFoldersOnly.count)] exist at the time job was terminated. `n folder list = [$roleFoldersOnly]" #note this will get the files and folders if they are non empty. if($roleFoldersOnly.count) { foreach( $folderPath in $roleFoldersOnly.FullName) { $tempFiles = Get-ChildItem -Path $folderPath -Recurse -ErrorAction SilentlyContinue -Verbose Trace-Progress -message "$functionName_$($JobToTerminate.Name) : $($tempFiles.count) files/folders to copy" #Copy-Item -Path $RoleFoldersInTemp1 -Destination "$OutputPath\FilesCopiedOnTermination\$($JobToTerminate.Name)\" -Recurse -Force -ErrorAction SilentlyContinue -Verbose #Remove-Item -Path $RoleFoldersInTemp1 -Recurse -ErrorAction SilentlyContinue -verbose if($tempFiles) { #$folderPath if((split-path $folderPath -leaf) -match 'UnCompressed') { $CompressionLevel = [System.IO.Compression.CompressionLevel]::NoCompression } else { $CompressionLevel = [System.IO.Compression.CompressionLevel]::Optimal } $output = Get-FolderAsZip -SourceFolder $folderPath -DestinationFolder "$OutputPath\FilesCopiedOnTermination\$($JobToTerminate.Name)\" -RoleName $JobToTerminate.Name -CompressionLevel $CompressionLevel -SharePath $SharePath -SasUri $SasUri Trace-Progress -message "$functionName_$($JobToTerminate.Name) : created and copied zip [($output.Path)] - [$($output.success)]" } else { Trace-Progress -message "$functionName_$($JobToTerminate.Name) : [$folderPath] is empty, skipping copy" } } }else { Trace-Progress -message "$functionName_$($JobToTerminate.Name) : [$RoleFoldersInTemp1] is empty, nothing to copy upon termination" } #> } function Invoke-AzureStackLogCollection { [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingComputerNameHardcoded','')] [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [string[]] $FilterByRole, [Parameter(Mandatory=$false)] [string[]] $FilterByNode, [Parameter(Mandatory=$false)] [ValidateSet('File','CSV','WindowsEvent','ContainerState','Script')] [String[]] $FilterByLogType, [Parameter(Mandatory=$false)] [ValidateNotNullOrEmpty()] [string] $CustomerConfigurationFilePath, # This will get the logs starting from last hour [Parameter(Mandatory=$false)] [DateTime] $FromDate = ([DateTime]::UtcNow).AddHours(-1), # This will get the logs until the current date [Parameter(Mandatory=$false)] [DateTime] $ToDate = ([DateTime]::UtcNow), # Whether the log collection is autonomous [Parameter(Mandatory=$false)] [bool] $IsAutonomous = $false, # Log Collection Job type [Parameter(Mandatory=$false)] [string] $LogCollectionJobType = "OnDemand", # Log Collection output path [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $OutputPath, # Whether to only collect logs from the localhost [Parameter(Mandatory=$false)] [bool] $LocalMode = $false, # Whether we are savings logs to an output path / share path [Parameter(Mandatory=$false)] [bool] $SaveToPathSelected = $false ) Trace-EnteringMethod # Let start time be the first garbage collection time. $Global:LastGarbageCollectedTime = Get-Date $logCollectionStartDate = Get-Date $logCollectionPerfResult = @{} $Error.Clear() $functionName = $($MyInvocation.MyCommand.Name) $azsLogFolderPrefix = "DiagLogs-" + ((Get-Date).ToString('yyyyMMddHHmmss')) # Adding this OutputFolderName variable to have azsLogFolderPrefix and env name together. $OutputFolderName = "$azsLogFolderPrefix-$env:ComputerName" #User has provided the outputpath, lets add AzureStackLog-yyyyMMddHHmmss to the path. $OutputPath = Join-Path -Path $OutputPath -ChildPath "$OutputFolderName" $transcriptPath = "$OutputPath\Get-AzureStackLog_Output.log" Start-Transcript -Path $transcriptPath -Verbose Trace-Progress -Message "$functionName : Outputpath = [$OutputPath]" Trace-Progress -Message "$functionName : transcriptPath = [$transcriptPath]" $collectionDetails = @{} $collectionDetails.Reason = "User Initiated" $maxJobTimeInSec = 15600 # 260min = 15600sec $maxJobLimit = 6 Trace-Progress -Message "$functionName : Running on Machine = [$($env:COMPUTERNAME)] with max parallel jobs =[$maxJobLimit]" #$filterOnlyRoles = $FilterByRole -and -not $FilterByResourceProvider #$filterOnlyRps = $FilterByResourceProvider -and -not $FilterByRole # Prepare and send telemetry event $telActivityId = New-Guid if ($null -eq $FilterByRole) { $telRoles = "" } else { $telRoles = [string]$FilterByRole } $telParameters = $MyInvocation.BoundParameters.Keys $telParameters = $telParameters -join ',' $message = ("$functionName : Started new InvokeLogCollection instance with activity ID '{0}'" -f $telActivityId) Trace-Progress -Message $message Trace-Progress "Fromdate in UTC is $FromDate and ToDate in UTC is $ToDate" Trace-Progress "The local time zone is $(Get-Timezone)" # Convert UTC times to local time $ToDate = Convert-UtctoLocal -UtcdateTime $ToDate $FromDate = Convert-UtctoLocal -UtcdateTime $FromDate Trace-Progress "After conversion to the local timezone, FromDate is $FromDate and ToDate is $ToDate" if ($ToDate -gt (Get-Date).addMinutes(5)) { $OldToDate = $ToDate $ToDate = (Get-Date) $warningMessage = "$functionName : Log Collection To Date cannot be greater than current datetime. Changed Log Collection ToDate from $OldToDate to $ToDate" Trace-Progress -Message $warningMessage -Warning } if ($FromDate -gt (Get-Date)) { $errorMessage = "$functionName : Log Collection From Date cannot be greater than current datetime. Aborting log collection." Trace-Progress -Message $errorMessage -Error Stop-Transcript return } if ($ToDate -le $FromDate) { $errorMessage = "$functionName : ToDate cannot be less than or equal to FromDate. Aborting log collection." Trace-Progress -Message $errorMessage -Error Stop-Transcript return } $telDate = Get-Date # Send-InstanceStartEvent -ActivityId $telActivityId -Timestamp $telDate -Roles $telRoles -OutputParameters $telParameters -FromRange $FromDate -ToRange $ToDate -AlertBased $AlertBased $collectionDetails.computerName = $env:COMPUTERNAME $collectionDetails.computerOS = [environment]::OSVersion.VersionString $collectionDetails.activityId = $telActivityId $collectionDetails.telParams = $telParameters $collectionDetails.fromDate = $FromDate $collectionDetails.toDate = $ToDate $collectionDetails.rolesSelected = $telRoles $collectionDetails.outputFolderName = $OutputFolderName $collectionDetails.invokeTime = $telDate $collectionDetails.filterOnlyRoles = $filterOnlyRoles $collectionDetails.filterOnlyRps = $filterOnlyRps if (-not (Test-IsElevated)) { $errorMessage = "$functionName : Collecting logs requires elevation. Aborting." Trace-Progress -Message $errorMessage -Error #Send-InstanceFinishEvent -ActivityId $telActivityId -Timestamp (Get-Date) -TerminalError $errorMessage return } #Input object pipeline for parameters $argumentsObject = [PSCustomObject]@{} $isArcAEnv = $false # Get domain information $domain = $env:USERDNSDomain if ((Get-ItemProperty -Path "HKLM:\Software\Microsoft\ArcA\" -ErrorAction SilentlyContinue).IsArcAEnv) { $isArcAEnv = $true $localMode = $true } Trace-Progress -Message "$functionName : Domain name fetched from environment variable = $domain" Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "domain" -Value $domain Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "functionName" -Value $functionName Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "OutputPath" -Value $OutputPath Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "FromDate" -Value $FromDate Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "ToDate" -Value $ToDate Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "localMode" -Value $LocalMode Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "isArcAEnv" -Value $isArcAEnv Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "saveToPathSelected" -Value $SaveToPathSelected try { $FromDateG = $FromDate.ToString("G") $ToDateG = $ToDate.ToString("G") Trace-Progress -Message "$functionName : Collecting logs from $FromDateG to $ToDateG" Trace-Progress -Message "$functionName : Local time zone is $(([TimeZoneInfo]::Local).DisplayName)" Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "FromDateG" -Value $FromDateG Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "ToDateG" -Value $ToDateG if(-not $CustomerConfigurationFilePath) { if ($isArcAEnv) { $CustomerConfigurationFilePath = "$PSScriptRoot\ArcADiagnosticLogRoleConfiguration.json" } else { $CustomerConfigurationFilePath = "$PSScriptRoot\DiagnosticLogRoleConfiguration.json" } } $roles = Get-RoleInfoFromJsonConfig -RoleJsonFile $CustomerConfigurationFilePath Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "roles" -Value $roles $roleNames = Get-FilteredRoleName -Roles $roles -FilterByRole $FilterByRole -LogCollectionJobType $LogCollectionJobType $nodeNames = Get-NodeNamesFromClusterOrDefaultLocalHostName $vmRoleNames = Get-InfraVMNames -InfraHostNames $nodeNames # TODOTODO suppress node filtering for now. <# $nodeNames = Get-FilteredNodeName -Roles $roles -RoleNames $roleNames -FilterByNode $FilterByNode $roleNamesAfterNodeFilter = $roleNames | Where-Object { ($roles[$_].PublicConfiguration.Nodes.Node.Name | Where-Object { $nodeNames -contains $_}).Count -gt 0 } Trace-Progress "$functionName : Role names after applying node filter: $($roleNamesAfterNodeFilter -join ', ')" $collectionDetails.nodeFilteredRoles = $roleNamesAfterNodeFilter if ($FilterByRole -and $FilterByNode) { if (($null -eq $roleNamesAfterNodeFilter) -or ($null -ne (Compare-Object $roleNames $roleNamesAfterNodeFilter))) { $message = "Invalid role / node filters, please check the input and try again." Trace-Progress "$functionName : $message" -Error throw $message } } $roleNames = $roleNamesAfterNodeFilter Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "nodeNames" -Value $nodeNames #> # nodenames = list of baremetal nodes # if LocalMode, ensure we are only collecting logs on the local node. if ($LocalMode) { Trace-progress -Message "$functionName : In LocalMode. LocalMode will override filterByNode parameter to make sure it is the local node only." $filterByNode = @($env:COMPUTERNAME) } Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "filterByNode" -Value $FilterByNode Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "nodeNames" -Value $nodeNames Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "roleNames" -Value $roleNames Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "vmRoleNames" -Value $vmRoleNames if (-not $FilterByLogType) { Trace-Progress "$functionName : No log type filters specified, getting all log types." $FilterByLogType = @('File','CSV','WindowsEvent', 'Script') if ($isArcAEnv) { $FilterByLogType += 'ContainerState' } } Trace-Progress -Message "$functionName : Collecting $($FilterByLogType -join ', ') logs" Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "FilterByLogType" -Value $FilterByLogType <# Get cluster information (including multiple clusters) clusternames, fileservername, ismanagementcluster, diagnosticshare full path $allClusterInfo = Get-StorageClusterInformation -AllRoles $roles Add-Member -InputObject $argumentsObject -MemberType NoteProperty -Name "allClusterInfo" -Value $allClusterInfo #> # Job array to track background jobs created $jobArray = New-Object -TypeName System.Collections.ArrayList <# Creating scriptblock this way expands $psscriptroot, using powershell way of defining scriptblock delays expansion of $psscriptroot and fails if used inside a job init block #> Trace-Progress "$functionName : INITBLOCK - $PSScriptRoot" $initblock = [ScriptBlock]::Create("Import-Module -Name '$PSScriptRoot\LogCollectionHelper.psm1' -Force; Import-Module -Name '$PSScriptRoot\GenericHelper.psm1' -Force") # TimeTracker for each job $roleTimeLog = "$OutputPath\RoleTime_Output.csv" Trace-Progress -Message "Adding header to file [$roleTimeLog]" #Add header for csv file, makes it easier to open the file in excel Add-Content -Path $roleTimeLog -Value "RoleName, StartTime, EndTime, Minutes" ##TODO - try ?? or remove {} try { $logCollectionPerfResult.Add("SetupTime", ((Get-Date) - $logCollectionStartDate).TotalMinutes) # role count info $roleCount = $roleNames.Count Trace-Progress -Message "$functionName : Total number of roles $roleCount" foreach ($role in $roleNames) { $alljobs = Get-Job foreach($j in $alljobs) { $jobToBeRemoved = $null if($j.State -in @('Completed')) # let the failed, Suspended and Disconnected jobs be in the list { $j | Receive-Job $j |Stop-Job $jobToBeRemoved = $true } elseif (($jobTime = ((Get-date) - $j.PSBeginTime).Totalseconds) -gt $maxJobTimeInSec ) { #Time to terminate the job Trace-Progress -Message "$functionName : job [$($j.Name)] exceeded maximum time for individual job [$jobTime] seconds - Terminating Job forcefully...." -Error $jobToBeRemoved = $true # TerminateJobAndCopyTempFilesToOutputPath -JobToTerminate $j -TempFolderPath $destPath -OutputPath $OutputPath -SharePath $DestinationSharePath -SasUri $OutputSasUri } if ($jobToBeRemoved) { Trace-Progress -Message "$functionName : job $($j.Name) completed with state = $($j.State)" if ($j.State -ne 'Completed') { $errorMessage = $j.ChildJobs.JobStateInfo.Reason.Message if($errorMessage) { Trace-Progress -message "$functionName : child job- $($j.Name) errors = $errorMessage" } } $j |ForEach-Object {Add-Content -Path $roleTimeLog -Value "$($_.Name), $($_.PSBeginTime), $($_.PSEndTime), $(($_.PSEndTime - $_.PSBeginTime).TotalMinutes)"} $logCollectionPerfResult.Add($j.Name, ($j.PSEndTime - $j.PSBeginTime).TotalMinutes) $j | Remove-Job Invoke-GarbageCollector if ($j -in $jobArray) { Trace-Progress -Message "$functionName : Removing job [$($j.Name)] from job array" $jobArray.Remove($j) } } } while(@(Get-Job -State Running).Count -ge $maxJobLimit) { Trace-Progress -Message "$functionName : Running $maxJobLimit jobs already, waiting before starting next job" Start-Sleep 10 } # Background job that collects logs for each role $jobStatic = Start-Job -Scriptblock { param($argumentsObject) Trace-Progress -Message "$using:functionName : invoking script variables - using:PSScriptRoot = [$using:PSScriptRoot]" Set-Location $using:PSScriptRoot Trace-Progress -Message "$using:functionName : after set pwd, job pwd = [$pwd]" $transcriptPath = "$using:OutputPath\RoleOutput\Get-Roles_Output_$using:role.log" Start-Transcript -Path $transcriptPath -Verbose Get-RoleLogs -argumentsObject $argumentsObject -role $using:role Stop-Transcript } -Name $role -ArgumentList $argumentsObject ` -InitializationScript $initblock $jobArray.Add($jobStatic) Trace-Progress -Message "$functionName : Started Get-RoleLogs for role $role" } #foreach ($role in $roleNames) } catch { Trace-Progress -Message "$functionName : Failed to kick off jobs" } Trace-Progress -Message "$functionName : Background jobs for all roles and resource providers kicked off in parallel" while($jobArray.Count -gt 0) { $completedJob = $jobArray | Wait-Job -Any -Timeout 30 $jobToBeRemoved = $null if ($null -ne $completedJob) { # $completedJob will have exactly one job that completed. $jobToBeRemoved = $completedJob } else { # No job finished Continue to wait Trace-Progress -Message "$functionName : Waiting for $($jobArray.Count) jobs to finish.." # Check any job exceeded time foreach($j in $jobArray) { #Check if any of the job has exceeded the timeout if (($jobTime = ((Get-date) - $j.PSBeginTime).Totalseconds) -gt $maxJobTimeInSec ) { #Time to terminate the job Trace-Progress -Message "$functionName : job [$($j.Name)] exceeded maximum time for individual job [$jobTime] seconds - Terminating Job forcefully...." -Error #TerminateJobAndCopyTempFilesToOutputPath -JobToTerminate $j -TempFolderPath $destPath -OutputPath $OutputPath -SharePath $DestinationSharePath -SasUri $OutputSasUri $jobToBeRemoved = $j break; } } } if ($jobToBeRemoved) { $jobToBeRemoved | Stop-Job $jobToBeRemoved | Receive-Job Trace-Progress -Message "$functionName : job $($jobToBeRemoved.Name) completed with state = $($jobToBeRemoved.State)" if ($jobToBeRemoved.State -ne 'Completed') { $errorMessage = $jobToBeRemoved.ChildJobs.JobStateInfo.Reason.Message if($errorMessage) { Trace-Progress -message "$functionName : child job errors = $errorMessage" } } # Trace the time took by each job $jobToBeRemoved |ForEach-Object {Add-Content -Path $roleTimeLog -Value "$($_.Name), $($_.PSBeginTime), $($_.PSEndTime), $(($_.PSEndTime - $_.PSBeginTime).TotalMinutes)"} $logCollectionPerfResult.Add($jobToBeRemoved.Name, ($jobToBeRemoved.PSEndTime - $jobToBeRemoved.PSBeginTime).TotalMinutes) $jobToBeRemoved | Remove-Job Invoke-GarbageCollector if ($jobToBeRemoved -in $jobArray) { Trace-Progress -Message "$functionName : Removing job [$($jobToBeRemoved.Name)] from job array" $jobArray.Remove($jobToBeRemoved) } } } #while($jobArray.Count -gt 0) Write-Output "`r" Trace-Progress -Message "$functionName : Logs placed at : $OutputPath on $env:ComputerName" } finally { $logCollectionCleanUpStartDate = Get-Date if($jobArray.Count -gt 0) { Trace-Progress -Message "$functionName : Termination event triggered - job array has $($jobArray.Count) jobs in progress" $jobArray | Stop-job foreach($j in $jobArray) { Trace-Progress -Message "$functionName : processing $($j.Id) , $($j.Name) , $($j.State)" } Trace-Progress -Message "$functionName : terminated all jobs" Trace-Progress -Message "$functionName : Copying in progress files if any..." } $logCollectionPerfResult.Add("CleanUpTime", ((Get-Date) - $logCollectionCleanUpStartDate).TotalMinutes) $logCollectionPerfSortedResult = $logCollectionPerfResult.GetEnumerator() | Select-Object -Property Key,Value | Sort-Object -Property value -Descending $logCollectionPerfSortedResult | ConvertTo-Json | Set-Content "$OutputPath\Get-AzureStackLog_Performance.json" $LogCollectionUploadNumberOfFiles = ($telInventory | Measure-Object).Count $LogCollectionUploadFileSizeInMb = $telOutputSizeMb # dump current process metrics, we can compare at the end # Trace-InvokingProcessStats -Role "Main_End" if ($roleTimeLog -ne $null) { Show-RoleTime -roleTimesCsvFilePath $roleTimeLog } else { Trace-Progress "RoleTime CSV FilePath is null. Cannot show graph of time to collect logs for each role." } Write-ErrorsIfExist -Role "MainLogCollection" $logCollectionDuration = ($ToDate - $FromDate).Totalhours.ToString("0.0##") $logCollectionTime = ((Get-Date) - $logCollectionStartDate).TotalMinutes.ToString("0.0##") # Mandatory trace statements as output file is parsed by StampTools.psm1 in SupportBridge repo Trace-Progress -Message "$functionName : LogCollectionUploadFileSizeInMb : $LogCollectionUploadFileSizeInMb" Trace-Progress -Message "$functionName : LogCollectionUploadNumberOfFiles : $LogCollectionUploadNumberOfFiles" Trace-Progress -Message "$functionName : LogCollectionUploadDirectory : $OutputFolderName" Trace-Progress -Message "$functionName : logCollectionTime : $logCollectionTime" Trace-Progress -Message "$functionName : logCollectionDuration : $logCollectionDuration" $collectionDetails.logCollectionUploadFileSizeInMb = $LogCollectionUploadFileSizeInMb $collectionDetails.logCollectionUploadNumberOfFiles = $LogCollectionUploadNumberOfFiles $collectionDetails.outputFolderName = $OutputFolderName $collectionDetails.logCollectionTime = $logCollectionTime $collectionDetails.logCollectionDuration = $logCollectionDuration # This has to be the last line before the transcipt ends, other wise LogCollection BVT validation would fail. Trace-Progress "$functionName : Completed collecting information for all roles - Get-AzureStackLog_Output for [$logCollectionDuration] hours took [$logCollectionTime] minutes, followed up by copying out to assigned destination." # Transcript needs tobe stopped as move-item would fail moving get-azurestacklog. Stop-Transcript $totalTime = ((Get-Date) - $logCollectionStartDate).TotalMinutes.ToString("0.0##") Trace-Progress "$functionName : Log collection completed duration [$totalTime] minutes" } } Export-ModuleMember -Function Invoke-AzureStackLogCollection # SIG # Begin signature block # MIIoKgYJKoZIhvcNAQcCoIIoGzCCKBcCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAVSe/NNxNH9tQW # uNsd0QFYuMjNqY87Dc+MDl2WZp4ENKCCDXYwggX0MIID3KADAgECAhMzAAADrzBA # DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA # hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG # 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN # xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL # go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB # tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd # mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ # 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY # 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp # XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn # TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT # e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG # OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O # PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk # ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx # HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt # CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGgowghoGAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIAQaCpajxIL1TYg9yKGzf9oj # DfylmBeyGlcdccXwm9R/MEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAVB4DfbgSoQOX396+r72maGkq4yx14LUtu96WD4eLnd290GWTfgrX97XB # U2f48rmDPkAfcrr9UpWOBGtj3Soz3X4vKAorYlXchA/J9XQRcN9u+ad+lNZcwRiM # hFNMht6UraahBXToFre86MRC1YW7Jz1KhwD3ZFuK5E3Pk1/t0TVA+QXsELL8iA60 # 43ClhUmfKIayxG8iQEzXo4g/oMbELi1H7Pk6EGJRf3DlJVEhg0N7lVyTOEzr1VBW # ekfm7Uak6AUhGXUygUeGduO/odzU/yTg9TqfVlwd1gbZMPH9DpWT/GBpRxH/nZ3S # qMuSpmb2WUYVS+57TUb5tPIgBZDq1aGCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCC # F3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq # hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCAA9IXTTp9PAqCfi0p48Xx915NRXnsc59moNz18AjB0egIGZpVeMrwn # GBMyMDI0MDcyMzExMDE0OS40NThaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l # cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTIwMC0w # NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg # ghHqMIIHIDCCBQigAwIBAgITMwAAAecujy+TC08b6QABAAAB5zANBgkqhkiG9w0B # AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzEyMDYxODQ1 # MTlaFw0yNTAzMDUxODQ1MTlaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z # MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTIwMC0wNUUwLUQ5NDcxJTAjBgNV # BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQDCV58v4IuQ659XPM1DtaWMv9/HRUC5kdiEF89YBP6/ # Rn7kjqMkZ5ESemf5Eli4CLtQVSefRpF1j7S5LLKisMWOGRaLcaVbGTfcmI1vMRJ1 # tzMwCNIoCq/vy8WH8QdV1B/Ab5sK+Q9yIvzGw47TfXPE8RlrauwK/e+nWnwMt060 # akEZiJJz1Vh1LhSYKaiP9Z23EZmGETCWigkKbcuAnhvh3yrMa89uBfaeHQZEHGQq # dskM48EBcWSWdpiSSBiAxyhHUkbknl9PPztB/SUxzRZjUzWHg9bf1mqZ0cIiAWC0 # EjK7ONhlQfKSRHVLKLNPpl3/+UL4Xjc0Yvdqc88gOLUr/84T9/xK5r82ulvRp2A8 # /ar9cG4W7650uKaAxRAmgL4hKgIX5/0aIAsbyqJOa6OIGSF9a+DfXl1LpQPNKR79 # 2scF7tjD5WqwIuifS9YUiHMvRLjjKk0SSCV/mpXC0BoPkk5asfxrrJbCsJePHSOE # blpJzRmzaP6OMXwRcrb7TXFQOsTkKuqkWvvYIPvVzC68UM+MskLPld1eqdOOMK7S # bbf2tGSZf3+iOwWQMcWXB9gw5gK3AIYK08WkJJuyzPqfitgubdRCmYr9CVsNOuW+ # wHDYGhciJDF2LkrjkFUjUcXSIJd9f2ssYitZ9CurGV74BQcfrxjvk1L8jvtN7mul # IwIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFM/+4JiAnzY4dpEf/Zlrh1K73o9YMB8G # A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG # Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy # MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w # XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy # dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG # A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD # AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQB0ofDbk+llWi1cC6nsfie5Jtp09o6b6ARC # pvtDPq2KFP+hi+UNNP7LGciKuckqXCmBTFIhfBeGSxvk6ycokdQr3815pEOaYWTn # HvQ0+8hKy86r1F4rfBu4oHB5cTy08T4ohrG/OYG/B/gNnz0Ol6v7u/qEjz48zXZ6 # ZlxKGyZwKmKZWaBd2DYEwzKpdLkBxs6A6enWZR0jY+q5FdbV45ghGTKgSr5ECAOn # LD4njJwfjIq0mRZWwDZQoXtJSaVHSu2lHQL3YHEFikunbUTJfNfBDLL7Gv+sTmRi # DZky5OAxoLG2gaTfuiFbfpmSfPcgl5COUzfMQnzpKfX6+FkI0QQNvuPpWsDU8sR+ # uni2VmDo7rmqJrom4ihgVNdLaMfNUqvBL5ZiSK1zmaELBJ9a+YOjE5pmSarW5sGb # n7iVkF2W9JQIOH6tGWLFJS5Hs36zahkoHh8iD963LeGjZqkFusKaUW72yMj/yxTe # GEDOoIr35kwXxr1Uu+zkur2y+FuNY0oZjppzp95AW1lehP0xaO+oBV1XfvaCur/B # 5PVAp2xzrosMEUcAwpJpio+VYfIufGj7meXcGQYWA8Umr8K6Auo+Jlj8IeFS6lSv # KhqQpmdBzAMGqPOQKt1Ow3ZXxehK7vAiim3ZiALlM0K546k0sZrxdZPgpmz7O8w9 # gHLuyZAQezCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI # hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy # MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC # AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg # M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF # dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6 # GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp # Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu # yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E # XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0 # lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q # GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ # +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA # PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw # EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG # NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV # MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj # cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK # BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC # AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX # zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v # cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI # KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG # 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x # M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC # VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449 # xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM # nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS # PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d # Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn # GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs # QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL # jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL # 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNN # MIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn # MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjkyMDAtMDVFMC1EOTQ3MSUwIwYDVQQD # ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQCz # cgTnGasSwe/dru+cPe1NF/vwQ6CBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6km/UDAiGA8yMDI0MDcyMzA1MzQw # OFoYDzIwMjQwNzI0MDUzNDA4WjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDqSb9Q # AgEAMAcCAQACAhWFMAcCAQACAhPGMAoCBQDqSxDQAgEAMDYGCisGAQQBhFkKBAIx # KDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZI # hvcNAQELBQADggEBAD6Uowihps4SbN1tYbfBvzR4xxiZTqWTHSqmtrmWNg5sUg67 # uZgcJ/Z42MY6vMyO1Fxivat6PTtLNayC2K38b0vsdZ6pP+usOd+y6iCTGvjlBCB4 # ca/Ml8iLBku5FVd6FGBvvphDLnDck8CPI5Mjx6u9TntOSTKHZ4GlYaGJlbSDxcok # Bryk85VZmBCU5HnB6lo7MEM5sA9guj5beOIXjnL9/6IfJBdgC+kFKYdPnNstIAwp # LQRubtb47YweUeoO0u2rcVn4RfaiuTiIE+o8F8pQVwKUZBSEvi56JkbD0xUVVnYn # WJyIKNPTu5GEs7HB3R1+tQgB8fiqCFFoqi3vN9oxggQNMIIECQIBATCBkzB8MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNy # b3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAecujy+TC08b6QABAAAB5zAN # BglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8G # CSqGSIb3DQEJBDEiBCB0tZ5IgaTcAolv5SLE4AauBlPNEZEm6n+IUJ2q5W0bFDCB # +gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIOU2XQ12aob9DeDFXM9UFHeEX74F # v0ABvQMG7qC51nOtMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh # c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD # b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw # MTACEzMAAAHnLo8vkwtPG+kAAQAAAecwIgQgDlbALYs/w6dxBn8RluP88Rppxee3 # KfxN7Wt0j1aV+hgwDQYJKoZIhvcNAQELBQAEggIAeQx9rY9xdr7++mxt3kt4BWfq # 4p5MT2I6vZF73mk3FOB2Nbynwkvx6luvdResk3VsJsApHNS4LDJGXqsOCDtMiKoN # OLx7jIyykbU1OIfjLqNFHkV00btnvkGNTFcPb30Ea4DLljHpAFa3osY0rWduDfHj # Hcv0Y75f9cgBcvYWz6HNkDyICzfx061P6IzcNmd+gSk0AbuwueBwI2udf+ORreYm # 7NMKYh4uGsqMYdZaPRBvb4xNd9YX+aRQ0XtyQOglZA3e/X+NKcIRUn70/mXtIJ4x # kIAt8ZI6cOUGJ1acPd7aNGCawnbLIrDE371+OpEeiXeKwyblwUpjiw+fF0Qp1LIY # 9bOeOQTlYmXPsISL/26+iOI4ND5nnlXYuTew5IFEx/CmIzdPXbgU8Rw8V6exgCF9 # QoVu8amOB9lIyNNKPn0IQW7X1C7btGt3cEYxxIR9oRSUUIQzgZl90ZdpuVXowPQS # TrQ2MnB3t7Ir0/3RiczrALyTApNQ/0cy+/bPnWorAGBO447PpgCuUEfBWuIRwyIz # 3UrPcSoM1FcFV57tfGXemVFZ2VfxvFo9F5flHg415wcMwxXfS+0CnpjQV13i7TNh # /NI35BJoi3+0y8n2z6TWB1uIZ0Zq3dCPC5oMnWWfUC0nGwWCMrXTKqe2PtPEXYaR # ZoDeAyB11WzhCgwvRlg= # SIG # End signature block |