Private/AzStackHci.VMCheckpointHealth.Assessment.ps1
|
Set-StrictMode -Version 1.0 function Get-HyperVEventPolicy { [OutputType([pscustomobject])] param() [pscustomobject]@{ CriticalIds = @(3216) OperationFailureIds = @(18012, 19100, 16300) LowSignalIds = @(3280, 12240, 15268, 19090, 32510) ContextIds = @(18500, 18510, 19070, 19080) MergeFailureIds = @(19090, 19100, 32510) MergeSuccessIds = @(19080) ForkCommitHResults = @('0x80048102', '0x800703EE') LeadingHResults = @('0x800480BD', '0x800480BC') SymptomHResults = @('0x80070020', '0x80070002') } } function Get-HyperVEventSignalAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][int]$EventId, [AllowEmptyString()][string]$Log, [AllowEmptyString()][string]$Message, [Parameter(Mandatory)]$Policy ) $hasCheckpointContext = ($Message -match '(?i)checkpoint|differencing|fork|virtual\s+hard\s+disk|\bvhdx?\b') $hasCommitForkError = ($Message -match [regex]::Escape('0x80048102')) $hasFileInvalid = ($Message -match [regex]::Escape('0x800703EE')) $isContextual3216 = (($EventId -eq 3216) -and ($Log -eq 'Worker') -and $hasCheckpointContext) $isConfirming = ($hasCommitForkError -or $isContextual3216 -or ($hasFileInvalid -and $hasCheckpointContext)) $hasLeadingCode = @($Policy.LeadingHResults | Where-Object { $Message -match [regex]::Escape($_) }).Count -gt 0 $hasSymptomCode = @($Policy.SymptomHResults | Where-Object { $Message -match [regex]::Escape($_) }).Count -gt 0 $role = if ($isConfirming) { 'Confirming' } elseif ($hasLeadingCode) { 'Leading' } elseif (($Policy.OperationFailureIds -contains $EventId) -or ($Policy.LowSignalIds -contains $EventId) -or $hasSymptomCode) { 'Operational' } elseif ($Policy.ContextIds -contains $EventId) { 'Context' } else { 'Other' } [pscustomobject]@{ Role = $role; IsConfirmingFork = [bool]$isConfirming; HasCheckpointContext = [bool]$hasCheckpointContext } } function Resolve-HyperVOperationRecovery { [OutputType([pscustomobject])] param( [object[]]$Events = @(), [int[]]$FailureIds = @(18012, 19100, 16300), [int[]]$CompletionIds = @(19080), [int[]]$CompletionEligibleFailureIds = @(19100), [ValidateRange(1, 1440)][int]$MaxMinutes = 30 ) $failures = @($Events | Where-Object { $FailureIds -contains [int]$_.Id } | Sort-Object 'Time (UTC)') $completions = @($Events | Where-Object { $CompletionIds -contains [int]$_.Id } | Sort-Object 'Time (UTC)') if ($failures.Count -eq 0) { return [pscustomobject]@{ Status = 'NotApplicable'; FailureCount = 0; CompletionCount = $completions.Count; CausalMatchCount = 0; ApparentMatchCount = 0; UnresolvedCount = 0 } } $causalMatchCount = 0 $apparentMatchCount = 0 $unresolvedCount = 0 $evidencePattern = '(?i)(?:[a-z]:\\[^\r\n|"''<>]+?\.(?:avhdx|vhdx|vhd)|(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f]))' foreach ($failure in $failures) { if ($CompletionEligibleFailureIds -notcontains [int]$failure.Id) { $unresolvedCount++; continue } try { $failureTime = [datetime]::ParseExact([string]$failure.'Time (UTC)', 'yyyy-MM-dd HH:mm:ss', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal) } catch { $unresolvedCount++; continue } $boundedCompletions = @($completions | Where-Object { try { $completionTime = [datetime]::ParseExact([string]$_.'Time (UTC)', 'yyyy-MM-dd HH:mm:ss', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal) ($completionTime -ge $failureTime) -and ($completionTime -le $failureTime.AddMinutes($MaxMinutes)) } catch { $false } }) if ($boundedCompletions.Count -eq 0) { $unresolvedCount++; continue } $failureKeys = @([regex]::Matches([string]$failure.FullMessage, $evidencePattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) $causalCompletion = @($boundedCompletions | Where-Object { $completionKeys = @([regex]::Matches([string]$_.FullMessage, $evidencePattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) @($failureKeys | Where-Object { $completionKeys -contains $_ }).Count -gt 0 } | Select-Object -First 1) if ($failureKeys.Count -gt 0 -and $causalCompletion.Count -gt 0) { $causalMatchCount++ } else { $apparentMatchCount++ } } $status = if ($unresolvedCount -gt 0) { 'Unresolved' } elseif ($causalMatchCount -eq $failures.Count) { 'ConfirmedRecovered' } else { 'ApparentlyRecovered' } [pscustomobject]@{ Status = $status; FailureCount = $failures.Count; CompletionCount = $completions.Count; CausalMatchCount = $causalMatchCount; ApparentMatchCount = $apparentMatchCount; UnresolvedCount = $unresolvedCount } } function Get-HyperVEventCsvDisposition { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][object]$Event, [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Events, [Parameter(Mandatory)][object]$Policy, [AllowEmptyCollection()][object[]]$CompletionEvents ) $eventId = [int]$Event.Id $signalRole = if ($Event.PSObject.Properties['SignalRole']) { [string]$Event.SignalRole } else { '' } $isConfirmingFork = [bool]($Event.PSObject.Properties['IsConfirmingFork'] -and $Event.IsConfirmingFork) if ($isConfirmingFork -or $signalRole -eq 'Confirming') { return [pscustomobject]@{ EventClassification = 'High-signal'; VerdictDriver = $true; RecoveryDisposition = 'Unresolved' DispositionReason = 'Confirming checkpoint fork-commit or rollback evidence contributes to the VM verdict.' } } if ($Policy.OperationFailureIds -contains $eventId) { if (-not $PSBoundParameters.ContainsKey('CompletionEvents')) { $CompletionEvents = @($Events | Where-Object { $Policy.MergeSuccessIds -contains [int]$_.Id }) } $recovery = Resolve-HyperVOperationRecovery -Events (@($Event) + $completionEvents) ` -FailureIds $Policy.OperationFailureIds -CompletionIds $Policy.MergeSuccessIds $verdictDriver = ($recovery.Status -eq 'Unresolved') return [pscustomobject]@{ EventClassification = 'High-signal'; VerdictDriver = [bool]$verdictDriver RecoveryDisposition = [string]$recovery.Status DispositionReason = if ($verdictDriver) { 'The VM-attributed operation failure has no eligible bounded recovery evidence and contributes to the VM verdict.' } else { 'Bounded merge-completion evidence reduces this operation failure to recovered context.' } } } if ($Policy.LowSignalIds -contains $eventId) { return [pscustomobject]@{ EventClassification = 'Low-signal'; VerdictDriver = $false; RecoveryDisposition = 'ContextOnly' DispositionReason = 'The event is retained as low-signal operational context and does not drive the VM verdict by itself.' } } if (($Policy.ContextIds -contains $eventId) -or ($Policy.MergeSuccessIds -contains $eventId) -or $signalRole -eq 'Context') { return [pscustomobject]@{ EventClassification = 'Corroborating'; VerdictDriver = $false; RecoveryDisposition = 'ContextOnly' DispositionReason = 'The event is retained as lifecycle or recovery context and does not drive the VM verdict.' } } [pscustomobject]@{ EventClassification = 'Informational'; VerdictDriver = $false; RecoveryDisposition = 'NotApplicable' DispositionReason = 'The event is informational and has no checkpoint-operation recovery disposition.' } } function ConvertTo-HyperVEventCsvRows { [OutputType([object[]])] param( [AllowEmptyCollection()][object[]]$Events = @(), [Parameter(Mandatory)][object]$Policy, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId, [AllowEmptyString()][string]$DefaultNode, [ValidateSet('StandardLookback', 'HistoricOrphanWindow', 'HistoricActiveCheckpointWindow')] [string]$DefaultEvidenceScope = 'StandardLookback' ) $projected = [System.Collections.Generic.List[object]]::new() $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) foreach ($eventRow in @($Events)) { if (-not $eventRow) { continue } $timeUtc = if ($eventRow.PSObject.Properties['Time (UTC)']) { [string]$eventRow.'Time (UTC)' } elseif ($eventRow.PSObject.Properties['Time']) { [string]$eventRow.Time } else { '' } $node = if ($eventRow.PSObject.Properties['Node'] -and $eventRow.Node) { [string]$eventRow.Node } else { $DefaultNode } $recordId = if ($eventRow.PSObject.Properties['RecordId']) { [long]$eventRow.RecordId } else { 0L } $fullMessage = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } elseif ($eventRow.PSObject.Properties['Message']) { [string]$eventRow.Message } else { '' } $identityKey = if ($recordId -gt 0) { '{0}|{1}|{2}' -f $node, [string]$eventRow.Log, $recordId } else { '{0}|{1}|{2}|{3}|{4}' -f $node, [string]$eventRow.Log, $timeUtc, [int]$eventRow.Id, $fullMessage } if (-not $seen.Add($identityKey)) { continue } $attribution = if ($eventRow.PSObject.Properties['VmAttributed']) { [pscustomobject]@{ Attributed = [bool]$eventRow.VmAttributed Method = if ($eventRow.PSObject.Properties['AttributionMethod']) { [string]$eventRow.AttributionMethod } else { 'ExistingAssessment' } Confidence = if ($eventRow.PSObject.Properties['AttributionConfidence']) { [string]$eventRow.AttributionConfidence } else { 'Unknown' } } } else { Resolve-HyperVEventAttribution -Message $fullMessage -VMName $VMName -VMId $VMId } $signal = Get-HyperVEventSignalAssessment -EventId ([int]$eventRow.Id) -Log ([string]$eventRow.Log) -Message $fullMessage -Policy $Policy $concern = if ($eventRow.PSObject.Properties['Concern']) { [string]$eventRow.Concern } elseif ($signal.Role -in @('Confirming', 'Leading', 'Operational')) { 'YES' } else { '' } $scope = if ($eventRow.PSObject.Properties['EvidenceScope'] -and $eventRow.EvidenceScope) { [string]$eventRow.EvidenceScope } else { $DefaultEvidenceScope } $row = [pscustomobject][ordered]@{ 'Time (UTC)' = $timeUtc AuditedVMName = $VMName AuditedVMId = $VMId Node = $node RecordId = $recordId Id = [int]$eventRow.Id Level = if ($eventRow.PSObject.Properties['Level']) { [string]$eventRow.Level } else { '' } Log = [string]$eventRow.Log Concern = $concern CollectedAsConcern = ($concern -eq 'YES') VmAttributed = [bool]$attribution.Attributed AttributionMethod = [string]$attribution.Method AttributionConfidence = [string]$attribution.Confidence EvidenceScope = $scope CorrelationAnchor = if ($eventRow.PSObject.Properties['CorrelationAnchor']) { [string]$eventRow.CorrelationAnchor } else { '' } CorrelationWindowStartUtc = if ($eventRow.PSObject.Properties['CorrelationWindowStartUtc']) { [string]$eventRow.CorrelationWindowStartUtc } else { '' } CorrelationWindowEndUtc = if ($eventRow.PSObject.Properties['CorrelationWindowEndUtc']) { [string]$eventRow.CorrelationWindowEndUtc } else { '' } EventClassification = '' VerdictDriver = $false IsConfirmingFork = [bool]$signal.IsConfirmingFork RecoveryDisposition = '' DispositionReason = '' FullMessage = $fullMessage } $hasDisposition = $eventRow.PSObject.Properties['EventClassification'] -and $eventRow.EventClassification -and $eventRow.PSObject.Properties['RecoveryDisposition'] -and $eventRow.RecoveryDisposition if ($hasDisposition) { $row.EventClassification = [string]$eventRow.EventClassification $row.VerdictDriver = [bool]$eventRow.VerdictDriver $row.RecoveryDisposition = [string]$eventRow.RecoveryDisposition $row.DispositionReason = [string]$eventRow.DispositionReason } else { $disposition = Get-HyperVEventCsvDisposition -Event $row -Events $Events -Policy $Policy $row.EventClassification = [string]$disposition.EventClassification $row.VerdictDriver = [bool]$disposition.VerdictDriver $row.RecoveryDisposition = [string]$disposition.RecoveryDisposition $row.DispositionReason = [string]$disposition.DispositionReason } [void]$projected.Add($row) } $projected.ToArray() } function Compare-VMCollectionStateToken { [OutputType([pscustomobject])] param([Parameter(Mandatory)]$StartToken, [Parameter(Mandatory)]$EndToken) $reasons = [System.Collections.Generic.List[string]]::new() if (-not ([string]$StartToken.OwnerNode).Equals([string]$EndToken.OwnerNode, [StringComparison]::OrdinalIgnoreCase)) { [void]$reasons.Add('OwnerNode') } if (-not ([string]$StartToken.State).Equals([string]$EndToken.State, [StringComparison]::OrdinalIgnoreCase)) { [void]$reasons.Add('State') } if ([int]$StartToken.CheckpointCount -ne [int]$EndToken.CheckpointCount) { [void]$reasons.Add('CheckpointCount') } $startPaths = @($StartToken.DiskPaths | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) $endPaths = @($EndToken.DiskPaths | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) if (($startPaths -join "`n") -ne ($endPaths -join "`n")) { [void]$reasons.Add('DiskPaths') } if ([string]$StartToken.ConfigLastWriteUtc -ne [string]$EndToken.ConfigLastWriteUtc) { [void]$reasons.Add('ConfigLastWriteUtc') } [pscustomobject]@{ Changed = ($reasons.Count -gt 0); Reasons = $reasons.ToArray() } } function Get-VMCollectionStateImpact { [OutputType([string])] param( [Parameter(Mandatory)][ValidateSet('Stable', 'Changed', 'Unavailable')][string]$Status, [string[]]$Reasons = @(), [bool]$ReplicationEnabled = $false, [AllowEmptyString()][string]$ReplicaProductSeverity, [AllowEmptyString()][string]$ReplicaState ) if ($Status -eq 'Stable') { return 'Stable' } $healthyReplicaConfigWrite = ($Status -eq 'Changed' -and @($Reasons).Count -eq 1 -and $Reasons[0] -eq 'ConfigLastWriteUtc' -and $ReplicationEnabled -and $ReplicaProductSeverity -eq 'Healthy' -and $ReplicaState.Equals('Replicating', [StringComparison]::OrdinalIgnoreCase)) if ($healthyReplicaConfigWrite) { return 'Advisory' } 'Inconclusive' } function Get-HyperVReplicationAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][bool]$Enabled, [AllowEmptyString()][string]$State, [AllowEmptyString()][string]$Health, [AllowEmptyString()][string]$Mode, [bool]$MeasurementsAvailable = $false, [datetime]$LastReplicationTimeUtc = [datetime]::MinValue, [long]$PendingBytes = 0, [double]$LatencySeconds = 0, [long]$MissedCount = 0, [double]$FrequencySeconds = 0, [long]$AverageReplicationBytes = 0, [long]$SuccessfulCount = -1, [double]$MonitoringIntervalSeconds = 0, [datetime]$NowUtc = [datetime]::UtcNow, [int]$MaxAgeMinutes = 60, [long]$MaxPendingMB = 1024, [int]$MaxLatencySeconds = 300, [int]$MaxMissedCount = 0, [double]$MaxAgeCycles = 12, [double]$MaxPendingCycles = 2, [double]$MaxLatencyCycles = 2, [double]$MaxMissedRatePercent = 10, [long]$MinMissedCountForConcern = 3 ) if (-not $Enabled) { return [pscustomobject]@{ Severity = 'NotApplicable'; ProductSeverity = 'NotApplicable'; MeasurementStatus = 'NotApplicable' IsConcern = $false; HasAdvisory = $false; IsCritical = $false State = $State; Health = $Health; Mode = $Mode; Reason = 'Hyper-V Replica is disabled.' ThresholdBreaches = @(); ConcernBreaches = @(); AdvisoryBreaches = @(); MeasurementsAvailable = $false } } $normalizedHealth = $Health.Trim() $normalizedState = $State.Trim() $productSeverity = switch ($normalizedHealth.ToLowerInvariant()) { 'critical' { 'Critical'; break } 'warning' { 'Warning'; break } 'normal' { if ($normalizedState) { 'Healthy' } else { 'Unknown' }; break } default { 'Unknown' } } $thresholdBreaches = [System.Collections.Generic.List[string]]::new() $concernBreaches = [System.Collections.Generic.List[string]]::new() $advisoryBreaches = [System.Collections.Generic.List[string]]::new() $effectiveAgeMinutes = [double]$MaxAgeMinutes $effectivePendingBytes = [long]($MaxPendingMB * 1MB) $effectiveLatencySeconds = [double]$MaxLatencySeconds if ($FrequencySeconds -gt 0) { $effectiveAgeMinutes = [math]::Max($effectiveAgeMinutes, (($FrequencySeconds * $MaxAgeCycles) / 60.0)) $effectiveLatencySeconds = [math]::Max($effectiveLatencySeconds, ($FrequencySeconds * $MaxLatencyCycles)) } if ($AverageReplicationBytes -gt 0) { $relativePendingBytes = [double]$AverageReplicationBytes * $MaxPendingCycles if ($relativePendingBytes -gt $effectivePendingBytes) { $effectivePendingBytes = [long][math]::Ceiling($relativePendingBytes) } } $lastReplicationAgeMinutes = $null $missedRatePercent = $null if ($MeasurementsAvailable) { $totalMeasuredCount = $SuccessfulCount + $MissedCount if ($SuccessfulCount -ge 0 -and $totalMeasuredCount -gt 0) { $missedRatePercent = (100.0 * $MissedCount) / $totalMeasuredCount } if ($LastReplicationTimeUtc -ne [datetime]::MinValue) { $lastReplicationAgeMinutes = ($NowUtc.ToUniversalTime() - $LastReplicationTimeUtc.ToUniversalTime()).TotalMinutes if ($lastReplicationAgeMinutes -gt $MaxAgeMinutes) { [void]$thresholdBreaches.Add('LastReplicationAge') if ($lastReplicationAgeMinutes -gt $effectiveAgeMinutes) { [void]$concernBreaches.Add('LastReplicationAge') } else { [void]$advisoryBreaches.Add('LastReplicationAge') } } } if ($PendingBytes -gt ($MaxPendingMB * 1MB)) { [void]$thresholdBreaches.Add('PendingBytes') if ($PendingBytes -gt $effectivePendingBytes) { [void]$concernBreaches.Add('PendingBytes') } else { [void]$advisoryBreaches.Add('PendingBytes') } } if ($LatencySeconds -gt $MaxLatencySeconds) { [void]$thresholdBreaches.Add('Latency') if ($LatencySeconds -gt $effectiveLatencySeconds) { [void]$concernBreaches.Add('Latency') } else { [void]$advisoryBreaches.Add('Latency') } } if ($MissedCount -gt $MaxMissedCount) { [void]$thresholdBreaches.Add('MissedCount') $missedIsConcern = ($MissedCount -ge $MinMissedCountForConcern) -and (($null -eq $missedRatePercent) -or ($missedRatePercent -gt $MaxMissedRatePercent)) if ($missedIsConcern) { [void]$concernBreaches.Add('MissedCount') } else { [void]$advisoryBreaches.Add('MissedCount') } } } $measurementStatus = if ($concernBreaches.Count -gt 0) { 'Concern' } elseif ($advisoryBreaches.Count -gt 0) { 'Advisory' } elseif ($MeasurementsAvailable) { 'Healthy' } else { 'Unavailable' } $severity = if ($productSeverity -eq 'Healthy' -and $measurementStatus -eq 'Concern') { 'Warning' } else { $productSeverity } $isConcern = ($productSeverity -in @('Critical', 'Warning', 'Unknown')) -or ($measurementStatus -eq 'Concern') [pscustomobject]@{ Severity = $severity; ProductSeverity = $productSeverity; MeasurementStatus = $measurementStatus IsConcern = $isConcern; HasAdvisory = ($measurementStatus -eq 'Advisory'); IsCritical = ($productSeverity -eq 'Critical') State = $normalizedState; Health = $normalizedHealth; Mode = $Mode.Trim() MeasurementsAvailable = $MeasurementsAvailable; LastReplicationTimeUtc = $LastReplicationTimeUtc LastReplicationAgeMinutes = $lastReplicationAgeMinutes PendingBytes = $PendingBytes; LatencySeconds = $LatencySeconds; MissedCount = $MissedCount FrequencySeconds = $FrequencySeconds; AverageReplicationBytes = $AverageReplicationBytes SuccessfulCount = $SuccessfulCount; MissedRatePercent = $missedRatePercent MonitoringIntervalSeconds = $MonitoringIntervalSeconds EffectiveMaxAgeMinutes = $effectiveAgeMinutes; EffectiveMaxPendingBytes = $effectivePendingBytes EffectiveMaxLatencySeconds = $effectiveLatencySeconds; MaxMissedRatePercent = $MaxMissedRatePercent ThresholdBreaches = $thresholdBreaches.ToArray() ConcernBreaches = $concernBreaches.ToArray(); AdvisoryBreaches = $advisoryBreaches.ToArray() Reason = if ($productSeverity -eq 'Critical') { 'Hyper-V Replica health is Critical.' } elseif ($productSeverity -eq 'Warning') { 'Hyper-V Replica health is Warning.' } elseif ($productSeverity -eq 'Unknown') { 'Hyper-V Replica is enabled but health or state evidence is unavailable.' } elseif ($measurementStatus -eq 'Concern') { "Hyper-V Replica measurements significantly exceed the limits calculated for this VM's replication frequency." } elseif ($measurementStatus -eq 'Advisory') { 'One Hyper-V Replica measurement is outside its expected range while product health remains Normal.' } else { 'Hyper-V Replica reports Normal health with an available state.' } } } function Resolve-HyperVEventAttribution { [OutputType([pscustomobject])] param([AllowEmptyString()][string]$Message, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId) $normalizedTargetId = $VMId.Trim().Trim('{', '}', '(', ')') $guidPattern = '(?i)(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f])' $guidMatches = [regex]::Matches($Message, $guidPattern) if ($guidMatches.Count -gt 0) { $attributed = $false foreach ($guidMatch in $guidMatches) { if ($normalizedTargetId -and $guidMatch.Value.Equals($normalizedTargetId, [StringComparison]::OrdinalIgnoreCase)) { $attributed = $true; break } } return [pscustomobject]@{ Attributed = $attributed; Method = 'StructuredGuid'; Confidence = 'High'; StructuredIdentifierPresent = $true } } $namePattern = '(?i)\b(?:virtual\s+machine|vm)\s+(?:name\s*[:=]?\s*)?[''\"](?<Name>[^''\"]+)[''\"]' $nameMatches = [regex]::Matches($Message, $namePattern) if ($nameMatches.Count -gt 0) { $attributed = $false foreach ($nameMatch in $nameMatches) { if ($VMName -and $nameMatch.Groups['Name'].Value.Equals($VMName, [StringComparison]::OrdinalIgnoreCase)) { $attributed = $true; break } } return [pscustomobject]@{ Attributed = $attributed; Method = 'StructuredName'; Confidence = 'High'; StructuredIdentifierPresent = $true } } $fallbackAttributed = $false if ($VMName) { $boundedNamePattern = '(?i)(?<![\p{L}\p{N}_\\/-])' + [regex]::Escape($VMName) + '(?![\p{L}\p{N}_\\/-])'; $fallbackAttributed = [regex]::IsMatch($Message, $boundedNamePattern) } [pscustomobject]@{ Attributed = $fallbackAttributed; Method = 'BoundedNameFallback'; Confidence = if ($fallbackAttributed) { 'Low' } else { 'None' }; StructuredIdentifierPresent = $false } } function New-HyperVEventIdentityIndex { [OutputType([pscustomobject])] param([AllowEmptyCollection()][object[]]$Events = @()) $byGuid = @{} $byName = @{} $unstructured = [System.Collections.Generic.List[object]]::new() $guidPattern = '(?i)(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f])' $namePattern = '(?i)\b(?:virtual\s+machine|vm)\s+(?:name\s*[:=]?\s*)?[''\"](?<Name>[^''\"]+)[''\"]' foreach ($eventRow in @($Events)) { if (-not $eventRow) { continue } $message = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } else { [string]$eventRow.Message } $guidKeys = @([regex]::Matches($message, $guidPattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) if ($guidKeys.Count -gt 0) { foreach ($key in $guidKeys) { if (-not $byGuid.ContainsKey($key)) { $byGuid[$key] = [System.Collections.Generic.List[object]]::new() } [void]$byGuid[$key].Add($eventRow) } continue } $nameKeys = @([regex]::Matches($message, $namePattern) | ForEach-Object { $_.Groups['Name'].Value.ToLowerInvariant() } | Sort-Object -Unique) if ($nameKeys.Count -gt 0) { foreach ($key in $nameKeys) { if (-not $byName.ContainsKey($key)) { $byName[$key] = [System.Collections.Generic.List[object]]::new() } [void]$byName[$key].Add($eventRow) } continue } [void]$unstructured.Add($eventRow) } [pscustomobject]@{ ByGuid = $byGuid; ByName = $byName; Unstructured = $unstructured.ToArray(); TotalCount = @($Events).Count } } function Select-HyperVEventsForVM { [OutputType([object[]])] param( [Parameter(Mandatory)][object]$Index, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId ) $selected = [System.Collections.Generic.List[object]]::new() $normalizedId = $VMId.Trim().Trim('{', '}', '(', ')').ToLowerInvariant() $normalizedName = $VMName.ToLowerInvariant() if ($normalizedId -and $Index.ByGuid.ContainsKey($normalizedId)) { foreach ($eventRow in $Index.ByGuid[$normalizedId]) { [void]$selected.Add($eventRow) } } if ($normalizedName -and $Index.ByName.ContainsKey($normalizedName)) { foreach ($eventRow in $Index.ByName[$normalizedName]) { [void]$selected.Add($eventRow) } } foreach ($eventRow in @($Index.Unstructured)) { $message = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } else { [string]$eventRow.Message } $attribution = Resolve-HyperVEventAttribution -Message $message -VMName $VMName -VMId $VMId if ($attribution.Attributed) { [void]$selected.Add($eventRow) } } $selected.ToArray() } function Get-ClusterRoleVMAbsenceAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][ValidateNotNullOrEmpty()][string]$RoleOwner, [AllowEmptyString()][string]$FoundNode, [ValidateRange(0, 1024)][int]$FailedNodeCount = 0 ) if ($FoundNode) { return [pscustomobject]@{ Category = 'OwnerMismatch' Detail = "The cluster role records owner '$RoleOwner', but the Hyper-V VM was found on '$FoundNode'. The role ownership and Hyper-V inventory are inconsistent; verify the clustered role and VM resources." } } if ($FailedNodeCount -gt 0) { return [pscustomobject]@{ Category = 'VerificationIncomplete' Detail = "The cluster role exists and records owner '$RoleOwner', but the Hyper-V VM was not found there. Cluster-wide verification was incomplete because $FailedNodeCount node(s) could not be queried." } } [pscustomobject]@{ Category = 'StaleClusterRoleCandidate' Detail = "The cluster role exists and records owner '$RoleOwner', but no Hyper-V VM with this name was found on any cluster node. This can occur when a VM is deleted in Hyper-V but its Failover Clustering role is not removed; verify the role and its resources." } } function Resolve-EventCoverage { [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$CoverageRows, [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedNodes, [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedChannels, [Parameter(Mandatory)][datetime]$EarliestWindowStart ) $rowsByKey = @{} foreach ($row in @($CoverageRows)) { if (-not $row) { continue }; $node = [string]$row.Node; $channel = [string]$row.Channel; if (-not $node -or -not $channel) { continue }; $rowsByKey[("{0}|{1}" -f $node.ToLowerInvariant(), $channel.ToLowerInvariant())] = $row } $assessmentRows = [System.Collections.Generic.List[object]]::new() foreach ($node in @($ExpectedNodes | Where-Object { $_ } | Sort-Object -Unique)) { foreach ($channel in @($ExpectedChannels | Where-Object { $_ } | Sort-Object -Unique)) { $key = "{0}|{1}" -f $node.ToLowerInvariant(), $channel.ToLowerInvariant() $row = if ($rowsByKey.ContainsKey($key)) { $rowsByKey[$key] } else { $null } $querySucceeded = [bool]($row -and $row.QuerySucceeded) $enablementKnown = [bool]($row -and $row.PSObject.Properties['IsEnabled'] -and ($row.IsEnabled -is [bool])) $isEnabled = if ($enablementKnown) { [bool]$row.IsEnabled } else { $false } $oldest = if ($row -and $row.OldestAvailable) { [datetime]$row.OldestAvailable } else { $null } $status = if (-not $row -or -not $querySucceeded) { 'Unavailable' } elseif (-not $enablementKnown) { 'Unavailable' } elseif (-not $isEnabled) { 'Disabled' } elseif (-not $oldest) { 'EnabledEmpty' } elseif ($oldest.ToUniversalTime() -gt $EarliestWindowStart.ToUniversalTime()) { 'Wrapped' } else { 'Covered' } $sufficient = ($status -in @('Covered', 'EnabledEmpty')) [void]$assessmentRows.Add([pscustomobject]@{ Node = [string]$node; Channel = [string]$channel; Status = $status; Sufficient = [bool]$sufficient; QuerySucceeded = $querySucceeded; IsEnabled = if ($enablementKnown) { $isEnabled } else { $null }; OldestAvailable = $oldest; Error = if ($row -and $row.Error) { [string]$row.Error } elseif (-not $row) { 'Coverage row was not returned.' } elseif (-not $enablementKnown) { 'Channel enablement state was not returned.' } else { '' } }) } } $rows = $assessmentRows.ToArray() $complete = ($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.Sufficient }).Count -eq 0) [pscustomobject]@{ Complete = [bool]$complete; OverallStatus = if ($complete) { 'Covered' } else { 'Incomplete' }; Rows = $rows; CoveredCount = @($rows | Where-Object Status -eq 'Covered').Count; WrappedCount = @($rows | Where-Object Status -eq 'Wrapped').Count; EnabledEmptyCount = @($rows | Where-Object Status -eq 'EnabledEmpty').Count; DisabledCount = @($rows | Where-Object Status -eq 'Disabled').Count; UnavailableCount = @($rows | Where-Object Status -eq 'Unavailable').Count } } function Get-VMCheckpointVerdictAssessment { [OutputType([pscustomobject])] param( [bool]$ConfirmingForkSignature, [bool]$HasAttachedLayers, [bool]$HasIncompleteChain, [bool]$HasStaleEvidence, [bool]$SnapshotLayerMismatch, [bool]$HasOrphans, [bool]$VssUnhealthy, [bool]$ReplicationConcern, [bool]$StorageConcern, [int]$EscalatingEventCount, [bool]$RequiredEvidenceUnavailable, [bool]$StateInconclusive ) $holdState = ($ConfirmingForkSignature -and $HasAttachedLayers) $investigate = ((-not $holdState) -and ($HasIncompleteChain -or $HasStaleEvidence -or $SnapshotLayerMismatch -or $HasOrphans -or $VssUnhealthy -or $ReplicationConcern -or $StorageConcern -or ($EscalatingEventCount -gt 0) -or $RequiredEvidenceUnavailable -or $StateInconclusive)) [pscustomobject]@{ HoldState = [bool]$holdState Investigate = [bool]$investigate Recommendation = if ($holdState) { 'HOLD STATE' } elseif ($investigate) { 'INVESTIGATE' } else { 'OK' } } } function Select-DiscoveredVMsForAudit { [OutputType([pscustomobject])] param( [object[]]$Candidates, [Nullable[int]]$Maximum ) if ($null -ne $Maximum -and ($Maximum -lt 1 -or $Maximum -gt 1000)) { throw 'Maximum must be between 1 and 1000 when specified.' } $byName = @{} foreach ($candidate in @($Candidates)) { if (-not $candidate -or -not $candidate.Name) { continue } $name = [string]$candidate.Name $key = $name.ToLowerInvariant() if (-not $byName.ContainsKey($key)) { $byName[$key] = [pscustomobject]@{ Name = $name Reasons = [System.Collections.Generic.List[string]]::new() Score = 0 } } $reason = [string]$candidate.Reason if ($reason -and -not $byName[$key].Reasons.Contains($reason)) { [void]$byName[$key].Reasons.Add($reason) } $reasonScore = if ($reason -match 'fork|3216|0x80048102') { 400 } elseif ($reason -match '19100|16300') { 300 } elseif ($reason -match '0x80070020|sharing violation') { 200 } elseif ($reason -match '19090') { 100 } else { 0 } if ($reasonScore -gt $byName[$key].Score) { $byName[$key].Score = $reasonScore } } $ranked = @($byName.Values | ForEach-Object { $orderedReasons = @($_.Reasons | Sort-Object { if ($_ -match 'fork|3216|0x80048102') { 0 } elseif ($_ -match '19100|16300') { 1 } elseif ($_ -match '0x80070020|sharing violation') { 2 } elseif ($_ -match '19090') { 3 } else { 4 } }, { $_ }) [pscustomobject]@{ Name = $_.Name Reason = if ($orderedReasons.Count -gt 0) { $orderedReasons[0] } else { 'High-risk checkpoint/merge signal' } Reasons = $orderedReasons Score = $_.Score } } | Sort-Object @{ Expression = { $_.Score }; Descending = $true }, Name) $audit = $ranked $deferred = @() if ($null -ne $Maximum) { $audit = @($ranked | Select-Object -First $Maximum) $deferred = @($ranked | Select-Object -Skip $Maximum) } [pscustomobject]@{ EligibleCount = $ranked.Count Audit = @($audit) Deferred = @($deferred) Cap = $Maximum } } function Resolve-ActiveCheckpointHistoricVerdict { [OutputType([object])] param( [bool]$HoldState, [bool]$Investigate, [bool]$LowSignalOnly, [int]$SeverityScore, [bool]$ForkConfirmed, [bool]$CoverageIncomplete ) if ($ForkConfirmed) { $HoldState = $true $Investigate = $false $LowSignalOnly = $false $SeverityScore = 100 } elseif ($CoverageIncomplete -and -not $HoldState) { $Investigate = $true $LowSignalOnly = $false if ($SeverityScore -lt 55) { $SeverityScore = 55 } } [pscustomobject]@{ HoldState = $HoldState Investigate = $Investigate LowSignalOnly = $LowSignalOnly SeverityScore = $SeverityScore } } function Complete-CheckpointHealthPassThruResult { [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory, ValueFromPipeline)] [object]$Result, [Parameter(Mandatory)] [object]$RunData ) process { $reportData = if ($Result.PSObject.Properties['ReportData']) { $Result.ReportData } else { $null } $source = if ($Result.PSObject.Properties['Source'] -and $Result.Source) { [string]$Result.Source } else { 'Input' } $recommendation = if ($Result.PSObject.Properties['Recommendation'] -and $Result.Recommendation) { [string]$Result.Recommendation } else { 'ERROR' } $detail = if ($Result.PSObject.Properties['Detail']) { [string]$Result.Detail } else { '' } if (-not $detail -and $recommendation -eq 'INVESTIGATE' -and $reportData -and $reportData.PSObject.Properties['InvestigationDrivers'] -and $reportData.InvestigationDrivers -and $reportData.InvestigationDrivers.PSObject.Properties['AssessmentText']) { $detail = [string]$reportData.InvestigationDrivers.AssessmentText } $assessmentConfidence = if ($reportData -and $reportData.PSObject.Properties['AssessmentConfidence']) { switch ([string]$reportData.AssessmentConfidence) { 'High' { 'High' } 'Moderate' { 'Moderate' } 'Complete' { 'High' } default { 'Low' } } } else { 'Low' } $nestedStatus = if ($reportData -and $reportData.PSObject.Properties['CollectionStatus']) { $reportData.CollectionStatus } else { $null } $notCollected = [pscustomobject]@{ Status = 'NotCollected' } $collectionStatus = [pscustomobject][ordered]@{ Outcome = [pscustomobject]@{ Status = $recommendation; Detail = $detail } VhdChains = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VhdChains']) { $nestedStatus.VhdChains } else { $notCollected } VirtualDiskInventory = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VirtualDiskInventory']) { $nestedStatus.VirtualDiskInventory } else { $notCollected } EventLogs = if ($nestedStatus -and $nestedStatus.PSObject.Properties['EventLogs']) { $nestedStatus.EventLogs } else { $notCollected } HistoricEvents = if ($nestedStatus -and $nestedStatus.PSObject.Properties['HistoricEvents']) { $nestedStatus.HistoricEvents } else { $notCollected } StateConsistency = if ($nestedStatus -and $nestedStatus.PSObject.Properties['StateConsistency']) { $nestedStatus.StateConsistency } else { $notCollected } VssWriters = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VssWriters']) { $nestedStatus.VssWriters } else { $notCollected } Artifacts = if ($nestedStatus -and $nestedStatus.PSObject.Properties['Artifacts']) { $nestedStatus.Artifacts } else { $notCollected } } [pscustomobject][ordered]@{ VMName = if ($Result.PSObject.Properties['VMName']) { [string]$Result.VMName } else { '' } Cluster = if ($Result.PSObject.Properties['Cluster']) { [string]$Result.Cluster } else { '' } OwningNode = if ($Result.PSObject.Properties['OwningNode']) { [string]$Result.OwningNode } else { '' } Source = $source Recommendation = $recommendation HoldState = [bool]($Result.PSObject.Properties['HoldState'] -and $Result.HoldState) HasAttachedCheckpoints = [bool]($Result.PSObject.Properties['HasAttachedCheckpoints'] -and $Result.HasAttachedCheckpoints) HasStaleCheckpoints = [bool]($Result.PSObject.Properties['HasStaleCheckpoints'] -and $Result.HasStaleCheckpoints) HasOrphanedCheckpoints = [bool]($Result.PSObject.Properties['HasOrphanedCheckpoints'] -and $Result.HasOrphanedCheckpoints) AttachedCheckpointCount = if ($Result.PSObject.Properties['AttachedCheckpointCount']) { [int]$Result.AttachedCheckpointCount } else { 0 } StaleCheckpointCount = if ($Result.PSObject.Properties['StaleCheckpointCount']) { [int]$Result.StaleCheckpointCount } else { 0 } StaleAttachedLayerCount = if ($Result.PSObject.Properties['StaleAttachedLayerCount']) { [int]$Result.StaleAttachedLayerCount } else { 0 } SnapshotLayerMismatch = [bool]($Result.PSObject.Properties['SnapshotLayerMismatch'] -and $Result.SnapshotLayerMismatch) ConcernEventCount = if ($Result.PSObject.Properties['ConcernEventCount']) { [int]$Result.ConcernEventCount } else { 0 } AssessmentConfidence = $assessmentConfidence CollectionStatus = $collectionStatus ReportFile = if ($Result.PSObject.Properties['ReportFile']) { $Result.ReportFile } else { $null } Detail = $detail ReportData = $reportData RunData = $RunData } } } # SIG # Begin signature block # MIInQQYJKoZIhvcNAQcCoIInMjCCJy4CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBsRrAN67Z/ETeD # plXlQqPxo9thTwSMIA1jTAM6um9t76CCDLowggX1MIID3aADAgECAhMzAAACHU0Z # yE7XD1dIAAAAAAIdMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNVBAYTAlVTMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBD # b2RlIFNpZ25pbmcgUENBIDIwMjQwHhcNMjYwNDE2MTg1OTQzWhcNMjcwNDE1MTg1 # OTQzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYD # VQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IB # DwAwggEKAoIBAQDQvewXxx9gZZFC6Ys1WBay8BJ8kGA4JQnH5CMafqOASlTpK9H8 # o5ZXTXt0caVQTNMUPt445wXYD+dFtaKWTwDn1I52oUSrC9vJin1Gsqt+zyKJL5Dg # 3eQXbQNR61DmMy20GLTIO3SFed9Rfi/ophgCLGFLDR3r0KvHjwMb/jYWS0celV/4 # Lz27LfAekm8v9E5IXaeiXbAUYZKK090n4CVl3JBtbN+9DtI9SNu/yjvozW52/u7R # X/Ttpa/KDlpuokZ+Zcbvmtd9ur9gFLvZzh41o9MsE/clQtdaFWGvuo6Jua/ntpgk # ey3E5/vBFe+MJPG6phdnuo6r57ZudCudiI1bAgMBAAGjggGbMIIBlzAOBgNVHQ8B # Af8EBAMCB4AwHwYDVR0lBBgwFgYKKwYBBAGCN0wIAQYIKwYBBQUHAwMwHQYDVR0O # BBYEFH6QuMwqcPG0hQlQ6c5jCtTTLrVeMEUGA1UdEQQ+MDykOjA4MR4wHAYDVQQL # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xFjAUBgNVBAUTDTIzMDAxMis1MDc1NTkw # HwYDVR0jBBgwFoAUf1k/VCHarU/vBeXmo9ctBpQSCDEwYAYDVR0fBFkwVzBVoFOg # UYZPaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0 # JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNybDBtBggrBgEFBQcBAQRh # MF8wXQYIKwYBBQUHMAKGUWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y2VydHMvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNy # dDAMBgNVHRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4ICAQBKTbYOjzwTG/DXGaz9 # s6+fQeaTtDcFmMY+5UyVFCyj7Pv+5i37qfX8lSL/tBIfYQfWsMuBQlfZurJD6r4H # VJ2CeH+1fgiq8dcHdVKoZ3Sa2qXoX3cq9iS8cVb06B7+5/XJ7I0OxHH9fDsvJ3T3 # w5V/ZtAIFmLrl+P0CtG+92uzRsn0nTbdFjOkLMLWPLAU3THohKRlSEMgFJpPkm5n # 5UAZ35xX6FWCrDLsSKb555bTifwa8mJBwdlof0bmfYidH+dxZ1FdDxvLnNl9zeKs # A4kejaaIqqIPguhwAti5Ql7BlTNoJNwxCvBmqW2MQLnCkYN/VVUsR3V2x/rcTNzo # Bf/Z/SpROvdaA2ZOOd1uioXJt3tdLQ7vHpqpib0KfWr/FWXW10q38VxfCnRQBqzb # SuztR7nEMuzX7Ck+B/XaPDXd1qh72+QYyB0Z2VzWmO9zsnb9Uq/dwu8LGeQqnyu6 # 7SDGACvnXii2fb9+US492VTnXSnFKyqwgzUyFMtZK1/sHYTv6bG4TtQUygQxTN+Z # V+aJIlKO2MqZ7bKrAnOzS9m6NgoTdWOq11bTOZwKlIEV/EhV9SWkDmdpR/hPPT2v # 6TEj4F8PT/zHjRezIU5c/DGlt/VhY/pK0XkJtEyMmmS1BMtjU/rqBZVMIm3dnxQs # /TBByr+Cf8Z1r7aifQVQ+WSqzjCCBr0wggSloAMCAQICEzMAAAA5O7Y3Gb8GHWcA # AAAAADkwDQYJKoZIhvcNAQEMBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDExMB4XDTI0MDgwODIwNTQxOFoXDTM2MDMyMjIyMTMwNFow # VzELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEo # MCYGA1UEAxMfTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAyNDCCAiIwDQYJ # KoZIhvcNAQEBBQADggIPADCCAgoCggIBANgBnB7jOMeqlRYHNa265v4IY9fH8TKh # emHfPINe1gpLaV3dhg324WwH06LcHbpnsBukCDNitryo0dtS/EW6I/yEL/bLSY8h # KpbfQuWusBPr9qazYcDxCW/qnjb5JsI1s8bNOg3bVATvQVL4tcf03aTycsz8QeCd # M0l/yHRObJ9QqazM1r6VPEOJ7LL+uEEb73w6QCuhs89a1uv1zerOYMnsneRRwCbp # yW11IcggU0cRKDDq1pjVJzIbIF6+oiXXbReOsgeI8zu1FyQfK0fVkaya8SmVHQ/t # Of23mZ4W9k0Ri22QW9p3UgSC5OUDktKxxcCmGL6tXLfOGSWHIIV4YrTJTT6PNty5 # REojHJuZHArkF9VnHTERWoTjAzfI3kP+5b4alUdhgAZ7ttOu1bVnXfHaqPYl2rPs # 20ji03LOVWsh/radgE17es5hL+t6lV0eVHrVhsssROWJuz2MXMCt7iw7lFPG9LXK # Gjsmonn2gotGdHIuEg5JnJMJVmixd5LRlkmgYRZKzhxSCwyoGIq0PhaA7Y+VPct5 # pCHkijcIIDm0nlkK+0KyepolcqGm0T/GYQRMhHJlGOOmVQop36wUVUYklUy++vDW # eEgEo4s7hxN6mIbf2MSIQ/iIfMZgJxC69oukMUXCrOC3SkE/xIkgpfl22MM1itkZ # 35nNXkMolU1lAgMBAAGjggFOMIIBSjAOBgNVHQ8BAf8EBAMCAYYwEAYJKwYBBAGC # NxUBBAMCAQAwHQYDVR0OBBYEFH9ZP1Qh2q1P7wXl5qPXLQaUEggxMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # ci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0fBFMwUTBPoE2gS4ZJaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcBAQRSMFAwTgYIKwYBBQUHMAKGQmh0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNydDANBgkqhkiG9w0BAQwFAAOCAgEAFJQfOChP7onn6fLI # MKrSlN1WYKwDFgAddymOUO3FrM8d7B/W/iQ6DxXsDn7D5W4wMwYeLystcEqfkjz4 # NURRgazyMu5yRzQh4LqjA4tStTcJh1opExo7nn5PuPBYnbu0+THSuVHTe0VTTPVh # ily/piFrDo3axQ9P4C+Ol5yet+2gTfekICS5xS+cYfSIvgn0JksVBVMYVI5QFu/q # hnLhsEFEUzG8fvv0hjgkO+lkpV9ty6GkN4vdnd7ya6Q6aR9y34aiM1qmxaxBi6OU # nyNl6fkuun/diTFnYDLTppOkr/mg5WSfCiDVMNCxtj4wPKC5OmHm1DQIt/MNokbb # H3UGsFP1QbzsLocuSqLCvH09Io3fDPTmscR9Y75G4qX7RTX8AdBPo0I6OEojf39z # uFZt0qOHm65YWQE69cZM2ueE1MB05dNNgHK9gTE7zKvK/fg8B2qjW88MT/WF5V5u # vZGtqa9FSL2RazArA+rDPuf6JGYz4HpgMZHB4S6szWSKYBv0VisCzfxgeU+dquXW # 9bd0auYlOB58DPcOYKdc3Se94g+xL4pcEhbB54JOgAkwYTu/9dLeH2pDqeJZAABV # DWRQCaXfO5LgyKwKCLYXpigrZYCjUSBcr+Ve8PFWMhVTQl0v4q8J/AUmQN5W4n10 # 1cY2L4A7GTQG1h32HHAvfQESWP0xghndMIIZ2QIBATBuMFcxCzAJBgNVBAYTAlVT # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jv # c29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMjQCEzMAAAIdTRnITtcPV0gAAAAAAh0w # DQYJYIZIAWUDBAIBBQCggZAwGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwLwYJ # KoZIhvcNAQkEMSIEIErLwtWq5MCvRS9PvAfNCRT7l7HATGBK0Dd88kxQzUOEMEIG # CisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEBBQAEggEAtGq2wNgJteYphuHn # Ci4po4g5FKB+SDZRLdfPbr/dNIYzry3IqifdBHCTOyuwuWkL1jwCbrcouVac7MLo # 2efmwa9LH3Qy+Unx0viCg3llFHxNcKGI7wfekg0dZ1VNUwIpiU4kVKrrgx0yjc1q # 7iFXnGMQ44VxEJJPGsOYvTboQrQlmcX2ac4sEwTBYQ6t98M9efFqDKgEiLqG9V79 # NWaPKtxqrjICbbN+yO9xtkDk0Zv+NAx+UfEm63ZgnlZbq6IINDqUDCWC7IXF/nfL # 4KR1m0WEbGnPBNxVNHu978Z5IWGqhWutCVrDuxgffB5ZA0p25Ean0hIeqAMZcBoL # L74IEqGCF60wghepBgorBgEEAYI3AwMBMYIXmTCCF5UGCSqGSIb3DQEHAqCCF4Yw # gheCAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsqhkiG9w0BCRABBKCCAUkEggFF # MIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFlAwQCAQUABCDzxFRIgFB28wZA # lZDCPfGQ5cN2zJXeIQqgw2zXtnXq7QIGaq1s1AZnGBMyMDI2MDkyNTE3MTEwNS44 # MDJaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExp # bWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjozNjA1LTA1RTAtRDk0NzEl # MCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZaCCEfswggcoMIIF # EKADAgECAhMzAAACE7BDNWbPr5XoAAEAAAITMA0GCSqGSIb3DQEBCwUAMHwxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jv # c29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI1MDgxNDE4NDgxN1oXDTI2MTEx # MzE4NDgxN1owgdMxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # LTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEn # MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjM2MDUtMDVFMC1EOTQ3MSUwIwYDVQQD # ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNlMIICIjANBgkqhkiG9w0BAQEF # AAOCAg8AMIICCgKCAgEA9Jl64LoZxDINSFgz+9KS5Ozv5m548ePVzc9RXWe4T4/M # plfga4eq12RGdp5cVvnjde5vxfq2ax/jnu7vUW4rZN4mOUm5vh+kcYsQlYQ53Fwg # IB3nEjcQHomrG3mZe/ozjFSAr6JbglKtIeAySPzAcFzyAer5lLNUHBEvQMM8BOjM # yapCvh0xsg4xKFcVEJQLKEfCGBffMZI/amutHFb3CUTZ7aVpG2KHEFUNlZ1vwMKv # xXTPRDnbwPGzyyqJJznfsLNHQ4vXt2ttS1PeCoGI0hN1Peq8yGsIXM9oocwC06DG # NSM/4LAx2uKvwmUn6NwLc0+tmvny6w28rZLejskRfnVWofEv1mWY0jHUnHrwSGBS # 8gVP9gcBs6P5g0OpJPMfxdUkHXRkcMPPW0hIP8NbW8W5Sup8HuwnSKbjpyAlGBUd # M/V5rZb0sZmkn714r6ULGK+cLLAN6R3FhX6N0nj64F27LTK2BbS0pJZaXjo0eDNz # 1QcxeIFLUgF+RBsLYDn8E8cCkexK8Nlt3Gi9zJf55w6UfTZ+kwTMxMqFxh7+Tfx7 # +aBObZ+nx961AtiqAy7zVV69o/LWRdKPZdvZn9ESyGbTnPfjkBERv22prSlETlRw # zP6bmEVOKWLWVwxuwh7bUWUuUb1cj93zvttQYGQat5E9ALLJNmlvLKCskB7raLsC # AwEAAaOCAUkwggFFMB0GA1UdDgQWBBQTnhBKx+FryphQWMRipH49sMFAOjAfBgNV # HSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBfBgNVHR8EWDBWMFSgUqBQhk5o # dHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNyb3NvZnQlMjBU # aW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmwwbAYIKwYBBQUHAQEEYDBeMFwG # CCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRz # L01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNydDAMBgNV # HRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMIMA4GA1UdDwEB/wQEAwIH # gDANBgkqhkiG9w0BAQsFAAOCAgEAgmxaJrGqQ2D6UJhZ6Ql2SZFOaNuGbW3LzB+E # S+l2BB1MJtBRSFdi/hVY33NpxsJQhQ5TLVp0DXYOkIoPQc17rH+IVhemO8jCt+U6 # I1TIw6cR7c+tEo/Jjp6EqEU1c4/mraMjgHhQ+raC/OUAm98A1r4bIPHtsBmLROGm # eE5XLIFaBIZWHvh2COXITKObXVd5wGtJ1dZZdwaHACXF506jta+uoUdyzAeuNlTP # LTrZ8nyhxGwk9Vh6eiDQ7CQMWSSa8DJS9PUXjeoi9vTdS7ZMXqu+tv6Qz3xtoBF5 # +YFK4uE+miGs90Fxm0VK2lWrmFhjkRl5zyoHOdwG7spNYkDomCPNWIudUQmQYKpt # /Hsspfcb+xpnWIDQdMzgE8pj1vpwLgWEnH7LtT4dZCeoDo9PK40RxBD8kKJ769ng # kEwfwCD2EX/MQk79eIvOhpnH12GuVByvaKZk5XZvqtPONNwr8q/qA3877IuWwWgn # aeX+prpw0dZ/QLtbGGVrgP+TRQjt+2dcZA5P3X4LwANhiPsy0Ol4XCdj7OxBLFvO # zsCPDPaVnkp+dfDFG+NOBir7aqTJ68622pymg1V+6gc/1RvxC/wgvYyG033ecJqv # 0On0ZRNYr+i/OkwgA3HP1aLD0aHrEpw6lt0263iRkCvrcdcOW8w3jC8TJuaGWyC2 # S9jEjzgwggdxMIIFWaADAgECAhMzAAAAFcXna54Cm0mZAAAAAAAVMA0GCSqGSIb3 # DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIw # MAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAx # MDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMyMjVaMHwxCzAJBgNVBAYTAlVT # MRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQK # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1l # LVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA # 5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51yMo1V/YBf2xK4OK9uT4XYDP/ # XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY6GB9alKDRLemjkZrBxTzxXb1 # hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9cmmvHaus9ja+NSZk2pg7uhp7 # M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN7928jaTjkY+yOSxRnOlwaQ3K # Ni1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDuaRr3tpK56KTesy+uDRedGbsoy # 1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74kpEeHT39IM9zfUGaRnXNxF80 # 3RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2K26oElHovwUDo9Fzpk03dJQc # NIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5TI4CvEJoLhDqhFFG4tG9ahha # YQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZki1ugpoMhXV8wdJGUlNi5UPkL # iWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9QBXpsxREdcu+N+VLEhReTwDwV # 2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3PmriLq0CAwEAAaOCAd0wggHZMBIG # CSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUCBBYEFCqnUv5kxJq+gpE8RjUp # zxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJlpxtTNRnpcjBcBgNVHSAEVTBT # MFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIBFjNodHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9yeS5odG0wEwYDVR0lBAwwCgYI # KwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGG # MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186a # GMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3Br # aS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsG # AQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwDQYJKoZIhvcN # AQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/ypb+pcFLY+TkdkeLEGk5c9MTO1 # OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulmZzpTTd2YurYeeNg2LpypglYA # A7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM9W0jVOR4U3UkV7ndn/OOPcbz # aN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECWOKz3+SmJw7wXsFSFQrP8DJ6L # GYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4FOmRsqlb30mjdAy87JGA0j3m # Sj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3UwxTSwethQ/gpY3UA8x1RtnWN0 # SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPXfx5bRAGOWhmRaw2fpCjcZxko # JLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVXVAmxaQFEfnyhYWxz/gq77EFm # PWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGConsXHRWJjXD+57XQKBqJC482 # 2rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU5nR0W2rRnj7tfqAxM328y+l7 # vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEGahC0HVUzWLOhcGbyoYIDVjCC # Aj4CAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExp # bWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjozNjA1LTA1RTAtRDk0NzEl # MCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsO # AwIaAxUAmBE8SCjxgjacmy8/VEdk7NxpR6aggYMwgYCkfjB8MQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt # ZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsFAAIFAO5hJRgwIhgPMjAyNjA5 # MjUxNjUxMzZaGA8yMDI2MDkyNjE2NTEzNlowdDA6BgorBgEEAYRZCgQBMSwwKjAK # AgUA7mElGAIBADAHAgEAAgIBhjAHAgEAAgITZzAKAgUA7mJ2mAIBADA2BgorBgEE # AYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYag # MA0GCSqGSIb3DQEBCwUAA4IBAQA8Ie3fiwlH429tqQ1dPF4rdTPM+t+ARqsqGjAh # LDd7auLO6tCCAUzkaaDJmDOLtIfFZ0akoMYBBZR1ztV/WAtmxu+EGdOAIx5i7TGD # CmUWpLwxm3M+taRPkaRWtCEjvIuoAL4T7CEOq1wha78vnK68q3pqp9+FHHEFESgy # VV5zVccqzWI2R44uoT+mUnLzbHr3oJ7TYwTZ81psqizgOpDbtZB+yt8ZNCkJnFrF # r6P7EQn0D3pHCBty/jwoUCE73Lz1Vu7JEUYmXjbrxmLw4PpsgaW0TR30bWhkq8kJ # V7eyeONrqrMSlSOv2iVDtmiF3ruvc+t1IQ4S375j3sg1OAb5MYIEDTCCBAkCAQEw # gZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAITsEM1Zs+vlegA # AQAAAhMwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0B # CRABBDAvBgkqhkiG9w0BCQQxIgQgSpg8SkzcxtUHkioVilc/ipoJa6aWhl/DYRcc # souAP3gwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9BCDM4QltFIUz8J4DjAzP # 4nVodZvQxYGleUIfp86Oa5xYaDCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwAhMzAAACE7BDNWbPr5XoAAEAAAITMCIEINLGyACvGzeKaQXWu3TV # AXijSHZqlywbiErvD/5EoXuZMA0GCSqGSIb3DQEBCwUABIICAF/8zVB8Jy30GsFO # CCE9tfkAjoiiSdE87ExJkfrvO3KAGsgDYG/xCU/jUke7JNwuIL4n6TybD2XTXdLA # 6zzyat8Ke3b6Y/gmOGSNUZSkCXN6TJJnYMbLlI9XRoJKwA5vTCmbCtg50pGJ1Hso # egW1FoGlyegHN+ysBDfhVsGceSJY7jMcepfsbGYWiMdiNmTD84IC9q9jl06nsOw6 # uyja+fa2nQPQJX/ngeP09zWEH93TlKgltCanNCQ4CPdMS2Gqo4qLBSGjdcTK/5Yy # ezfQE4IHV18pThGbuViem8H/Q+QG2Z8P8/+tUbEEpDUw3NWDzqd7+/FkKsEex4TO # 3jyMjbn4IxAbaXg4y16vi7447G2ViomeTcyNdBPob9S/jL8cs616FNDXXngy3aml # sv48i16zCWRqgX+utnK12TNDQIhxQ31Js38q0bH52gAjgAtf2TwHT6fZoV8l+0+a # 4boCEUnIVd35IfqK+8aSdZRwyVkfjdMz2bm9V0Mi2yiLfWiaXOI37Yo9iGUVc1+9 # NpRqX9Qz1/qtXTetH+QaA8ewpaq7zrQvdRsrWfywC+N9G4kpsawxCILivlE5043k # pekZJtLqO5dildbAuUMC9eMnb5UpubyjaOqVrBt7OQDydCKOWtqdMzS0o5Nf52xd # MKqRS+mmedJL4rcY31pIQ922Fdkb # SIG # End signature block |