Gutcheck

0.28.0

Private/Kinds/Stability.ps1

# The Stability Kind: what the event logs remember about this machine.
#
# Everything else Gutcheck does describes the machine as it is now. This describes what it
# has been doing, which is usually the Check that answers "it crashes sometimes" - a
# complaint no reading taken at rest can address.
#
# Every selection here is by provider name and numeric event id, never by message text. A
# German Windows returns German messages and invariant ids, and the entire estate this was
# built for is German. Payload fields are read positionally, the way the Windows event
# schema defines them, for the same reason.

# How far back a Run looks. A Check Definition parameter, because a Customer with a machine
# that misbehaves monthly and one that misbehaves hourly want different windows.
$script:StabilityDefaultDays = 30

# Which providers and ids make up each scan. Data rather than code so the selection can be
# read at a glance and so a provider that was renamed is one line to fix.
$script:StabilityScans = @(
    @{ Key = 'UnexpectedShutdowns'; Tag = 'Unexpected shutdown'; Log = 'System'
       Provider = @('Microsoft-Windows-Kernel-Power'); Id = @(41) }
    @{ Key = 'BlueScreenEvents';    Tag = 'Blue screen';         Log = 'System'
       Provider = @('Microsoft-Windows-WER-SystemErrorReporting'); Id = @(1001) }
    @{ Key = 'HardwareErrors';      Tag = 'WHEA hardware error'; Log = 'System'
       Provider = @('Microsoft-Windows-WHEA-Logger'); Id = @() }
    @{ Key = 'FirmwareThrottles';   Tag = 'Firmware CPU limit';  Log = 'System'
       Provider = @('Microsoft-Windows-Kernel-Processor-Power'); Id = @(37) }
    @{ Key = 'DiskDeviceErrors';    Tag = 'Disk / file system';  Log = 'System'
       Provider = @('disk', 'stornvme', 'storahci', 'iaStorAC', 'iaStorAVC', 'iaStorVD')
       Id = @(7, 11, 51, 129, 153) }
    @{ Key = 'NtfsErrors';          Tag = 'Disk / file system';  Log = 'System'
       # 98 only when it reports a volume that needs repair. See Select-NtfsError.
       Provider = @('Microsoft-Windows-Ntfs', 'Ntfs'); Id = @(55, 98) }
    @{ Key = 'DisplayResets';       Tag = 'Display driver reset'; Log = 'System'
       Provider = @('Display'); Id = @(4101) }
    @{ Key = 'ResourceExhaustion';  Tag = 'Low memory';          Log = 'System'
       Provider = @('Microsoft-Windows-Resource-Exhaustion-Detector'); Id = @(2004) }
    @{ Key = 'ServiceFailureEvents'; Tag = 'Service crash';      Log = 'System'
       Provider = @('Service Control Manager'); Id = @(7031, 7034) }
    @{ Key = 'CrashEvents';         Tag = 'Application crash';   Log = 'Application'
       Provider = @('Application Error'); Id = @(1000) }
    @{ Key = 'HangEvents';          Tag = 'Application hang';    Log = 'Application'
       Provider = @('Application Hang'); Id = @(1002) }
    @{ Key = 'RuntimeFaultEvents';  Tag = '.NET crash';          Log = 'Application'
       Provider = @('.NET Runtime'); Id = @(1026) }
    # Gathered but deliberately untagged: Windows Error Reporting 1001 covers every report
    # WER has ever filed, thousands on an ordinary machine, and only the LiveKernelEvent
    # entries are a Finding. Tagging the lot would bury the blue screens in events.csv
    # under the noise a Technician opened the file to see past.
    @{ Key = 'ErrorReports';        Tag = '';                    Log = 'Application'
       Provider = @('Windows Error Reporting'); Id = @(1001) }
)

# What Get-StabilityData hands its Judge. See Private/DataShape.ps1.
$script:StabilityDataShape = @{
    Days                = 'Answered'
    UnexpectedShutdowns = 'List'
    BlueScreens         = 'List'
    HardwareErrors      = 'List'
    FirmwareThrottles   = 'List'
    DiskErrors          = 'List'
    Disks               = 'Value'
    DiskHistory         = 'List'
    DisplayResets       = 'List'
    ResourceExhaustion  = 'List'
    Crashes             = 'List'
    Hangs               = 'List'
    RuntimeFaults       = 'List'
    CoveredPrograms     = 'List'
    Incidents           = 'List'
    LiveKernelEvents    = 'List'
    ServiceFailures     = 'List'
    ReliabilityIndex    = 'Value'
    UnreadableLogs      = 'List'
    Events              = 'List'
    NetworkDisruption   = 'Value'
    NetworkDependency   = 'Value'
    Adapters            = 'Value'
    Correlation         = 'Value'
    Change              = 'Value'
}

function Get-StabilityData {
    <#
    .SYNOPSIS
        Reads the event logs and returns what is in them. Judges none of it.
    .DESCRIPTION
        A log this Run's rights could not open is recorded by name rather than swallowed,
        because a Technician must be able to tell an empty log from a log nobody read.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [hashtable]$Parameters = @{},

        # What the Checks before this one gathered, keyed by Kind, and what the Elevated
        # Part gathered. The Judge is handed this Kind's data and nothing else, so what it
        # needs of the network Checks to hold a crash against is copied in here.
        [AllowNull()][hashtable]$Observed = @{}
    )

    $days  = [int](Get-Parameter $Parameters 'Days' $script:StabilityDefaultDays)
    $since = (Get-Date).AddDays(-$days)

    $unreadable = @{}
    $gathered   = @{}
    # Not List[object]: @($x) throws on one of those. See Public/Invoke-Gutcheck.ps1.
    $events     = New-Object System.Collections.Generic.List[psobject]

    foreach ($scan in $script:StabilityScans) {
        $found = @(Get-StabilityEvent -Log $scan.Log -Provider $scan.Provider -Id $scan.Id `
                                      -Since $since -Unreadable $unreadable)
        if ($scan.Key -eq 'NtfsErrors') { $found = @($found | Select-NtfsError) }
        $gathered[$scan.Key] = $found
        if ($scan.Tag) {
            foreach ($entry in $found) { $events.Add((ConvertTo-EventRow -LogEntry $entry -Tag $scan.Tag)) }
        }
    }

    # The driver hangs Windows recovered from, picked out of everything WER reported.
    $liveKernel = @(ConvertTo-LiveKernelIncident -Since $since -LogEntry @($gathered['ErrorReports'] | Where-Object {
        @($_.Properties).Count -gt 2 -and "$($_.Properties[2].Value)" -match 'LiveKernelEvent'
    }))
    foreach ($incident in $liveKernel) {
        $events.Add([pscustomobject]@{
            Time = $incident.Time; Category = 'LiveKernelEvent'; Provider = 'Windows Error Reporting'
            Id   = 1001; Message = ('{0} {1} ({2} reports)' -f $incident.Code, $incident.Dump, $incident.Reports).Trim()
        })
    }

    # Materialised before it is sorted: piping a hashtable's live KeyCollection straight
    # into Sort-Object throws inside the enumerable binder when the hashtable is empty,
    # which is exactly the case a clean machine produces.
    $unreadableLogs = @($unreadable.Keys)

    $reliability = $null
    try {
        $metric = Get-CimInstance Win32_ReliabilityStabilityMetrics -ErrorAction Stop |
            Sort-Object TimeGenerated -Descending | Select-Object -First 1
        if ($metric) { $reliability = [math]::Round($metric.SystemStabilityIndex, 1) }
    }
    catch { }

    $crashes = @($gathered['CrashEvents']        | ForEach-Object { ConvertTo-CrashRow -LogEntry $_ })
    $hangs   = @($gathered['HangEvents']         | ForEach-Object { ConvertTo-HangRow -LogEntry $_ })
    $faults  = @($gathered['RuntimeFaultEvents'] | ForEach-Object { ConvertTo-RuntimeFaultRow -LogEntry $_ })

    # Whether a program path is on the network has to be asked of the machine - a drive
    # letter says nothing about itself - so it is asked here, once per path, the way the
    # App Kind asks, and the answer travels on the incident as data.
    $remotePaths = @($crashes | ForEach-Object { $_.AppPath } | Where-Object { $_ } | Sort-Object -Unique |
        Where-Object { Test-RemoteLocation -Path $_ })

    # How far apart two events of one program may be and still be one crash. A Check
    # Definition parameter, because it decides how many crashes a Report counts.
    #
    # Read here, on the Gatherer's side of ADR-0002, and carried in the data with the
    # incidents and the correlation settings below, because a Section builder receives no
    # parameters: the data must carry what the Section shows.
    $window = Get-IncidentWindowSetting -Parameters $Parameters

    [pscustomobject]@{
        PSTypeName          = 'Gutcheck.Data.Stability'
        Days                = $days
        UnexpectedShutdowns = @($gathered['UnexpectedShutdowns'] | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        BlueScreens         = @($gathered['BlueScreenEvents']    | ForEach-Object { ConvertTo-BlueScreenRow -LogEntry $_ })
        HardwareErrors      = @($gathered['HardwareErrors']      | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        FirmwareThrottles   = @($gathered['FirmwareThrottles']   | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        DiskErrors          = @(@($gathered['DiskDeviceErrors']) + @($gathered['NtfsErrors']) |
                                Where-Object { $_ } | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        # Which disks and volumes the machine has now, and which drives the Elevated Part
        # saw attached: a disk error is held against both, because an event says
        # "Datentraeger 1" and not that it was a memory stick somebody has taken home.
        Disks               = Get-StabilityCurrentDisk
        DiskHistory         = (Get-DataCollection (Get-ObservedKindData -Observed $Observed -Kind 'DiskHealth') 'DiskHistory')
        DisplayResets       = @($gathered['DisplayResets']       | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        ResourceExhaustion  = @($gathered['ResourceExhaustion']  | ForEach-Object { ConvertTo-EventRow -LogEntry $_ })
        Crashes             = $crashes
        Hangs               = $hangs
        RuntimeFaults       = $faults
        # The programs an application's own Check judges in this Run, each with the
        # pattern of its processes: what the Run chose, before the first Check.
        CoveredPrograms     = @((Get-DataCollection (Get-ObservedKindData -Observed $Observed -Kind 'AppSelection') 'Selected') | ForEach-Object {
            [pscustomobject]@{ App = "$(Get-DataProperty $_ 'App')"; ProcessPattern = "$(Get-DataProperty $_ 'ProcessPattern')" } })
        # The three lists above as the crashes behind them. The lists stay: other Kinds
        # read them, and events.csv is every event, not every crash.
        Incidents           = @(ConvertTo-CrashIncident -Crash $crashes -RuntimeFault $faults -Hang $hangs `
                                    -WindowSeconds $window -RemotePath $remotePaths)
        LiveKernelEvents    = @($liveKernel)
        ServiceFailures     = @($gathered['ServiceFailureEvents'] | ForEach-Object { ConvertTo-ServiceFailureRow -LogEntry $_ })
        ReliabilityIndex    = $reliability
        UnreadableLogs      = @($unreadableLogs | Sort-Object)
        Events              = @($events)
        # What the network Checks gathered, as far as a crash is held against it. Each
        # says whether it was gathered at all: a Check that was skipped, or an Elevated
        # Part that did not come back, is not a network on which nothing happened.
        NetworkDisruption   = Copy-ObservedNetworkDisruption -Observed $Observed
        NetworkDependency   = Copy-ObservedNetworkDependency -Observed $Observed
        Adapters            = Copy-ObservedAdapter -Observed $Observed
        # The window and the events the Check Definition holds a crash against, for the
        # Section: a Section is built from data alone, and must show the disruptions the
        # Finding counted.
        Correlation         = Get-CorrelationSetting -Parameters $Parameters
        # What the Change Check before this one gathered, every Change of it: which of
        # them a crash is tied to is the Judge's to decide. See Get-CrashChangeTie.
        Change              = Copy-ObservedChange -Observed $Observed
    }
}

function Get-ObservedKindData {
    <#
    .SYNOPSIS
        What an earlier Check gathered for one Kind, or $null when none did. Pure.
    #>
    [CmdletBinding()]
    param([AllowNull()][hashtable]$Observed, [Parameter(Mandatory)][string]$Kind)

    if (-not $Observed -or -not $Observed.ContainsKey($Kind)) { return $null }
    $Observed[$Kind]
}

function Test-DataCarried {
    <#
    .SYNOPSIS
        Whether gathered data carries a property at all, and with a value. Pure.
    .DESCRIPTION
        An empty list is carried: a Check that looked and listed nothing has said
        something, and a Check older than the property has not.
    #>
    [CmdletBinding()]
    [OutputType([bool])]
    param([AllowNull()]$InputObject, [Parameter(Mandatory)][string]$Name)

    if ($null -eq $InputObject) { return $false }
    if ($InputObject -is [hashtable]) { return ($InputObject.ContainsKey($Name) -and $null -ne $InputObject[$Name]) }
    $property = $InputObject.PSObject.Properties[$Name]
    [bool]($property -and $null -ne $property.Value)
}

function Copy-ObservedList {
    <#
    .SYNOPSIS
        A list of names another Check gathered, as strings - or $null when that Check
        carried no such list. Pure, and decides nothing.
    #>
    [CmdletBinding()]
    [OutputType([string[]])]
    param([AllowNull()]$InputObject, [Parameter(Mandatory)][string]$Name)

    if (-not (Test-DataCarried -InputObject $InputObject -Name $Name)) { return $null }
    , [string[]]@((Get-DataCollection $InputObject $Name) | ForEach-Object { "$_" })
}

function Get-NumberParameter {
    <#
    .SYNOPSIS
        A numeric Check Definition parameter, or its default when the Definition is silent
        about it or says something that is no number. Pure.
    .DESCRIPTION
        A Definition is a document somebody edits. "x" where a number belongs must not
        make a Judge throw: the Run would lose every Finding of the Check over a typing
        mistake in one threshold. The default is what the Kind does unasked.
    #>
    [CmdletBinding()]
    [OutputType([double])]
    param(
        [Parameter(Position = 0)][AllowNull()][hashtable]$Parameters,
        [Parameter(Mandatory, Position = 1)][string]$Name,
        [Parameter(Mandatory, Position = 2)][double]$Default
    )

    $number = ConvertTo-Number (Get-Parameter $Parameters $Name $Default)
    if ($null -eq $number -or [double]::IsNaN($number)) { return $Default }
    $number
}

function Get-ListParameter {
    <#
    .SYNOPSIS
        A Check Definition parameter that is a list of names, trimmed and without blanks -
        or its default when the Definition is silent about it or names nothing. Pure.
    #>
    [CmdletBinding()]
    [OutputType([string[]])]
    param(
        [Parameter(Position = 0)][AllowNull()][hashtable]$Parameters,
        [Parameter(Mandatory, Position = 1)][string]$Name,
        [Parameter(Mandatory, Position = 2)][string[]]$Default
    )

    $named = @(@(Get-Parameter $Parameters $Name $Default) | ForEach-Object { "$_".Trim() } | Where-Object { $_ })
    if (-not $named.Count) { $named = @($Default) }
    , [string[]]$named
}

function Get-StabilityEvent {
    <#
    .SYNOPSIS
        Reads one provider-and-id selection out of one log. Records a log it cannot open.
    .DESCRIPTION
        Get-WinEvent throws rather than returning nothing when a filter matches no events,
        which is indistinguishable from a real failure by its type alone. A log that was
        refused is recorded by name so the Judge can say so; a log that simply held nothing
        is not, because that is an answer rather than a gap.
    #>
    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$Log,
        [AllowEmptyCollection()][string[]]$Provider = @(),
        [AllowEmptyCollection()][int[]]$Id = @(),
        [Parameter(Mandatory)][datetime]$Since,
        [Parameter(Mandatory)][hashtable]$Unreadable
    )

    if (-not (Test-EventLogReadable -Log $Log -Unreadable $Unreadable)) { return }

    $result = @()

    # One query for all of a scan's providers, and not one each. Asked for alone, a
    # provider this machine has never registered - an Intel storage driver on an AMD
    # board - makes Get-WinEvent throw "the parameter is incorrect" whatever it is told
    # about errors; asked for among others, it is passed over.
    $filter = @{ LogName = $Log; StartTime = $Since }
    if ($Provider.Count) { $filter.ProviderName = $Provider }
    if ($Id.Count)       { $filter.Id = $Id }

    # Asked not to stop, and the errors collected instead of caught. "No events matched"
    # is an error to Get-WinEvent and the ordinary answer on a clean machine; stopping on
    # it writes a TerminatingError line into the transcript for every scan that found
    # nothing, and the console log a Technician hands on then reads like a tool that is
    # broken. The catch stays for what throws regardless.
    $problems = @()
    try {
        $result = @(Get-WinEvent -FilterHashtable $filter -ErrorAction SilentlyContinue -ErrorVariable problems)
    }
    catch { $problems = @($_) }

    foreach ($problem in @($problems)) {
        if (Test-AccessDenied -ErrorRecord $problem) { $Unreadable[$Log] = $true }
        # Anything else is "no events matched" or a provider this machine does not have.
        # Reporting that as a gap would make every clean machine look unread.
    }
    $result
}

function Test-EventLogReadable {
    <#
    .SYNOPSIS
        Whether this Run's rights may read a log. Records a log it may not.
    .DESCRIPTION
        Asked by name, before any filtered query, because a filtered query never says it
        was refused: Get-WinEvent -FilterHashtable drops a log the caller may not read and
        reports that no events matched - exactly what an empty log reports. Only reading
        the log by name raises the refusal. Without this, a Run without admin rights reads
        Diagnostics-Performance or the Group Policy log as a machine with nothing in it.
 
        Asked once per log per Gatherer: the answer is recorded in $Unreadable, and a log
        already recorded there is not asked about again.
    #>
    [CmdletBinding()]
    [OutputType([bool])]
    param(
        [Parameter(Mandatory)][string]$Log,
        [Parameter(Mandatory)][hashtable]$Unreadable
    )

    if ($Unreadable.ContainsKey($Log)) { return $false }
    $problems = @()
    try { $null = Get-WinEvent -LogName $Log -MaxEvents 1 -ErrorAction SilentlyContinue -ErrorVariable problems }
    catch { $problems = @($_) }

    foreach ($problem in @($problems)) {
        # An empty log raises "no events" here too; only a refusal makes it unreadable.
        if (Test-AccessDenied -ErrorRecord $problem) { $Unreadable[$Log] = $true; return $false }
    }
    $true
}

function Test-AccessDenied {
    <#
    .SYNOPSIS
        Whether an error record says the caller was not allowed, rather than not lucky.
    #>
    [CmdletBinding()]
    [OutputType([bool])]
    param([Parameter(Mandatory)]$ErrorRecord)

    $exception = $ErrorRecord.Exception
    while ($exception) {
        if ($exception -is [UnauthorizedAccessException]) { return $true }
        $exception = $exception.InnerException
    }

    # The message is matched only as a fallback, and in both languages the estate runs,
    # because the typed exception is not always what surfaces.
    "$($ErrorRecord.FullyQualifiedErrorId) $($ErrorRecord.Exception.Message)" -match
        'Unauthorized|unauthori|Zugriff|nicht autorisiert'
}

# How much of an event's message is kept. See ConvertTo-EventRow.
$script:StabilityEventMessageLength = 4000

function ConvertTo-EventRow {
    <#
    .SYNOPSIS
        One event, reduced to what a Report and a Technician need from it.
    #>
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry, [string]$Tag)

    # Whole, where it used to be cut at 500 characters: the cut fell in the middle of a
    # sentence, and what an SMB or a .NET event says last - which share, which method - is
    # what a Technician opened it for. The Report shortens what it shows and lets it be
    # opened; events.csv holds all of it. The limit left is against an event that carries
    # a dump, and it falls between words and says that it fell.
    $message = "$($LogEntry.Message)"
    if ($message) {
        $message = ($message -replace '\s+', ' ').Trim()
        if ($message.Length -gt $script:StabilityEventMessageLength) {
            $cut = $message.LastIndexOf(' ', $script:StabilityEventMessageLength)
            if ($cut -lt 1) { $cut = $script:StabilityEventMessageLength }
            $message = $message.Substring(0, $cut) + ' [...]'
        }
    }

    $row = [ordered]@{
        Time     = $LogEntry.TimeCreated
        Provider = "$($LogEntry.ProviderName)"
        Id       = $LogEntry.Id
        Message  = $message
    }
    # Tagged rows are the ones bound for events.csv, where the tag is the column that lets
    # a Technician find the blue screens among four hundred lines.
    if ($Tag) { $row.Insert(1, 'Category', $Tag) }
    [pscustomobject]$row
}

function Get-EventPayload {
    <#
    .SYNOPSIS
        One positional field out of an event's payload, or $null when it is not there.
    .DESCRIPTION
        Positional because the names are localised and the positions are not. Bounds are
        checked because a real machine logs events with short payloads, and a Judge handed
        a row built by blind indexing would throw on data Windows considers valid.
    #>
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry, [Parameter(Mandatory)][int]$Index)

    $properties = @($LogEntry.Properties)
    if ($Index -lt 0 -or $Index -ge $properties.Count) { return $null }
    "$($properties[$Index].Value)"
}

function Select-NtfsError {
    <#
    .SYNOPSIS
        Passes on the NTFS events that report something wrong, and drops the ones that do not.
    .DESCRIPTION
        Ntfs 98 is logged at Information level whatever it says. Its third field,
        CorruptionActionState, is 0 for "volume is healthy, no action is needed" - written
        every time a volume or a shadow copy is mounted - and anything else for a volume
        that needs a scan or chkdsk. A 98 whose state cannot be read is kept: showing one
        event too many is the cheaper mistake.
    #>
    [CmdletBinding()]
    param([Parameter(ValueFromPipeline)]$LogEntry)

    process {
        if ($null -eq $LogEntry) { return }
        if ($LogEntry.Id -eq 98 -and (Get-EventPayload -LogEntry $LogEntry -Index 2) -eq '0') { return }
        $LogEntry
    }
}

function ConvertTo-LiveKernelIncident {
    <#
    .SYNOPSIS
        Reduces Windows Error Reporting's LiveKernelEvent reports to the incidents behind them.
    .DESCRIPTION
        WER logs event 1001 again each time it retries a queued report, so one driver hang
        is reported a hundred times over the following months. The incident is the dump:
        reports are grouped by the dump file they attach, or by report id when there is
        none, and dated by the timestamp in the dump's name rather than by when WER last
        mentioned it. An incident from before the period is not in the period, however
        recently it was reported again.
    #>
    [CmdletBinding()]
    param(
        [AllowNull()][AllowEmptyCollection()]$LogEntry,
        [Parameter(Mandatory)][datetime]$Since
    )

    $incidents = [ordered]@{}
    foreach ($entry in @($LogEntry | Where-Object { $_ })) {
        $dump = ''
        if ("$(Get-EventPayload -LogEntry $entry -Index 15)" -match '([^\\\s]+\.dmp)') { $dump = $Matches[1] }
        $reportId = "$(Get-EventPayload -LogEntry $entry -Index 19)"

        # No dump and no report id: nothing says two events are the same incident.
        $key = if ($dump) { $dump } elseif ($reportId) { $reportId } else { [guid]::NewGuid().ToString() }

        if (-not $incidents.Contains($key)) {
            $code = "$(Get-EventPayload -LogEntry $entry -Index 5)"
            $incidents[$key] = [pscustomobject]@{
                Time    = $entry.TimeCreated
                Code    = if ($code) { '0x{0}' -f $code } else { '' }
                Dump    = $dump
                Reports = 0
            }
        }
        $incident = $incidents[$key]
        $incident.Reports++
        if ($entry.TimeCreated -lt $incident.Time) { $incident.Time = $entry.TimeCreated }
    }

    foreach ($incident in $incidents.Values) {
        # WATCHDOG-20260915-1448.dmp: when the hang happened, in local time.
        if ($incident.Dump -match '(\d{8})-(\d{4})\.dmp$') {
            $taken = [datetime]::MinValue
            if ([datetime]::TryParseExact($Matches[1] + $Matches[2], 'yyyyMMddHHmm',
                    [cultureinfo]::InvariantCulture, [Globalization.DateTimeStyles]::None, [ref]$taken)) {
                $incident.Time = $taken
            }
        }
        if ($incident.Time -ge $Since) { $incident }
    }
}

function ConvertTo-BlueScreenRow {
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry)

    # The bugcheck line reads "0x00000133 (0x0000...)"; only the stop code is the lead.
    $code = Get-EventPayload -LogEntry $LogEntry -Index 0
    if ($code) { $code = $code -replace '\s*\(.*$', '' }

    [pscustomobject]@{ Time = $LogEntry.TimeCreated; StopCode = $code }
}

function ConvertTo-CrashRow {
    <#
    .SYNOPSIS
        An Application Error 1000 payload, in the order the schema defines it.
    #>
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry)

    [pscustomobject]@{
        Time      = $LogEntry.TimeCreated
        App       = Get-EventPayload -LogEntry $LogEntry -Index 0
        Version   = Get-EventPayload -LogEntry $LogEntry -Index 1
        Module    = Get-EventPayload -LogEntry $LogEntry -Index 3
        ModuleVer = Get-EventPayload -LogEntry $LogEntry -Index 4
        Exception = Get-EventPayload -LogEntry $LogEntry -Index 6
        AppPath   = Get-EventPayload -LogEntry $LogEntry -Index 10
    }
}

function ConvertTo-HangRow {
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry)
    [pscustomobject]@{ Time = $LogEntry.TimeCreated; App = Get-EventPayload -LogEntry $LogEntry -Index 0 }
}

function ConvertTo-RuntimeFaultRow {
    <#
    .SYNOPSIS
        A .NET Runtime 1026, whose payload is one blob of text rather than named fields.
    .DESCRIPTION
        App is absent when the text names no executable, as it always was. The framework
        version and the method are empty strings then, and the exception types an empty
        list: a later Check asks what the row says without asking first whether it does.
    #>
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry)

    # Read from the whole text, here, because nothing downstream has it: the row bound for
    # events.csv is cut at 500 characters, which is before most stacks reach the program's
    # own code. The payload's single field is the same text, for the event whose message
    # Windows could not render.
    $message = "$($LogEntry.Message)"
    if (-not $message.Trim()) { $message = "$(Get-EventPayload -LogEntry $LogEntry -Index 0)" }
    $read = ConvertFrom-RuntimeFaultMessage -Message $message

    [pscustomobject]@{
        Time             = $LogEntry.TimeCreated
        App              = $read.App
        FrameworkVersion = $read.FrameworkVersion
        ExceptionTypes   = $read.ExceptionTypes
        Method           = $read.Method
    }
}

# What an exception type looks like: a dotted name whose last part ends in "Exception". The
# plus sign is how the runtime writes a type nested in another.
$script:RuntimeFaultTypePattern = [regex]'(?:\w+[.+])+\w*Exception\b'

# What a stack line looks like: indented, one word ("bei", "at"), then a dotted name up to
# its parameter list. The word is whatever the language makes it, so it is only required to
# be one, and of letters - which leaves out the arrow before an inner exception.
$script:RuntimeFaultStackPattern = [regex]'^\s+[^\W\d_]+\s+([^\s(]+\.[^\s(]+)\('

function ConvertFrom-RuntimeFaultMessage {
    <#
    .SYNOPSIS
        What the text of a .NET Runtime 1026 says: application, framework, exception types
        and the first method of the stack.
    .DESCRIPTION
        The labels are localised - "Ausnahmeinformationen" on a German Windows, "Exception
        Info" on an English one - so nothing is found by its label. An exception type is
        matched on the type name itself, and a stack line by its shape.
 
        Types are taken from every line that is not a stack line, because an event names
        one per exception it unwound and a stack line names types of its own: every
        WinForms crash passes through OnThreadException(System.Exception).
 
        Reads text and nothing else, and returns every field whatever the text was: a
        message that names no exception leaves the types an empty list and the method
        empty.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param([AllowNull()][AllowEmptyString()][string]$Message)

    $lines = @("$Message" -split '\r?\n')

    # The first line is the application. Failing that, an executable named anywhere, which
    # is all this looked for before it read the rest.
    $app = $null
    if ($lines[0] -match '^[^:]+:\s*(\S.*?\.exe)\s*$') { $app = ($Matches[1] -split '[\\/]')[-1] }
    elseif ("$Message" -match '([^\\/\s:]+\.exe)')     { $app = $Matches[1] }

    $framework = ''
    $method    = ''
    $types     = New-Object System.Collections.Generic.List[string]

    for ($index = 0; $index -lt $lines.Count; $index++) {
        $line = $lines[$index]

        $frame = $script:RuntimeFaultStackPattern.Match($line)
        if ($frame.Success) {
            if (-not $method) { $method = $frame.Groups[1].Value }
            continue
        }

        # Not the line that named the application: a program may be called anything.
        if ($index -eq 0) { continue }

        # v4.0.30319 for the .NET Framework, 6.0.522.21309 for what came after it.
        if (-not $framework -and $line -match '^[^:]*:\s*(v?\d+(?:\.\d+)+)\s*$') { $framework = $Matches[1] }

        foreach ($match in $script:RuntimeFaultTypePattern.Matches($line)) {
            if (-not $types.Contains($match.Value)) { $types.Add($match.Value) }
        }
    }

    [pscustomobject]@{
        App              = $app
        FrameworkVersion = $framework
        ExceptionTypes   = [string[]]$types.ToArray()
        Method           = $method
    }
}

function ConvertTo-ServiceFailureRow {
    [CmdletBinding()]
    param([Parameter(Mandatory)]$LogEntry)
    [pscustomobject]@{ Time = $LogEntry.TimeCreated; Service = Get-EventPayload -LogEntry $LogEntry -Index 0 }
}

function ConvertTo-StabilityFinding {
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [AllowNull()]$Data,
        [hashtable]$Parameters = @{},
        # What a Hint may assume about the machine: what it is connected by, and whether
        # it is docked. Only the Hints of a crashing program go by it.
        [AllowNull()]$Situation
    )

    $days = Get-DataProperty $Data 'Days'
    if ($null -eq $days) { $days = $script:StabilityDefaultDays }

    New-EventCountFinding -Data $Data -Parameters $Parameters -Property 'UnexpectedShutdowns' `
        -Check (Get-Text 'Check.Stability.UnexpectedShutdownsKernelPower') -WarnParameter 'UnexpectedShutdownWarnAbove' `
        -FailParameter 'UnexpectedShutdownFailAbove' -WarnAbove 0 -FailAbove 2 `
        -Meaning (Get-Text 'Meaning.Stability.UnexpectedShutdowns') `
        -Hint (Get-Text 'Hint.Stability.UnexpectedShutdowns')

    New-BlueScreenFinding -Data $Data -Parameters $Parameters

    New-HardwareErrorFinding -Data $Data -Parameters $Parameters

    New-EventCountFinding -Data $Data -Parameters $Parameters -Property 'FirmwareThrottles' `
        -Check (Get-Text 'Check.Stability.CPUSpeedLimitedBy') -WarnParameter 'FirmwareThrottleWarnAbove' `
        -FailParameter 'FirmwareThrottleFailAbove' -WarnAbove 0 -FailAbove ([double]::MaxValue) `
        -Meaning (Get-Text 'Meaning.Stability.FirmwareThrottles') `
        -Hint (Get-Text 'Hint.Stability.ThermalPowerThrottlingHappenedIn')

    New-DiskErrorFinding -Data $Data -Parameters $Parameters

    New-EventCountFinding -Data $Data -Parameters $Parameters -Property 'DisplayResets' `
        -Check (Get-Text 'Check.Stability.GraphicsDriverResetsTDR') -WarnParameter 'DisplayResetWarnAbove' `
        -FailParameter 'DisplayResetFailAbove' -WarnAbove 0 -FailAbove ([double]::MaxValue) `
        -Hint (Get-Text 'Hint.Stability.UpdateGraphicsDriverVendorOEM')

    New-EventCountFinding -Data $Data -Parameters $Parameters -Property 'ResourceExhaustion' `
        -Check (Get-Text 'Check.Stability.LowVirtualMemoryWarnings') -WarnParameter 'ResourceExhaustionWarnAbove' `
        -FailParameter 'ResourceExhaustionFailAbove' -WarnAbove 0 -FailAbove 0 `
        -Meaning (Get-Text 'Meaning.Stability.ResourceExhaustion') `
        -Hint (Get-Text 'Hint.Stability.RAMPageFileExhaustedPrograms')

    $crashFindings = @(New-ApplicationCrashFinding -Data $Data -Parameters $Parameters -Situation $Situation)
    $crashFindings

    New-LiveKernelFinding -Data $Data -Parameters $Parameters

    New-ServiceFailureFinding  -Data $Data -Parameters $Parameters
    New-ReliabilityFinding     -Data $Data -Parameters $Parameters
    New-UnreadableLogFinding   -Data $Data -Parameters $Parameters

    # The crashes and hangs on the 30 days, beside the network's events. ADR-0010.
    ConvertTo-StabilityTimedEvent -Data $Data -Parameters $Parameters -Finding $crashFindings
}

function ConvertTo-StabilityTimedEvent {
    <#
    .SYNOPSIS
        Each crash and each hang of a program as a marker of the timeline, with the program
        and the Finding that counted it. Pure. ADR-0010.
    .DESCRIPTION
        A crash is counted by the program's own Finding where it crashed often enough for
        one, and by the Finding about the scattered crashes otherwise: the same split
        Get-CrashingProgram makes for the Findings. A program that only stopped answering
        is counted by no Finding (see Select-CrashIncident), and its hang is drawn grey.
 
        One marker per incident, not per event: one crash is logged up to three times. An
        incident without a time cannot be placed and is left out.
    .PARAMETER Finding
        What New-ApplicationCrashFinding made of the same incidents.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param([AllowNull()]$Data, [hashtable]$Parameters = @{}, [AllowNull()][AllowEmptyCollection()]$Finding)

    # The Findings New-ApplicationCrashFinding made: one per program that crashes
    # repeatedly, about that program, and one about the rest.
    $scattered = @($Finding | Where-Object { $_ -and "$(Get-DataProperty $_ 'Check')" -eq (Get-Text 'Check.Stability.ScatteredCrashes') })[0]
    $byProgram = @{}
    foreach ($one in @($Finding | Where-Object { $_ -and "$(Get-DataProperty $_ 'Subject')" })) {
        $byProgram["$(Get-DataProperty $one 'Subject')".ToLowerInvariant()] = $one
    }

    $window = Get-IncidentWindowSetting -Parameters $Parameters
    foreach ($program in @(Get-CrashingProgram -Data $Data -Parameters $Parameters)) {
        $by = $scattered
        $key = "$($program.App)".ToLowerInvariant()
        if ($program.Repeated -and $byProgram.ContainsKey($key)) { $by = $byProgram[$key] }
        foreach ($incident in @($program.Incidents)) {
            $at = ConvertTo-DataTime (Get-DataProperty $incident 'Time')
            if ($null -eq $at) { continue }
            New-TimedEvent -Time $at -Lane Marker -Mark Crash -Subject "$(Get-DataProperty $incident 'App')" -Log 'Application' -Finding $by
        }
    }

    $incidents = Get-StabilityIncident -Data $Data -WindowSeconds $window
    foreach ($incident in @($incidents | Where-Object { $null -ne $_ -and (Test-HangOnlyIncident -Incident $_) })) {
        $at = ConvertTo-DataTime (Get-DataProperty $incident 'Time')
        if ($null -eq $at) { continue }
        New-TimedEvent -Time $at -Lane Marker -Mark Hang -Subject "$(Get-DataProperty $incident 'App')" -Log 'Application'
    }
}

function Get-StabilityCurrentDisk {
    <#
    .SYNOPSIS
        The numbers of the disks and the letters of the volumes this machine has now, and
        which of them are external: on USB, or removable. Readable without admin rights.
        Never throws.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param()

    $drives  = @(Get-CimInstance Win32_DiskDrive -ErrorAction SilentlyContinue)
    $numbers = @($drives | ForEach-Object { "$($_.Index)" })
    $logical = @(Get-CimInstance Win32_LogicalDisk -ErrorAction SilentlyContinue | Where-Object { $_.DriveType -in 2, 3 })
    $volumes = @($logical | ForEach-Object { "$($_.DeviceID)".ToUpperInvariant() })

    # A USB hard disk is a fixed disk to Windows and a USB stick a removable one: the
    # disk says which bus it is on, the volume only that it can be taken out.
    $external = @($drives | Where-Object { "$($_.InterfaceType)" -eq 'USB' -or "$($_.MediaType)" -match '(?i)removable|external' } |
        ForEach-Object { "$($_.Index)" })
    $externalVolumes = @($logical | Where-Object { $_.DriveType -eq 2 } | ForEach-Object { "$($_.DeviceID)".ToUpperInvariant() })
    try {
        if ($external.Count -and (Get-Command -Name Get-Partition -ErrorAction SilentlyContinue)) {
            $externalVolumes += @(Get-Partition -ErrorAction Stop | Where-Object { $external -contains "$($_.DiskNumber)" -and $_.DriveLetter } |
                ForEach-Object { ('{0}:' -f $_.DriveLetter).ToUpperInvariant() })
        }
    }
    catch { }

    [pscustomobject]@{
        Gathered = [bool]$numbers.Count; Numbers = $numbers; Volumes = $volumes
        External = $external; ExternalVolumes = @($externalVolumes | Sort-Object -Unique)
    }
}

function New-EventCountFinding {
    <#
    .SYNOPSIS
        The Finding shape shared by every scan judged purely on how many events there were.
    .DESCRIPTION
        Eight of these Checks differ only in their name, their thresholds, their Hint and
        what a count above none means.
        Writing them out eight times would make eight places for a threshold to stop being
        a parameter, which is the thing this rebuild exists to prevent.
    #>
    [CmdletBinding()]
    param(
        [AllowNull()]$Data, [hashtable]$Parameters,
        [Parameter(Mandatory)][string]$Property,
        [Parameter(Mandatory)][string]$Check,
        [Parameter(Mandatory)][string]$WarnParameter,
        [Parameter(Mandatory)][string]$FailParameter,
        [Parameter(Mandatory)][double]$WarnAbove,
        [Parameter(Mandatory)][double]$FailAbove,
        [string]$Hint = '',

        # What the events mean, where the name of the Check does not say it.
        [string]$Meaning = '',

        # What the events are, where one of them is an observation: declared from the first.
        [AllowNull()][string[]]$Signal
    )

    $warn  = Get-NumberParameter $Parameters $WarnParameter $WarnAbove
    $fail  = Get-NumberParameter $Parameters $FailParameter $FailAbove
    $count = (Get-DataCollection $Data $Property).Count

    $observed = @()
    if ($count) { $observed = @($Signal | Where-Object { $_ }) }
    New-Finding -Category Stability -Check $Check -Severity (Get-Severity $count $warn $fail) `
        -Value (Get-EventCountText -Count $count -Data $Data) -Meaning $Meaning -Hint $Hint -Signal $observed
}

function Get-EventCountText {
    <#
    .SYNOPSIS
        A number of events as a Value: how many, and in how many days. Pure.
    .DESCRIPTION
        The bare number was the whole Value, and "120" beside the name of a Check does
        not say 120 of what.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([int]$Count, [AllowNull()]$Data)

    $days = Get-DataProperty $Data 'Days'
    if ($null -eq $days) { $days = $script:StabilityDefaultDays }
    if ($Count -le 0) { return (Get-Text 'Value.Stability.EventCountNone') -f $days }
    (Get-Text 'Value.Stability.EventCount') -f $Count, $days
}

function New-ServiceFailureFinding {
    [CmdletBinding()]
    param([AllowNull()]$Data, [hashtable]$Parameters)

    $warn = Get-NumberParameter $Parameters 'ServiceFailureWarnAbove' 10

    $failures = (Get-DataCollection $Data 'ServiceFailures')
    if (-not $failures.Count) {
        return New-Finding -Category Stability -Check (Get-Text 'Check.Stability.CrashedServices') -Severity OK -Value (Get-Text 'Value.Shared.None')
    }

    $named = $failures | Where-Object { $_.Service } | Group-Object Service |
        Sort-Object Count -Descending | Select-Object -First 5 |
        ForEach-Object { '{0} x{1}' -f $_.Name, $_.Count }

    # A handful of service restarts is ordinary on Windows and the script never failed on
    # them; only an unusual number is worth a Technician's attention.
    $severity = 'INFO'
    if ($failures.Count -gt $warn) { $severity = 'WARN' }

    New-Finding -Category Stability -Check (Get-Text 'Check.Stability.CrashedServices') -Severity $severity `
        -Value ($named -join ', ') `
        -Meaning (Get-Text 'Meaning.Stability.ServiceFailures') `
        -Hint (Get-Text 'Hint.Stability.AServiceThatKeepsDying')
}

function New-ReliabilityFinding {
    [CmdletBinding()]
    param([AllowNull()]$Data, [hashtable]$Parameters)

    $warnBelow = Get-NumberParameter $Parameters 'ReliabilityIndexWarnBelow' 6
    $failBelow = Get-NumberParameter $Parameters 'ReliabilityIndexFailBelow' 3

    $index = ConvertTo-Number (Get-DataProperty $Data 'ReliabilityIndex')
    if ($null -eq $index) {
        # Absent on a machine whose Reliability Monitor task has never run, which says
        # nothing about how stable the machine is.
        return New-UnavailableFinding -Category Stability -Check (Get-Text 'Check.Stability.ReliabilityIndex110') `
            -Hint (Get-Text 'Hint.Stability.WindowsHasNotComputedA')
    }

    New-Finding -Category Stability -Check (Get-Text 'Check.Stability.ReliabilityIndex110') `
        -Severity (Get-SeverityBelow $index $warnBelow $failBelow) -Value $index `
        -Hint (Get-Text 'Hint.Stability.OpenPerfmonRelForThe')
}

function New-UnreadableLogFinding {
    <#
    .SYNOPSIS
        One Finding per log this Run's rights could not open.
    #>
    [CmdletBinding()]
    param([AllowNull()]$Data, [hashtable]$Parameters)

    foreach ($log in (Get-DataCollection $Data 'UnreadableLogs')) {
        New-Finding -Category Access -Check ((Get-Text 'Check.Stability.EventLog') -f $log) -Severity INFO `
            -Value (Get-Text 'Value.Stability.NotReadableWithRights') `
            -Hint (Get-Text 'Hint.Stability.NeedsAdminRightsThisLog')
    }
}

function ConvertTo-StabilitySection {
    [CmdletBinding()]
    [OutputType([psobject])]
    param([AllowNull()]$Data)

    # The column names are what the Report shows, so they are German and live with the
    # other text. An exception code stands with what it means behind it, here and in the
    # incidents below: a code is never shown bare.
    $crashes = (Get-DataCollection $Data 'Crashes')
    if ($crashes.Count) {
        New-Section -Title (Get-Text 'Title.Stability.ApplicationCrashesByApp') -Help (Get-Text 'Help.Stability.ApplicationCrashesByApp') -Row @(
            $crashes | Group-Object App | Sort-Object Count -Descending | ForEach-Object {
                $row = [ordered]@{}
                $row[(Get-Text 'Column.Stability.Crashes.Count')] = $_.Count
                $row[(Get-Text 'Column.Stability.Crashes.App')]   = $_.Name
                $row[(Get-Text 'Column.Stability.Crashes.Last')]  = (@($_.Group | Sort-Object Time -Descending)[0]).Time
                $row[(Get-Text 'Column.Stability.Crashes.TopModules')] = (
                    $_.Group | Group-Object { '{0} / {1}' -f $_.Module, $_.Exception } |
                        Sort-Object Count -Descending | Select-Object -First 3 |
                        ForEach-Object { '{0} / {1} x{2}' -f $_.Group[0].Module, (Get-ExceptionCodeText "$($_.Group[0].Exception)"), $_.Count }
                ) -join '; '
                [pscustomobject]$row
            }
        )
    }

    # Every incident, for whoever has to hand them to a vendor.
    $incidents = Get-StabilityIncident -Data $Data
    if ($incidents.Count) {
        $setting    = Get-SectionCorrelationParameter -Data $Data
        $disruption = Get-IncidentDisruptionColumn -Incident $incidents -Data $Data -Parameters $setting
        $windows    = Get-CorrelationSetting -Parameters $setting
        $heading    = Get-DisruptionColumnHeading -Seconds $windows.WindowSeconds -BeforeSeconds $windows.BeforeSeconds

        $index = -1
        New-Section -Title (Get-Text 'Title.Stability.CrashIncidents') -Help (Get-Text 'Help.Stability.CrashIncidents') -Row @(
            foreach ($incident in $incidents) {
                $index++
                $code = "$(Get-DataProperty $incident 'Exception')"
                if ($code) { $code = Get-ExceptionCodeText $code }
                elseif (-not (Select-CrashIncident -Incident $incident).Count) { $code = Get-Text 'Value.Stability.IncidentHang' }

                $time = ConvertTo-DataTime (Get-DataProperty $incident 'Time')

                $row = [ordered]@{}
                $row[(Get-Text 'Column.Stability.Incident.Time')]          = $time
                $row[(Get-Text 'Column.Stability.Incident.App')]           = Get-DataProperty $incident 'App'
                $row[(Get-Text 'Column.Stability.Incident.Version')]       = Get-DataProperty $incident 'Version'
                $row[(Get-Text 'Column.Stability.Incident.Exception')]     = $code
                $row[(Get-Text 'Column.Stability.Incident.Module')]        = Get-DataProperty $incident 'Module'
                $row[(Get-Text 'Column.Stability.Incident.ExceptionType')] = (Get-DataCollection $incident 'ExceptionTypes') -join ', '
                $row[(Get-Text 'Column.Stability.Incident.Method')]        = Get-DataProperty $incident 'Method'
                $row[(Get-Text 'Column.Stability.Incident.Path')]          = Get-DataProperty $incident 'AppPath'
                $row[$heading]                                             = $disruption[$index]
                [pscustomobject]$row
            }
        )
    }

    # What Windows said when memory ran low: it names the programs that held the most,
    # which is what the Finding sends a Technician to read.
    $lowMemory = @((Get-DataCollection $Data 'ResourceExhaustion') | Sort-Object { ConvertTo-DataTime (Get-DataProperty $_ 'Time') } -Descending | Select-Object -First 50)
    if ($lowMemory.Count) {
        New-Section -Title (Get-Text 'Title.Stability.ResourceExhaustion') -Help (Get-Text 'Help.Stability.ResourceExhaustion') -Row @(
            $lowMemory | ForEach-Object {
                $row = [ordered]@{}
                $row[(Get-Text 'Column.Stability.Incident.Time')]              = Get-DataProperty $_ 'Time'
                $row[(Get-Text 'Column.Stability.ResourceExhaustion.Message')] = Get-DataProperty $_ 'Message'
                [pscustomobject]$row
            }
        )
    }

    $hangs = (Get-DataCollection $Data 'Hangs')
    if ($hangs.Count) {
        New-Section -Title (Get-Text 'Title.Stability.ApplicationHangsByApp') -Help (Get-Text 'Help.Stability.ApplicationHangsByApp') -Row @(
            $hangs | Group-Object App | Sort-Object Count -Descending | ForEach-Object {
                $row = [ordered]@{}
                $row[(Get-Text 'Column.Stability.Crashes.Count')] = $_.Count
                $row[(Get-Text 'Column.Stability.Crashes.App')]   = $_.Name
                [pscustomobject]$row
            }
        )
    }

    # Every disk, controller and file system error, with the drive it was about.
    New-DiskErrorSection -Data $Data
}

function ConvertTo-StabilityEvent {
    <#
    .SYNOPSIS
        The events this Check gathered, for the Report's events.csv.
    .DESCRIPTION
        The optional fourth half of the Kind interface. A Kind that reads events returns
        them here and the Run collects them; nothing is written to shared state and no
        later Check reaches back into this one.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param([AllowNull()]$Data)

    (Get-DataCollection $Data 'Events')
}