Gutcheck

0.28.0

Private/Kinds/StabilityCrash.ps1

# A program's crashes: the Findings about them, the indicators read off them and the
# verdict drawn from those. A part of the Stability Kind, and no Kind.

# Where each exception type sends a Technician. Keyed by the type's own name without its
# namespace, because SqlException lives in more than one and means the same in each. A
# type that is not here is the application's own business. See Get-ExceptionTypeHint.
$script:ExceptionTypeDirections = @{
    'SEHException'                = 'Native'
    'AccessViolationException'    = 'Native'
    'FileLoadException'           = 'Load'
    'FileNotFoundException'       = 'Load'
    'BadImageFormatException'     = 'Load'
    'IOException'                 = 'FileAccess'
    'UnauthorizedAccessException' = 'FileAccess'
    'SqlException'                = 'Database'
    'TimeoutException'            = 'Database'
    'OutOfMemoryException'        = 'Memory'
}

function Get-ExceptionTypeDirection {
    <#
    .SYNOPSIS
        The exception types in these .NET rows, by where they point: Native, Load,
        FileAccess, Database, Memory, and Other for a type that points nowhere. Pure.
    .DESCRIPTION
        What Get-ExceptionTypeMeaning and Get-ExceptionTypeHint both go by, so that what
        a type is said to mean and what is said to do about it are about the same types.
    #>
    [CmdletBinding()]
    [OutputType([hashtable])]
    param([AllowNull()][AllowEmptyCollection()]$RuntimeFault)

    $found = @{}
    foreach ($row in @($RuntimeFault)) {
        foreach ($type in (Get-DataCollection $row 'ExceptionTypes')) {
            $name = @("$type" -split '[.+]')[-1]
            if (-not $name) { continue }

            $direction = $script:ExceptionTypeDirections[$name]
            if (-not $direction) { $direction = 'Other' }

            if (-not $found.ContainsKey($direction)) { $found[$direction] = @() }
            if ($found[$direction] -notcontains $name) { $found[$direction] += $name }
        }
    }
    $found
}

function Get-ExceptionTypeMeaning {
    <#
    .SYNOPSIS
        What the exception types in these .NET rows mean, one sentence per direction.
        Empty when they name none. Pure.
    .DESCRIPTION
        The counterpart of Get-ExceptionTypeHint, with the same rule: "the application's
        own fault" is said only when no type points anywhere else.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyCollection()]$RuntimeFault)

    $found = Get-ExceptionTypeDirection -RuntimeFault $RuntimeFault
    if (-not $found.Count) { return '' }

    $means = @()
    if ($found.ContainsKey('Native'))     { $means += Get-Text 'Meaning.Stability.ExceptionNative'     ($found['Native'] -join ', ') }
    if ($found.ContainsKey('Load'))       { $means += Get-Text 'Meaning.Stability.ExceptionLoad'       ($found['Load'] -join ', ') }
    if ($found.ContainsKey('FileAccess')) { $means += Get-Text 'Meaning.Stability.ExceptionFileAccess' ($found['FileAccess'] -join ', ') }
    if ($found.ContainsKey('Database'))   { $means += Get-Text 'Meaning.Stability.ExceptionDatabase'   ($found['Database'] -join ', ') }
    if ($found.ContainsKey('Memory'))     { $means += Get-Text 'Meaning.Stability.ExceptionMemory'     ($found['Memory'] -join ', ') }

    if (-not $means.Count) {
        $means += Get-Text 'Meaning.Stability.ExceptionOther' (@($found['Other'] | Select-Object -First 3) -join ', ')
    }
    $means -join ' | '
}

function Get-ExceptionTypeHint {
    <#
    .SYNOPSIS
        Where the exception types in these .NET rows point, as a Hint. Empty when they
        name none.
    .DESCRIPTION
        An exception code says how a program died; the type says where to look. One
        sentence per direction, each naming the types that point in it.
 
        "The application's own fault" is said only when no type points anywhere else: the
        runtime names the wrapper beside the cause, and a TypeInitializationException
        around a FileNotFoundException is a file that was not there.
 
        Part of the Judge, and used by the App Kind's as well: it is handed rows and reads
        nothing.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyCollection()]$RuntimeFault)

    $found = Get-ExceptionTypeDirection -RuntimeFault $RuntimeFault
    if (-not $found.Count) { return '' }

    $hints = @()
    if ($found.ContainsKey('Native'))     { $hints += Get-Text 'Hint.Stability.ExceptionNative'     ($found['Native'] -join ', ') }
    if ($found.ContainsKey('Load'))       { $hints += Get-Text 'Hint.Stability.ExceptionLoad'       ($found['Load'] -join ', ') }
    if ($found.ContainsKey('FileAccess')) { $hints += Get-Text 'Hint.Stability.ExceptionFileAccess' ($found['FileAccess'] -join ', ') }
    if ($found.ContainsKey('Database'))   { $hints += Get-Text 'Hint.Stability.ExceptionDatabase'   ($found['Database'] -join ', ') }
    if ($found.ContainsKey('Memory'))     { $hints += Get-Text 'Hint.Stability.ExceptionMemory'     ($found['Memory'] -join ', ') }

    if (-not $hints.Count) {
        # The first few: a machine with many applications names many types of its own.
        $hints += Get-Text 'Hint.Stability.ExceptionOther' (@($found['Other'] | Select-Object -First 3) -join ', ')
    }
    $hints -join ' | '
}

function New-ApplicationCrashFinding {
    <#
    .SYNOPSIS
        One Finding for each program that crashes repeatedly, one for the rest, and one OK
        Finding for a machine on which nothing crashed.
    .DESCRIPTION
        A count over all applications names no program, and a Technician had to open the
        event log to learn that all nine crashes were one. Each program now speaks for
        itself, whether or not anybody wrote a Check Definition for it.
    #>
    [CmdletBinding()]
    param([AllowNull()]$Data, [hashtable]$Parameters, [AllowNull()]$Situation)

    $programs = @(Get-CrashingProgram -Data $Data -Parameters $Parameters)
    if (-not $programs.Count) {
        return New-Finding -Category Stability -Check (Get-Text 'Check.Stability.ApplicationCrashesAllApps') `
            -Severity OK -Value 0
    }

    $days = Get-DataProperty $Data 'Days'
    if ($null -eq $days) { $days = $script:StabilityDefaultDays }

    # An access violation in one program is that program. In several that have nothing to
    # do with each other it is what they share: the memory they run in.
    $spreadOver = Get-NumberParameter $Parameters 'AccessViolationSpreadPrograms' 3
    $violating  = @($programs | Where-Object { @($_.Incidents | Where-Object { "$(Get-DataProperty $_ 'Exception')" -match '(?i)c0000005' }).Count })
    $spread     = $violating.Count -ge $spreadOver

    foreach ($program in @($programs | Where-Object { $_.Repeated })) {
        # A program that an application's own Check judges in this Run is said here, and
        # judged there. Seen: a helper's four crashes as a WARN of its own with "0 of 3
        # fell together with a disruption of the network", beside the Check that knew
        # they fell together with its host going away - one fault, two alarms, and the
        # one that knew less sent the Technician to look at the network.
        $covered = Get-CrashCoveringApp -App $program.App -Covered (Get-DataCollection $Data 'CoveredPrograms')
        if ($covered) {
            New-ProgramCrashFinding -App $program.App -Incident $program.Incidents -Days $days -Parameters $Parameters -CoveredBy $covered
            continue
        }
        $evidence = Get-ProgramCrashEvidence -App $program.App -Incident $program.Incidents -Data $Data -Parameters $Parameters -Situation $Situation
        $signal = @()
        if ($spread -and $violating -contains $program) { $signal = @('access-violations-spread') }
        # Held to what the Finding itself says of it: a program known to be local whose
        # crashes fell near a disruption once is not said to have crashed with the network.
        if (Test-CorrelationWorthALook -Correlation $evidence.Correlation) { $signal += 'crash-with-network-disruption' }

        # Whether this program's crashes began shortly after a Change. No part of the
        # verdict, which weighs what the crashes themselves say: it is a date beside them.
        $tie = Get-CrashChangeTie -Time @($program.Incidents | ForEach-Object { Get-DataProperty $_ 'Time' }) `
            -Change (Get-DataProperty $Data 'Change') -App $program.App -Log 'Application' -Days $days -Parameters $Parameters
        $began = @()
        if ($null -ne $tie) { $began = @(Get-Text 'Value.Stability.Change.FirstIncident' $tie.First (Get-ChangeTieText -Tie $tie)) }

        # The verdict last of what it is drawn from, because it is what the lines before it
        # add up to; and its Hint first, because it is what to do. What it means goes in
        # the order of the Hints. The Change after all of it.
        New-ProgramCrashFinding -App $program.App -Incident $program.Incidents -Days $days -Parameters $Parameters `
            -Network $evidence.Network -Severity $evidence.Verdict.Severity `
            -Line (@(Get-ProgramCorrelationLine -Correlation $evidence.Correlation) + @(Get-ProgramVerdictLine -Verdict $evidence.Verdict) + $began) `
            -Meaning (@($evidence.Verdict.Meaning) + @(Get-ProgramCorrelationMeaning -Correlation $evidence.Correlation) + @(Get-ChangeTieMeaning -Tie $tie)) `
            -Hint (@($evidence.Verdict.Hint) + @(Get-ProgramCorrelationHint -Correlation $evidence.Correlation -Situation $Situation) + @(Get-ChangeTieHint -Tie $tie)) `
            -Signal $signal -Reference @(Get-ChangeTieReference -Tie $tie)
    }

    $scattered = @($programs | Where-Object { -not $_.Repeated })
    $signal = @()
    if ($spread -and @($scattered | Where-Object { $violating -contains $_ }).Count) { $signal = @('access-violations-spread') }
    New-ScatteredCrashFinding -Program $scattered -Parameters $Parameters -Signal $signal
}

function Get-CrashCoveringApp {
    <#
    .SYNOPSIS
        The application whose own Check judges a program in this Run, by the pattern of
        its processes; nothing where none does. Pure.
    .DESCRIPTION
        An event names a program with its extension and a pattern is written without, as
        Select-AppRow reads them. A pattern that is no regular expression covers nothing.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyString()][string]$App, [AllowNull()][AllowEmptyCollection()]$Covered)

    $name = "$App".Trim() -replace '\.exe$', ''
    if (-not $name) { return '' }
    foreach ($one in @($Covered | Where-Object { $_ })) {
        $pattern = "$(Get-DataProperty $one 'ProcessPattern')"
        $owner   = "$(Get-DataProperty $one 'App')".Trim()
        if (-not $pattern -or -not $owner) { continue }
        try { if ($name -match $pattern) { return $owner } } catch { continue }
    }
    ''
}

function Get-CrashingProgram {
    <#
    .SYNOPSIS
        The programs that crashed, each with its incidents, the one that crashed most
        first. Pure.
    .DESCRIPTION
        Repeated says whether the program reached the number of incidents from which it
        gets a Finding of its own. Three by default: one or two crashes in a month happen
        to healthy programs, and three is the smallest number at which crashes can be said
        to be spread over several places - so every program that can earn that statement
        has a Finding to carry it.
 
        A crash whose program no event named is kept, under no name, and never counts as
        repeated: nothing says two of those were the same program.
 
        Also for a later Check that knows more about these crashes than this one does, so
        that it judges the same programs and the same incidents. The rows are written to
        the pipeline one by one: collect them with @( ).
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param([AllowNull()]$Data, [hashtable]$Parameters = @{})

    $threshold = Get-NumberParameter $Parameters 'ProgramIncidentThreshold' 3
    $window    = Get-IncidentWindowSetting -Parameters $Parameters

    $crashes = Select-CrashIncident -Incident (Get-StabilityIncident -Data $Data -WindowSeconds $window)

    $byProgram = @{}
    $order     = @()
    foreach ($incident in $crashes) {
        $key = "$(Get-DataProperty $incident 'App')".Trim().ToLowerInvariant()
        if (-not $byProgram.ContainsKey($key)) { $byProgram[$key] = @(); $order += $key }
        $byProgram[$key] += $incident
    }

    $programs = foreach ($key in $order) {
        $incidents = @($byProgram[$key] | Sort-Object @{ Expression = { Get-IncidentTimeKey $_ } } -Descending)
        [pscustomobject]@{
            # As the newest incident writes it.
            App       = "$(Get-DataProperty $incidents[0] 'App')".Trim()
            Incidents = $incidents
            Repeated  = [bool]($key -and $incidents.Count -ge $threshold)
        }
    }

    $programs | Sort-Object @{ Expression = { $_.Incidents.Count }; Descending = $true },
                            @{ Expression = { Get-IncidentTimeKey $_.Incidents[0] }; Descending = $true }
}

function Get-IncidentTimeKey {
    <#
    .SYNOPSIS
        An incident's time as something to sort by: the earliest date there is when it has none.
    #>
    [CmdletBinding()]
    [OutputType([datetime])]
    param([AllowNull()]$Incident)

    $time = ConvertTo-DataTime (Get-DataProperty $Incident 'Time')
    if ($null -eq $time) { return [datetime]::MinValue }
    $time
}

function Get-CountedText {
    <#
    .SYNOPSIS
        Names, counted: "6x this, 2x that", the most frequent first. Pure.
    .DESCRIPTION
        Names that are equally frequent stay in the order they were handed over, which
        Sort-Object alone does not promise on Windows PowerShell 5.1.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyCollection()][string[]]$Name, [int]$First = 0)

    $counts = @{}
    $order  = @()
    foreach ($item in @($Name | Where-Object { $_ })) {
        if (-not $counts.ContainsKey($item)) { $counts[$item] = 0; $order += $item }
        $counts[$item]++
    }

    if (-not $order.Count) { return '' }

    $ranked = @(for ($index = 0; $index -lt $order.Count; $index++) {
        [pscustomobject]@{ Name = $order[$index]; Count = $counts[$order[$index]]; Index = $index }
    }) | Sort-Object @{ Expression = { $_.Count }; Descending = $true }, @{ Expression = { $_.Index } }
    $ranked = @($ranked)

    $shown = $ranked
    if ($First -gt 0 -and $ranked.Count -gt $First) { $shown = @($ranked | Select-Object -First $First) }

    $text = @($shown | ForEach-Object { Get-Text 'Value.Stability.Program.Counted' $_.Count $_.Name }) -join ', '
    if ($shown.Count -lt $ranked.Count) {
        $text = Get-Text 'Value.Stability.ScatteredMore' $text ($ranked.Count - $shown.Count)
    }
    $text
}

function Get-ExceptionCodeText {
    <#
    .SYNOPSIS
        An exception code as a Technician reads it: 0xc0000006, and what it means where
        that is known. Pure.
    .DESCRIPTION
        The codes that name a cause outright. c0000006 above all: a page of the program
        could not be read back in, which is what a program on a share does when the
        connection drops.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyString()][string]$Code)

    $plain = ("$Code".Trim() -replace '^0x', '').ToLowerInvariant()
    if (-not $plain) { return '' }

    $meaning = switch ($plain) {
        'c0000005' { Get-Text 'Value.Stability.Code.AccessViolation' }
        'c0000006' { Get-Text 'Value.Stability.Code.InPage' }
        'c0000017' { Get-Text 'Value.Stability.Code.NoMemory' }
        'c00000fd' { Get-Text 'Value.Stability.Code.StackOverflow' }
        'c0000374' { Get-Text 'Value.Stability.Code.HeapCorruption' }
        'c0000409' { Get-Text 'Value.Stability.Code.FailFast' }
        # STATUS_FATAL_USER_CALLBACK_EXCEPTION: an exception nobody caught inside a
        # callback of the window system, as ntstatus.h names it.
        'c000041d' { Get-Text 'Value.Stability.Code.UserCallback' }
        'e0434352' { Get-Text 'Value.Stability.Code.DotNet' }
        'e06d7363' { Get-Text 'Value.Stability.Code.Cpp' }
        # Any other status of Windows, as Windows says it. Not the codes programs make
        # up for themselves (0xE...): those are no status and have no sentence.
        default    { if ($plain -match '^[c8]0[0-9a-f]{6}$') { Get-NtStatusText -Code "0x$plain" } else { '' } }
    }

    if ($meaning) { return '0x{0} ({1})' -f $plain, $meaning }
    '0x{0}' -f $plain
}

function Get-FaultSite {
    <#
    .SYNOPSIS
        The distinct places in a program at which these incidents happened. Pure.
    .DESCRIPTION
        The place is the first method of the stack where the .NET runtime named one, and
        the faulting module where it did not: every .NET crash faults in clr.dll or
        KERNELBASE.dll, so the module says nothing there, and a native program has no
        method. An incident that names neither has no place and is not counted as one.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyCollection()]$Incident)

    $sites = @()
    foreach ($row in @($Incident | Where-Object { $null -ne $_ })) {
        $site = Get-IncidentFaultSite -Incident $row
        if ($site -and $sites -notcontains $site) { $sites += $site }
    }
    $sites
}

function Get-IncidentFaultSite {
    <#
    .SYNOPSIS
        The place in the program at which one incident happened, or nothing when no event
        of it named one. Pure. See Get-FaultSite.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()]$Incident)

    $method = "$(Get-DataProperty $Incident 'Method')".Trim()
    if ($method) { return ('method:{0}' -f $method) }

    $module = "$(Get-DataProperty $Incident 'Module')".Trim()
    if ($module) { return ('module:{0}' -f $module.ToLowerInvariant()) }
    ''
}

function Test-FaultSiteSpread {
    <#
    .SYNOPSIS
        Whether these incidents are spread over too many places to be one bug. Pure.
    .DESCRIPTION
        A bug crashes a program in the same place every time. A program that dies in a
        different place each time is being failed by something beneath it: the share it
        runs from, the disk, the memory, a driver, a virus scanner. From three incidents at
        three places by default; both numbers are Check Definition parameters.
 
        And no one place may hold more than half of the incidents that have one, which is
        the third parameter. Twenty-five crashes at one method and three elsewhere are
        three places, and a program that dies in one place: a bug, with three strays.
    #>
    [CmdletBinding()]
    [OutputType([bool])]
    param([AllowNull()][AllowEmptyCollection()]$Incident, [hashtable]$Parameters = @{})

    $needIncidents = Get-NumberParameter $Parameters 'FaultSiteSpreadIncidents' 3
    $needSites     = Get-NumberParameter $Parameters 'FaultSiteSpreadSites' 3

    $dominance     = Get-NumberParameter $Parameters 'FaultSiteDominancePercent' 50

    $incidents = @($Incident | Where-Object { $null -ne $_ })
    if ($incidents.Count -lt $needIncidents) { return $false }

    $perSite = @{}
    $placed  = 0
    $most    = 0
    foreach ($row in $incidents) {
        $site = Get-IncidentFaultSite -Incident $row
        if (-not $site) { continue }
        if (-not $perSite.ContainsKey($site)) { $perSite[$site] = 0 }
        $perSite[$site]++
        $placed++
        if ($perSite[$site] -gt $most) { $most = $perSite[$site] }
    }
    if (-not $placed -or $perSite.Count -lt $needSites) { return $false }

    ($most * 100 / $placed) -le $dominance
}

function ConvertTo-ProgramKey {
    <#
    .SYNOPSIS
        A program's name as something to compare by. Pure.
    .DESCRIPTION
        An event names a program acmesync.exe or ACMESYNC.EXE, and the inventory of what
        depends on the network names the process, acmesync.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyString()][string]$App)

    "$App".Trim().ToLowerInvariant() -replace '\.exe$', ''
}

function ConvertTo-IndicatorWeight {
    <#
    .SYNOPSIS
        A weight as a Check Definition writes it, as one of high, medium, low and off. Pure.
    .DESCRIPTION
        "off" takes an indicator out of the verdict altogether. Anything that is none of
        the four is the default: a mistyped weight must not switch an indicator off.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()]$Value, [Parameter(Mandatory)][string]$Default)

    $weight = "$Value".Trim().ToLowerInvariant()
    if (@('high', 'medium', 'low', 'off') -contains $weight) { return $weight }
    $Default
}

function New-CrashIndicator {
    <#
    .SYNOPSIS
        One indicator of a verdict: what it is, how much it weighs, and whether it was
        present, absent or could not be looked at. Pure.
    .PARAMETER Text
        What the Finding lists when it is present.
    .PARAMETER Name
        What the Finding lists when it could not be assessed. Empty for an indicator that
        always can be, because it is read off the crashes themselves.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [Parameter(Mandatory)][string]$Key,
        [Parameter(Mandatory)][string]$Weight,
        [Parameter(Mandatory)][ValidateSet('Present', 'Absent', 'NotAssessed')][string]$State,
        [AllowEmptyString()][string]$Text = '',
        [AllowEmptyString()][string]$Name = ''
    )

    [pscustomobject]@{ Key = $Key; Weight = $Weight; State = $State; Text = $Text; Name = $Name }
}

function Get-ProgramCrashIndicator {
    <#
    .SYNOPSIS
        The indicators a program's verdict is drawn from, each weighed and each said to be
        present, absent or not assessed. Pure.
    .DESCRIPTION
        Three weigh heavily by default, because each on its own is what a program on a
        share that lost its connection looks like:
 
          NetworkPath the program, or a module the inventory lists for it, lives on
                            a network path
          NetworkException an incident shows an exception code or a .NET exception type
                            that says a page or a file could not be read: 0xc0000006,
                            SEHException, and the three ways an assembly fails to load
          Disruption an incident coincided with a network disruption: one in its
                            window, or a reconnect to the program's Server shortly after
                            it. See Get-ProgramCorrelation.
 
        Two weigh less, because they are true of machines on which nothing is wrong:
 
          FaultSiteSpread the crashes are spread over too many places to be one bug
          UsbAdapter an adapter that is up is on USB - a dock, a dongle. Its being
                            on USB is weighed, and not its power saving: that is on by
                            default nearly everywhere, and says so in the text when it is.
 
        Not assessed is not absent. NetworkPath is not assessed when nothing places the
        program on the network and nobody took the inventory, or when a path of it is on
        a drive that is neither mapped nor a disk of the machine; Disruption when nobody
        could look for disruptions, or not at every incident - a log not read, an incident
        from before the logs begin; UsbAdapter when the Network Check gathered no
        adapters, or gathered them before it knew which are on USB.
 
        Disruption is present when the correlation says what coincided weighs. For a
        program on the network one incident is enough; for one known to be local it takes
        a share of them. See Test-CorrelationWeight.
 
        Every weight is a Check Definition parameter, and so is what stands behind each
        indicator. A weight of "off" leaves an indicator out.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [AllowNull()][AllowEmptyCollection()]$Incident,
        [AllowNull()]$Network,
        [AllowNull()]$Correlation,
        [AllowNull()]$Adapters,
        [hashtable]$Parameters = @{}
    )

    $incidents = @($Incident | Where-Object { $null -ne $_ })

    # The program or one of its modules on a network path.
    $weight = ConvertTo-IndicatorWeight (Get-Parameter $Parameters 'IndicatorWeightNetworkPath' 'high') -Default 'high'
    $places = Get-DataCollection $Network 'Places'
    $drives = Get-DataCollection $Network 'UnknownDrives'
    $state  = 'Absent'
    $text   = ''
    $name   = Get-Text 'Value.Stability.Indicator.Name.NetworkPath'
    if ($places.Count) { $state = 'Present'; $text = Get-Text 'Value.Stability.Indicator.NetworkPath' ($places -join ', ') }
    elseif ($drives.Count) {
        # A drive letter that is neither mapped nor a disk of this machine: nothing says
        # the program is local, and it is not judged as if something did.
        $state = 'NotAssessed'
        $name  = Get-Text 'Value.Stability.Indicator.Name.NetworkPathDrive' $name ($drives -join ', ')
    }
    elseif (-not [bool](Get-DataProperty $Network 'Inventoried')) { $state = 'NotAssessed' }
    New-CrashIndicator -Key 'NetworkPath' -Weight $weight -State $state -Text $text -Name $name

    # An exception that is what a lost share looks like. An empty list in a Check
    # Definition is a list left out, not "none": see Get-ListParameter.
    $weight = ConvertTo-IndicatorWeight (Get-Parameter $Parameters 'IndicatorWeightNetworkException' 'high') -Default 'high'
    $codes  = @((Get-ListParameter $Parameters 'NetworkExceptionCodes' @('c0000006')) |
        ForEach-Object { ($_ -replace '^0x', '').ToLowerInvariant() } | Where-Object { $_ })
    $types  = (Get-ListParameter $Parameters 'NetworkExceptionTypes' @(
        'SEHException', 'FileLoadException', 'FileNotFoundException', 'BadImageFormatException'))
    $needed = Get-NumberParameter $Parameters 'NetworkExceptionIncidents' 1

    $showing    = 0
    $seenCodes  = @()
    $seenTypes  = @()
    foreach ($row in $incidents) {
        $shows = $false
        $code  = ("$(Get-DataProperty $row 'Exception')".Trim() -replace '^0x', '').ToLowerInvariant()
        if ($code -and @($codes) -contains $code) {
            $shows = $true
            if ($seenCodes -notcontains ('0x{0}' -f $code)) { $seenCodes += '0x{0}' -f $code }
        }
        foreach ($type in (Get-DataCollection $row 'ExceptionTypes')) {
            $short = @("$type" -split '[.+]')[-1]
            if ($short -and @($types) -contains $short) {
                $shows = $true
                if ($seenTypes -notcontains $short) { $seenTypes += $short }
            }
        }
        if ($shows) { $showing++ }
    }
    $state = 'Absent'
    $text  = ''
    if ($showing -gt 0 -and $showing -ge $needed) {
        $state = 'Present'
        $text  = Get-Text 'Value.Stability.Indicator.NetworkException' ((@($seenCodes) + @($seenTypes)) -join ', ')
    }
    New-CrashIndicator -Key 'NetworkException' -Weight $weight -State $state -Text $text

    # A network disruption in an incident's window.
    # Whether what coincided is enough is the correlation's to say: of a program known to
    # be local it takes more than one chance coincidence. See Test-CorrelationWeight.
    $weight     = ConvertTo-IndicatorWeight (Get-Parameter $Parameters 'IndicatorWeightDisruption' 'high') -Default 'high'
    $correlated = ConvertTo-Number (Get-DataProperty $Correlation 'Correlated')
    if ($null -eq $correlated) { $correlated = 0 }
    $total = ConvertTo-Number (Get-DataProperty $Correlation 'Total')
    if ($null -eq $total) { $total = 0 }

    $weighs = [bool](Get-DataProperty $Correlation 'Weighs')
    if (-not (Test-DataCarried -InputObject $Correlation -Name 'Weighs')) {
        $weighs = Test-CorrelationWeight -Correlated $correlated -Total $total `
            -KnownLocal ([bool](Get-DataProperty $Network 'KnownLocal')) -Parameters $Parameters
    }

    # Not assessed wherever an incident was not, or not wholly, looked at: one that lies
    # before the logs begin might have been the one that coincided.
    $open = [bool]($null -eq $Correlation -or (Get-DataProperty $Correlation 'State') -eq 'NotAssessed' -or
        [bool](Get-DataProperty $Correlation 'LogsUnread') -or
        (ConvertTo-Number (Get-DataProperty $Correlation 'Before')) -gt 0 -or
        (ConvertTo-Number (Get-DataProperty $Correlation 'Untimed')) -gt 0)
    # Of a program the network does not concern, a disruption nobody could look for is
    # not a gap in the evidence. See Test-ProgramNetworkConcern.
    if ([bool](Get-DataProperty $Correlation 'SetAside')) { $open = $false }

    $state = 'Absent'
    $text  = ''
    if ($weighs) {
        $state = 'Present'
        $text  = Get-Text 'Value.Stability.Indicator.Disruption' $correlated $total
    }
    elseif ($open) { $state = 'NotAssessed' }
    New-CrashIndicator -Key 'Disruption' -Weight $weight -State $state -Text $text `
        -Name (Get-Text 'Value.Stability.Indicator.Name.Disruption')

    # Fault sites spread too widely to be one bug.
    $weight = ConvertTo-IndicatorWeight (Get-Parameter $Parameters 'IndicatorWeightFaultSiteSpread' 'medium') -Default 'medium'
    $state  = 'Absent'
    $text   = ''
    if (Test-FaultSiteSpread -Incident $incidents -Parameters $Parameters) {
        $state = 'Present'
        $text  = Get-Text 'Value.Stability.Indicator.FaultSiteSpread' (@(Get-FaultSite -Incident $incidents).Count)
    }
    New-CrashIndicator -Key 'FaultSiteSpread' -Weight $weight -State $state -Text $text

    # An adapter on USB.
    $weight  = ConvertTo-IndicatorWeight (Get-Parameter $Parameters 'IndicatorWeightUsbAdapter' 'medium') -Default 'medium'
    $rows    = Get-DataCollection $Adapters 'Adapters'
    $onUsb   = @($rows | Where-Object { (Get-DataProperty $_ 'IsUsb') -eq $true })
    $unknown = @($rows | Where-Object { $null -eq (Get-DataProperty $_ 'IsUsb') })
    $state   = 'Absent'
    $text    = ''
    if ($onUsb.Count) {
        $state = 'Present'
        $names = @($onUsb | ForEach-Object {
            $name = "$(Get-DataProperty $_ 'Description')".Trim()
            if (-not $name) { $name = "$(Get-DataProperty $_ 'Name')".Trim() }
            $name
        } | Where-Object { $_ }) -join ', '

        if (@($onUsb | Where-Object { (Get-DataProperty $_ 'PowerSavingOn') -eq $true }).Count) {
            $text = Get-Text 'Value.Stability.Indicator.UsbAdapterSaving' $names
        }
        else { $text = Get-Text 'Value.Stability.Indicator.UsbAdapter' $names }

        if ((Get-DataProperty $Adapters 'UsbSelectiveSuspend') -eq $true) { $text = Get-Text 'Value.Stability.Indicator.UsbSuspend' $text }
    }
    elseif (-not [bool](Get-DataProperty $Adapters 'Gathered') -or $unknown.Count) { $state = 'NotAssessed' }
    New-CrashIndicator -Key 'UsbAdapter' -Weight $weight -State $state -Text $text `
        -Name (Get-Text 'Value.Stability.Indicator.Name.UsbAdapter')
}

function Get-VerdictOutcome {
    <#
    .SYNOPSIS
        What a number of heavy and of lighter indicators amounts to. Pure.
    .DESCRIPTION
        Cause enough heavy indicators to name the cause: two by default
        Suspicion one heavy indicator and at least one lighter one
        Lead one heavy indicator, and nothing beside it
        Environment no heavy indicator, and fault sites spread too widely to be one bug:
                     something around the program, and nothing says it is the network
        Application none of the above: the application's own fault
 
        Only the spread makes "Environment" of lighter indicators without a heavy one. An
        adapter on USB is a reason to suspect the network of a program that depends on
        it, and none to doubt a program that does not.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param(
        [double]$High, [double]$Medium, [bool]$Spread,
        [double]$FailHigh, [double]$WarnHigh, [double]$SuspicionMedium
    )

    if ($High -gt 0 -and $High -ge $FailHigh) { return 'Cause' }
    if ($High -gt 0 -and $High -ge $WarnHigh) {
        if ($Medium -gt 0 -and $Medium -ge $SuspicionMedium) { return 'Suspicion' }
        return 'Lead'
    }
    if ($Spread) { return 'Environment' }
    'Application'
}

function Get-MostFrequent {
    <#
    .SYNOPSIS
        The value that occurs most often, the first of them when several do. Empty when
        there is none. Pure.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowNull()][AllowEmptyCollection()][string[]]$Value)

    $counts = @{}
    $best   = ''
    foreach ($item in @($Value | Where-Object { $_ })) {
        if (-not $counts.ContainsKey($item)) { $counts[$item] = 0 }
        $counts[$item]++
        # Only a value that overtakes replaces the one found first.
        if (-not $best -or $counts[$item] -gt $counts[$best]) { $best = $item }
    }
    $best
}

function Get-ProgramVendorText {
    <#
    .SYNOPSIS
        What the vendor of a program needs to find its fault: the program and its version,
        the module that faulted most often with its version, and the method. Pure.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([AllowEmptyString()][string]$App = '', [AllowNull()][AllowEmptyCollection()]$Incident)

    $incidents = @($Incident | Where-Object { $null -ne $_ })

    # The version of the newest incident that names one: the incidents come newest first.
    $version = @($incidents | ForEach-Object { "$(Get-DataProperty $_ 'Version')".Trim() } | Where-Object { $_ } | Select-Object -First 1)
    $parts   = @(('{0} {1}' -f $App, "$version").Trim())

    $module = Get-MostFrequent -Value @($incidents | ForEach-Object { "$(Get-DataProperty $_ 'Module')".Trim() })
    if ($module) {
        $moduleVersion = @($incidents | Where-Object { "$(Get-DataProperty $_ 'Module')".Trim() -eq $module } |
            ForEach-Object { "$(Get-DataProperty $_ 'ModuleVer')".Trim() } | Where-Object { $_ } | Select-Object -First 1)
        $parts += (Get-Text 'Value.Stability.Verdict.VendorModule' $module "$moduleVersion").Trim()
    }

    $method = Get-MostFrequent -Value @($incidents | ForEach-Object { "$(Get-DataProperty $_ 'Method')".Trim() })
    if ($method) { $parts += Get-Text 'Value.Stability.Verdict.VendorMethod' $method }

    @($parts | Where-Object { $_ }) -join ', '
}

function Get-ProgramCrashVerdict {
    <#
    .SYNOPSIS
        The verdict on one program's crashes, from a chain of evidence. Pure.
    .DESCRIPTION
        Returns Severity, Sentence, Present and NotAssessed, and beside them Outcome,
        Incomplete, Hint and Meaning. The Sentence is the verdict and part of the Value;
        Meaning is what follows from it and the Sentence does not say.
 
        Two heavy indicators are a FAIL whose Sentence names the cause, and the share when
        there is one. One heavy indicator is a WARN: a suspicion when a lighter one stands
        beside it, and otherwise a single lead that is said to be one. None is the
        application's own fault, with what its vendor needs - or, when the crashes are
        spread over too many places to be one bug, something around the program. For
        those two Severity is empty: the Finding stays graded on the number of crashes.
 
        The three numbers are Check Definition parameters, as every weight is.
 
        An indicator that could not be looked at is listed in NotAssessed and counted as
        nothing - neither present nor absent. When the ones that are missing could have
        made another verdict of it, the Sentence says the evidence is incomplete: a
        "this is the application's fault" on a machine whose network nobody could look at
        is a guess, and must not read like a finding.
 
        Present and NotAssessed list the indicators with their weight, heaviest first, so
        that second level can check the reasoning.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [AllowEmptyString()][string]$App = '',
        [AllowNull()][AllowEmptyCollection()]$Incident,
        [AllowNull()]$Network,
        [AllowNull()]$Correlation,
        [AllowNull()]$Adapters,
        [hashtable]$Parameters = @{},
        [AllowNull()]$Situation
    )

    $failHigh        = Get-NumberParameter $Parameters 'VerdictFailHighIndicators' 2
    $warnHigh        = Get-NumberParameter $Parameters 'VerdictWarnHighIndicators' 1
    $suspicionMedium = Get-NumberParameter $Parameters 'VerdictSuspicionMediumIndicators' 1

    $indicators = @(Get-ProgramCrashIndicator -Incident $Incident -Network $Network -Correlation $Correlation `
        -Adapters $Adapters -Parameters $Parameters)

    # Heaviest first, and within one weight in the order they were looked at.
    $present = @()
    $open    = @()
    $count   = @{ high = 0; medium = 0; openHigh = 0; openMedium = 0 }
    $spread  = $false
    $usb     = $false
    foreach ($weight in 'high', 'medium', 'low') {
        $label = switch ($weight) {
            'high'   { Get-Text 'Value.Stability.Weight.High' }
            'medium' { Get-Text 'Value.Stability.Weight.Medium' }
            default  { Get-Text 'Value.Stability.Weight.Low' }
        }
        foreach ($indicator in @($indicators | Where-Object { $_.Weight -eq $weight })) {
            if ($indicator.State -eq 'Present') {
                $present += Get-Text 'Value.Stability.Indicator.Weighted' $indicator.Text $label
                if ($weight -eq 'high')   { $count.high++ }
                if ($weight -eq 'medium') { $count.medium++ }
                if ($indicator.Key -eq 'FaultSiteSpread') { $spread = $true }
                if ($indicator.Key -eq 'UsbAdapter')      { $usb = $true }
            }
            elseif ($indicator.State -eq 'NotAssessed') {
                $open += Get-Text 'Value.Stability.Indicator.Weighted' $indicator.Name $label
                if ($weight -eq 'high')   { $count.openHigh++ }
                if ($weight -eq 'medium') { $count.openMedium++ }
            }
        }
    }

    $outcome = Get-VerdictOutcome -High $count.high -Medium $count.medium -Spread $spread `
        -FailHigh $failHigh -WarnHigh $warnHigh -SuspicionMedium $suspicionMedium

    # What it would have been had everything nobody could look at been there.
    $atMost = Get-VerdictOutcome -High ($count.high + $count.openHigh) -Medium ($count.medium + $count.openMedium) -Spread $spread `
        -FailHigh $failHigh -WarnHigh $warnHigh -SuspicionMedium $suspicionMedium
    $incomplete = $atMost -ne $outcome

    $severity = ''
    $hints    = @()
    $means    = @()
    switch ($outcome) {
        'Cause' {
            $severity = 'FAIL'
            $places   = Get-DataCollection $Network 'Places'
            if ($places.Count) { $sentence = Get-Text 'Value.Stability.Verdict.CauseShare' ($places -join ', ') }
            else               { $sentence = Get-Text 'Value.Stability.Verdict.Cause' }
            $means += Get-Text 'Meaning.Stability.VerdictNetworkCause'
            $hints += Get-StabilityNetworkHint -Kind Cause -Situation $Situation
        }
        'Suspicion' {
            $severity = 'WARN'
            $sentence = Get-Text 'Value.Stability.Verdict.Suspicion'
            $hints   += Get-StabilityNetworkHint -Kind Suspected -Situation $Situation
        }
        'Lead' {
            $severity = 'WARN'
            $sentence = Get-Text 'Value.Stability.Verdict.Lead'
            $hints   += Get-StabilityNetworkHint -Kind Suspected -Situation $Situation
        }
        'Environment' { $sentence = Get-Text 'Value.Stability.Verdict.Environment' }
        default       { $sentence = Get-Text 'Value.Stability.Verdict.Application' (Get-ProgramVendorText -App $App -Incident $Incident) }
    }
    if ($incomplete) { $sentence = Get-Text 'Value.Stability.Verdict.Incomplete' $sentence }

    # The dock is worth a look only where something points at the network at all.
    if ($usb -and $count.high -gt 0) { $hints += Get-StabilityUsbAdapterHint -Situation $Situation }

    [pscustomobject]@{
        Severity    = $severity
        Sentence    = $sentence
        Present     = [string[]]$present
        NotAssessed = [string[]]$open
        Outcome     = $outcome
        Incomplete  = $incomplete
        Hint        = [string[]]$hints
        Meaning     = [string[]]$means
    }
}

function Get-ProgramVerdictLine {
    <#
    .SYNOPSIS
        The lines a verdict adds to a program's Finding: the indicators that were present,
        the ones that could not be assessed, and the sentence. Pure.
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param([Parameter(Mandatory)]$Verdict)

    if (@($Verdict.Present).Count) { Get-Text 'Value.Stability.Verdict.Present' (@($Verdict.Present) -join '; ') }
    else                           { Get-Text 'Value.Stability.Verdict.PresentNone' }

    if (@($Verdict.NotAssessed).Count) { Get-Text 'Value.Stability.Verdict.NotAssessed' (@($Verdict.NotAssessed) -join '; ') }

    $Verdict.Sentence
}

function Test-ProgramNetworkConcern {
    <#
    .SYNOPSIS
        Whether what the network did around a program's crashes is worth a word in its
        Finding. Pure.
    .DESCRIPTION
        The question "did it crash because its Server dropped?" is one about a program
        that has a Server. Asked of every program, a monitoring agent installed on C:
        that throws NullReferenceException was told that the SMB event logs do not reach
        back far enough, to enlarge them, and that the evidence against it is incomplete
        - advice about a network it does not use, ahead of the advice that applies.
 
        So the network concerns a program when something says so:
 
          - it is not known to be off the network: it or a module of it is on a network
            path, a path of it is on a drive nobody could place, or nobody took the
            inventory that would have listed what it loads (the NetworkPath indicator
            is present, or not assessed)
          - an incident shows an exception that is what a lost share looks like
          - an incident did coincide with a disruption: what was found is always said
 
        Otherwise the correlation is set aside: no line, no Hint, and no open indicator
        that would call the verdict incomplete. The incidents' own table still shows a
        disruption beside each incident it was found for.
    #>
    [CmdletBinding()]
    [OutputType([bool])]
    param(
        [AllowNull()][AllowEmptyCollection()]$Incident,
        [AllowNull()]$Network,
        [AllowNull()]$Correlation,
        [hashtable]$Parameters = @{}
    )

    if ("$(Get-DataProperty $Correlation 'State')" -eq 'Correlated') { return $true }
    $indicators = @(Get-ProgramCrashIndicator -Incident $Incident -Network $Network -Correlation $null -Adapters $null -Parameters $Parameters)
    # An indicator switched off in the Check Definition is not there, and says nothing.
    if (@($indicators | Where-Object { $_.Key -eq 'NetworkPath' -and $_.State -ne 'Absent' }).Count) { return $true }
    [bool]@($indicators | Where-Object { $_.Key -eq 'NetworkException' -and $_.State -eq 'Present' }).Count
}

function Get-ProgramCrashEvidence {
    <#
    .SYNOPSIS
        Everything the Judge knows about one program's crashes beyond the crashes: where
        it runs from, what the network did around them, and the verdict. Pure.
    .DESCRIPTION
        Read from this Kind's own data, into which the Gatherer copied what the network
        Checks before it gathered. Data from before it did carries none of that, and then
        reads as it should: nobody looked.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [AllowEmptyString()][string]$App = '',
        [AllowNull()][AllowEmptyCollection()]$Incident,
        [AllowNull()]$Data,
        [hashtable]$Parameters = @{},
        [AllowNull()]$Situation
    )

    $network     = Get-ProgramNetwork -App $App -Incident $Incident -NetworkDependency (Get-DataProperty $Data 'NetworkDependency')
    $correlation = Get-ProgramCorrelation -Incident $Incident -Network $network `
        -NetworkDisruption (Get-DataProperty $Data 'NetworkDisruption') -Parameters $Parameters
    if ($null -ne $correlation -and
        -not (Test-ProgramNetworkConcern -Incident $Incident -Network $network -Correlation $correlation -Parameters $Parameters)) {
        $correlation | Add-Member -NotePropertyName 'SetAside' -NotePropertyValue $true -Force
    }
    $verdict     = Get-ProgramCrashVerdict -App $App -Incident $Incident -Network $network -Correlation $correlation `
        -Adapters (Get-DataProperty $Data 'Adapters') -Parameters $Parameters -Situation $Situation

    [pscustomobject]@{
        Network     = $network
        Correlation = $correlation
        Verdict     = $verdict
    }
}

function Get-ProgramCrashLine {
    <#
    .SYNOPSIS
        What is known about one program's crashes, a line for each thing known. Pure.
    .DESCRIPTION
        How often and when last; where the program runs from, and that this is on the
        network when it is; the exception codes and types, counted; and over how many
        places the crashes are spread. A line is left out when nothing is known for it.
 
        A list, and not one sentence, because more is known than the crashes say: whether
        they coincided with a network disruption, and what all of it adds up to, are lines
        of their own that New-ProgramCrashFinding is handed. See Get-ProgramCrashEvidence.
        The lines are written to the pipeline one by one: collect them with @( ).
    #>
    [CmdletBinding()]
    [OutputType([string])]
    param(
        [AllowNull()][AllowEmptyCollection()]$Incident,
        $Days = $script:StabilityDefaultDays,
        [hashtable]$Parameters = @{},

        # What Get-ProgramNetwork returned for the program, where a Check gathered what it
        # is built from. Without it a path is named as the event wrote it.
        [AllowNull()]$Network
    )

    $incidents = @($Incident | Where-Object { $null -ne $_ })
    if ($null -eq $Network) { $Network = Get-ProgramNetwork -Incident $incidents -NetworkDependency $null }
    $lines = @()

    $count = Get-Text 'Value.Stability.Program.Incidents' $incidents.Count $Days
    if ($incidents.Count -eq 1) { $count = Get-Text 'Value.Stability.Program.IncidentOne' $Days }
    $last = @($incidents | ForEach-Object { ConvertTo-DataTime (Get-DataProperty $_ 'Time') } |
        Where-Object { $null -ne $_ } | Sort-Object -Descending)
    if ($last.Count) { $count = '{0}, {1}' -f $count, (Get-Text 'Value.Stability.Program.Last' $last[0]) }
    $lines += $count

    # A program can have been started from more than one place in a month. A mapped drive
    # is named as the share it stands for, because the share is what names the Server and
    # K: is a letter on one desk.
    foreach ($location in (Get-DataCollection $Network 'Locations')) {
        if ($location.Share -and $location.Drive) { $lines += Get-Text 'Value.Stability.Program.PathShare' $location.Share $location.Drive }
        elseif ($location.Remote)                 { $lines += Get-Text 'Value.Stability.Program.PathRemote' $location.Path }
        elseif ("$(Get-DataProperty $location 'Placement')" -eq 'Unknown') {
            # Neither mapped nor a disk of this machine: not said to be local.
            $lines += Get-Text 'Value.Stability.Program.PathUnknownDrive' $location.Path $location.Drive
        }
        else                                      { $lines += Get-Text 'Value.Stability.Program.Path' $location.Path }
    }

    $codes = Get-CountedText -Name @($incidents | ForEach-Object { Get-ExceptionCodeText (Get-DataProperty $_ 'Exception') })
    # By incident: a type the runtime named twice for one crash is one crash.
    $types = Get-CountedText -Name @($incidents | ForEach-Object {
        (Get-DataCollection $_ 'ExceptionTypes') | ForEach-Object { @("$_" -split '[.+]')[-1] } | Select-Object -Unique
    })
    $exceptions = @()
    if ($codes) { $exceptions += $codes }
    if ($types) { $exceptions += Get-Text 'Value.Stability.Program.DotNetTypes' $types }
    if ($exceptions.Count) { $lines += Get-Text 'Value.Stability.Program.Exceptions' ($exceptions -join '; ') }

    $sites = @(Get-FaultSite -Incident $incidents).Count
    if (Test-FaultSiteSpread -Incident $incidents -Parameters $Parameters) {
        $lines += Get-Text 'Value.Stability.Program.SitesOutside' $sites
    }
    elseif ($sites) { $lines += Get-Text 'Value.Stability.Program.Sites' $sites }

    $lines
}

# What stands between the lines of a Finding's Value. The Report shows a Value in one table
# cell and findings.csv in one field, so a line break would be lost in the one and awkward
# in the other.
$script:StabilityLineSeparator = ' | '

function New-ProgramCrashFinding {
    <#
    .SYNOPSIS
        The Finding for one program that crashes repeatedly. Pure.
    .DESCRIPTION
        Graded on the number of incidents: WARN from the third, because a program that
        gets this Finding has crashed repeatedly, and FAIL above twenty, which is where the
        count over all applications failed. Never OK, because it is a crash; below the
        WARN threshold - a Check Definition can ask for a Finding from the first crash -
        it is INFO.
 
        -Line, -Hint, -Meaning and -Severity are for what is known beyond the crashes -
        what the network did around them, and the verdict: those lines follow the ones
        built here, those Hints and Meanings lead, and that Severity counts when it is
        worse. Nothing makes the Finding better than the number of crashes makes it.
    #>
    [CmdletBinding()]
    [OutputType([psobject])]
    param(
        [AllowNull()][AllowEmptyCollection()]$Incident,
        [AllowEmptyString()][string]$App = '',
        $Days = $script:StabilityDefaultDays,
        [hashtable]$Parameters = @{},
        [AllowNull()][AllowEmptyCollection()][string[]]$Line = @(),
        [AllowNull()][AllowEmptyCollection()][string[]]$Hint = @(),
        [AllowNull()][AllowEmptyCollection()][string[]]$Meaning = @(),
        [AllowEmptyString()][string]$Severity = '',

        # What Get-ProgramNetwork returned for the program. Built from the incidents alone
        # when it is left out, and then knows no drive mapping.
        [AllowNull()]$Network,

        # What is known of this program's crashes from looking at all programs together.
        [AllowNull()][AllowEmptyCollection()][string[]]$Signal,

        # What a Technician should read with it: the timeline, where the crashes began
        # after a Change.
        [AllowNull()][AllowEmptyCollection()]$Reference,

        # The application whose own Check judges this program in this Run. The crashes
        # are then counted here and no more: what they mean and what to do is said there.
        [AllowEmptyString()][string]$CoveredBy = ''
    )

    $warn = Get-NumberParameter $Parameters 'ProgramIncidentWarnAbove' 2
    $fail = Get-NumberParameter $Parameters 'ProgramIncidentFailAbove' 20

    $incidents = @($Incident | Where-Object { $null -ne $_ })
    if (-not $App -and $incidents.Count) { $App = "$(Get-DataProperty $incidents[0] 'App')".Trim() }
    if ($null -eq $Network) { $Network = Get-ProgramNetwork -App $App -Incident $incidents -NetworkDependency $null }

    if ($CoveredBy) {
        return New-Finding -Category Stability -Check (Get-Text 'Check.Stability.ProgramCrashes' $App) -Severity INFO `
            -Value ((@(Get-ProgramCrashLine -Incident $incidents -Days $Days -Parameters $Parameters -Network $Network)[0], (Get-Text 'Value.Stability.ProgramCovered' $CoveredBy)) -join $script:StabilityLineSeparator) `
            -Meaning (Get-Text 'Meaning.Stability.ProgramCovered' $CoveredBy) -Hint (Get-Text 'Hint.Stability.ProgramCovered' $CoveredBy) `
            -Signal @($Signal | Where-Object { $_ }) -Subject $App
    }

    $graded = Get-Severity $incidents.Count $warn $fail
    if ($graded -eq 'OK') { $graded = 'INFO' }

    $lines = @(Get-ProgramCrashLine -Incident $incidents -Days $Days -Parameters $Parameters -Network $Network) +
             @($Line | Where-Object { $_ })

    # What to do, the most specific first: what another Check added, that the program is
    # on the network, that the crashes are spread, where the exception types point - and
    # last where the evidence is, which is all there is to say when nothing else is known.
    #
    # What it means beside it, in the same order, where the lines do not say it: that
    # the crashes are spread, and what that speaks for, is a line of its own.
    $hints = @($Hint | Where-Object { $_ })
    $means = @($Meaning | Where-Object { $_ })
    if ([bool](Get-DataProperty $Network 'OnNetwork')) {
        $means += Get-Text 'Meaning.Stability.ProgramOnNetwork'
        $hints += Get-Text 'Hint.Stability.ProgramOnNetwork'
    }
    $drives = Get-DataCollection $Network 'UnknownDrives'
    if ($drives.Count) { $hints += Get-Text 'Hint.Stability.ProgramDriveUnknown' ($drives -join ', ') }
    if (Test-FaultSiteSpread -Incident $incidents -Parameters $Parameters) {
        $hints += Get-Text 'Hint.Stability.ProgramFaultSitesSpread'
    }
    $direction = Get-ExceptionTypeHint -RuntimeFault $incidents
    if ($direction) { $hints += $direction }
    $typeMeans = Get-ExceptionTypeMeaning -RuntimeFault $incidents
    if ($typeMeans) { $means += $typeMeans }
    $hints += Get-Text 'Hint.Stability.ProgramCrashes'

    # A page that could not be read from a share is the network; from a local drive it
    # is the drive.
    $signal = @($Signal | Where-Object { $_ })
    $unread = @($incidents | Where-Object { "$(Get-DataProperty $_ 'Exception')" -match '(?i)c0000006' })
    # Known to be local, and not merely not known to be on the network: a drive nobody
    # could place is no evidence about the drives of this machine.
    if ($unread.Count -and [bool](Get-DataProperty $Network 'KnownLocal')) { $signal += 'program-not-paged-in' }

    New-Finding -Category Stability -Check (Get-Text 'Check.Stability.ProgramCrashes' $App) `
        -Severity (Get-WorstSeverity $graded $Severity) `
        -Value ($lines -join $script:StabilityLineSeparator) `
        -Meaning ($means -join $script:StabilityLineSeparator) `
        -Hint ($hints -join $script:StabilityLineSeparator) `
        -Signal $signal -Subject $App -Reference @($Reference | Where-Object { $null -ne $_ })
}

function New-ScatteredCrashFinding {
    <#
    .SYNOPSIS
        One Finding for the programs that crashed, but not often enough for one of their
        own. Nothing when there is no such program. Pure.
    .DESCRIPTION
        Graded as the count over all applications was, on the incidents left over: many
        programs each crashing a little is still a machine on which programs crash. Below
        the WARN threshold it is INFO, not OK, because it reports crashes.
    #>
    [CmdletBinding()]
    param([AllowNull()][AllowEmptyCollection()]$Program, [hashtable]$Parameters = @{}, [AllowNull()][AllowEmptyCollection()][string[]]$Signal)

    $programs = @($Program | Where-Object { $null -ne $_ })
    if (-not $programs.Count) { return }

    $warn = Get-NumberParameter $Parameters 'ApplicationCrashWarnAbove' 5
    $fail = Get-NumberParameter $Parameters 'ApplicationCrashFailAbove' 20

    $names = @(foreach ($program in $programs) {
        $name = $program.App
        if (-not $name) { $name = Get-Text 'Value.Stability.UnknownProgram' }
        foreach ($incident in @($program.Incidents)) { $name }
    })

    $severity = Get-Severity $names.Count $warn $fail
    if ($severity -eq 'OK') { $severity = 'INFO' }

    New-Finding -Category Stability -Check (Get-Text 'Check.Stability.ScatteredCrashes') -Severity $severity `
        -Value (Get-Text 'Value.Stability.Scattered' $names.Count (Get-CountedText -Name $names -First 5)) `
        -Hint (Get-Text 'Hint.Stability.CrashingApplicationsSeeTheSection') -Signal @($Signal | Where-Object { $_ })
}