Private/Kinds/AppLog.ps1
|
# The AppLog Kind: what an application wrote into its own log. # # Windows' event log knows that a program crashed. It does not know that the program has # been failing to load a screen forty times a day, because the program caught that, wrote # a line into its own log file, and carried on. That file is where the vendor's support # looks first, and a Technician rarely knows it exists. A Check Definition names it and # says how its lines are built; this Kind counts the errors in it and groups them by where # they came from. # # Built for, and observed on 2026-10-09 with, a .NET application logging through NLog to a # semicolon-separated file with a header line - Number;Level;Date;Logger;Message; # Exception;Location - one file per day, the day in the file's name and only the time of # day in the line. One day's file held 629 lines, 29 of them errors, 28 from one place. # # Two ways a log is read: # # - As delimited text with a header (the default): the Check Definition names the # delimiter and which columns hold the level, the time, the source, the message and # the exception. A quoted field may run over several lines, as a stack trace does. # - Line by line against LinePattern, a regular expression with the named groups # time, level, source and message, for a log that is plain text. A line that does not # match belongs to the entry before it and is not read. # # What is kept. Counts for every level, and of the entries at an error level the time, # the source, the first line of the exception and the message cut to a length - enough to # group by and to show one example. A log holds whatever the application put there, a # Customer's data included; nothing of an entry that is not an error leaves the file. # How much of a message and of an exception's first line is kept, and how many error # entries: what is shown is a grouping and an example, not the log. $script:AppLogMessageLength = 300 $script:AppLogExceptionLength = 200 $script:AppLogMaximumErrors = 2000 function ConvertTo-AppLogTime { <# .SYNOPSIS When an entry was written, from what the line says and the day of its file. Pure. .DESCRIPTION A time of day alone ("00:02:42.2502") is put on the file's day. A whole timestamp is read as ISO first and then as the machine writes dates. $null for what is neither: an entry without a time is still counted. #> [CmdletBinding()] [OutputType([datetime])] param([AllowNull()][AllowEmptyString()][string]$Value, [AllowNull()]$FileDate) $text = "$Value".Trim() if (-not $text) { return $null } if ($text -match '^\d{1,2}:\d{2}(:\d{2}([.,]\d+)?)?$') { if ($null -eq $FileDate) { return $null } $span = [timespan]::Zero if ([timespan]::TryParse(($text -replace ',', '.'), [Globalization.CultureInfo]::InvariantCulture, [ref]$span)) { return ([datetime]$FileDate).Date + $span } return $null } $when = [datetime]::MinValue $styles = [Globalization.DateTimeStyles]::AssumeLocal if ([datetime]::TryParse(($text -replace ',(\d+)$', '.$1'), [Globalization.CultureInfo]::InvariantCulture, $styles, [ref]$when)) { return $when } if ([datetime]::TryParse($text, [Globalization.CultureInfo]::CurrentCulture, $styles, [ref]$when)) { return $when } $null } function Get-AppLogFileDate { <# .SYNOPSIS The day a log file is for: from its name when the name carries one, else the day it was last written. Pure. #> [CmdletBinding()] [OutputType([datetime])] param([AllowNull()][string]$Name, [AllowNull()]$LastWriteTime, [AllowNull()][AllowEmptyString()][string]$Pattern) if ("$Pattern".Trim()) { try { if ("$Name" -match $Pattern -and $Matches.Count -gt 1) { $day = [datetime]::MinValue if ([datetime]::TryParse($Matches[1], [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::None, [ref]$day)) { return $day.Date } } } catch { } } if ($null -ne $LastWriteTime) { return ([datetime]$LastWriteTime).Date } $null } function ConvertTo-AppLogCut { <# .SYNOPSIS Text on one line, cut to a length between words. Pure. #> [CmdletBinding()] [OutputType([string])] param([AllowNull()][AllowEmptyString()][string]$Text, [int]$Length) $line = ("$Text" -replace '\s+', ' ').Trim() if ($line.Length -le $Length) { return $line } $cut = $line.LastIndexOf(' ', $Length) if ($cut -lt 1) { $cut = $Length } $line.Substring(0, $cut) + ' [...]' } function ConvertFrom-AppLogText { <# .SYNOPSIS One log file's text as entries: level, time, source, message, exception. Pure. .DESCRIPTION -Layout carries what the Check Definition says about the file: Delimiter and the five column names, or LinePattern. Every entry is returned with its level; the caller keeps the errors. An exception is its first line - the type and what it said - without the stack trace. #> [CmdletBinding()] [OutputType([psobject])] param([AllowNull()][AllowEmptyString()][string]$Text, [hashtable]$Layout = @{}, [AllowNull()]$FileDate) if (-not "$Text".Trim()) { return } $pattern = "$($Layout.LinePattern)".Trim() if ($pattern) { foreach ($line in ("$Text" -split "`r?`n")) { $match = $null try { $match = [regex]::Match($line, $pattern) } catch { return } if (-not $match.Success) { continue } [pscustomobject]@{ Level = $match.Groups['level'].Value.Trim().ToUpperInvariant() Time = ConvertTo-AppLogTime -Value $match.Groups['time'].Value -FileDate $FileDate Source = $match.Groups['source'].Value.Trim() Message = $match.Groups['message'].Value Exception = '' } } return } $delimiter = "$($Layout.Delimiter)" if (-not $delimiter) { $delimiter = ';' } $rows = @() try { $rows = @(ConvertFrom-Csv -InputObject $Text -Delimiter $delimiter -ErrorAction Stop) } catch { return } foreach ($row in $rows) { $level = "$(Get-DataProperty $row $Layout.LevelColumn)".Trim().ToUpperInvariant() if (-not $level) { continue } [pscustomobject]@{ Level = $level Time = ConvertTo-AppLogTime -Value "$(Get-DataProperty $row $Layout.TimeColumn)" -FileDate $FileDate Source = "$(Get-DataProperty $row $Layout.SourceColumn)".Trim() Message = "$(Get-DataProperty $row $Layout.MessageColumn)" Exception = @("$(Get-DataProperty $row $Layout.ExceptionColumn)" -split "`r?`n")[0] } } } function Get-AppLogLayout { <# .SYNOPSIS What a Check Definition says about how the log is built, with the defaults. Pure. #> [CmdletBinding()] [OutputType([hashtable])] param([hashtable]$Parameters = @{}) @{ LinePattern = "$(Get-Parameter $Parameters 'LinePattern' '')" Delimiter = "$(Get-Parameter $Parameters 'Delimiter' ';')" LevelColumn = "$(Get-Parameter $Parameters 'LevelColumn' 'Level')" TimeColumn = "$(Get-Parameter $Parameters 'TimeColumn' 'Date')" SourceColumn = "$(Get-Parameter $Parameters 'SourceColumn' 'Logger')" MessageColumn = "$(Get-Parameter $Parameters 'MessageColumn' 'Message')" ExceptionColumn = "$(Get-Parameter $Parameters 'ExceptionColumn' 'Exception')" } } function Read-AppLogFile { <# .SYNOPSIS A log file's text, read while the application has it open for writing. #> [CmdletBinding()] [OutputType([string])] param([Parameter(Mandatory)][string]$Path) $stream = [IO.File]::Open($Path, [IO.FileMode]::Open, [IO.FileAccess]::Read, ([IO.FileShare]::ReadWrite -bor [IO.FileShare]::Delete)) try { $reader = New-Object System.IO.StreamReader($stream, [Text.Encoding]::UTF8, $true) try { $reader.ReadToEnd() } finally { $reader.Dispose() } } finally { $stream.Dispose() } } function Get-AppLogData { [CmdletBinding()] [OutputType([psobject])] param([hashtable]$Parameters = @{}) $configured = "$(Get-Parameter $Parameters 'Path' '')".Trim() $days = ConvertTo-Number (Get-Parameter $Parameters 'Days' 7) if ($null -eq $days -or $days -le 0) { $days = 7 } $maximumMB = ConvertTo-Number (Get-Parameter $Parameters 'MaxFileMB' 20) if ($null -eq $maximumMB -or $maximumMB -le 0) { $maximumMB = 20 } $errorLevels = @(@(Get-Parameter $Parameters 'ErrorLevels' @('ERROR', 'FATAL', 'CRITICAL')) | ForEach-Object { "$_".Trim().ToUpperInvariant() } | Where-Object { $_ }) $datePattern = "$(Get-Parameter $Parameters 'FileDatePattern' '(\d{4}-\d{2}-\d{2})')" $layout = Get-AppLogLayout -Parameters $Parameters $gatheredAt = Get-Date $since = $gatheredAt.Date.AddDays(-($days - 1)) # The last part of the path may hold wildcards: Log*.csv is one log in many files. $path = '' $found = @() if ($configured) { $path = [Environment]::ExpandEnvironmentVariables($configured) $folder = Split-Path -Path $path -Parent $leaf = Split-Path -Path $path -Leaf if ($folder -and (Test-Path -LiteralPath $folder -PathType Container)) { $found = @(Get-ChildItem -LiteralPath $folder -Filter $leaf -File -Force -ErrorAction SilentlyContinue) } } $levels = @{} $files = New-Object 'System.Collections.Generic.List[psobject]' $errors = New-Object 'System.Collections.Generic.List[psobject]' $errorCount = 0 foreach ($file in $found) { $fileDate = Get-AppLogFileDate -Name $file.Name -LastWriteTime $file.LastWriteTime -Pattern $datePattern # A file is in the period by the day it is for, and by when it was last written. if ($file.LastWriteTime -lt $since -and ($null -eq $fileDate -or $fileDate -lt $since)) { continue } $record = [ordered]@{ Name = $file.Name; Length = $file.Length; LastWriteTime = $file.LastWriteTime; Entries = $null; Errors = $null; Skipped = $null } if ($file.Length -gt $maximumMB * 1MB) { $record.Skipped = 'TooLarge' } else { try { $entries = @(ConvertFrom-AppLogText -Text (Read-AppLogFile -Path $file.FullName) -Layout $layout -FileDate $fileDate) $record.Entries = $entries.Count $record.Errors = 0 foreach ($entry in $entries) { if ($null -ne $entry.Time -and $entry.Time -lt $since) { continue } $levels[$entry.Level] = 1 + [int]$levels[$entry.Level] if ($errorLevels -notcontains $entry.Level) { continue } $errorCount++ $record.Errors++ if ($errors.Count -ge $script:AppLogMaximumErrors) { continue } $errors.Add([pscustomobject]@{ Time = $entry.Time Level = $entry.Level Source = $entry.Source Exception = ConvertTo-AppLogCut -Text $entry.Exception -Length $script:AppLogExceptionLength Message = ConvertTo-AppLogCut -Text $entry.Message -Length $script:AppLogMessageLength File = $file.Name }) } } catch { $record.Skipped = $_.Exception.GetBaseException().Message } } $files.Add([pscustomobject]$record) } [pscustomobject]@{ PSTypeName = 'Gutcheck.Data.AppLog' App = "$(Get-Parameter $Parameters 'App' '')" ConfiguredPath = $configured Path = $path Days = $days GatheredAt = $gatheredAt Files = @($files) Levels = @($levels.Keys | Sort-Object | ForEach-Object { [pscustomobject]@{ Level = $_; Count = $levels[$_] } }) ErrorCount = $errorCount Errors = @($errors) } } function Get-AppLogErrorGroup { <# .SYNOPSIS The errors by where they came from and what they were: count, last time, one example. Most frequent first. Pure. .DESCRIPTION "Where" is the last part of the source - a logger is named for its class, and the namespaces before it are the same on every line - and "what" is the exception's type, the part before the first colon. #> [CmdletBinding()] [OutputType([psobject])] param([AllowNull()][AllowEmptyCollection()]$ErrorEntry) $order = New-Object 'System.Collections.Generic.List[string]' $groups = @{} foreach ($entry in @($ErrorEntry | Where-Object { $_ })) { $source = @("$(Get-DataProperty $entry 'Source')" -split '\.')[-1].Trim() $type = @("$(Get-DataProperty $entry 'Exception')" -split ':', 2)[0].Trim() $type = @($type -split '\.')[-1] if ($type -notmatch '^[A-Za-z_][A-Za-z0-9_`]*$') { $type = '' } $key = '{0}|{1}' -f $source, $type if (-not $groups.ContainsKey($key)) { $order.Add($key) $groups[$key] = [pscustomobject]@{ Count = 0; Source = $source; Exception = $type; Last = $null; Example = "$(Get-DataProperty $entry 'Message')" } } $group = $groups[$key] $group.Count++ $time = Get-DataProperty $entry 'Time' if ($null -ne $time) { try { if ($null -eq $group.Last -or [datetime]$time -gt [datetime]$group.Last) { $group.Last = [datetime]$time } } catch { } } } # Sort-Object is not stable on Windows PowerShell 5.1: the order of first appearance # breaks a tie. $index = @{} for ($i = 0; $i -lt $order.Count; $i++) { $index[$order[$i]] = $i } $order | Sort-Object @{ Expression = { -$groups[$_].Count } }, @{ Expression = { $index[$_] } } | ForEach-Object { $groups[$_] } } function ConvertTo-AppLogFinding { [CmdletBinding()] [OutputType([psobject])] param( [AllowNull()]$Data, [hashtable]$Parameters = @{} ) $warn = Get-Parameter $Parameters 'ErrorWarnAbove' 20 $fail = Get-Parameter $Parameters 'ErrorFailAbove' ([double]::MaxValue) $check = Get-Text 'Check.AppLog.Errors' $path = "$(Get-DataProperty $Data 'Path')" if (-not "$(Get-DataProperty $Data 'ConfiguredPath')") { return New-Finding -Category Apps -Check $check -Severity INFO ` -Value (Get-Text 'Value.AppLog.NotConfigured') -Hint (Get-Text 'Hint.AppLog.NotConfigured') } $days = Get-DataProperty $Data 'Days' $files = Get-DataCollection $Data 'Files' if (-not $files.Count) { return New-Finding -Category Apps -Check $check -Severity INFO ` -Value ((Get-Text 'Value.AppLog.NoFiles') -f $path, $days) -Hint (Get-Text 'Hint.AppLog.NoFiles') } $skipped = @($files | Where-Object { Get-DataProperty $_ 'Skipped' }) $read = @($files | Where-Object { -not (Get-DataProperty $_ 'Skipped') }) if (-not $read.Count) { return New-Finding -Category Apps -Check $check -Severity INFO ` -Value ((Get-Text 'Value.AppLog.NoneRead') -f $skipped.Count, (Get-DataProperty $skipped[0] 'Name'), (Get-AppLogSkippedText -Skipped (Get-DataProperty $skipped[0] 'Skipped'))) ` -Hint (Get-Text 'Hint.AppLog.NoneRead') } $count = ConvertTo-Number (Get-DataProperty $Data 'ErrorCount') if ($null -eq $count) { $count = 0 } $lines = @() if (-not $count) { $lines += (Get-Text 'Value.AppLog.NoErrors') -f $days, $read.Count } else { $groups = @(Get-AppLogErrorGroup -ErrorEntry (Get-DataCollection $Data 'Errors')) $last = @($groups | ForEach-Object { $_.Last } | Where-Object { $null -ne $_ } | Sort-Object -Descending) $line = (Get-Text 'Value.AppLog.ErrorCount') -f $count, $days if ($last.Count) { $line = (Get-Text 'Value.AppLog.ErrorCountLast') -f $count, $days, $last[0] } $lines += $line $top = @($groups | Select-Object -First 3 | ForEach-Object { $what = $_.Source if ($_.Exception) { $what = '{0} ({1})' -f $_.Source, $_.Exception } (Get-Text 'Value.AppLog.GroupCount') -f $_.Count, $what }) $lines += (Get-Text 'Value.AppLog.Most') -f ($top -join '; ') } if ($skipped.Count) { $lines += (Get-Text 'Value.AppLog.Skipped') -f $skipped.Count } $severity = Get-Severity $count $warn $fail $hint = '' if ($count) { if ($severity -eq 'OK') { $severity = 'INFO' } $hint = (Get-Text 'Hint.AppLog.Errors') -f (Get-AppLogSectionTitle -Data $Data), $path } New-Finding -Category Apps -Check $check -Severity $severity -Value ($lines -join ' | ') -Hint $hint } function Get-AppLogSectionTitle { <# .SYNOPSIS What the Section of one application's errors is called. Pure. .DESCRIPTION One place, because the Finding's Hint names the Section by this title and the Report makes a link of it only when the two are the same. #> [CmdletBinding()] [OutputType([string])] param([AllowNull()]$Data) $app = "$(Get-DataProperty $Data 'App')".Trim() if ($app) { return (Get-Text 'Title.AppLog.ErrorsOf') -f $app } Get-Text 'Title.AppLog.Errors' } function Get-AppLogSkippedText { <# .SYNOPSIS Why a file was not read, in German. Pure. .DESCRIPTION The Gatherer says TooLarge of a file over MaxFileMB; anything else it says is what Windows said when the file would not open, and stays in Windows' words. #> [CmdletBinding()] [OutputType([string])] param([AllowNull()][AllowEmptyString()][string]$Skipped) if ("$Skipped" -eq 'TooLarge') { return Get-Text 'Value.AppLog.Skipped.TooLarge' } "$Skipped" } function ConvertTo-AppLogSection { [CmdletBinding()] [OutputType([psobject])] param([AllowNull()]$Data) # The column names are what the Report shows, so they are German and live with the # other text. New-Section -Title (Get-AppLogSectionTitle -Data $Data) -Row @( Get-AppLogErrorGroup -ErrorEntry (Get-DataCollection $Data 'Errors') | ForEach-Object { $row = [ordered]@{} $row[(Get-Text 'Column.AppLog.Count')] = $_.Count $row[(Get-Text 'Column.AppLog.Source')] = $_.Source $row[(Get-Text 'Column.AppLog.Exception')] = $_.Exception $row[(Get-Text 'Column.AppLog.Last')] = $(if ($null -ne $_.Last) { '{0:yyyy-MM-dd HH:mm}' -f $_.Last } else { '' }) $row[(Get-Text 'Column.AppLog.Example')] = $_.Example [pscustomobject]$row } ) $app = "$(Get-DataProperty $Data 'App')".Trim() $title = Get-Text 'Title.AppLog.Files' if ($app) { $title = (Get-Text 'Title.AppLog.FilesOf') -f $app } New-Section -Title $title -Row @( foreach ($file in (Get-DataCollection $Data 'Files')) { $row = [ordered]@{} $row[(Get-Text 'Column.AppLog.File')] = Get-DataProperty $file 'Name' $row[(Get-Text 'Column.AppLog.LastWriteTime')] = Get-DataProperty $file 'LastWriteTime' $row[(Get-Text 'Column.AppLog.Length')] = Get-DataProperty $file 'Length' $row[(Get-Text 'Column.AppLog.Entries')] = Get-DataProperty $file 'Entries' $row[(Get-Text 'Column.AppLog.Errors')] = Get-DataProperty $file 'Errors' $row[(Get-Text 'Column.AppLog.Skipped')] = Get-AppLogSkippedText -Skipped (Get-DataProperty $file 'Skipped') [pscustomobject]$row } ) } function ConvertTo-AppLogEvent { <# .SYNOPSIS The errors as rows for events.csv, beside what Windows logged. #> [CmdletBinding()] [OutputType([psobject])] param([AllowNull()]$Data) $app = "$(Get-DataProperty $Data 'App')".Trim() foreach ($entry in (Get-DataCollection $Data 'Errors')) { $message = "$(Get-DataProperty $entry 'Message')" $exception = "$(Get-DataProperty $entry 'Exception')" if ($exception) { $message = '{0} | {1}' -f $message, $exception } [pscustomobject][ordered]@{ Time = Get-DataProperty $entry 'Time' Category = 'AppLog' Provider = ('{0} {1}' -f $app, (Get-DataProperty $entry 'Source')).Trim() Id = Get-DataProperty $entry 'Level' Message = $message } } } |