Private/Aggregation.ps1
|
#Requires -Version 5.1 function Read-SMBeatSampleFiles { param( [Parameter(Mandatory = $true)] [string]$DataRoot, [datetime]$StartUtc, [datetime]$EndUtc ) $dir = Get-SMBeatSampleDirectory -DataRoot $DataRoot $records = New-Object System.Collections.Generic.List[object] if (-not (Test-Path -LiteralPath $dir)) { return @() } $startDay = $StartUtc.Date.AddDays(-1) $endDay = $EndUtc.Date.AddDays(1) Get-ChildItem -LiteralPath $dir -Filter '*.jsonl' -ErrorAction SilentlyContinue | Sort-Object Name | ForEach-Object { $dayPart = [System.IO.Path]::GetFileNameWithoutExtension($_.Name) $fileDay = [datetime]::MinValue if (-not [datetime]::TryParseExact($dayPart, 'yyyy-MM-dd', $script:SMBeatInvariant, [System.Globalization.DateTimeStyles]::AssumeUniversal, [ref]$fileDay)) { return } if ($fileDay -lt $startDay -or $fileDay -gt $endDay) { return } $lines = [System.IO.File]::ReadAllLines($_.FullName, $script:SMBeatUtf8NoBom) foreach ($line in $lines) { if ([string]::IsNullOrWhiteSpace($line)) { continue } $obj = ConvertFrom-SMBeatJson -Json $line $ts = ConvertFrom-SMBeatUtcString -Value $obj.ts if ($ts -lt $StartUtc -or $ts -ge $EndUtc) { continue } $records.Add($obj) | Out-Null } } return $records.ToArray() } function Get-SMBeatExpectedSampleCount { param( [datetime]$StartUtc, [datetime]$EndUtc, [int]$IntervalSec ) if ($IntervalSec -le 0) { $IntervalSec = 60 } $seconds = ($EndUtc - $StartUtc).TotalSeconds if ($seconds -le 0) { return 1 } [math]::Max(1, [int][math]::Floor($seconds / $IntervalSec)) } function Add-SMBeatAccumulator { param( [hashtable]$Map, [string]$Key, [int64]$Read, [int64]$Write, [int64]$Sent, [int64]$Received ) if (-not $Map.ContainsKey($Key)) { $Map[$Key] = @{ Read = [int64]0 Write = [int64]0 Sent = [int64]0 Received = [int64]0 } } $Map[$Key].Read += $Read $Map[$Key].Write += $Write $Map[$Key].Sent += $Sent $Map[$Key].Received += $Received } function ConvertTo-SMBeatNamedTotals { param( [hashtable]$Map ) $list = New-Object System.Collections.Generic.List[object] foreach ($key in $Map.Keys) { $v = $Map[$key] $list.Add([PSCustomObject]@{ Name = $key ReadBytes = [int64]$v.Read WriteBytes = [int64]$v.Write SentBytes = [int64]$v.Sent ReceivedBytes = [int64]$v.Received ReadGiB = (ConvertTo-SMBeatGiB -Bytes $v.Read) WriteGiB = (ConvertTo-SMBeatGiB -Bytes $v.Write) }) | Out-Null } $list | Sort-Object -Property ReadBytes -Descending } function New-SMBeatEmptyTotals { [PSCustomObject]@{ ReadBytes = [int64]0 WriteBytes = [int64]0 SentBytes = [int64]0 ReceivedBytes = [int64]0 NicBytesIn = [int64]0 NicBytesOut = [int64]0 ReadGiB = 0 WriteGiB = 0 } } function ConvertTo-SMBeatTotalsObject { param( [int64]$Read, [int64]$Write, [int64]$Sent, [int64]$Received, [int64]$NicIn, [int64]$NicOut ) [PSCustomObject]@{ ReadBytes = $Read WriteBytes = $Write SentBytes = $Sent ReceivedBytes = $Received NicBytesIn = $NicIn NicBytesOut = $NicOut ReadGiB = (ConvertTo-SMBeatGiB -Bytes $Read) WriteGiB = (ConvertTo-SMBeatGiB -Bytes $Write) NicInGiB = (ConvertTo-SMBeatGiB -Bytes $NicIn) NicOutGiB = (ConvertTo-SMBeatGiB -Bytes $NicOut) } } function Get-SMBeatSeries { param( [object[]]$Records, [datetime]$StartUtc, [datetime]$EndUtc, [TimeZoneInfo]$TimeZone, [string]$Granularity ) $buckets = @{} foreach ($rec in $Records) { $ts = ConvertFrom-SMBeatUtcString -Value $rec.ts $local = Convert-SMBeatUtcToTimeZone -Utc $ts -TimeZone $TimeZone $start = Get-SMBeatBucketStart -LocalTime $local -Granularity $Granularity $key = $start.ToString('o', $script:SMBeatInvariant) if (-not $buckets.ContainsKey($key)) { $buckets[$key] = @{ Start = $start Read = [int64]0 Write = [int64]0 NicIn = [int64]0 NicOut = [int64]0 } } $kind = [string]$rec.kind if ($kind -eq 'server') { $buckets[$key].Read += [int64]$rec.readBytesDelta $buckets[$key].Write += [int64]$rec.writeBytesDelta } elseif ($kind -eq 'nic') { $buckets[$key].NicIn += [int64]$rec.receivedBytesDelta $buckets[$key].NicOut += [int64]$rec.sentBytesDelta } } $result = New-Object System.Collections.Generic.List[object] foreach ($key in ($buckets.Keys | Sort-Object)) { $b = $buckets[$key] $end = Get-SMBeatBucketEnd -BucketStart $b.Start -Granularity $Granularity $result.Add([PSCustomObject]@{ Granularity = $Granularity PeriodStart = $b.Start PeriodEnd = $end ReadBytes = $b.Read WriteBytes = $b.Write NicBytesIn = $b.NicIn NicBytesOut = $b.NicOut ReadGiB = (ConvertTo-SMBeatGiB -Bytes $b.Read) WriteGiB = (ConvertTo-SMBeatGiB -Bytes $b.Write) }) | Out-Null } return $result.ToArray() } function New-SMBeatReportFromSamples { param( [object[]]$Records, [datetime]$StartUtc, [datetime]$EndUtc, [TimeZoneInfo]$TimeZone, [string[]]$Granularity, [int]$IntervalSec, $PreviousTotals ) if ($null -eq $Records) { $Records = @() } $warnings = New-Object System.Collections.Generic.List[string] $serverRecords = @($Records | Where-Object { $_.kind -eq 'server' }) $expected = Get-SMBeatExpectedSampleCount -StartUtc $StartUtc -EndUtc $EndUtc -IntervalSec $IntervalSec $actual = $serverRecords.Count $coverage = 0 if ($expected -gt 0) { $coverage = [math]::Min(100, [math]::Round(100.0 * $actual / $expected, 1)) } if ($coverage -lt 90) { $warnings.Add(("Coverage {0}% — collector gaps are possible (expected ~{1} server samples, found {2})." -f $coverage, $expected, $actual)) | Out-Null } $read = [int64]0 $write = [int64]0 $sent = [int64]0 $recv = [int64]0 $nicIn = [int64]0 $nicOut = [int64]0 $byServer = @{} $byShare = @{} $byClient = @{} $byUser = @{} foreach ($rec in $Records) { $r = [int64]0 $w = [int64]0 $s = [int64]0 $v = [int64]0 if ($rec.readBytesDelta) { $r = [int64]$rec.readBytesDelta } if ($rec.writeBytesDelta) { $w = [int64]$rec.writeBytesDelta } if ($rec.sentBytesDelta) { $s = [int64]$rec.sentBytesDelta } if ($rec.receivedBytesDelta) { $v = [int64]$rec.receivedBytesDelta } $kind = [string]$rec.kind $serverName = [string]$rec.server if ($kind -eq 'server') { $read += $r $write += $w $sent += $s $recv += $v Add-SMBeatAccumulator -Map $byServer -Key $serverName -Read $r -Write $w -Sent $s -Received $v } elseif ($kind -eq 'share') { Add-SMBeatAccumulator -Map $byShare -Key $rec.name -Read $r -Write $w -Sent $s -Received $v } elseif ($kind -eq 'session') { $clientKey = $rec.client if ([string]::IsNullOrWhiteSpace([string]$clientKey)) { $clientKey = [string]$rec.name } $userKey = $rec.user if ([string]::IsNullOrWhiteSpace([string]$userKey)) { $userKey = '(unknown)' } Add-SMBeatAccumulator -Map $byClient -Key ([string]$clientKey) -Read $r -Write $w -Sent $s -Received $v Add-SMBeatAccumulator -Map $byUser -Key ([string]$userKey) -Read $r -Write $w -Sent $s -Received $v } elseif ($kind -eq 'nic') { $nicIn += $v $nicOut += $s } } $hourly = @() $daily = @() $weekly = @() if ($Granularity -contains 'Hour') { $hourly = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Hour) } if ($Granularity -contains 'Day') { $daily = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Day) } if ($Granularity -contains 'Week') { $weekly = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Week) } [PSCustomObject]@{ PSTypeName = 'Merlin.SMBeat.Report' StartUtc = $StartUtc EndUtc = $EndUtc TimeZone = $TimeZone.Id CoveragePct = $coverage IntervalSec = $IntervalSec Warnings = $warnings.ToArray() Totals = (ConvertTo-SMBeatTotalsObject -Read $read -Write $write -Sent $sent -Received $recv -NicIn $nicIn -NicOut $nicOut) ByServer = @(ConvertTo-SMBeatNamedTotals -Map $byServer) ByShare = @(ConvertTo-SMBeatNamedTotals -Map $byShare) ByClient = @(ConvertTo-SMBeatNamedTotals -Map $byClient) ByUser = @(ConvertTo-SMBeatNamedTotals -Map $byUser) Hourly = $hourly Daily = $daily Weekly = $weekly PreviousTotals = $PreviousTotals SampleCount = $Records.Count } } function New-SMBeatReportFromDataRoot { param( [string]$DataRoot, [datetime]$WindowStart, [datetime]$WindowEnd, [datetime]$PrevStart, [datetime]$PrevEnd, [TimeZoneInfo]$TimeZone, [string[]]$Granularity ) $config = Read-SMBeatConfig -DataRoot $DataRoot $interval = [int]$config.IntervalSec if ($interval -le 0) { $interval = 60 } $records = @(Read-SMBeatSampleFiles -DataRoot $DataRoot -StartUtc $WindowStart -EndUtc $WindowEnd) $prevRecords = @(Read-SMBeatSampleFiles -DataRoot $DataRoot -StartUtc $PrevStart -EndUtc $PrevEnd) $prevTotals = $null if ($prevRecords.Count -gt 0) { $prevReport = New-SMBeatReportFromSamples -Records $prevRecords -StartUtc $PrevStart -EndUtc $PrevEnd -TimeZone $TimeZone -Granularity @() -IntervalSec $interval $prevTotals = $prevReport.Totals } New-SMBeatReportFromSamples -Records $records -StartUtc $WindowStart -EndUtc $WindowEnd -TimeZone $TimeZone -Granularity $Granularity -IntervalSec $interval -PreviousTotals $prevTotals } function Resolve-SMBeatReportWindow { param( [datetime]$Start, [datetime]$End, [int]$LastHours, [int]$LastDays, [int]$LastWeeks ) $endUtc = Get-SMBeatUtcNow if ($PSBoundParameters.ContainsKey('End') -and $End) { $endUtc = $End.ToUniversalTime() } $startUtc = $null if ($PSBoundParameters.ContainsKey('Start') -and $Start) { $startUtc = $Start.ToUniversalTime() } elseif ($LastHours -gt 0) { $startUtc = $endUtc.AddHours(-1 * $LastHours) } elseif ($LastDays -gt 0) { $startUtc = $endUtc.AddDays(-1 * $LastDays) } elseif ($LastWeeks -gt 0) { $startUtc = $endUtc.AddDays(-7 * $LastWeeks) } else { $startUtc = $endUtc.AddDays(-7) } if ($startUtc -ge $endUtc) { throw 'Start must be earlier than End.' } [PSCustomObject]@{ StartUtc = $startUtc EndUtc = $endUtc } } function Merge-SMBeatNamedTotalLists { param( [object[]]$Lists ) $map = @{} foreach ($list in $Lists) { if ($null -eq $list) { continue } foreach ($row in @($list)) { Add-SMBeatAccumulator -Map $map -Key ([string]$row.Name) -Read ([int64]$row.ReadBytes) -Write ([int64]$row.WriteBytes) -Sent ([int64]$row.SentBytes) -Received ([int64]$row.ReceivedBytes) } } @(ConvertTo-SMBeatNamedTotals -Map $map) } function Merge-SMBeatSeriesLists { param( [object[]]$Lists, [string]$Granularity ) $map = @{} foreach ($list in $Lists) { if ($null -eq $list) { continue } foreach ($row in @($list)) { $key = $row.PeriodStart.ToString('o') if (-not $map.ContainsKey($key)) { $map[$key] = @{ Start = $row.PeriodStart End = $row.PeriodEnd Read = [int64]0 Write = [int64]0 NicIn = [int64]0 NicOut = [int64]0 } } $map[$key].Read += [int64]$row.ReadBytes $map[$key].Write += [int64]$row.WriteBytes if ($row.NicBytesIn) { $map[$key].NicIn += [int64]$row.NicBytesIn } if ($row.NicBytesOut) { $map[$key].NicOut += [int64]$row.NicBytesOut } } } $result = New-Object System.Collections.Generic.List[object] foreach ($key in ($map.Keys | Sort-Object)) { $b = $map[$key] $result.Add([PSCustomObject]@{ Granularity = $Granularity PeriodStart = $b.Start PeriodEnd = $b.End ReadBytes = $b.Read WriteBytes = $b.Write NicBytesIn = $b.NicIn NicBytesOut = $b.NicOut ReadGiB = (ConvertTo-SMBeatGiB -Bytes $b.Read) WriteGiB = (ConvertTo-SMBeatGiB -Bytes $b.Write) }) | Out-Null } $result.ToArray() } function Merge-SMBeatReports { param( [Parameter(Mandatory = $true)] [object[]]$Reports, $PreviousTotals ) if ($null -eq $Reports -or $Reports.Count -eq 0) { return $null } if ($Reports.Count -eq 1) { $one = $Reports[0] if ($PreviousTotals) { $one.PreviousTotals = $PreviousTotals } return $one } $first = $Reports[0] $read = [int64]0 $write = [int64]0 $sent = [int64]0 $recv = [int64]0 $nicIn = [int64]0 $nicOut = [int64]0 $samples = 0 $covSum = 0.0 $warnings = New-Object System.Collections.Generic.List[string] $serverLists = New-Object System.Collections.Generic.List[object] $shareLists = New-Object System.Collections.Generic.List[object] $clientLists = New-Object System.Collections.Generic.List[object] $userLists = New-Object System.Collections.Generic.List[object] $hourlyLists = New-Object System.Collections.Generic.List[object] $dailyLists = New-Object System.Collections.Generic.List[object] $weeklyLists = New-Object System.Collections.Generic.List[object] foreach ($r in $Reports) { $read += [int64]$r.Totals.ReadBytes $write += [int64]$r.Totals.WriteBytes $sent += [int64]$r.Totals.SentBytes $recv += [int64]$r.Totals.ReceivedBytes $nicIn += [int64]$r.Totals.NicBytesIn $nicOut += [int64]$r.Totals.NicBytesOut $samples += [int]$r.SampleCount $covSum += [double]$r.CoveragePct foreach ($w in @($r.Warnings)) { if ($w) { $warnings.Add([string]$w) | Out-Null } } $serverLists.Add(@($r.ByServer)) | Out-Null $shareLists.Add(@($r.ByShare)) | Out-Null $clientLists.Add(@($r.ByClient)) | Out-Null $userLists.Add(@($r.ByUser)) | Out-Null $hourlyLists.Add(@($r.Hourly)) | Out-Null $dailyLists.Add(@($r.Daily)) | Out-Null $weeklyLists.Add(@($r.Weekly)) | Out-Null } [PSCustomObject]@{ PSTypeName = 'Merlin.SMBeat.Report' StartUtc = $first.StartUtc EndUtc = $first.EndUtc TimeZone = $first.TimeZone CoveragePct = [math]::Round($covSum / $Reports.Count, 1) IntervalSec = $first.IntervalSec Warnings = $warnings.ToArray() Totals = (ConvertTo-SMBeatTotalsObject -Read $read -Write $write -Sent $sent -Received $recv -NicIn $nicIn -NicOut $nicOut) ByServer = (Merge-SMBeatNamedTotalLists -Lists $serverLists.ToArray()) ByShare = (Merge-SMBeatNamedTotalLists -Lists $shareLists.ToArray()) ByClient = (Merge-SMBeatNamedTotalLists -Lists $clientLists.ToArray()) ByUser = (Merge-SMBeatNamedTotalLists -Lists $userLists.ToArray()) Hourly = @(Merge-SMBeatSeriesLists -Lists $hourlyLists.ToArray() -Granularity Hour) Daily = @(Merge-SMBeatSeriesLists -Lists $dailyLists.ToArray() -Granularity Day) Weekly = @(Merge-SMBeatSeriesLists -Lists $weeklyLists.ToArray() -Granularity Week) PreviousTotals = $PreviousTotals SampleCount = $samples } } |