SMBeat

0.4.8

Private/Aggregation.ps1

#Requires -Version 5.1

function Read-SMBeatSampleFiles {
    param(
        [Parameter(Mandatory = $true)]
        [string]$DataRoot,
        [datetime]$StartUtc,
        [datetime]$EndUtc
    )

    $dir = Get-SMBeatSampleDirectory -DataRoot $DataRoot
    $records = New-Object System.Collections.Generic.List[object]
    if (-not (Test-Path -LiteralPath $dir)) {
        return @()
    }

    $startDay = $StartUtc.Date.AddDays(-1)
    $endDay = $EndUtc.Date.AddDays(1)

    Get-ChildItem -LiteralPath $dir -Filter '*.jsonl' -ErrorAction SilentlyContinue | Sort-Object Name | ForEach-Object {
        $dayPart = [System.IO.Path]::GetFileNameWithoutExtension($_.Name)
        $fileDay = [datetime]::MinValue
        if (-not [datetime]::TryParseExact($dayPart, 'yyyy-MM-dd', $script:SMBeatInvariant, [System.Globalization.DateTimeStyles]::AssumeUniversal, [ref]$fileDay)) {
            return
        }
        if ($fileDay -lt $startDay -or $fileDay -gt $endDay) {
            return
        }

        $lines = [System.IO.File]::ReadAllLines($_.FullName, $script:SMBeatUtf8NoBom)
        foreach ($line in $lines) {
            if ([string]::IsNullOrWhiteSpace($line)) {
                continue
            }
            $obj = ConvertFrom-SMBeatJson -Json $line
            $ts = ConvertFrom-SMBeatUtcString -Value $obj.ts
            if ($ts -lt $StartUtc -or $ts -ge $EndUtc) {
                continue
            }
            $records.Add($obj) | Out-Null
        }
    }

    return $records.ToArray()
}

function Get-SMBeatExpectedSampleCount {
    param(
        [datetime]$StartUtc,
        [datetime]$EndUtc,
        [int]$IntervalSec
    )

    if ($IntervalSec -le 0) {
        $IntervalSec = 60
    }

    $seconds = ($EndUtc - $StartUtc).TotalSeconds
    if ($seconds -le 0) {
        return 1
    }

    [math]::Max(1, [int][math]::Floor($seconds / $IntervalSec))
}

function Add-SMBeatAccumulator {
    param(
        [hashtable]$Map,
        [string]$Key,
        [int64]$Read,
        [int64]$Write,
        [int64]$Sent,
        [int64]$Received
    )

    if (-not $Map.ContainsKey($Key)) {
        $Map[$Key] = @{
            Read     = [int64]0
            Write    = [int64]0
            Sent     = [int64]0
            Received = [int64]0
        }
    }

    $Map[$Key].Read += $Read
    $Map[$Key].Write += $Write
    $Map[$Key].Sent += $Sent
    $Map[$Key].Received += $Received
}

function Test-SMBeatVolumeNear {
    param(
        [int64]$Left,
        [int64]$Right,
        [double]$MinRatio = 0.8
    )

    $hi = $Left
    if ($Right -gt $hi) {
        $hi = $Right
    }
    $lo = $Left
    if ($Right -lt $lo) {
        $lo = $Right
    }
    if ($hi -le 0) {
        return $true
    }

    return (([double]$lo / [double]$hi) -ge $MinRatio)
}

function Get-SMBeatEtwRetrievedBytes {
    param(
        [hashtable]$ByUser,
        [int64]$ShareSent
    )

    $userSent = [int64]0
    if ($null -ne $ByUser) {
        foreach ($uk in @($ByUser.Keys)) {
            $userSent += [int64]$ByUser[$uk].Sent
        }
    }
    if ($userSent -gt $ShareSent) {
        return $userSent
    }

    return $ShareSent
}

function ConvertTo-SMBeatNamedTotals {
    param(
        [hashtable]$Map
    )

    $list = New-Object System.Collections.Generic.List[object]
    foreach ($key in $Map.Keys) {
        $v = $Map[$key]
        $list.Add([PSCustomObject]@{
            Name           = $key
            ReadBytes      = [int64]$v.Read
            WriteBytes     = [int64]$v.Write
            SentBytes      = [int64]$v.Sent
            ReceivedBytes  = [int64]$v.Received
            ReadGB        = (ConvertTo-SMBeatGB -Bytes $v.Read)
            WriteGB       = (ConvertTo-SMBeatGB -Bytes $v.Write)
            SentGB        = (ConvertTo-SMBeatGB -Bytes $v.Sent)
            ReceivedGB    = (ConvertTo-SMBeatGB -Bytes $v.Received)
        }) | Out-Null
    }

    $list | Sort-Object -Property @{
        Expression = {
            $outB = [int64]0
            $inB = [int64]0
            if ($_.SentBytes) { $outB = [int64]$_.SentBytes }
            if ($_.ReceivedBytes) { $inB = [int64]$_.ReceivedBytes }
            if ($outB -gt $inB) { $outB } else { $inB }
        }
    } -Descending
}

function New-SMBeatEmptyTotals {
    [PSCustomObject]@{
        ReadBytes     = [int64]0
        WriteBytes    = [int64]0
        SentBytes     = [int64]0
        ReceivedBytes = [int64]0
        NicBytesIn    = [int64]0
        NicBytesOut   = [int64]0
        ReadGB       = 0
        WriteGB      = 0
        SentGB       = 0
        ReceivedGB   = 0
        NicInGB      = 0
        NicOutGB     = 0
        RetrievedBytes = [int64]0
        RetrievedGB  = 0
        WrittenBytes   = [int64]0
        WrittenGB     = 0
    }
}

function ConvertTo-SMBeatTotalsObject {
    param(
        [int64]$Read,
        [int64]$Write,
        [int64]$Sent,
        [int64]$Received,
        [int64]$NicIn,
        [int64]$NicOut,
        [int64]$Retrieved,
        [int64]$Written
    )

    if (-not $PSBoundParameters.ContainsKey('Retrieved')) {
        $Retrieved = $NicOut
    }
    if (-not $PSBoundParameters.ContainsKey('Written')) {
        $Written = $Received
    }

    [PSCustomObject]@{
        ReadBytes      = $Read
        WriteBytes     = $Write
        SentBytes      = $Sent
        ReceivedBytes  = $Received
        NicBytesIn     = $NicIn
        NicBytesOut    = $NicOut
        ReadGB        = (ConvertTo-SMBeatGB -Bytes $Read)
        WriteGB       = (ConvertTo-SMBeatGB -Bytes $Write)
        SentGB        = (ConvertTo-SMBeatGB -Bytes $Sent)
        ReceivedGB    = (ConvertTo-SMBeatGB -Bytes $Received)
        NicInGB       = (ConvertTo-SMBeatGB -Bytes $NicIn)
        NicOutGB      = (ConvertTo-SMBeatGB -Bytes $NicOut)
        RetrievedBytes = $Retrieved
        RetrievedGB   = (ConvertTo-SMBeatGB -Bytes $Retrieved)
        WrittenBytes   = $Written
        WrittenGB     = (ConvertTo-SMBeatGB -Bytes $Written)
    }
}

function Get-SMBeatPeakStats {
    param(
        [object[]]$Hourly,
        [int64]$RetrievedBytes,
        [int64]$WrittenBytes,
        [datetime]$StartUtc,
        [datetime]$EndUtc
    )

    $peakStart = $null
    $peakBytes = [int64]0
    $peakWriteStart = $null
    $peakWriteBytes = [int64]0
    foreach ($row in @($Hourly)) {
        if ($null -eq $row) { continue }
        $b = [int64]0
        if ($row.PSObject.Properties['RetrievedBytes'] -and $row.RetrievedBytes) {
            $b = [int64]$row.RetrievedBytes
        }
        if ($b -gt $peakBytes) {
            $peakBytes = $b
            $peakStart = $row.PeriodStart
        }

        $w = [int64]0
        if ($row.PSObject.Properties['WrittenBytes'] -and $row.WrittenBytes) {
            $w = [int64]$row.WrittenBytes
        }
        elseif ($row.PSObject.Properties['ReceivedBytes'] -and $row.ReceivedBytes) {
            $w = [int64]$row.ReceivedBytes
        }
        if ($w -gt $peakWriteBytes) {
            $peakWriteBytes = $w
            $peakWriteStart = $row.PeriodStart
        }
    }

    $hours = ($EndUtc - $StartUtc).TotalHours
    if ($hours -le 0) {
        $hours = 1
    }

    [PSCustomObject]@{
        PeakStart                  = $peakStart
        PeakRetrievedBytes         = $peakBytes
        PeakRetrievedGB           = (ConvertTo-SMBeatGB -Bytes $peakBytes)
        AverageRetrievedGBPerHour = (ConvertTo-SMBeatGB -Bytes ([int64][math]::Round($RetrievedBytes / $hours)))
        PeakWrittenStart           = $peakWriteStart
        PeakWrittenBytes           = $peakWriteBytes
        PeakWrittenGB             = (ConvertTo-SMBeatGB -Bytes $peakWriteBytes)
        AverageWrittenGBPerHour   = (ConvertTo-SMBeatGB -Bytes ([int64][math]::Round($WrittenBytes / $hours)))
    }
}

function Get-SMBeatSeries {
    param(
        [object[]]$Records,
        [datetime]$StartUtc,
        [datetime]$EndUtc,
        [TimeZoneInfo]$TimeZone,
        [string]$Granularity
    )

    $buckets = @{}
    foreach ($rec in $Records) {
        $ts = ConvertFrom-SMBeatUtcString -Value $rec.ts
        $local = Convert-SMBeatUtcToTimeZone -Utc $ts -TimeZone $TimeZone
        $start = Get-SMBeatBucketStart -LocalTime $local -Granularity $Granularity
        $key = $start.ToString('o', $script:SMBeatInvariant)
        if (-not $buckets.ContainsKey($key)) {
            $buckets[$key] = @{
                Start        = $start
                Read         = [int64]0
                Write        = [int64]0
                Sent         = [int64]0
                Received     = [int64]0
                ServerRead   = [int64]0
                ServerWrite  = [int64]0
                ServerSent   = [int64]0
                ServerRecv   = [int64]0
                HasShare     = $false
                NicIn        = [int64]0
                NicOut       = [int64]0
                Tcp445Sent   = [int64]0
                Tcp445Recv   = [int64]0
                HasTcp445    = $false
            }
        }

        $kind = [string]$rec.kind
        if ($kind -eq 'server') {
            $buckets[$key].ServerRead += [int64]$rec.readBytesDelta
            $buckets[$key].ServerWrite += [int64]$rec.writeBytesDelta
            $buckets[$key].ServerSent += [int64]$rec.sentBytesDelta
            $buckets[$key].ServerRecv += [int64]$rec.receivedBytesDelta
        }
        elseif ($kind -eq 'share') {
            $buckets[$key].Read += [int64]$rec.readBytesDelta
            $buckets[$key].Write += [int64]$rec.writeBytesDelta
            $buckets[$key].Sent += [int64]$rec.sentBytesDelta
            $buckets[$key].Received += [int64]$rec.receivedBytesDelta
            $buckets[$key].HasShare = $true
        }
        elseif ($kind -eq 'tcp445') {
            $name = [string]$rec.name
            if ($name -eq '_listen445') {
                $buckets[$key].Tcp445Sent += [int64]$rec.sentBytesDelta
                $buckets[$key].Tcp445Recv += [int64]$rec.receivedBytesDelta
                $buckets[$key].HasTcp445 = $true
            }
        }
        elseif ($kind -eq 'nic') {
            $buckets[$key].NicIn += [int64]$rec.receivedBytesDelta
            $buckets[$key].NicOut += [int64]$rec.sentBytesDelta
        }
    }

    $result = New-Object System.Collections.Generic.List[object]
    foreach ($key in ($buckets.Keys | Sort-Object)) {
        $b = $buckets[$key]
        $end = Get-SMBeatBucketEnd -BucketStart $b.Start -Granularity $Granularity
        $useShare = [bool]$b.HasShare
        $readB = $b.ServerRead
        $writeB = $b.ServerWrite
        $sentB = $b.ServerSent
        $recvB = $b.ServerRecv
        if ($useShare) {
            $readB = $b.Read
            $writeB = $b.Write
            $sentB = $b.Sent
            $recvB = $b.Received
        }
        $retrievedB = $b.NicOut
        $writtenB = $recvB
        if ([bool]$b.HasTcp445) {
            $retrievedB = $b.Tcp445Sent
            $writtenB = $b.Tcp445Recv
            if ([bool]$b.HasShare -and $b.Received -gt $writtenB -and $b.Received -gt (10 * [math]::Max($writtenB, [int64]1))) {
                $writtenB = $b.Received
            }
            if ($b.NicOut -gt 1048576 -and $b.Tcp445Sent -gt $b.NicOut -and $b.Tcp445Sent -gt [int64]([double]$b.NicOut * 1.1)) {
                $etwRead = [int64]0
                if ([bool]$b.HasShare) {
                    $etwRead = [int64]$b.Sent
                }
                if ($etwRead -gt 1048576 -and (Test-SMBeatVolumeNear -Left $etwRead -Right $b.NicOut)) {
                    $retrievedB = $etwRead
                }
            }
        }
        $result.Add([PSCustomObject]@{
            Granularity    = $Granularity
            PeriodStart    = $b.Start
            PeriodEnd      = $end
            ReadBytes      = $readB
            WriteBytes     = $writeB
            SentBytes      = $sentB
            ReceivedBytes  = $recvB
            NicBytesIn     = $b.NicIn
            NicBytesOut    = $b.NicOut
            RetrievedBytes = $retrievedB
            WrittenBytes   = $writtenB
            ReadGB        = (ConvertTo-SMBeatGB -Bytes $readB)
            WriteGB       = (ConvertTo-SMBeatGB -Bytes $writeB)
            SentGB        = (ConvertTo-SMBeatGB -Bytes $sentB)
            ReceivedGB    = (ConvertTo-SMBeatGB -Bytes $recvB)
            NicInGB       = (ConvertTo-SMBeatGB -Bytes $b.NicIn)
            NicOutGB      = (ConvertTo-SMBeatGB -Bytes $b.NicOut)
            RetrievedGB   = (ConvertTo-SMBeatGB -Bytes $retrievedB)
            WrittenGB     = (ConvertTo-SMBeatGB -Bytes $writtenB)
        }) | Out-Null
    }

    return $result.ToArray()
}

function New-SMBeatReportFromSamples {
    param(
        [object[]]$Records,
        [datetime]$StartUtc,
        [datetime]$EndUtc,
        [TimeZoneInfo]$TimeZone,
        [string[]]$Granularity,
        [int]$IntervalSec,
        $PreviousTotals
    )

    if ($null -eq $Records) {
        $Records = @()
    }

    $warnings = New-Object System.Collections.Generic.List[string]
    $serverRecords = @($Records | Where-Object { $_.kind -eq 'tcp445' -and $_.name -eq '_listen445' })
    if ($serverRecords.Count -eq 0) {
        $serverRecords = @($Records | Where-Object { $_.kind -eq 'server' })
    }
    $expected = Get-SMBeatExpectedSampleCount -StartUtc $StartUtc -EndUtc $EndUtc -IntervalSec $IntervalSec
    $actual = $serverRecords.Count
    $coverage = 0
    if ($expected -gt 0) {
        $coverage = [math]::Min(100, [math]::Round(100.0 * $actual / $expected, 1))
    }
    if ($coverage -lt 90) {
        $warnings.Add(("Coverage {0}% - collector gaps are possible (expected ~{1} server samples, found {2})." -f $coverage, $expected, $actual)) | Out-Null
    }

    $serverRead = [int64]0
    $serverWrite = [int64]0
    $serverSent = [int64]0
    $serverRecv = [int64]0
    $shareRead = [int64]0
    $shareWrite = [int64]0
    $shareSent = [int64]0
    $shareRecv = [int64]0
    $nicIn = [int64]0
    $nicOut = [int64]0
    $tcp445Sent = [int64]0
    $tcp445Recv = [int64]0
    $hasTcp445 = $false
    $byServerFromServer = @{}
    $byServerFromShares = @{}
    $byServerFromUsers = @{}
    $byServerFromTcp = @{}
    $byShare = @{}
    $byClient = @{}
    $byClientTcp = @{}
    $byClientEtw = @{}
    $byUser = @{}

    foreach ($rec in $Records) {
        $r = [int64]0
        $w = [int64]0
        $s = [int64]0
        $v = [int64]0
        if ($rec.readBytesDelta) { $r = [int64]$rec.readBytesDelta }
        if ($rec.writeBytesDelta) { $w = [int64]$rec.writeBytesDelta }
        if ($rec.sentBytesDelta) { $s = [int64]$rec.sentBytesDelta }
        if ($rec.receivedBytesDelta) { $v = [int64]$rec.receivedBytesDelta }

        $kind = [string]$rec.kind
        $serverName = [string]$rec.server

        if ($kind -eq 'server') {
            $serverRead += $r
            $serverWrite += $w
            $serverSent += $s
            $serverRecv += $v
            Add-SMBeatAccumulator -Map $byServerFromServer -Key $serverName -Read $r -Write $w -Sent $s -Received $v
        }
        elseif ($kind -eq 'share') {
            $shareRead += $r
            $shareWrite += $w
            $shareSent += $s
            $shareRecv += $v
            Add-SMBeatAccumulator -Map $byShare -Key $rec.name -Read $r -Write $w -Sent $s -Received $v
            Add-SMBeatAccumulator -Map $byServerFromShares -Key $serverName -Read $r -Write $w -Sent $s -Received $v
        }
        elseif ($kind -eq 'session') {
            $clientKey = $rec.client
            if ([string]::IsNullOrWhiteSpace([string]$clientKey)) {
                $clientKey = [string]$rec.name
            }
            $userKey = $rec.user
            if ([string]::IsNullOrWhiteSpace([string]$userKey)) {
                $userKey = '(unknown)'
            }
            if (-not [string]::IsNullOrWhiteSpace([string]$clientKey)) {
                Add-SMBeatAccumulator -Map $byClient -Key ([string]$clientKey) -Read $r -Write $w -Sent $s -Received $v
            }
            Add-SMBeatAccumulator -Map $byUser -Key ([string]$userKey) -Read $r -Write $w -Sent $s -Received $v
        }
        elseif ($kind -eq 'client') {
            $clientKey = $rec.client
            if ([string]::IsNullOrWhiteSpace([string]$clientKey)) {
                $clientKey = [string]$rec.name
            }
            if (-not [string]::IsNullOrWhiteSpace([string]$clientKey)) {
                Add-SMBeatAccumulator -Map $byClientEtw -Key ([string]$clientKey) -Read $r -Write $w -Sent $s -Received $v
            }
        }
        elseif ($kind -eq 'user') {
            $userKey = $rec.user
            if ([string]::IsNullOrWhiteSpace([string]$userKey)) {
                $userKey = [string]$rec.name
            }
            if ([string]::IsNullOrWhiteSpace([string]$userKey)) {
                $userKey = '(unknown)'
            }
            Add-SMBeatAccumulator -Map $byUser -Key ([string]$userKey) -Read $r -Write $w -Sent $s -Received $v
            if (-not [string]::IsNullOrWhiteSpace($serverName)) {
                Add-SMBeatAccumulator -Map $byServerFromUsers -Key $serverName -Read $r -Write $w -Sent $s -Received $v
            }
        }
        elseif ($kind -eq 'tcp445') {
            $hasTcp445 = $true
            $name = [string]$rec.name
            if ($name -eq '_listen445') {
                $tcp445Sent += $s
                $tcp445Recv += $v
                Add-SMBeatAccumulator -Map $byServerFromTcp -Key $serverName -Read 0 -Write 0 -Sent $s -Received $v
            }
            else {
                $clientKey = $rec.client
                if ([string]::IsNullOrWhiteSpace([string]$clientKey)) {
                    $clientKey = $name
                }
                Add-SMBeatAccumulator -Map $byClientTcp -Key ([string]$clientKey) -Read 0 -Write 0 -Sent $s -Received $v
            }
        }
        elseif ($kind -eq 'nic') {
            $nicIn += $v
            $nicOut += $s
        }
    }

    $useShareTotals = ($shareSent + $shareRecv) -gt ($serverSent + $serverRecv)
    $read = $serverRead
    $write = $serverWrite
    $sent = $serverSent
    $recv = $serverRecv
    $byServer = $byServerFromServer
    if ($useShareTotals) {
        $read = $shareRead
        $write = $shareWrite
        $sent = $shareSent
        $recv = $shareRecv
        $byServer = $byServerFromShares
    }
    if ($hasTcp445) {
        $byServer = $byServerFromTcp
        if ($byClientTcp.Count -gt 0) {
            $byClient = $byClientTcp
        }
    }
    if ($byClientEtw.Count -gt 0) {
        if ($byClient.Count -eq 0) {
            $byClient = $byClientEtw
        }
        else {
            foreach ($ek in @($byClientEtw.Keys)) {
                $etw = $byClientEtw[$ek]
                if (-not $byClient.ContainsKey($ek)) {
                    $byClient[$ek] = $etw
                    continue
                }
                $cur = $byClient[$ek]
                $etwSent = [int64]$etw.Sent
                $curSent = [int64]$cur.Sent
                if ($etwSent -gt 1048576 -and $curSent -gt [int64]([double]$etwSent * 1.1)) {
                    $cur.Sent = $etwSent
                }
                elseif ($etwSent -gt $curSent) {
                    $cur.Sent = $etwSent
                }
                if ([int64]$etw.Received -gt [int64]$cur.Received) { $cur.Received = $etw.Received }
                if ([int64]$etw.Read -gt [int64]$cur.Read) { $cur.Read = $etw.Read }
                if ([int64]$etw.Write -gt [int64]$cur.Write) { $cur.Write = $etw.Write }
            }
        }
    }

    $retrieved = $nicOut
    $written = $recv
    if ($hasTcp445) {
        $retrieved = $tcp445Sent
        $written = $tcp445Recv
    }
    elseif ($nicOut -gt 104857600 -and $nicOut -gt (10 * [math]::Max($sent, [int64]1))) {
        $warnings.Add('NIC Out is much larger than SMB Sent. On some hosts SMB share counters miss large reads; retrieved volume falls back to NIC Out.') | Out-Null
    }

    $etwRetrieved = Get-SMBeatEtwRetrievedBytes -ByUser $byUser -ShareSent $shareSent
    $etwWritten = $shareRecv
    foreach ($uk in @($byUser.Keys)) {
        $uRecv = [int64]$byUser[$uk].Received
        if ($uRecv -gt $etwWritten) {
            $etwWritten = $uRecv
        }
    }

    $retrievedFromEtw = $false
    if ($hasTcp445 -and $nicOut -gt 1048576 -and $retrieved -gt $nicOut -and $retrieved -gt [int64]([double]$nicOut * 1.1)) {
        if ($etwRetrieved -gt 1048576 -and (Test-SMBeatVolumeNear -Left $etwRetrieved -Right $nicOut)) {
            $retrieved = $etwRetrieved
            $retrievedFromEtw = $true
            $warnings.Add('TCP 445 outbound is larger than NIC Out and ETW retrieved. Headline uses ETW so it matches users and shares; ESTATA can overcount long reads.') | Out-Null
        }
    }

    if ($hasTcp445 -and $retrieved -gt 10485760) {
        $shareFloor = [int64]([math]::Floor($retrieved * 0.1))
        if ($shareSent -lt $shareFloor) {
            $warnings.Add('Share bytes are much smaller than TCP 445 retrieved. Sessions open before the collector started map to (unknown) until reconnect; ETW buffer loss is also possible.') | Out-Null
        }
    }

    if ($hasTcp445 -and $etwWritten -gt 1048576 -and $etwWritten -gt (10 * [math]::Max($written, [int64]1))) {
        $written = $etwWritten
        $warnings.Add('TCP 445 inbound is much smaller than ETW written. Short-lived writes are counted per user and share; ESTATA often misses them.') | Out-Null
    }

    foreach ($sk in @($byServer.Keys)) {
        $etwRecv = [int64]0
        $etwSent = [int64]0
        if ($byServerFromShares.ContainsKey($sk)) {
            $etwRecv = [int64]$byServerFromShares[$sk].Received
            $etwSent = [int64]$byServerFromShares[$sk].Sent
        }
        if ($byServerFromUsers.ContainsKey($sk)) {
            $uRecv = [int64]$byServerFromUsers[$sk].Received
            $uSent = [int64]$byServerFromUsers[$sk].Sent
            if ($uRecv -gt $etwRecv) {
                $etwRecv = $uRecv
            }
            if ($uSent -gt $etwSent) {
                $etwSent = $uSent
            }
        }
        $curRecv = [int64]$byServer[$sk].Received
        if ($etwRecv -gt 1048576 -and $etwRecv -gt (10 * [math]::Max($curRecv, [int64]1))) {
            $byServer[$sk].Received = $etwRecv
        }
        $curSent = [int64]$byServer[$sk].Sent
        if ($retrievedFromEtw -and $etwSent -gt 1048576) {
            $byServer[$sk].Sent = $etwSent
        }
        elseif ($etwSent -gt 1048576 -and $curSent -gt [int64]([double]$etwSent * 1.1) -and $nicOut -gt 1048576 -and (Test-SMBeatVolumeNear -Left $etwSent -Right $nicOut)) {
            $byServer[$sk].Sent = $etwSent
        }
    }

    $hourly = @()
    $daily = @()
    $weekly = @()
    if ($Granularity -contains 'Hour') {
        $hourly = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Hour)
    }
    if ($Granularity -contains 'Day') {
        $daily = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Day)
    }
    if ($Granularity -contains 'Week') {
        $weekly = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Week)
    }

    $hourlyForPeak = $hourly
    if ($hourlyForPeak.Count -eq 0) {
        $hourlyForPeak = @(Get-SMBeatSeries -Records $Records -StartUtc $StartUtc -EndUtc $EndUtc -TimeZone $TimeZone -Granularity Hour)
    }
    $peak = Get-SMBeatPeakStats -Hourly $hourlyForPeak -RetrievedBytes $retrieved -WrittenBytes $written -StartUtc $StartUtc -EndUtc $EndUtc

    [PSCustomObject]@{
        PSTypeName                 = 'Merlin.SMBeat.Report'
        StartUtc                   = $StartUtc
        EndUtc                     = $EndUtc
        TimeZone                   = $TimeZone.Id
        CoveragePct                = $coverage
        IntervalSec                = $IntervalSec
        Warnings                   = $warnings.ToArray()
        Totals                     = (ConvertTo-SMBeatTotalsObject -Read $read -Write $write -Sent $sent -Received $recv -NicIn $nicIn -NicOut $nicOut -Retrieved $retrieved -Written $written)
        ByServer                   = @(ConvertTo-SMBeatNamedTotals -Map $byServer)
        ByShare                    = @(ConvertTo-SMBeatNamedTotals -Map $byShare)
        ByClient                   = @(ConvertTo-SMBeatNamedTotals -Map $byClient)
        ByUser                     = @(ConvertTo-SMBeatNamedTotals -Map $byUser)
        Hourly                     = $hourly
        Daily                      = $daily
        Weekly                     = $weekly
        PeakStart                  = $peak.PeakStart
        PeakRetrievedGB           = $peak.PeakRetrievedGB
        AverageRetrievedGBPerHour = $peak.AverageRetrievedGBPerHour
        PeakWrittenStart           = $peak.PeakWrittenStart
        PeakWrittenGB             = $peak.PeakWrittenGB
        AverageWrittenGBPerHour   = $peak.AverageWrittenGBPerHour
        PreviousTotals             = $PreviousTotals
        SampleCount                = $Records.Count
        ModuleVersion              = (Get-SMBeatModuleVersion)
        GeneratedUtc               = (Get-SMBeatUtcNow)
    }
}

function New-SMBeatReportFromDataRoot {
    param(
        [string]$DataRoot,
        [datetime]$WindowStart,
        [datetime]$WindowEnd,
        [datetime]$PrevStart,
        [datetime]$PrevEnd,
        [TimeZoneInfo]$TimeZone,
        [string[]]$Granularity
    )

    $config = Read-SMBeatConfig -DataRoot $DataRoot
    $interval = [int]$config.IntervalSec
    if ($interval -le 0) { $interval = 60 }
    $records = @(Read-SMBeatSampleFiles -DataRoot $DataRoot -StartUtc $WindowStart -EndUtc $WindowEnd)
    $prevRecords = @(Read-SMBeatSampleFiles -DataRoot $DataRoot -StartUtc $PrevStart -EndUtc $PrevEnd)
    $prevTotals = $null
    if ($prevRecords.Count -gt 0) {
        $prevReport = New-SMBeatReportFromSamples -Records $prevRecords -StartUtc $PrevStart -EndUtc $PrevEnd -TimeZone $TimeZone -Granularity @() -IntervalSec $interval
        $prevTotals = $prevReport.Totals
    }

    New-SMBeatReportFromSamples -Records $records -StartUtc $WindowStart -EndUtc $WindowEnd -TimeZone $TimeZone -Granularity $Granularity -IntervalSec $interval -PreviousTotals $prevTotals
}

function Resolve-SMBeatReportWindow {
    param(
        [datetime]$Start,
        [datetime]$End,
        [int]$LastHours,
        [int]$LastDays,
        [int]$LastWeeks
    )

    $endUtc = Get-SMBeatUtcNow
    if ($PSBoundParameters.ContainsKey('End') -and $End) {
        $endUtc = $End.ToUniversalTime()
    }

    $startUtc = $null
    if ($PSBoundParameters.ContainsKey('Start') -and $Start) {
        $startUtc = $Start.ToUniversalTime()
    }
    elseif ($LastHours -gt 0) {
        $startUtc = $endUtc.AddHours(-1 * $LastHours)
    }
    elseif ($LastDays -gt 0) {
        $startUtc = $endUtc.AddDays(-1 * $LastDays)
    }
    elseif ($LastWeeks -gt 0) {
        $startUtc = $endUtc.AddDays(-7 * $LastWeeks)
    }
    else {
        $startUtc = $endUtc.AddDays(-7)
    }

    if ($startUtc -ge $endUtc) {
        throw 'Start must be earlier than End.'
    }

    [PSCustomObject]@{
        StartUtc = $startUtc
        EndUtc   = $endUtc
    }
}

function Merge-SMBeatNamedTotalLists {
    param(
        [object[]]$Lists
    )

    $map = @{}
    foreach ($list in $Lists) {
        if ($null -eq $list) { continue }
        foreach ($row in @($list)) {
            Add-SMBeatAccumulator -Map $map -Key ([string]$row.Name) -Read ([int64]$row.ReadBytes) -Write ([int64]$row.WriteBytes) -Sent ([int64]$row.SentBytes) -Received ([int64]$row.ReceivedBytes)
        }
    }
    @(ConvertTo-SMBeatNamedTotals -Map $map)
}

function Merge-SMBeatSeriesLists {
    param(
        [object[]]$Lists,
        [string]$Granularity
    )

    $map = @{}
    foreach ($list in $Lists) {
        if ($null -eq $list) { continue }
        foreach ($row in @($list)) {
            $key = $row.PeriodStart.ToString('o')
            if (-not $map.ContainsKey($key)) {
                $map[$key] = @{
                    Start    = $row.PeriodStart
                    End      = $row.PeriodEnd
                    Read     = [int64]0
                    Write    = [int64]0
                    Sent     = [int64]0
                    Received = [int64]0
                    NicIn     = [int64]0
                    NicOut    = [int64]0
                    Retrieved = [int64]0
                    Written   = [int64]0
                }
            }
            $map[$key].Read += [int64]$row.ReadBytes
            $map[$key].Write += [int64]$row.WriteBytes
            if ($row.SentBytes) { $map[$key].Sent += [int64]$row.SentBytes }
            if ($row.ReceivedBytes) { $map[$key].Received += [int64]$row.ReceivedBytes }
            if ($row.NicBytesIn) { $map[$key].NicIn += [int64]$row.NicBytesIn }
            if ($row.NicBytesOut) { $map[$key].NicOut += [int64]$row.NicBytesOut }
            if ($row.RetrievedBytes) { $map[$key].Retrieved += [int64]$row.RetrievedBytes }
            elseif ($row.NicBytesOut) { $map[$key].Retrieved += [int64]$row.NicBytesOut }
            if ($row.PSObject.Properties['WrittenBytes'] -and $row.WrittenBytes) {
                $map[$key].Written += [int64]$row.WrittenBytes
            }
            elseif ($row.ReceivedBytes) {
                $map[$key].Written += [int64]$row.ReceivedBytes
            }
        }
    }

    $result = New-Object System.Collections.Generic.List[object]
    foreach ($key in ($map.Keys | Sort-Object)) {
        $b = $map[$key]
        $result.Add([PSCustomObject]@{
            Granularity  = $Granularity
            PeriodStart  = $b.Start
            PeriodEnd    = $b.End
            ReadBytes     = $b.Read
            WriteBytes    = $b.Write
            SentBytes     = $b.Sent
            ReceivedBytes = $b.Received
            NicBytesIn     = $b.NicIn
            NicBytesOut    = $b.NicOut
            RetrievedBytes = $b.Retrieved
            WrittenBytes   = $b.Written
            ReadGB        = (ConvertTo-SMBeatGB -Bytes $b.Read)
            WriteGB       = (ConvertTo-SMBeatGB -Bytes $b.Write)
            SentGB        = (ConvertTo-SMBeatGB -Bytes $b.Sent)
            ReceivedGB    = (ConvertTo-SMBeatGB -Bytes $b.Received)
            NicInGB       = (ConvertTo-SMBeatGB -Bytes $b.NicIn)
            NicOutGB      = (ConvertTo-SMBeatGB -Bytes $b.NicOut)
            RetrievedGB   = (ConvertTo-SMBeatGB -Bytes $b.Retrieved)
            WrittenGB     = (ConvertTo-SMBeatGB -Bytes $b.Written)
        }) | Out-Null
    }
    $result.ToArray()
}

function Merge-SMBeatReports {
    param(
        [Parameter(Mandatory = $true)]
        [object[]]$Reports,
        $PreviousTotals
    )

    if ($null -eq $Reports -or $Reports.Count -eq 0) {
        return $null
    }
    if ($Reports.Count -eq 1) {
        $one = $Reports[0]
        if ($PreviousTotals) {
            $one.PreviousTotals = $PreviousTotals
        }
        return $one
    }

    $first = $Reports[0]
    $read = [int64]0
    $write = [int64]0
    $sent = [int64]0
    $recv = [int64]0
    $nicIn = [int64]0
    $nicOut = [int64]0
    $retrieved = [int64]0
    $written = [int64]0
    $samples = 0
    $covSum = 0.0
    $warnings = New-Object System.Collections.Generic.List[string]
    $serverLists = New-Object System.Collections.Generic.List[object]
    $shareLists = New-Object System.Collections.Generic.List[object]
    $clientLists = New-Object System.Collections.Generic.List[object]
    $userLists = New-Object System.Collections.Generic.List[object]
    $hourlyLists = New-Object System.Collections.Generic.List[object]
    $dailyLists = New-Object System.Collections.Generic.List[object]
    $weeklyLists = New-Object System.Collections.Generic.List[object]

    foreach ($r in $Reports) {
        $read += [int64]$r.Totals.ReadBytes
        $write += [int64]$r.Totals.WriteBytes
        $sent += [int64]$r.Totals.SentBytes
        $recv += [int64]$r.Totals.ReceivedBytes
        $nicIn += [int64]$r.Totals.NicBytesIn
        $nicOut += [int64]$r.Totals.NicBytesOut
        if ($r.Totals.RetrievedBytes) {
            $retrieved += [int64]$r.Totals.RetrievedBytes
        }
        else {
            $retrieved += [int64]$r.Totals.NicBytesOut
        }
        if ($r.Totals.PSObject.Properties['WrittenBytes'] -and $r.Totals.WrittenBytes) {
            $written += [int64]$r.Totals.WrittenBytes
        }
        else {
            $written += [int64]$r.Totals.ReceivedBytes
        }
        $samples += [int]$r.SampleCount
        $covSum += [double]$r.CoveragePct
        foreach ($w in @($r.Warnings)) { if ($w) { $warnings.Add([string]$w) | Out-Null } }
        $serverLists.Add(@($r.ByServer)) | Out-Null
        $shareLists.Add(@($r.ByShare)) | Out-Null
        $clientLists.Add(@($r.ByClient)) | Out-Null
        $userLists.Add(@($r.ByUser)) | Out-Null
        $hourlyLists.Add(@($r.Hourly)) | Out-Null
        $dailyLists.Add(@($r.Daily)) | Out-Null
        $weeklyLists.Add(@($r.Weekly)) | Out-Null
    }

    $hourlyMerged = @(Merge-SMBeatSeriesLists -Lists $hourlyLists.ToArray() -Granularity Hour)
    $peak = Get-SMBeatPeakStats -Hourly $hourlyMerged -RetrievedBytes $retrieved -WrittenBytes $written -StartUtc $first.StartUtc -EndUtc $first.EndUtc

    [PSCustomObject]@{
        PSTypeName                 = 'Merlin.SMBeat.Report'
        StartUtc                   = $first.StartUtc
        EndUtc                     = $first.EndUtc
        TimeZone                   = $first.TimeZone
        CoveragePct                = [math]::Round($covSum / $Reports.Count, 1)
        IntervalSec                = $first.IntervalSec
        Warnings                   = $warnings.ToArray()
        Totals                     = (ConvertTo-SMBeatTotalsObject -Read $read -Write $write -Sent $sent -Received $recv -NicIn $nicIn -NicOut $nicOut -Retrieved $retrieved -Written $written)
        ByServer                   = (Merge-SMBeatNamedTotalLists -Lists $serverLists.ToArray())
        ByShare                    = (Merge-SMBeatNamedTotalLists -Lists $shareLists.ToArray())
        ByClient                   = (Merge-SMBeatNamedTotalLists -Lists $clientLists.ToArray())
        ByUser                     = (Merge-SMBeatNamedTotalLists -Lists $userLists.ToArray())
        Hourly                     = $hourlyMerged
        Daily                      = @(Merge-SMBeatSeriesLists -Lists $dailyLists.ToArray() -Granularity Day)
        Weekly                     = @(Merge-SMBeatSeriesLists -Lists $weeklyLists.ToArray() -Granularity Week)
        PeakStart                  = $peak.PeakStart
        PeakRetrievedGB           = $peak.PeakRetrievedGB
        AverageRetrievedGBPerHour = $peak.AverageRetrievedGBPerHour
        PeakWrittenStart           = $peak.PeakWrittenStart
        PeakWrittenGB             = $peak.PeakWrittenGB
        AverageWrittenGBPerHour   = $peak.AverageWrittenGBPerHour
        PreviousTotals             = $PreviousTotals
        SampleCount                = $samples
        ModuleVersion              = (Get-SMBeatModuleVersion)
        GeneratedUtc               = (Get-SMBeatUtcNow)
    }
}