ExchangeLogs.psm1
|
$script:ModuleRoot = $PSScriptRoot $script:ModuleVersion = (Import-PowerShellDataFile -Path "$($script:ModuleRoot)\ExchangeLogs.psd1").ModuleVersion # Detect whether at some level dotsourcing was enforced $script:doDotSource = Get-PSFConfigValue -FullName ExchangeLogs.Import.DoDotSource -Fallback $false if ($ExchangeLogs_dotsourcemodule) { $script:doDotSource = $true } <# Note on Resolve-Path: All paths are sent through Resolve-Path/Resolve-PSFPath in order to convert them to the correct path separator. This allows ignoring path separators throughout the import sequence, which could otherwise cause trouble depending on OS. Resolve-Path can only be used for paths that already exist, Resolve-PSFPath can accept that the last leaf my not exist. This is important when testing for paths. #> # Detect whether at some level loading individual module files, rather than the compiled module was enforced $importIndividualFiles = Get-PSFConfigValue -FullName ExchangeLogs.Import.IndividualFiles -Fallback $false if ($ExchangeLogs_importIndividualFiles) { $importIndividualFiles = $true } if (Test-Path (Resolve-PSFPath -Path "$($script:ModuleRoot)\..\.git" -SingleItem -NewChild)) { $importIndividualFiles = $true } if ("<was compiled>" -eq '<was not compiled>') { $importIndividualFiles = $true } function Import-ModuleFile { <# .SYNOPSIS Loads files into the module on module import. .DESCRIPTION This helper function is used during module initialization. It should always be dotsourced itself, in order to proper function. This provides a central location to react to files being imported, if later desired .PARAMETER Path The path to the file to load .EXAMPLE PS C:\> . Import-ModuleFile -File $function.FullName Imports the file stored in $function according to import policy #> [CmdletBinding()] Param ( [string] $Path ) $resolvedPath = $ExecutionContext.SessionState.Path.GetResolvedPSPathFromPSPath($Path).ProviderPath if ($doDotSource) { . $resolvedPath } else { $ExecutionContext.InvokeCommand.InvokeScript($false, ([scriptblock]::Create([io.file]::ReadAllText($resolvedPath))), $null, $null) } } #region Load individual files if ($importIndividualFiles) { # Execute Preimport actions foreach ($path in (& "$ModuleRoot\internal\scripts\preimport.ps1")) { . Import-ModuleFile -Path $path } # Import all internal functions foreach ($function in (Get-ChildItem "$ModuleRoot\internal\functions" -Filter "*.ps1" -Recurse -ErrorAction Ignore)) { . Import-ModuleFile -Path $function.FullName } # Import all public functions foreach ($function in (Get-ChildItem "$ModuleRoot\functions" -Filter "*.ps1" -Recurse -ErrorAction Ignore)) { . Import-ModuleFile -Path $function.FullName } # Execute Postimport actions foreach ($path in (& "$ModuleRoot\internal\scripts\postimport.ps1")) { . Import-ModuleFile -Path $path } # End it here, do not load compiled code below return } #endregion Load individual files #region Load compiled code <# This file loads the strings documents from the respective language folders. This allows localizing messages and errors. Load psd1 language files for each language you wish to support. Partial translations are acceptable - when missing a current language message, it will fallback to English or another available language. #> Import-PSFLocalizedString -Path "$($script:ModuleRoot)\en-us\*.psd1" -Module 'ExchangeLogs' -Language 'en-US' function Wait-JobCompleteWithOutput { <# .SYNOPSIS Waits until job(s) are completed and has delivered the output .DESCRIPTION Waits until job(s) are completed and has delivered the output .PARAMETER Job Job(s) to wait for .EXAMPLE PS C:\> Assert-RSJobCompleteWithOutput -Job $jobs Returns true when all jobs are finished #> [CmdletBinding()] param ( $Job ) foreach ($item in $Job) { if($item.State -like "Completed" -and $item.HasMoreData -and -not $item.Output.Count) { Write-PSFMessage -Level Debug -Message "Runspace job '$($item.name)' is in state '$($item.state)' but did not delivered output data. Waiting for output" while (-not $item.Output) { # do nothing, check again. tooks usally arround 50-100ms } } } } function global:Expand-LogRecordPopImap { <# .SYNOPSIS Expand the data from records group into a flat data record .DESCRIPTION Expand the data from records group into a flat data record .PARAMETER InputObject The dataset to expand .PARAMETER SessionIdName The name of the session grouping attribute .PARAMETER ShowProgress If specified, progress information on record processing is showed .PARAMETER IncludeActivityContextData Include full exchange internals in POP3 log expansion process. There is a lot of internal system information in the logfiles, with unneccessary content for statistical output. Suppressing this data is reducing the log footage. .EXAMPLE PS C:\> Expand-LogRecordPop -InputObject $DataSet Expand the data from records group into a flat data record #> [CmdletBinding()] param ( $InputObject, [string] $SessionIdName = "sessionId", [switch] $ShowProgress, [switch] $IncludeActivityContextData ) begin { $Error.Clear() } process { if ($ShowProgress) { $i = 0 if ($InputObject.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($InputObject.count / 100) } } foreach ($record in $InputObject) { # assure qualified session in log $_startIndicator = $record.Group[0] | Where-Object command -like "OpenSession" if (-not $_startIndicator) { Write-PSFMessage -Level Warning -Message "Detect fragmented record! Missing previous logfile with partital records. Skip processing $($SessionIdName) '$($record.$SessionIdName)' in $($record.LogFolder)\$($record.LogFileName)" continue } # data text record in array, avoids parsing full data array $commands = $record.Group.command $context = $record.Group.context #| Select-Object -Unique # full text log $logLines = New-Object -TypeName "System.Collections.ArrayList" foreach ($item in $record.Group) { $logline = $item.seqNumber + " " + $item.command if ($item.parameters.Length -gt 0) { $logline = $logline + " | parameters: " + $item.parameters } if ($item.context.Length -gt 0) { if (-not $IncludeActivityContextData) { if ($item.context -like "*;ActivityContextData=*") { $logline = $logline + ([string]::Join(";", ($item.context.Split(";") -notlike "*ActivityContextData=*"))) } else { $logline = $logline + " | context: " + $item.context } } else { $logline = $logline + " | context: " + $item.context } } $null = $logLines.add($logLine) } if ($logLines) { $logText = [string]::Join("`n", ($logLines | ForEach-Object { $_ }) ) } else { $logtext = "" } # LocalEndpoint [string]$localEndpoint = $record.Group[-1].sIP # RemoteEndpoint [string]$remoteEndpoint = $record.Group[-1].cIp # Errors #Log-type - IMAP4 Log $errMsgs = $context | Where-Object { $_ -like 'R="*' -or $_ -like 'ErrMsg=*' } if ($errMsgs) { $hasError = $true $errorMessage = foreach ($errMsg in $errMsgs) { if($errMsg.StartsWith("ErrMsg=")) { $errMsg -replace "ErrMsg=" } elseif($errMsg.StartsWith("R=")) { $_msg = $errMsg.split(";") | Where-Object { $_ -like "R=*" } if($_msg) { (([string]($errMsg.split(";")[0] -replace "R=").Trim('"')) -replace "z NO ") -replace "-ERR " } else { $errMsg } } else { $errMsg } } if ($errorMessage.count -gt 1) { $errorMessage = [string]::Join(" | ", $errorMessage) } } else { $hasError = $false $errorMessage = "" } # IsProxysession $proxycontext = $context -like "*Proxy:*" if ("proxy" -in $commands) { $isProxysession = $true $proxyServer = $record.Group | Where-Object { $_.command -like "proxy" } | Select-Object -ExpandProperty parameters $proxyStatus = "" } elseif ($proxycontext) { $isProxysession = $true $proxycontext = ((($proxycontext -split "Msg=") | Where-Object { $_ -like "*Proxy:*" }) -split '"' | Where-Object { $_ -like "*Proxy:*" }) -replace "Proxy:" $proxyServer = ($proxycontext -split ";")[0] $proxyStatus = ($proxycontext -split ";")[1] } else { $isProxysession = $false $proxyServer = "" $proxyStatus = "" } # Authentication if ("auth" -in $commands -or "user" -in $commands -or "authenticate" -in $commands -or "login" -in $commands) { $authenticationEnabled = $true } else { $authenticationEnabled = $false } if ($authenticationEnabled) { $userName = $record.Group | Where-Object { $_.command -like "auth" -or $_.command -like "user" -or $_.command -like "authenticate" -or $_.command -like "login" } | Select-Object -ExpandProperty User -Unique if ($userName.count -gt 1) { $userName = [string]::Join(", ", $userName) } } else { $userName = "" } if ($authenticationEnabled -and ("auth" -in $commands -or "pass" -in $commands -or "authenticate" -in $commands -or "login" -in $commands)) { $authDetails = ((($record.Group | Where-Object { $_.command -like "auth" -or $_.command -like "pass" -or $_.command -like "authenticate" -or $_.command -like "login"}).context -split ";")[1] -replace 'Msg="').TrimEnd('"') -Split ", " $authHash = @{ "RecipientType" = ($authDetails | Where-Object { $_ -like "RecipientType: *" }) -replace "RecipientType: " "RecipientTypeDetails" = ($authDetails | Where-Object { $_ -like "RecipientTypeDetails: *" }) -replace "RecipientTypeDetails: " "DisplayName" = ($authDetails | Where-Object { $_ -like "Selected Mailbox: Display Name: *" }) -replace "Selected Mailbox: Display Name: " "MailboxGuid" = ($authDetails | Where-Object { $_ -like "Mailbox Guid: *" }) -replace "Mailbox Guid: " "DatabaseGuid" = ($authDetails | Where-Object { $_ -like "Database: *" }) -replace "Database: " "ServerFqdn" = ($authDetails | Where-Object { $_ -like "Location: ServerFqdn: *" }) -replace "Location: ServerFqdn: " "ServerVersion" = ($authDetails | Where-Object { $_ -like "ServerVersion: *" }) -replace "ServerVersion: " "DatabaseName" = ($authDetails | Where-Object { $_ -like "DatabaseName: *" }) -replace "DatabaseName: " "HomePublicFolderDatabaseGuid" = ($authDetails | Where-Object { $_ -like "HomePublicFolderDatabaseGuid: *" }) -replace "HomePublicFolderDatabaseGuid: " } } else { $authDetails = "" $authHash = @{} } # Statistics $stats = (($record.Group | Where-Object { $_.command -like "stat" }).Context -split ";") | Where-Object { $_ -like "Rows=*" -or $_ -like "TotalSize=*" } if ($stats) { $rows = ($stats -like "Rows=*") -replace "Rows=" if ($rows) { $rows = [int]::Parse($rows) } else { [int]$rows = 0 } $totalSize = ($stats -like "TotalSize=*") -replace "TotalSize=" if ($totalSize) { $totalSize = $totalSize = [int]::Parse($totalSize) } else { [int]$totalSize = 0 } } else { $rows = 0 $totalSize = 0 } $budgetinfo = ((([array]$context -like "*Budget=*") | Select-Object -Last 1) -split 'Budget="')[-1] if ($budgetinfo) { $budgetDetails = $budgetinfo.TrimEnd('"') -Split "," $budgetHash = @{ "OwnerSID" = (($budgetDetails | Where-Object { $_ -like "Owner:Sid~*" }) -split "~")[1] "Conn" = [int](($budgetDetails | Where-Object { $_ -like "Conn:*" }) -replace "Conn:") "MaxConn" = ($budgetDetails | Where-Object { $_ -like "MaxConn:*" }) -replace "MaxConn:" "MaxBurst" = ($budgetDetails | Where-Object { $_ -like "MaxBurst:*" }) -replace "MaxBurst:" "Balance" = ($budgetDetails | Where-Object { $_ -like "Balance:*" }) -replace "Balance:" "Cutoff" = ($budgetDetails | Where-Object { $_ -like "Cutoff:*" }) -replace "Cutoff:" "RechargeRate" = ($budgetDetails | Where-Object { $_ -like "RechargeRate:*" }) -replace "RechargeRate:" "Policy" = ($budgetDetails | Where-Object { $_ -like "Policy:*" }) -replace "Policy:" "IsServiceAccount" = [bool]::Parse( (($budgetDetails | Where-Object { $_ -like "IsServiceAccount:*" }) -replace "IsServiceAccount:") ) "LiveTime" = [timespan]::Parse( (($budgetDetails | Where-Object { $_ -like "LiveTime:*" }) -replace "LiveTime:") ) } } else { $budgetDetails = "" $budgetHash = @{} } $stls = ($record.group.command | Where-Object { $_ -like "stls" -or $_ -like "starttls"}).count $auth = ($record.group.command | Where-Object { $_ -like "auth" -or $_ -like "authenticate"}).count $user = ($record.group.command | Where-Object { $_ -like "user" -or $_ -like "login"}).count $pass = ($record.group.command | Where-Object { $_ -like "pass" }).count $stat = ($record.group.command | Where-Object { $_ -like "stat*"}).count $uidl = ($record.group.command | Where-Object { $_ -like "uidl" }).count $list = ($record.group.command | Where-Object { $_ -like "list" }).count $retr = ($record.group.command | Where-Object { $_ -like "retr" }).count $dele = ($record.group.command | Where-Object { $_ -like "dele" }).count # construct output object $outputRecord = [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($record.metadataHash['Log-type'].Replace(' ','')).Record" "LogFolder" = $record.LogFolder "LogFileName" = $record.LogFileName $SessionIdName = $record.$SessionIdName "DateStart" = ($record.Group | Sort-Object 'dateTime')[0].'dateTime' -as [datetime] "DateEnd" = ($record.Group | Sort-Object 'dateTime')[-1].'dateTime' -as [datetime] "SequenceCount" = $record.Group.count "LocalIP" = $localEndpoint -replace ":$([string]$localEndpoint.split(":")[-1])", "" "LocalPort" = $localEndpoint.split(":")[-1] "RemoteIP" = $remoteEndpoint -replace ":$([string]$remoteEndpoint.split(":")[-1])", "" "RemotePort" = $remoteEndpoint.split(":")[-1] "AuthenticationEnabled" = $authenticationEnabled "User" = $userName "RecipientType" = (. { if ($authDetails) { $authHash["RecipientType" ] } }) "RecipientTypeDetails" = (. { if ($authDetails) { $authHash["RecipientTypeDetails" ] } }) "DisplayName" = (. { if ($authDetails) { $authHash["DisplayName" ] } }) "MailboxGuid" = (. { if ($authDetails) { $authHash["MailboxGuid" ] } }) "DatabaseGuid" = (. { if ($authDetails) { $authHash["DatabaseGuid" ] } }) "ServerFqdn" = (. { if ($authDetails) { $authHash["ServerFqdn" ] } }) "ServerVersion" = (. { if ($authDetails) { $authHash["ServerVersion" ] } }) "DatabaseName" = (. { if ($authDetails) { $authHash["DatabaseName" ] } }) "HomePublicFolderDatabaseGuid" = (. { if ($authDetails) { $authHash["HomePublicFolderDatabaseGuid"] } }) "OwnerSID" = (. { if ($budgetDetails) { $budgetHash["OwnerSID"] } }) "ConnectionCount" = (. { if ($budgetDetails) { $budgetHash["Conn"] } }) "ConnectionMax" = (. { if ($budgetDetails) { $budgetHash["MaxConn"] } }) "MaxBurst" = (. { if ($budgetDetails) { $budgetHash["MaxBurst"] } }) "Balance" = (. { if ($budgetDetails) { $budgetHash["Balance"] } }) "Cutoff" = (. { if ($budgetDetails) { $budgetHash["Cutoff"] } }) "RechargeRate" = (. { if ($budgetDetails) { $budgetHash["RechargeRate"] } }) "Policy" = (. { if ($budgetDetails) { $budgetHash["Policy"] } }) "IsServiceAccount" = (. { if ($budgetDetails) { $budgetHash["IsServiceAccount"] } }) "LiveTime" = (. { if ($budgetDetails) { $budgetHash["LiveTime"] } }) "TotalObjects" = $rows "TotalSize" = $totalSize "HasError" = $hasError "ErrorMessage" = $errorMessage "IsProxysession" = $isProxysession "ProxyServer" = $proxyServer "ProxyStatus" = $proxyStatus "CmdCountStls" = $stls "CmdCountAuth" = $auth "CmdCountUser" = $user "CmdCountPass" = $pass "CmdCountStat" = $stat "CmdCountUidl" = $uidl "CmdCountList" = $list "CmdCountRetr" = $retr "CmdCountDele" = $dele "LogText" = $logText } # add metadata attributes foreach ($key in $record.metadataHash.Keys) { if ($key -like "Date") { $value = $record.metadataHash[$key] -as [datetime] } else { $value = $record.metadataHash[$key] } $outputRecord | Add-Member -MemberType NoteProperty -Name $key -Value $value -Force } # output data $outputRecord # report in detail if errors occur (for debugging because the processing in operating in runspaces) if ($Error) { Write-Warning "Error detected while processing $($outputRecord.LogFolder)\$($outputRecord.LogFileName) with $($record.$SessionIdName)" $Error.Clear() } # output progress of switch is set (only debugging purpose) if ($ShowProgress) { if (($i % $refreshInterval) -eq 0) { Write-Progress -Activity "Process logfile record " -Status "$($record.LogFileName) - $($SessionIdName): $($record.$SessionIdName) ($($i) / $($InputObject.count))" -PercentComplete ($i / $InputObject.count * 100) } $i = $i + 1 } } } end { } } (Get-Command Expand-LogRecordPopImap).Visibility = "Private" function global:Expand-LogRecordSmtp { <# .SYNOPSIS Expand the data from records group into a flat data record .DESCRIPTION Expand the data from records group into a flat data record .PARAMETER InputObject The dataset to expand .PARAMETER SessionIdName The name of the session grouping attribute .PARAMETER ShowProgress If specified, progress information on record processing is showed .EXAMPLE PS C:\> Expand-LogRecordSmtp -InputObject $DataSet Expand the data from records group into a flat data record #> [CmdletBinding()] param ( $InputObject, [string] $SessionIdName = "session-Id", [switch] $ShowProgress ) begin { $Error.Clear() } process { if ($ShowProgress) { $i = 0 if ($InputObject.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($InputObject.count / 100) } } foreach ($record in $InputObject) { # assure qualified session in log $_startIndicator = $record.Group | Where-Object Event -eq "+" $_stopIndicator = $record.Group | Where-Object Event -eq "-" if ((-not $_startIndicator) -or (-not $_stopIndicator)) { Write-PSFMessage -Level Warning -Message "Detect fragmented record! Skip processing $($SessionIdName) '$($record.$SessionIdName)' in $($record.LogFolder)\$($record.LogFileName)" continue } # data text record in array, avoids parsing full data array $groupData = $record.Group.data # full text log $logtext = "" foreach ($item in $record.Group) { $logtext = $logtext + "$(if($logtext){"`n"})" + $item.event + " " + $item.data if ($item.context.Length -gt 0) { $logtext = $logtext + " (context: " + $item.context + ")" } } # ServerName $serverName = $record.Group[0].'connector-id'.split("\")[0] # ServerNameHELO [string]$_serverNameHELO = $groupData | Where-Object { $_ -like "220 * Microsoft*" } | Select-Object -First 1 if ($_serverNameHELO) { [String]$serverNameHELO = $_serverNameHELO.TrimStart("220 ").Split(" ")[0] } else { [String]$serverNameHELO = "" } # ServerOptions $_serverOptions = foreach ($item in $groupData) { if ($item -match "^250\s\s\S+\sHello\s\[\S+]\s(?'ServerOptions'(\S|\s)+)") { $Matches['ServerOptions'] } } if ($_serverOptions) { [string]$serverOptions = [string]::Join(",", $_serverOptions) } else { [string]$serverOptions = "" } # ClientNameHELO $_clientNameHELO = foreach ($item in ($groupData -like "EHLO *" | Select-Object -Unique)) { ([string]$item).trim("EHLO ") } if ($_clientNameHELO) { [string]$clientNameHELO = [string]::Join(",", $_clientNameHELO) } else { [string]$clientNameHELO = "" } # MailFrom [string[]]$_mailFrom = ([array]$groupData -like "MAIL FROM:*") -Split 'MAIL FROM:' | Where-Object { $_ } if ($_mailFrom) { [string[]]$_mailFrom = ($_mailFrom.trim() -Replace '^<', '') | ForEach-Object { $_.split('>')[0] } | Select-Object -Unique | Where-Object { $_ } } if ($_mailFrom) { [string]$mailFrom = [string]::Join(",", $_mailFrom) } else { [string]$mailFrom = "" } # RcptTo [string[]]$_rcptTo = ([array]$groupData -like "RCPT TO:*") -Split 'RCPT TO:' | Where-Object { $_ } if ($_rcptTo) { [string[]]$_rcptTo = ($_rcptTo.trim() -Replace '^<', '') | ForEach-Object { $_.split('>')[0] } | Where-Object { $_ } } if ($_rcptTo) { [string]$rcptTo = [string]::Join(",", $_rcptTo) } else { [string]$rcptTo = "" } # XOOrg [string[]]$_xoorg = foreach ($item in $groupData) { if ($item -match "XOORG=(?'xoorg'\S+)") { $Matches['xoorg'] } } if ($_xoorg) { [string]$xoorg = [string]::Join(",", ($_xoorg.trim() | Select-Object -Unique)) } else { [string]$xoorg = "" } $smtpIdLine = $groupData -match "^250\s2.6.0\s<(?'SmtpId'\S+)" [timespan]$deliveryDuration = [timespan]::new(0) [double]$deliveryBandwidth = 0 [string]$remoteServerHostName = "" [string]$internalId = "" [int]$mailSize = 0 [String]$SmtpId = "" if ($smtpIdLine) { [string[]]$_smtpIdRecords = foreach ($line in $smtpIdLine) { $line.trim("250 2.6.0 <").split(">")[0] } if ($_smtpIdRecords) { $SmtpId = [string]::Join(",", $_smtpIdRecords.trim() ) } else { [string]$smtpId = "" } [string[]]$_remoteServerHostName = foreach ($item in $_smtpIdRecords) { $item.split("@")[1] } if ($_remoteServerHostName) { [string]$remoteServerHostName = [string]::Join(",", ($_remoteServerHostName | Select-Object -Unique) ) } else { [string]$remoteServerHostName = "" } [string[]]$_internalId = $smtpIdLine | ForEach-Object { ($_ -split "InternalId=")[1].split(",")[0] } if ($_internalId) { [string]$internalId = [string]::Join(",", $_internalId.trim() ) } else { [string]$internalId = "" } if ($smtpIdLine -like "*bytes in*") { [int[]]$_mailSize = $smtpIdLine | ForEach-Object { ($_ -split " bytes in ")[0].split(" ")[-1] } if ($_mailSize) { $mailSize = ($_mailSize | Measure-Object -Sum).Sum } else { [int]$mailSize = 0 } ForEach ($item in $smtpIdLine) { $deliveryDuration = $deliveryDuration + [timespan]::FromSeconds( [System.Convert]::ToDouble( (($item -split " bytes in ")[1].split(", ")[0]) , [cultureinfo]::GetCultureInfo('en-us') )) } ForEach ($item in $smtpIdLine) { $deliveryBandwidth = $deliveryBandwidth + [double]::Parse( (($item.TrimEnd(" KB/sec Queued mail for delivery") -split ", ")[-1]) ) $deliveryBandwidth = [math]::Round( ($deliveryBandwidth / $smtpIdLine.count), 3 ) } } } if ($record.Group | Where-Object data -like "Tarpit*") { $tarpitDetect = $true } else { $tarpitDetect = $false } $tarpitDuration = [timespan]::new(0) $tarpitMessage = "" if ($tarpitDetect) { $tarpitDuration = [timespan]::FromSeconds( (($record.Group | where-Object data -like "Tarpit*").data.replace("Tarpit for '", "") | ForEach-Object { $_.split("'")[0] -as [timespan] } | Measure-Object Seconds -Sum).Sum ) $tarpitMessage = (($record.Group | Where-Object data -like "Tarpit*").data -split "(due\sto\s')")[-1].trim("'") } [string]$connectorID = $record.Group[0].'connector-id' if ($connectorID) { if ($connectorID -match "\\") { $connectorName = $connectorID.split("\")[1] } else { $connectorName = $connectorID } } else { $connectorName = "" } if ($connectorID) { $connectorNameWithoutServerName = $connectorName.replace($serverName, "").trim() } else { $connectorNameWithoutServerName = "" } [string]$localEndpoint = $record.Group[-1].'local-endpoint' [string]$remoteEndpoint = $record.Group[-1].'remote-endpoint' if ($groupData -clike "AUTH *") { $authenticationEnabled = $true } else { $authenticationEnabled = $false } $authenticationType = "" $authenticationUser = "" $authenticationMessage = "" if ($authenticationEnabled) { $null = $record.Group | Where-Object data -Match "^AUTH\s(?'Method'\S+)" $authenticationType = $Matches['Method'] $authenticationUser = ($record.Group | Where-Object context -like "authenticated").data $text = @( "235 2.7.0 Authentication successful", "504 5.7.4 Unrecognized authentication type", "535 5.7.3 Authentication unsuccessful" ) [string]$authenticationMessage = ($record.Group | Where-Object data -in $text)[-1].data } # TLS records if ($groupData -clike " CN=*") { $tlsEnabled = $true } else { $tlsEnabled = $false } $tlsAlgorithmEncryption = "" $tlsAlgorithmKeyExchange = "" $tlsAlgorithmMacHash = "" $tlsCertificateRemote = "" $tlsCertificateRemoteIssuer = "" $tlsCertificateRemoteNotAfter = "" $tlsCertificateRemoteNotBefore = "" $tlsCertificateRemoteSAN = "" $tlsCertificateRemoteSerial = "" $tlsCertificateRemoteThumbprint = "" $TlsCertificateServer = "" $tlsCertificateServerIssuer = "" $tlsCertificateServerNotAfter = "" $tlsCertificateServerNotBefore = "" $tlsCertificateServerSAN = "" $tlsCertificateServerSerial = "" $tlsCertificateServerThumbprint = "" $tlsCrypto = "" $tlsDomain = "" $tlsDomainCapabilities = "" $tlsProtocol = "" $tlsStatus = "" $tlsStatusRecord = "" if ($tlsEnabled) { # gather TLS related records $tlsRecords = $record.Group | Where-Object { ($_.event -eq '*') -and ($_.context -like "Sending certificate*" -or $_.context -like "Remote certificate*" -or $_.context -like "TLS protocol*" -or $_.context -like "*TlsDomainCapabilities=*") } | Select-Object 'sequence-number', context, data if ($tlsRecords) { # TLS Crypto string $_tlsCrypto = $TlsRecords | Where-Object context -like "TLS *" | Select-Object -ExpandProperty context -Unique if ($_tlsCrypto) { $tlsCrypto = [string]::Join(",", $_tlsCrypto) } else { $tlsCrypto = "" } if ($tlsCrypto) { # TLS protocol $_tlsProtocol = foreach ($item in $tlsCrypto) { ([string]$item).Replace('TLS protocol ', '').Split(" ")[0] } if ($_tlsProtocol) { $tlsProtocol = [string]::Join(",", $_tlsProtocol) } else { $tlsProtocol = "" } # TLS Algorithm Encryption $_tlsAlgorithmEncryption = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "encryption algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmEncryption) { $tlsAlgorithmEncryption = [string]::Join(",", $_tlsAlgorithmEncryption) } else { $tlsAlgorithmEncryption = "" } # TLS Algorithm MacHash $_tlsAlgorithmMacHash = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "hash algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmMacHash) { $tlsAlgorithmMacHash = [string]::Join(",", $_tlsAlgorithmMacHash) } else { $tlsAlgorithmMacHash = "" } # TLS Algorithm KeyExchange $_tlsAlgorithmKeyExchange = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "exchange algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmKeyExchange) { $tlsAlgorithmKeyExchange = [string]::Join(",", $_tlsAlgorithmKeyExchange) } else { $tlsAlgorithmKeyExchange = "" } } # TLS Server certificate $_tlsCertificateServerRecord = $tlsRecords | where-Object context -Like "Sending certificat*" | Select-Object -First 1 -ExpandProperty data if ($_tlsCertificateServerRecord) { $_tlsCertificateServerRecord = $_tlsCertificateServerRecord.trim() [string]$certText = $_tlsCertificateServerRecord # TLS Server certificate - Subject alternate names [String]$tlsCertificateServerSAN = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerSAN).TrimEnd() # TLS Server certificate - Not after [String]$_tlsCertificateServerNotAfter = $certText.split(" ")[-1] $tlsCertificateServerNotAfter = [datetime]::Parse($_tlsCertificateServerNotAfter) $certText = $certText.TrimEnd($_tlsCertificateServerNotAfter).TrimEnd() # TLS Server certificate - Not before [String]$_tlsCertificateServerNotBefore = $certText.split(" ")[-1] $tlsCertificateServerNotBefore = [datetime]::Parse($_tlsCertificateServerNotBefore) $certText = $certText.TrimEnd($_tlsCertificateServerNotBefore).TrimEnd() # TLS Server certificate - Thumbprint [String]$tlsCertificateServerThumbprint = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerThumbprint).TrimEnd() # TLS Server certificate - Serial number [String]$tlsCertificateServerSerial = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerSerial).TrimEnd() # TLS Server certificate - Issuer name [String]$tlsCertificateServerIssuer = "CN=" + ($certText -split (" CN="))[1] $certText = $certText.TrimEnd($tlsCertificateServerIssuer).TrimEnd() # TLS Server certificate - Subject [String]$tlsCertificateServerIssuer = $certText } [String]$tlsCertificateServer = $_tlsCertificateServerRecord # TLS Remote certificate $_tlsCertificateRemoteRecord = $tlsRecords | where-Object context -Like "Remote certificat*" | Select-Object -First 1 -ExpandProperty data if ($_tlsCertificateRemoteRecord) { $_tlsCertificateRemoteRecord = $_tlsCertificateRemoteRecord.trim() [string]$certText = $_tlsCertificateRemoteRecord # TLS Remote certificate - Subject alternate names [String]$tlsCertificateRemoteSAN = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteSAN).TrimEnd() # TLS Remote certificate - Not after [String]$_tlsCertificateRemoteNotAfter = $certText.split(" ")[-1] $tlsCertificateRemoteNotAfter = [datetime]::Parse($_tlsCertificateRemoteNotAfter) $certText = $certText.TrimEnd($_tlsCertificateRemoteNotAfter).TrimEnd() # TLS Remote certificate - Not before [String]$_tlsCertificateRemoteNotBefore = $certText.split(" ")[-1] $tlsCertificateRemoteNotBefore = [datetime]::Parse($_tlsCertificateRemoteNotBefore) $certText = $certText.TrimEnd($_tlsCertificateRemoteNotBefore).TrimEnd() # TLS Remote certificate - Thumbprint [String]$tlsCertificateRemoteThumbprint = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteThumbprint).TrimEnd() # TLS Remote certificate - Serial number [String]$tlsCertificateRemoteSerial = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteSerial).TrimEnd() # TLS Remote certificate - Issuer name [String]$tlsCertificateRemoteIssuer = "CN=" + ($certText -split (" CN="))[1] $certText = $certText.TrimEnd($tlsCertificateRemoteIssuer).TrimEnd() # TLS Remote certificate - Subject [String]$tlsCertificateRemoteIssuer = $certText } [String]$tlsCertificateRemote = $_tlsCertificateRemoteRecord # TLS Status Record $_tlsStatusRecord = ($tlsRecords | Where-Object context -Like "*; Status='*").context -Split "; " if ($_tlsStatusRecord) { # TLS Status Record [String]$tlsStatusRecord = [string]::Join("; ", $_tlsStatusRecord) # TLS Domain Capabilities $_tlsDomainCapabilities = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "TlsDomainCapabilities=*" }) -split "=")[1] ).trim("'") if ($_tlsDomainCapabilities) { $tlsDomainCapabilities = [string]::Join(",", $_tlsDomainCapabilities) } else { $tlsDomainCapabilities = "" } # TLS Status $_tlsStatus = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "Status=*" }) -split "=")[1] ).trim("'") if ($_tlsStatus) { $tlsStatus = [string]::Join(",", $_tlsStatus) } else { $tlsStatus = "" } # TLS Domain $_tlsDomain = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "Domain=*" }) -split "=")[1] ).trim("'") if ($_tlsDomain) { $tlsDomain = [string]::Join(",", $_tlsDomain) } else { $tlsDomain = "" } } else { [String]$_tlsStatusRecord = "" } } } # final status $_finalStatus = $record.Group[-1].context if ($_finalStatus -like "Local") { $finalStatus = "OK" } elseif ($_finalStatus) { $finalStatus = $_finalStatus } else { $finalStatus = "UNKNOWN" } # construct output object $outputRecord = [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($record.metadataHash['Log-type'].Replace(' ','')).Record" "LogFolder" = $record.LogFolder "LogFileName" = $record.LogFileName $SessionIdName = $record.$SessionIdName "DateStart" = ($record.Group | Sort-Object 'date-time')[0].'date-time' -as [datetime] "DateEnd" = ($record.Group | Sort-Object 'date-time')[-1].'date-time' -as [datetime] "SequenceCount" = $record.Group.count "ConnectorID" = $ConnectorID "ServerName" = $ServerName "ConnectorName" = $ConnectorName "ConnectorNameWithoutServerName" = $ConnectorNameWithoutServerName "LocalIP" = $localEndpoint -replace ":$([string]$localEndpoint.split(":")[-1])", "" "LocalPort" = $localEndpoint.split(":")[-1] "RemoteIP" = $remoteEndpoint -replace ":$([string]$remoteEndpoint.split(":")[-1])", "" "RemotePort" = $remoteEndpoint.split(":")[-1] "ServerNameHELO" = $ServerNameHELO "ServerOptions" = $ServerOptions "ClientNameHELO" = $clientNameHELO "TlsEnabled" = $TlsEnabled "AuthenticationEnabled" = $AuthenticationEnabled "AuthenticationType" = $AuthenticationType "AuthenticationUser" = $AuthenticationUser "AuthenticationMessage" = $AuthenticationMessage "TarpitDetect" = $TarpitDetect "TarpitDuration" = $TarpitDuration "TarpitMessage" = $TarpitMessage "MailFrom" = $MailFrom "RcptTo" = $rcptTo "XOOrg" = $XOOrg "SmtpId" = $SmtpId "RemoteServerHostName" = $RemoteServerHostName "InternalId" = $InternalId "MailSize" = $MailSize "DeliveryDuration" = $DeliveryDuration "DeliveryBandwidth" = $deliveryBandwidth "FinalSessionStatus" = $finalStatus "FinalizeMessage" = $groupData[-2] "TlsProtocol" = $tlsProtocol "TlsAlgorithmEncryption" = $tlsAlgorithmEncryption "TlsAlgorithmMacHash" = $tlsAlgorithmMacHash "TlsAlgorithmKeyExchange" = $tlsAlgorithmKeyExchange "TlsCertificateServer" = $tlsCertificateServer "TlsCertificateServerIssuer" = $tlsCertificateServerIssuer "TlsCertificateServerNotAfter" = $tlsCertificateServerNotAfter "TlsCertificateServerNotBefore" = $tlsCertificateServerNotBefore "TlsCertificateServerSAN" = $tlsCertificateServerSAN "TlsCertificateServerSerial" = $tlsCertificateServerSerial "TlsCertificateServerThumbprint" = $tlsCertificateServerThumbprint "TlsCertificateRemote" = $tlsCertificateRemote "TlsCertificateRemoteIssuer" = $tlsCertificateRemoteIssuer "TlsCertificateRemoteNotAfter" = $tlsCertificateRemoteNotAfter "TlsCertificateRemoteNotBefore" = $tlsCertificateRemoteNotBefore "TlsCertificateRemoteSAN" = $tlsCertificateRemoteSAN "TlsCertificateRemoteSerial" = $tlsCertificateRemoteSerial "TlsCertificateRemoteThumbprint" = $tlsCertificateRemoteThumbprint "TlsStatus" = $tlsStatus "TlsDomain" = $tlsDomain "TlsDomainCapabilities" = $tlsDomainCapabilities "TlsCrypto" = $tlsCrypto "TlsStatusRecord" = $tlsStatusRecord "LogText" = $logText } # add metadata attributes foreach ($key in $record.metadataHash.Keys) { if ($key -like "Date") { $value = $record.metadataHash[$key] -as [datetime] } else { $value = $record.metadataHash[$key] } $outputRecord | Add-Member -MemberType NoteProperty -Name $key -Value $value -Force } # output data $outputRecord # report in detail if errors occur (for debugging because the processing in operating in runspaces) if ($Error) { Write-Warning "Error detected while processing $($outputRecord.LogFolder)\$($outputRecord.LogFileName) with $($record.$SessionIdName)" $Error.Clear() } # output progress of switch is set (only debugging purpose) if ($ShowProgress) { if (($i % $refreshInterval) -eq 0) { Write-Progress -Activity "Process logfile record " -Status "$($record.LogFileName) - $($SessionIdName): $($record.$SessionIdName) ($($i) / $($InputObject.count))" -PercentComplete ($i / $InputObject.count * 100) } $i = $i + 1 } } } end { } } (Get-Command Expand-LogRecordSmtp).Visibility = "Private" function Import-LogData { <# .SYNOPSIS Get csv content from logfile output grouped records .DESCRIPTION Get csv content from logfile output grouped records .PARAMETER File The file to gather data .EXAMPLE PS C:\> Import-LogData -File RECV2020061300-1.LOG Return the csv records as grouped records by SessionID #> [CmdletBinding()] [OutputType([System.Collections.ArrayList])] param ( $File ) # Get content from logfile $File = Get-Item -Path $File -ErrorAction Stop Write-PSFMessage -Level Verbose -Message "Get content from logfile: $($File.Fullname)" $content = $File | Get-Content # split metadata and logcontent (first lines fo logfile) $metadata = $content -match '^#.*' | Select-Object -First 7 -Unique $header = $metadata[-1].split(": ")[-1] $logcontent = $content -match '^\d.*' # query meta data informations into hashtable $metadataHash = [ordered]@{} foreach ($metadatarecord in ($metadata[0 .. ($metadata.Count - 2)])) { $_data = $metadatarecord.TrimStart("#") -Split ': ' $metadataHash.Add($_data[0], $_data[1]) } Write-PSFMessage -Level VeryVerbose -Message "Detect $($metadataHash['Log-type'])" # convert filecontent to csv data and group records if know/supportet logfile type $records = $logcontent | ConvertFrom-Csv -Delimiter "," -Header $header.Split(",") $sessionIdName = Resolve-SessionIdName -LogType $metadataHash['Log-type'] $output = New-Object -TypeName "System.Collections.ArrayList" if ($sessionIdName) { Write-PSFMessage -Level VeryVerbose -Message "Going to group records by $($sessionIdName) from file '$($File.Fullname)'" foreach ($group in ($records | Group-Object $sessionIdName)) { $null = $output.Add( [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Group" $sessionIdName = $group.Name "Group" = $group.Group "Log-type" = $metadataHash["Log-type"] "metadataHash" = $metadataHash "LogFileName" = $File.Name "LogFolder" = $File.Directory } ) } } else { foreach ($record in $records) { $record.PSOBject.TypeNames.Insert(0, "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Record" ) $null = $output.Add( $record ) } # add metadata info to record groups foreach ($key in $metadataHash.Keys) { $output | Add-Member -MemberType NoteProperty -Name $key -Value $metadataHash[$key] -Force } $output | Add-Member -MemberType NoteProperty -Name "LogFileName" -Value $File.Name -Force $output | Add-Member -MemberType NoteProperty -Name "LogFolder" -Value $File.Directory -Force } Write-PSFMessage -Level VeryVerbose -Message "Finished processing $($output.count) records from file '$($File.Name)'" # output data to the pipeline $output } function Import-LogData_typed { <# .SYNOPSIS Get csv content from logfile output grouped records .DESCRIPTION Get csv content from logfile output grouped records .PARAMETER File The file to gather data .EXAMPLE PS C:\> Import-LogData -File RECV2020061300-1.LOG Return the csv records as grouped records by SessionID #> [CmdletBinding()] [OutputType([System.Collections.ArrayList])] param ( $File ) # Get content from logfile $File = Get-ChildItem -Path $File -File -ErrorAction Stop Write-PSFMessage -Level Verbose -Message "Get content from logfile: $($File.Fullname)" $content = $File | Get-Content # split metadata and logcontent (first lines fo logfile) $metadata = $content -match '^#.*' $header = $metadata[-1].split(": ")[-1] $logcontent = $content -notmatch $header # query meta data informations into hashtable $metadataHash = [ordered]@{} foreach ($metadatarecord in ($metadata[0 .. ($metadata.Count - 2)])) { $_data = $metadatarecord.TrimStart("#") -Split ': ' $metadataHash.Add($_data[0], $_data[1]) } # convert filecontent to csv data and group records if know/supportet logfile type $records = $logcontent | ConvertFrom-Csv -Delimiter "," -Header $header.Split(",") $sessionIdName = Resolve-SessionIdName -LogType $metadataHash['Log-type'] $output = New-Object -TypeName "System.Collections.ArrayList" if($sessionIdName) { foreach ($group in ($records | Group-Object $sessionIdName)) { $null = $output.Add( [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).TypedRecord" $sessionIdName = $group.Name "Group" = $group.Group } ) } } else { foreach ($record in $records) { $record.PSOBject.TypeNames.Insert(0, "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Record" ) $null = $output.Add( $record ) } } # add metadata info to record groups foreach ($key in $metadataHash.Keys) { $output | Add-Member -MemberType NoteProperty -Name $key -Value $metadataHash[$key] -Force } $output | Add-Member -MemberType NoteProperty -Name "LogFileName" -Value $File.Name -Force $output | Add-Member -MemberType NoteProperty -Name "LogFolder" -Value $File.Directory -Force Write-PSFMessage -Level VeryVerbose -Message "Finished logfile $($File.Name). Found $($output.count) recors" # output data to the pipeline $output } function Initialize-LogFile { <# .SYNOPSIS Test for valid file and set PSF Logging provider .DESCRIPTION Test for valid file and create logfil directory if needed .PARAMETER LogFile Name of the LogType from the meta data line in a logfile .PARAMETER AlternateLogName An alternative name, if the provided logfile is only a directory, without filename .PARAMETER LogInstanceName The name of the Instance for the PSFramework logfile logging provider .EXAMPLE PS C:\> Initialize-LogFile -LogFile $LogFile -AlternateLogName $MyInvocation.MyCommand Test for valid file and create logfil directory if needed #> [CmdletBinding()] param ( $LogFile, $AlternateLogName, $LogInstanceName ) if (Test-Path -Path $LogFile -IsValid) { if (Test-Path -Path $LogFile -PathType Container) { $LogFile = Join-Path -Path $LogFile -ChildPath $AlternateLogName } $logFilePath = Split-Path -Path $LogFile if (-not (Test-Path -Path $logFilePath -PathType Container)) { try { $null = New-Item -Path $logFilePath -ItemType Directory -Force -ErrorAction Stop } catch { Stop-PSFFunction -Message "Unable to create logfile folder '$($logFilePath)'" -ErrorAction Stop -Tag "ExchangeLogs" throw } } # enable logging provider to write logging information (only events with Warning, Critical, Output, Significant, VeryVerbose, Verbose, SomewhatVerbose, System) Set-PSFLoggingProvider -Name logfile -InstanceName $LogInstanceName -Enabled $true -FilePath $LogFile -IncludeTags "ExchangeLogs" -MinLevel 1 -MaxLevel 6 } else { Stop-PSFFunction -Message "Invalid Logfile '$($LogFile)' specified." -ErrorAction Stop -Tag "ExchangeLogs" throw } } function Resolve-SessionIdName { <# .SYNOPSIS Converts access tokens to readable objects .DESCRIPTION Converts access tokens to readable objects .PARAMETER LogType Name of the LogType from the meta data line in a logfile .EXAMPLE PS C:\> Resolve-SessionIdName -LogType "SMTP Receive Protocol Log" Returns the name of the grouping field for building record groups #> [CmdletBinding()] [OutputType([System.String])] param ( $LogType ) switch ($LogType) { {$_ -in "SMTP Receive Protocol Log", "SMTP Send Protocol Log"} { "session-Id" } {$_ -in "IMAP4 Log", "POP3 Log"} { "sessionId" } {$_ -in "Message Tracking Log"} { "message-id" } Default { Write-Warning "Unknown LogType"} } } function Get-ELExchangeLog { <# .SYNOPSIS Get records from exchange logfiles .DESCRIPTION Get records from exchange logfiles like SMTP Receive / SMTP Send / IMAP / POP /... Files are parsed and grouped by sessions as possible. .PARAMETER Path The folder to gather logfiles .PARAMETER Recurse If specified, the path will be gathered recursive .PARAMETER Filter Filter to be applied for files to parse .PARAMETER LogType Specifies the type of logfile to work through. There are multiple types supported and usually the command does an autodectect. Use tab completion on the parameter to see the supported logfile types. Use this parameter of your workload expects an explicit type of log (for example "SMTPReceiveProtocolLog") and you want to ensure, that no other logfiles are processed. .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" Return records from all files in the folder .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" -Recursive Return records from all files in the current and all subfolders. #> [CmdletBinding()] [Alias('gel')] param ( [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true, ValueFromPipelineByPropertyName = $true)] [Alias('FullName')] [String[]] $Path, [switch] $Recurse, [String] $Filter = "*.log", [ValidateSet("AutoDetect", "SMTPReceiveProtocolLog", "SMTPSendProtocolLog", "IMAP4Log", "POP3Log","MessageTrackingLog")] [string] $LogType = "AutoDetect" ) begin { $files = New-Object -TypeName "System.Collections.ArrayList" $batchJobId = $([guid]::NewGuid().ToString()) Write-PSFMessage -Level VeryVerbose -Message "Starting BatchId '$($batchJobId)' with LogType '$LogType'" } process { # get files from folder Write-PSFMessage -Level Verbose -Message "Gettings files$( if($Filter){" by filter '$($Filter)'"} ) in path '$path'" foreach ($pathItem in $Path) { $options = @{ "Path" = $pathItem "File" = $true "ErrorAction" = "Stop" } if ($Recurse) { $options.Add("Recurse", $true) } try { $ChildItemList = Get-ChildItem @options | Where-Object Length -ne 0 if ($Filter) { $ChildItemList = $ChildItemList | Where-Object Name -like $Filter } ($ChildItemList).FullName | Sort-Object | ForEach-Object { [void]$files.Add( $_ ) } } catch { Stop-PSFFunction -Message "Error, path '$($pathItem)' not found" } } } end { if (-not $files) { Stop-PSFFunction -Message "No file found to parse! ($([string]::Join(", ", $Path)))" break } $recordCount = 0 $files = $files | Sort-Object if ($files.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($files.count / 100) } Write-PSFMessage -Level Verbose -Message "$($files.count) file$(if($files.count -gt 1){"s"}) to process." $traceTimer = New-Object System.Diagnostics.Stopwatch $traceTimer.Start() # Import first file if ($files.Count -gt 1) { $filePrevious = $files[0] } else { $filePrevious = $files } $resultPreviousFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $filePrevious)) { [void]$resultPreviousFile.Add($record) } $sessionIdName = Resolve-SessionIdName -LogType $resultPreviousFile[0].'Log-type' # file validity check if($LogType -ne "AutoDetect") { if($LogType -notlike $resultPreviousFile[0].'Log-type'.Replace(" ", "")) { Stop-PSFFunction -Message "Invalid LogType detected/specified. Expect '$($LogType)', but found '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' in file '$($filePrevious)'." break } } else { $LogType = $resultPreviousFile[0].'Log-type'.Replace(" ", "") } if($LogType -notin (Get-PSFConfigValue -FullName 'ExchangeLogs.SupportedLogTypes')) { Stop-PSFFunction -Message "Invalid LogType detected. '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' from file '$($filePrevious)' is not a supported log type to process with this command." break } # Import remaining files Write-PSFMessage -Level Verbose -Message "Starting import on $($files.Count) remaining file(s)." for ($filecounter = 1; $filecounter -lt $files.Count; $filecounter++) { #region process # import next file $fileCurrent = $files[$filecounter] $resultCurrentFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $fileCurrent)) { [void]$resultCurrentFile.Add($record) } if ($resultCurrentFile[0].'Log-type' -ne $resultPreviousFile[0].'Log-type') { Stop-PSFFunction -Message "Incompatible logfile types ($($resultCurrentFile[0].'Log-type'), $($resultPreviousFile[0].'Log-type')) found! More then one type of logfile in folder '$($pathItem)'." break } if ($sessionIdName) { # loop through previous and current file to check on fragmented session records in both files (sessions over midnight) Write-PSFMessage -Level VeryVerbose -Message "Checking for overlapping log records on identifier '$($sessionIdName)' in file '$($filePrevious)' and '$($fileCurrent)'" $overlapSessionIDs = Compare-Object -ReferenceObject $resultPreviousFile[-1..-20] -DifferenceObject $resultCurrentFile[0..20] -Property $sessionIdName -ExcludeDifferent -IncludeEqual if ($overlapSessionIDs) { foreach ($overlapSessionId in $overlapSessionIDs.$sessionIdName) { Write-PSFMessage -Level VeryVerbose -Message "Found overlapping log record '$($overlapSessionId)'" # get the record fragments from both logfiles $overlapRecordCurrentFile = $resultCurrentFile | Where-Object $sessionIdName -like $overlapSessionId $overlapRecordPreviousFile = $resultPreviousFile | Where-Object $sessionIdName -like $overlapSessionId # merge records $overlapRecordPreviousFileMerged = $overlapRecordPreviousFile $overlapRecordPreviousFileMerged.Group = $overlapRecordPreviousFile.Group + $overlapRecordCurrentFile.Group # remove overlapping records from current logfile $resultCurrentFile.RemoveAt( $resultCurrentFile.IndexOf($overlapRecordCurrentFile) ) # remove overlapping records from previous logfile $resultPreviousFile.RemoveAt( $resultPreviousFile.IndexOf($overlapRecordPreviousFile) ) # add merged record into previous logfile [void]$resultPreviousFile.Add($overlapRecordPreviousFileMerged) } } } # invoke data transform processing in a runspace to parallize processing and continue to work through files $recordCount = $recordCount + $resultPreviousFile.count switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # put current file records in variable for previous file to check for further record fragments $filePrevious = $fileCurrent $resultPreviousFile = $resultCurrentFile # progress status info & receive completed runspaces if (($filecounter % $refreshInterval) -eq 0 -or $filecounter -eq $files.count) { Write-PSFMessage -Level System -Message "Procesed $refreshInterval files... Going to update progress status" $jobs = Get-RSJob -Batch $batchJobId -ErrorAction SilentlyContinue -Verbose:$false | Sort-Object ID $jobsCompleted = $jobs | Where-Object State -Like "Completed" # output remaining data in completed runspace if ($jobsCompleted) { Wait-JobCompleteWithOutput -Job $jobsCompleted $recordsProcessed = Receive-RSJob -Job $jobsCompleted -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Write-PSFMessage -Level VeryVerbose -Message "Receiving $($jobsCompleted.Count) completed runspace job(s) with $($recordsProcessed.count) processed records" Remove-RSJob -Job $jobsCompleted -Verbose:$false } # status update Write-Progress -Activity "Import logfiles in $($resultPreviousFile[0].LogFolder) | Currently working runspaces: $($jobs.count - $jobsCompleted.count) | Records in processing: $($recordCount) | Time elapsed: $($traceTimer.Elapsed)" -Status "$($resultPreviousFile[0].LogFileName) ($($filecounter) / $($files.count))" -PercentComplete ($filecounter / $files.count * 100) } #endregion process } # processing last remaining file switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } $recordCount = $recordCount + $resultPreviousFile.count Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # waiting for completion of all runspaces Write-PSFMessage -Level Verbose -Message "Finished processing $($files.Count) file(s) with overall $($recordCount) record(s). Awaiting running runspaces to complete record processing." do { $jobs = Get-RSJob -Batch $batchJobId -Verbose:$false -ErrorAction SilentlyContinue | Sort-Object ID $jobsOpen = $jobs | Where-Object State -NotLike "Completed" $jobsCompleted = $jobs | Where-Object State -Like "Completed" [string]$_names = $jobsCompleted.Name | Split-Path -Leaf -ErrorAction SilentlyContinue if (-not $_names) { $_names = "" } Write-PSFMessage -Level VeryVerbose -Message "Awaiting runspaces to complete processing: $($jobsOpen.count)/$($jobs.count) ($([string]::Join(", ", $_names)))" Start-Sleep -Milliseconds 200 } while ($jobsOpen) # output remaining data in runspace Wait-JobCompleteWithOutput -Job $jobs Write-PSFMessage -Level Verbose -Message "All runspaces completed. Gathering results" $recordsProcessed = Receive-RSJob -Job $jobs -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Remove-RSJob -Batch $batchJobId -Verbose:$false $traceTimer.Stop() Write-PSFMessage -Level Significant -Message "Duration on parsing $($files.count) file(s) with $($recordCount) records: $($traceTimer.Elapsed)" } } function Invoke-ELCentralizeLogging { <# .SYNOPSIS Copy function for centralizing log files out of exchange servers .DESCRIPTION This one is a utility function inside the ExchangeLogs module. It's intendet to create a centralized logging directory to gather logfiles (supported by the module) from all exchange servers into a single directory or fileshare and withing a folder structure eligible for further processing. The intented workflow provided by this function: - find a exchange server to connect on via active directory serviceconnectionpoitns - connect to exchange server - find all exchange servers and subsequently query logging path for supported services - copy the logfiles out of the exchange servers to a central logging share (usually a staging folder, due to further processing with Invoke-ELExchangeLogConvert. See more info in Invoke-ELExchangeLogConvert help.) - remembering last processed file, to reduce noise and load on next processing Requirements: - The executing user needs the permission to connect with powershell remoting into exchange server - Predefined roles like "Organization Management" or "Server Management" can be used - Find/ create a individual management role with Get-ManagementRole cmdlet - The executing user needs permission to access the administrative shares on the server - The users needs read/write permission on the central network share The workflow will be: - First, find all logs and centralize them into a staging-folder with 'Invoke-ELCentralizeLogging' - Second, process staging-folder with 'Invoke-ELExchangeLogConvert' and build read- and processable CSV files in a reporting-folder - Use other BI-/Analytical tools to process the CSV Files in the reporting folder .PARAMETER Destination The path to copy all logfiles from the exchange server(s) to. Can be a local directory, or a fileshare (recommended) .PARAMETER FilterServer Filter parameter to in-/exclude certain exchange server from processing. The filter is applied in a "like"-matching process, so wildcards and multiple values for inclusion are supportet. Default value: * .PARAMETER IncludeLog Filter parameter to specify the type of logs to query and process. This filter ofers a predefined set of values: "All", "HubTransport", "Frontendtransport", "IMAP4", "POP3" Multiple values are supported. Default value: All .PARAMETER DirectoryLastProcessedFile The folder where the function search for the information which file was processed as last file on the previous run. Should be a local directory, but can also be a share. Default value: C:\Administration\Logs\Exchange\CentralizedLogs .PARAMETER LogFile The logfile to archive processing information. Default value: -not specified- Recommended: C:\Administration\Logs\Exchange\CentralizeExchangeLogs.log .PARAMETER WhatIf If this switch is enabled, no actions are performed but informational messages will be displayed that explain what would happen if the command were to run. .PARAMETER Confirm If this switch is enabled, you will be prompted for confirmation before executing any operations that change state. .EXAMPLE PS C:\> Invoke-ELCentralizeLogging -Destination "\\SRV01\Logs\Exchange\Staging" Gathers all exchange servers with all supported logs and copy it to the share "\\SRV01\Logs\Exchange\Staging". No log is written, there are basic output information on the console and extended logging information in the session via "Get-PSFMessage". .EXAMPLE PS C:\> Invoke-ELCentralizeLogging -Destination "\\$($env:USERDNSDOMAIN)\system\Logs\Exchange\Staging" -LogFile "C:\Administration\Logs\CentralizedLogs\CentralizeExchangeLogs.log" Gathers all exchange servers with all supported logs and copy it to the share "\\$($env:USERDNSDOMAIN)\System\Logs\Exchange\Staging". This one assumes, there is a DFS share "System" in your domain with a folder "logs". Script actions are written to a logfile 'C:\Administration\Logs\CentralizedLogs\CentralizeExchangeLogs.log' for informationtracking of the process state. This one is usally recommended for usage in scheduled tasks/ automation. Additonally, there are basic output information on the console and extended logging information in the session via "Get-PSFMessage". .EXAMPLE PS C:\> Invoke-ELCentralizeLogging -Destination "\\SRV01\Logs\Exchange\Staging" -DirectoryLastProcessedFile "C:\Administration\Logs\CentralizedLogs" -LogFile "C:\Administration\Logs\CentralizedLogs\CentralizeExchangeLogs.log" -FilterServer "Ex01", "Ex02" -IncludeLog "HubTransport", "Frontendtransport" Gathers only exchange server "Ex01" and "Ex02" and only SMTP logs (Hub and Frontend transport service) and copy it to the share "\\SRV01\Logs\Exchange\Staging". Last processed files will be remembered in directory "C:\Administration\Logs\CentralizedLogs". This is the default used directory, but it can be changed to anything else. Script actions are written to a logfile 'C:\Administration\Logs\CentralizedLogs\CentralizeExchangeLogs.log' for informationtracking of the process state. This one is usally recommended for usage in scheduled tasks/ automation. Additonally, there are basic output information on the console and extended logging information in the session via "Get-PSFMessage". .EXAMPLE PS C:\> Invoke-ELCentralizeLogging -Path "\\$($env:USERDNSDOMAIN)\system\Logs\Exchange\Staging" -LastFileDirectory "C:\Administration\Logs\CentralizedLogs" Same result as in previous examples but with alias "Path" and "LastFileDirectory" on parameters "Destination" and "DirectoryLastProcessedFile". #> [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param( [Parameter(Mandatory = $true)] [Alias("Path")] [String] $Destination, [string[]] $FilterServer = "*", [ValidateSet("All", "HubTransport", "Frontendtransport", "IMAP4", "POP3")] [string[]] $IncludeLog = "All", [Alias("LastFileDirectory")] [String] $DirectoryLastProcessedFile = "C:\Administration\Logs\Exchange\CentralizedLogs", [String] $LogFile ) begin {} process {} end { #region Init and prerequisites if ($LogFile) { Initialize-LogFile -LogFile $LogFile -AlternateLogName $MyInvocation.MyCommand -LogInstanceName "ExchangeLogs" } Write-PSFMessage -Level Host -Message "----- Start script -----" -Tag "ExchangeLogs" # variables Write-PSFMessage -Level System -Message "Initialize variables" -Tag "ExchangeLogs" if ($DirectoryLastProcessedFile.EndsWith("\")) { $DirectoryLastProcessedFile = $DirectoryLastProcessedFile.TrimEnd("\") } if ($Destination.EndsWith("\")) { $Destination = $Destination.TrimEnd("\") } # find exchange server in active directory try { $adsiSearch = ([ADSISearcher]'(&(objectclass=serviceconnectionpoint)(|(keywords=77378F46-2C66-4aa9-A6A6-3E7A48B19596)(keywords="67661d7F-8FC4-4fa7-BFAC-E1D7794C1F68")))') $adsiSearch.SearchRoot = [adsi]"LDAP://CN=Services,CN=Configuration,$(([adsi]::new()).distinguishedName)" $exServerName = ($adsiSearch.FindOne()).Properties['Name'] } catch { Stop-PSFFunction -Message "Unable to query active directory for exchange serviceconnectionpoint (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } Write-PSFMessage -Level System -Message "Found exchange server: $($exServerName)" -Tag "ExchangeLogs" # check directories Write-PSFMessage -Level System -Message "Check directory to memorize last processed file ($($DirectoryLastProcessedFile))" -Tag "ExchangeLogs" if (-not (Test-Path -Path $DirectoryLastProcessedFile)) { try { $null = New-Item -Path $DirectoryLastProcessedFile -ItemType Directory -Force -ErrorAction Stop } catch { Stop-PSFFunction -Message "Error creating directory to memorize last processed file '$($DirectoryLastProcessedFile)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } } Write-PSFMessage -Level System -Message "Check destination directory ($($Destination))" -Tag "ExchangeLogs" if (-not (Test-Path -Path $Destination)) { try { $null = New-Item -Path $Destination -ItemType Directory -Force -ErrorAction Stop } catch { Stop-PSFFunction -Message "Error creating destination directory '$($DirectoryLastProcessedFile)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } } # connect to exchange server Write-PSFMessage -Level Verbose -Message "Init exchange session" -Tag "ExchangeLogs" try { $exSession = New-PSSession -ConfigurationName Microsoft.Exchange -ErrorAction Stop -ConnectionUri "http://$($exServerName)/PowerShell/" $null = Import-PSSession $exSession -DisableNameChecking -ErrorAction Stop } catch { Stop-PSFFunction -Message "Error connecting exchange session '$($exServerName)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } Write-PSFMessage -Level Verbose -Message "Created $($exSession.count) sessions. Server:$($exServerName)" -Tag "ExchangeLogs" #endregion Init and prereqs #region Query Logdata from exchange # query log paths $exServer = Get-ExchangeServer $exServer = foreach ($filterItem in $FilterServer) { $exServer | Where-Object -Property Name -Like $filterItem } $exServer = $exServer | Sort-Object -Property Name -Unique Write-PSFMessage -Level Verbose -Message "Found $($exServer.count) exchange server" -Tag "ExchangeLogs" if("All" -in $IncludeLog -or "HubTransport" -in $IncludeLog) { $logPathSmtpHub = Get-TransportService | Sort-Object -Property Name | Select-Object -Property Name, SendProtocolLogPath, ReceiveProtocolLogPath Write-PSFMessage -Level Verbose -Message "Found $($logPathSmtpHub.count) TransportService" -Tag "ExchangeLogs" } if("All" -in $IncludeLog -or "Frontendtransport" -in $IncludeLog) { $logPathSmtpFrontEnd = Get-FrontEndTransportService | Sort-Object -Property Name | Select-Object -Property Name, SendProtocolLogPath, ReceiveProtocolLogPath Write-PSFMessage -Level Verbose -Message "Found $($logPathSmtpFrontEnd.count) FrontEndTransportService" -Tag "ExchangeLogs" } if("All" -in $IncludeLog -or "IMAP4" -in $IncludeLog) { $logPathImap = $exServer | ForEach-Object { Get-ImapSettings -Server $_.name } | Sort-Object -Property Server | Select-Object -Property Server, LogFileLocation Write-PSFMessage -Level Verbose -Message "Found $($logPathImap.count) ImapSettings" -Tag "ExchangeLogs" } if("All" -in $IncludeLog -or "POP3" -in $IncludeLog) { $logPathPop = $exServer | ForEach-Object { Get-PopSettings -Server $_.name } | Sort-Object -Property Server | Select-Object -Property Server, LogFileLocation Write-PSFMessage -Level Verbose -Message "Found $($logPathPop.count) PopSettings" -Tag "ExchangeLogs" } # build server/path list $sourcePaths = @() # SMTP Hub Transportservice foreach ($item in $logPathSmtpHub) { $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Name "Path" = $item.SendProtocolLogPath "Type" = "Transport-Hub" } $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Name "Path" = $item.ReceiveProtocolLogPath "Type" = "Transport-Hub" } } # SMTP Frontend Transportservice foreach ($item in $logPathSmtpFrontEnd) { $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Name "Path" = $item.SendProtocolLogPath "Type" = "Transport-FrontEnd" } $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Name "Path" = $item.ReceiveProtocolLogPath "Type" = "Transport-FrontEnd" } } # IMAP service foreach ($item in $logPathImap) { $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Server "Path" = $item.LogFileLocation "Type" = "ClientAccess" } } # POP3 service foreach ($item in $logPathPop) { $sourcePaths += [PSCustomObject]@{ "ComputerName" = $item.Server "Path" = $item.LogFileLocation "Type" = "ClientAccess" } } $sourcePaths = $sourcePaths | Sort-Object ComputerName, Path Write-PSFMessage -Level Verbose -Message "$($sourcePaths.count) paths to process." -Tag "ExchangeLogs" #endregion Query Logdata from exchange #region process logfiles Write-PSFMessage -Level Host -Message "Start working through each server and each directory" -Tag "ExchangeLogs" # work through each server and each directory foreach ($server in $exServer.name) { Write-PSFMessage -Level Verbose -Message "Working with $($server)" | Out-File -FilePath $LogFile -Encoding default -Force -Append $sourcePathInServer = $sourcePaths | Where-Object ComputerName -like $server foreach ($source in $sourcePathInServer) { # variables for directory $counter = 0 $processedFiles = @() $lastFile = "$($DirectoryLastProcessedFile)\Last-$($source.Type)-$($source.Path.split("\")[-1])-$($server).xml" $destinationPath = "$($Destination)\$($source.Type)\$($source.Path.split("\")[-1])\$($server)" Write-PSFMessage -Level Verbose -Message "Processing: $($source) --to--> '$destinationPath'" -Tag "ExchangeLogs" # test and create destination directory Write-PSFMessage -Level VeryVerbose -Message "Check destination path '$($destinationPath)'" -Tag "ExchangeLogs" if (-not (Test-Path -Path $destinationPath)) { try { $null = New-Item -Path $destinationPath -ItemType Directory -Force } catch { Stop-PSFFunction -Message "Unable to create directory '$($destinationPath)' (Message: $($_.Exception.Message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } } # gathering files $sourcePath = "\\$($server)\$($source.path.Replace(":","$"))" $sourceFiles = Get-ChildItem -Path $sourcePath | Sort-Object -Property Name Write-PSFMessage -Level VeryVerbose -Message "There are $($sourceFiles.count) files in $($source)" -Tag "ExchangeLogs" # enumerate files in source to getting an filterable order foreach ($file in $sourceFiles) { $file | Add-Member -NotePropertyName "Order" -NotePropertyValue $counter -Force $counter++ } # if exist, query last processed file if (Test-Path -Path $LastFile) { $lastFileProcessed = Import-PSFClixml -Path $LastFile } else { $lastFileProcessed = $null } if ($lastFileProcessed) { $sourceFileStartFilter = $sourceFiles | Where-Object Name -like $lastFileProcessed.Name $sourceFiles = $sourceFiles | Where-Object order -gt $sourceFileStartFilter.Order } # ignore current logfile $sourceFiles = $sourceFiles[0 .. ($sourceFiles.count - 2)] Write-PSFMessage -Level VeryVerbose -Message "$($sourceFiles.count) files to process" -Tag "ExchangeLogs" # copy files to destination foreach ($file in $sourceFiles) { Write-PSFMessage -Level Debug -Message "Copy $($file.Name)" -Tag "ExchangeLogs" try { if ($pscmdlet.ShouldProcess("$($file.FullName) to $($destinationPath)", "Copy")) { Copy-Item -Path $file.FullName -Destination $destinationPath -ErrorAction Stop } $processedFiles += $file } catch { Stop-PSFFunction -Message "Unable to copy file $($file.name) to $($destinationPath). (Message: $($_.Exception.Message))" -ErrorRecord $_ -Exception $_.exception -Tag "ExchangeLogs" throw } } # export lastprocessed file for next run if ($processedFiles) { Write-PSFMessage -Level Host -Message "Processed $($processedFiles.count) files and remembering LastFileProcessed: $($processedFiles[-1].FullName)" -Tag "ExchangeLogs" if ($pscmdlet.ShouldProcess("Last processed file '$($processedFiles[-1].FullName)' to $($lastFile)", "Export")) { $processedFiles[-1] | Export-PSFClixml -Path $lastFile -Force } } else { Write-PSFMessage -Level Host -Message "No files processed from directory: $($sourcePath)" -Tag "ExchangeLogs" } } } #endregion process logfiles #region cleanup Write-PSFMessage -Level VeryVerbose -Message "Closing exchange session" -Tag "ExchangeLogs" $exSession | Remove-PSSession Write-PSFMessage -Level Host -Message "*** Finishing script ***" -Tag "ExchangeLogs" #endregion cleanup } } function Invoke-ELExchangeLogConvert { <# .SYNOPSIS Convert frunction to recursively process a folder structure and convert exchange logs to CSV files .DESCRIPTION This one is a utility function inside the ExchangeLogs module. It's intendet to parse through a folder structure (staging directory, filled by Invoke-ELCentralizeLogging), find all exchange logfiles (supported by the module) and convert the files into flatten and better read-/processable CSV files. The intented workflow provided by this function: - Parse through "Source" folder structure and find logfiles - convert logfiles to csv files and put them in the "Destination" folder. The folder structrue from source directory will be preserved and rebuild in destination folder - Processed files will be move to "Archive" folder. The folder structrue from source directory will be preserved and rebuild in archive folder. Requirements: - The users needs read/write permission on the central network share The workflow will be: - First, find all logs and centralize them into a staging-folder with 'Invoke-ELCentralizeLogging' - Second, process staging-folder with 'Invoke-ELExchangeLogConvert' and build read- and processable CSV files in a reporting-folder - Use other BI-/Analytical tools to process the CSV Files in the reporting folder .PARAMETER Source Path to folder where logfiles are stored for processing This one can also called 'staging directory' .PARAMETER Destination Path to store converted CSV files from exchange service logs This one can also called 'reporting directory' Default value: A folder "Reporting" next to the folder specified in source parameter .PARAMETER Archive Path fo archiving the processed exchange service log files out of the source directory This is only for archival reason. If no folder is specified, the logs in the source folder will be deleted! Default value: A folder "Archive" next to the folder specified in source parameter .PARAMETER Filter Name filter for files to be processed in the source directory Default value: *.log .PARAMETER MaxFileCount Amount of files processed in a processing cycle per directory. The higher the number, the more memory and time is consumed for processing. Assume usally at least 24 files per service (SMTP, IMAP, POP, ...) per exchange server Default value: 200 .PARAMETER LogFile The logfile to archive processing information. Default value: -not specified- Recommended: C:\Administration\Logs\Exchange\ExchangeLogConvert.log .PARAMETER WhatIf If this switch is enabled, no actions are performed but informational messages will be displayed that explain what would happen if the command were to run. .PARAMETER Confirm If this switch is enabled, you will be prompted for confirmation before executing any operations that change state. .EXAMPLE PS C:\> Invoke-ELExchangeLogConvert -Source "\\SRV01\Logs\Exchange\Staging" This will convert *.log files in all folders in path "\\SRV01\Logs\Exchange\Staging". The parameter "Source" can also be used with the alias "Path". Converted data will be put in CSV files in a folder "Reporting" in path "\\SRV01\Logs\Exchange". Processed files will be archived in a folder "Archive" in path "\\SRV01\Logs\Exchange". .EXAMPLE PS C:\> Invoke-ELExchangeLogConvert -Path "\\SRV01\Logs\Exchange\Staging" -Destination "\\SRV01\Logs\Exchange\Reporting" -Archive "\\SRV01\Logs\Exchange\Archive" Same behavior as in the first example, but the output path and the archival path is specified explicitly. Alias "Path" is used on the parameter "Source". .EXAMPLE PS C:\> Invoke-ELExchangeLogConvert -Source "\\$($env:USERDNSDOMAIN)\System\Logs\Exchange\Staging" -Destination "\\$($env:USERDNSDOMAIN)\System\Logs\Exchange\Reporting" -Archive "\\$($env:USERDNSDOMAIN)\system\Logs\Exchange\Archive" -MaxFileCount 1000 -LogFile "C:\Administration\Logs\Exchange\ExchangeLogConvert.log" In this case, the function will process a maxium of 1000 files (default is 200) and actions will be logged in logfile "C:\Administration\Logs\Exchange\ExchangeLogConvert.log" #> [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param( [Parameter(Mandatory = $true)] [Alias("Path")] [String] $Source, [String] $Destination = "$(Split-Path -Path $Source)\Reporting", [String] $Archive = "$(Split-Path -Path $Source)\Archive", [String] $Filter = "*.log", [int] $MaxFileCount = 200, [String] $LogFile = "C:\Administration\Logs\Exchange\ExchangeLogConvert.log" ) begin {} process {} end { #region Init and prerequisites if ($LogFile) { Initialize-LogFile -LogFile $LogFile -AlternateLogName $MyInvocation.MyCommand -LogInstanceName "ExchangeLogs" } Write-PSFMessage -Level Host -Message "----- Start script -----" -Tag "ExchangeLogs" # variables Write-PSFMessage -Level System -Message "Initialize variables" -Tag "ExchangeLogs" if ($Source.EndsWith("\")) { $Source = $Source.TrimEnd('\') } if ($Destination.EndsWith("\")) { $Destination = $Destination.TrimEnd('\') } if ($Archive.EndsWith("\")) { $Archive = $Archive.TrimEnd('\') } $dateStamp = Get-Date -Format "yyyy-MM-dd_HH-mm-ss" # check paths Write-PSFMessage -Level System -Message "Checking source-, destination- and archival-directory" -Tag "ExchangeLogs" # source try { $sourceDir = $Source | Get-Item -ErrorAction Stop } catch { Stop-PSFFunction -Message "Source directory not found! (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } # destination try { $destinationDir = $Destination | Get-Item -ErrorAction Stop } catch { Stop-PSFFunction -Message "Destination directory not found! (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } # archive try { $archiveDir = $Archive | Get-Item -ErrorAction Stop } catch { Stop-PSFFunction -Message "Archive directory not found! (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } #endregion Init and prerequisites #region recursing function function Invoke-SubDirectoryProcessing { [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param ( $SourceDir, $DestinationDir, $ArchiveDir, $Filter, $LogFile, [string] $DateStamp, [int] $MaxFileCount, [String] $NamePart ) $subDirectory = $SourceDir | Get-ChildItem -Directory foreach ($directory in $subDirectory) { Write-PSFMessage -Level Verbose -Message "Working on directory '$($directory.fullname)'" -Tag "ExchangeLogs" # variables $destPath = "$($DestinationDir)\$($directory.Name)" $archivePath = "$($ArchiveDir)\$($directory.Name)" if ($NamePart) { $NamePart = "$($NamePart)-$($directory.Name)" } else { $NamePart = $directory.Name } $tempFile = "$($env:TEMP)\ELExchangeLogConvert_$([guid]::NewGuid().tostring()).csv" # folder tests Write-PSFMessage -Level System -Message "Checking directory '$($destPath)' exists" -Tag "ExchangeLogs" if (-not (Test-Path -Path $destPath)) { try { $null = New-Item -Path $destPath -ItemType Directory -Force -ErrorAction Stop } catch { Stop-PSFFunction -Message "Error creating '$($destPath)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } } Write-PSFMessage -Level System -Message "Checking directory '$($archivePath)' exists" -Tag "ExchangeLogs" if (-not (Test-Path -Path $archivePath)) { try { $null = New-Item -Path $archivePath -ItemType Directory -Force -ErrorAction Stop } catch { Stop-PSFFunction -Message "Error creating '$($archivePath)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } } # file processing $filesToProcess = $directory | Get-ChildItem -Filter $Filter -File -Force | Select-Object -First $MaxFileCount if ($filesToProcess) { Write-PSFMessage -Level Verbose -Message "$($filesToProcess.count) files to process in directory '$($directory.name)'" -Tag "ExchangeLogs" # import and convert data from logfiles Write-PSFMessage -Level System -Message "Start ExchangeLog conversion in temp file '$($tempFile)'" -Tag "ExchangeLogs" try { if ($pscmdlet.ShouldProcess("$($filesToProcess.count) files in $($directory) and save results to $($tempFile)", "Convert")) { $filesToProcess | Get-ELExchangeLog -ErrorAction Stop | Export-Csv -Path $tempFile -Delimiter ";" -Encoding UTF8 -NoTypeInformation -Force -Append -ErrorAction Stop } } catch { Stop-PSFFunction -Message "Error importing and converting data from folder '$($directory.fullname)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } Write-PSFMessage -Level System -Message "Finsh conversion into temp file '$($tempFile)'" -Tag "ExchangeLogs" # explort data to csv file $exportFilePath = "$($destPath)\$($NamePart)_$($DateStamp).csv" Write-PSFMessage -Level System -Message "Copy temp file to destination '$($exportFilePath)'" -Tag "ExchangeLogs" try { if (Test-Path -Path $exportFilePath) { # unlikely, but possible if ($pscmdlet.ShouldProcess("Content from '$($tempFile)' into existing file $($exportFilePath)", "Add")) { Get-Content -Path $tempFile -ErrorAction Stop | Out-File -FilePath $exportFilePath -Encoding UTF8 -Append -ErrorAction Stop $exportFile = Get-Item -Path $exportFilePath -ErrorAction Stop } } else { # hopefully, the usual case if ($pscmdlet.ShouldProcess("'$($tempFile)' to '$($exportFilePath)'", "Copy")) { $exportFile = Copy-Item -Path $tempFile -Destination $exportFilePath -PassThru -ErrorAction Stop } } } catch { Stop-PSFFunction -Message "Error exporting data to csv file '$($exportFilePath)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } Write-PSFMessage -Level Verbose -Message "Done with data export. Filesize: $($exportFile.Length / 1KB) KB, File: $($exportFile.FullName)" -Tag "ExchangeLogs" # temp file cleanup Write-PSFMessage -Level System -Message "Going to clean up '$($tempFile)'" -Tag "ExchangeLogs" if ($pscmdlet.ShouldProcess("$($tempFile)", "Remove")) { Remove-Item -Path $tempFile -Force -Confirm:$false -WhatIf:$false } # move processed data to archive Write-PSFMessage -Level Verbose -Message "Moving processed files to '$($archivePath)'" -Tag "ExchangeLogs" try { if ($pscmdlet.ShouldProcess("$($filesToProcess.count) files from '$($destPath)' to '$($archivePath)'", "Move")) { $filesToProcess | Move-Item -Destination $archivePath -Force -ErrorAction Stop } } catch { Stop-PSFFunction -Message "Error moving files to archive directory '$($archivePath)' (Message: $($_.exception.message))" -ErrorRecord $_ -Exception $_.exception -ErrorAction Stop -Tag "ExchangeLogs" throw } } else { Write-PSFMessage -Level System -Message "No files to process in directory '$($directory.name)'" -Tag "ExchangeLogs" } # invoke recursion down the subdirectories Write-PSFMessage -Level System -Message "Calling recursive function for next directory" -Tag "ExchangeLogs" $paramsSubDirectoryProcessing = @{ "SourceDir" = $directory "DestinationDir" = $destPath "ArchiveDir" = $archivePath "Filter" = $Filter "MaxFileCount" = $MaxFileCount "Log" = $LogFile "DateStamp" = $DateStamp "NamePart" = $NamePart } if (Test-PSFParameterBinding -ParameterName "Whatif") { $paramsSubDirectoryProcessing.Add("Whatif", $true) } if (Test-PSFParameterBinding -ParameterName "Confirm") { $paramsSubDirectoryProcessing.Add("Confirm", $true) } Invoke-SubDirectoryProcessing @paramsSubDirectoryProcessing # trim name part if ($NamePart) { $NamePart = $NamePart.TrimEnd("-$($directory.Name)") } } } #endregion recursing function #region main script Write-PSFMessage -Level Host -Message "Start parsing '$($Source)' for log files to process." -Tag "ExchangeLogs" $paramsSubDirectoryProcessing = @{ "SourceDir" = $Source "DestinationDir" = $Destination "ArchiveDir" = $Archive "Filter" = $Filter "MaxFileCount" = $MaxFileCount "Log" = $LogFile "DateStamp" = $DateStamp "NamePart" = $NamePart } if (Test-PSFParameterBinding -ParameterName "Whatif") { $paramsSubDirectoryProcessing.Add("Whatif", $true) } if (Test-PSFParameterBinding -ParameterName "Confirm") { $paramsSubDirectoryProcessing.Add("Confirm", $true) } Invoke-SubDirectoryProcessing @paramsSubDirectoryProcessing Write-PSFMessage -Level Host -Message "*** Finishing script ***" -Tag "ExchangeLogs" #endregion main script } } <# This is an example configuration file By default, it is enough to have a single one of them, however if you have enough configuration settings to justify having multiple copies of it, feel totally free to split them into multiple files. #> <# # Example Configuration Set-PSFConfig -Module 'ExchangeLogs' -Name 'Example.Setting' -Value 10 -Initialize -Validation 'integer' -Handler { } -Description "Example configuration setting. Your module can then use the setting using 'Get-PSFConfigValue'" #> Set-PSFConfig -Module 'ExchangeLogs' -Name 'Import.DoDotSource' -Value $false -Initialize -Validation 'bool' -Description "Whether the module files should be dotsourced on import. By default, the files of this module are read as string value and invoked, which is faster but worse on debugging." Set-PSFConfig -Module 'ExchangeLogs' -Name 'Import.IndividualFiles' -Value $false -Initialize -Validation 'bool' -Description "Whether the module files should be imported individually. During the module build, all module code is compiled into few files, which are imported instead by default. Loading the compiled versions is faster, using the individual files is easier for debugging and testing out adjustments." Set-PSFConfig -Module 'ExchangeLogs' -Name 'SupportedLogTypes' -Value @( "SMTPReceiveProtocolLog", "SMTPSendProtocolLog", "IMAP4Log", "POP3Log", "MessageTrackingLog" ) -Initialize -Description "The name of the logfile types, specified in the header information of any exchange log file. This collection provides a list of supported formats to parsing for module ExchangeLogs." <# Stored scriptblocks are available in [PsfValidateScript()] attributes. This makes it easier to centrally provide the same scriptblock multiple times, without having to maintain it in separate locations. It also prevents lengthy validation scriptblocks from making your parameter block hard to read. Set-PSFScriptblock -Name 'ExchangeLogs.ScriptBlockName' -Scriptblock { } #> <# # Example: Register-PSFTeppScriptblock -Name "ExchangeLogs.alcohol" -ScriptBlock { 'Beer','Mead','Whiskey','Wine','Vodka','Rum (3y)', 'Rum (5y)', 'Rum (7y)' } #> <# # Example: Register-PSFTeppArgumentCompleter -Command Get-Alcohol -Parameter Type -Name ExchangeLogs.alcohol #> New-PSFLicense -Product 'ExchangeLogs' -Manufacturer 'andre' -ProductVersion $script:ModuleVersion -ProductType Module -Name MIT -Version "1.0.0.0" -Date (Get-Date "2020-06-13") -Text @" Copyright (c) 2020 Andreas Bellstedt Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. "@ #endregion Load compiled code |