functions/Get-ELExchangeLog.ps1
|
function Get-ELExchangeLog { <# .SYNOPSIS Get records from exchange logfiles .DESCRIPTION Get records from exchange logfiles like SMTP Receive / SMTP Send / IMAP / POP /... Files are parsed and grouped by sessions as possible. .PARAMETER Path The folder to gather logfiles .PARAMETER Recurse If specified, the path will be gathered recursive .PARAMETER Filter Filter to be applied for files to parse .PARAMETER LogType Specifies the type of logfile to work through. There are multiple types supported and usually the command does an autodectect. Use tab completion on the parameter to see the supported logfile types. Use this parameter of your workload expects an explicit type of log (for example "SMTPReceiveProtocolLog") and you want to ensure, that no other logfiles are processed. .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" Return records from all files in the folder .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" -Recursive Return records from all files in the current and all subfolders. #> [CmdletBinding()] [Alias('gel')] param ( [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true, ValueFromPipelineByPropertyName = $true)] [Alias('FullName')] [String[]] $Path, [switch] $Recurse, [String] $Filter = "*.log", [ValidateSet("AutoDetect", "SMTPReceiveProtocolLog", "SMTPSendProtocolLog", "IMAP4Log", "POP3Log","MessageTrackingLog")] [string] $LogType = "AutoDetect" ) begin { $files = New-Object -TypeName "System.Collections.ArrayList" $batchJobId = $([guid]::NewGuid().ToString()) Write-PSFMessage -Level VeryVerbose -Message "Starting BatchId '$($batchJobId)' with LogType '$LogType'" } process { # get files from folder Write-PSFMessage -Level Verbose -Message "Gettings files$( if($Filter){" by filter '$($Filter)'"} ) in path '$path'" foreach ($pathItem in $Path) { $options = @{ "Path" = $pathItem "File" = $true "ErrorAction" = "Stop" } if ($Recurse) { $options.Add("Recurse", $true) } try { $ChildItemList = Get-ChildItem @options | Where-Object Length -ne 0 if ($Filter) { $ChildItemList = $ChildItemList | Where-Object Name -like $Filter } ($ChildItemList).FullName | Sort-Object | ForEach-Object { [void]$files.Add( $_ ) } } catch { Stop-PSFFunction -Message "Error, path '$($pathItem)' not found" } } } end { if (-not $files) { Stop-PSFFunction -Message "No file found to parse! ($([string]::Join(", ", $Path)))" break } $recordCount = 0 $files = $files | Sort-Object if ($files.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($files.count / 100) } Write-PSFMessage -Level Verbose -Message "$($files.count) file$(if($files.count -gt 1){"s"}) to process." $traceTimer = New-Object System.Diagnostics.Stopwatch $traceTimer.Start() # Import first file if ($files.Count -gt 1) { $filePrevious = $files[0] } else { $filePrevious = $files } $resultPreviousFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $filePrevious)) { [void]$resultPreviousFile.Add($record) } $sessionIdName = Resolve-SessionIdName -LogType $resultPreviousFile[0].'Log-type' # file validity check if($LogType -ne "AutoDetect") { if($LogType -notlike $resultPreviousFile[0].'Log-type'.Replace(" ", "")) { Stop-PSFFunction -Message "Invalid LogType detected/specified. Expect '$($LogType)', but found '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' in file '$($filePrevious)'." break } } else { $LogType = $resultPreviousFile[0].'Log-type'.Replace(" ", "") } if($LogType -notin (Get-PSFConfigValue -FullName 'ExchangeLogs.SupportedLogTypes')) { Stop-PSFFunction -Message "Invalid LogType detected. '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' from file '$($filePrevious)' is not a supported log type to process with this command." break } # Import remaining files Write-PSFMessage -Level Verbose -Message "Starting import on $($files.Count) remaining file(s)." for ($filecounter = 1; $filecounter -lt $files.Count; $filecounter++) { #region process # import next file $fileCurrent = $files[$filecounter] $resultCurrentFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $fileCurrent)) { [void]$resultCurrentFile.Add($record) } if ($resultCurrentFile[0].'Log-type' -ne $resultPreviousFile[0].'Log-type') { Stop-PSFFunction -Message "Incompatible logfile types ($($resultCurrentFile[0].'Log-type'), $($resultPreviousFile[0].'Log-type')) found! More then one type of logfile in folder '$($pathItem)'." break } if ($sessionIdName) { # loop through previous and current file to check on fragmented session records in both files (sessions over midnight) Write-PSFMessage -Level VeryVerbose -Message "Checking for overlapping log records on identifier '$($sessionIdName)' in file '$($filePrevious)' and '$($fileCurrent)'" $overlapSessionIDs = Compare-Object -ReferenceObject $resultPreviousFile[-1..-20] -DifferenceObject $resultCurrentFile[0..20] -Property $sessionIdName -ExcludeDifferent -IncludeEqual if ($overlapSessionIDs) { foreach ($overlapSessionId in $overlapSessionIDs.$sessionIdName) { Write-PSFMessage -Level VeryVerbose -Message "Found overlapping log record '$($overlapSessionId)'" # get the record fragments from both logfiles $overlapRecordCurrentFile = $resultCurrentFile | Where-Object $sessionIdName -like $overlapSessionId $overlapRecordPreviousFile = $resultPreviousFile | Where-Object $sessionIdName -like $overlapSessionId # merge records $overlapRecordPreviousFileMerged = $overlapRecordPreviousFile $overlapRecordPreviousFileMerged.Group = $overlapRecordPreviousFile.Group + $overlapRecordCurrentFile.Group # remove overlapping records from current logfile $resultCurrentFile.RemoveAt( $resultCurrentFile.IndexOf($overlapRecordCurrentFile) ) # remove overlapping records from previous logfile $resultPreviousFile.RemoveAt( $resultPreviousFile.IndexOf($overlapRecordPreviousFile) ) # add merged record into previous logfile [void]$resultPreviousFile.Add($overlapRecordPreviousFileMerged) } } } # invoke data transform processing in a runspace to parallize processing and continue to work through files $recordCount = $recordCount + $resultPreviousFile.count switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # put current file records in variable for previous file to check for further record fragments $filePrevious = $fileCurrent $resultPreviousFile = $resultCurrentFile # progress status info & receive completed runspaces if (($filecounter % $refreshInterval) -eq 0 -or $filecounter -eq $files.count) { Write-PSFMessage -Level System -Message "Procesed $refreshInterval files... Going to update progress status" $jobs = Get-RSJob -Batch $batchJobId -ErrorAction SilentlyContinue -Verbose:$false | Sort-Object ID $jobsCompleted = $jobs | Where-Object State -Like "Completed" # output remaining data in completed runspace if ($jobsCompleted) { Wait-JobCompleteWithOutput -Job $jobsCompleted $recordsProcessed = Receive-RSJob -Job $jobsCompleted -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Write-PSFMessage -Level VeryVerbose -Message "Receiving $($jobsCompleted.Count) completed runspace job(s) with $($recordsProcessed.count) processed records" Remove-RSJob -Job $jobsCompleted -Verbose:$false } # status update Write-Progress -Activity "Import logfiles in $($resultPreviousFile[0].LogFolder) | Currently working runspaces: $($jobs.count - $jobsCompleted.count) | Records in processing: $($recordCount) | Time elapsed: $($traceTimer.Elapsed)" -Status "$($resultPreviousFile[0].LogFileName) ($($filecounter) / $($files.count))" -PercentComplete ($filecounter / $files.count * 100) } #endregion process } # processing last remaining file switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } $recordCount = $recordCount + $resultPreviousFile.count Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # waiting for completion of all runspaces Write-PSFMessage -Level Verbose -Message "Finished processing $($files.Count) file(s) with overall $($recordCount) record(s). Awaiting running runspaces to complete record processing." do { $jobs = Get-RSJob -Batch $batchJobId -Verbose:$false -ErrorAction SilentlyContinue | Sort-Object ID $jobsOpen = $jobs | Where-Object State -NotLike "Completed" $jobsCompleted = $jobs | Where-Object State -Like "Completed" [string]$_names = $jobsCompleted.Name | Split-Path -Leaf -ErrorAction SilentlyContinue if (-not $_names) { $_names = "" } Write-PSFMessage -Level VeryVerbose -Message "Awaiting runspaces to complete processing: $($jobsOpen.count)/$($jobs.count) ($([string]::Join(", ", $_names)))" Start-Sleep -Milliseconds 200 } while ($jobsOpen) # output remaining data in runspace Wait-JobCompleteWithOutput -Job $jobs Write-PSFMessage -Level Verbose -Message "All runspaces completed. Gathering results" $recordsProcessed = Receive-RSJob -Job $jobs -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Remove-RSJob -Batch $batchJobId -Verbose:$false $traceTimer.Stop() Write-PSFMessage -Level Significant -Message "Duration on parsing $($files.count) file(s) with $($recordCount) records: $($traceTimer.Elapsed)" } } |