Private/InvestigationCollectionHelpers.ps1
|
function Write-NSPInvestigationProgress { param([string]$Activity, [string]$Status) Write-Host ('[{0} UTC] {1}: {2}' -f [datetime]::UtcNow.ToString('HH:mm:ss'), $Activity, $Status) } # Offline validation and evidence persistence shared by scoped investigation commands. function New-NSPInvestigationOutput { param([Parameter(Mandatory)][string]$Path) $full = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) if (Test-Path -LiteralPath $full) { if (-not (Test-Path -LiteralPath $full -PathType Container)) { throw 'OutputPath must be a directory.' } if (@(Get-ChildItem -LiteralPath $full -Force -ErrorAction Stop).Count) { throw 'OutputPath is not empty. Use a new directory to preserve previous evidence.' } } else { $null = New-Item -ItemType Directory -Path $full -ErrorAction Stop -WhatIf:$false -Confirm:$false } $full } function Save-NSPInvestigationJson { param([Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][AllowNull()]$Data) $json = ConvertTo-Json -InputObject $Data -Depth 100 [IO.File]::WriteAllText($Path, $json, (New-Object Text.UTF8Encoding $false)) } function Assert-NSPInvestigationTenant { # Never authenticates or installs, even if the caller has not set NSP_NO_AUTOINSTALL. param([Parameter(Mandatory)][guid]$TenantId) $prior = $env:NSP_NO_AUTOINSTALL try { $env:NSP_NO_AUTOINSTALL = '1'; Import-NSPM365Graph } finally { $env:NSP_NO_AUTOINSTALL = $prior } if ((Get-NSPGraphTransport).Name -eq 'Fake') { $organizations = @((Invoke-NSPGraphRequest -Uri 'organization?$select=id')['value']) if ($organizations.Count -ne 1 -or [string]$organizations[0].id -ne $TenantId.ToString()) { throw 'Connected tenant does not match TenantId.' } } else { if (-not (Get-Command Get-MgContext -ErrorAction SilentlyContinue)) { throw 'No Graph context. Connect separately using an approved account and permissions.' } $context = Get-MgContext if (-not $context -or [string]$context.TenantId -ne $TenantId.ToString()) { throw 'Connected tenant does not match TenantId. Connect separately before collection.' } } } function Get-NSPInvestigationMailAddress { # Primary SMTP and smtp proxy addresses of one account, lower-case. Read-only. Throws when the # account cannot be read, so a caller never continues with a silently narrower address list. param([Parameter(Mandatory)][string]$UserPrincipalName) $uri = 'users/' + [uri]::EscapeDataString($UserPrincipalName) + '?$select=id,userPrincipalName,mail,proxyAddresses' $user = Invoke-NSPGraphRetry { Invoke-NSPGraphRequest -Uri $uri } -Activity 'Mail address lookup' if ([string]$user.userPrincipalName -ne $UserPrincipalName) { throw 'Resolved user does not match the requested account.' } $found = New-Object System.Collections.Generic.List[string] if ($user.mail) { $found.Add(([string]$user.mail).Trim().ToLowerInvariant()) } foreach ($proxy in @($user.proxyAddresses)) { $text = [string]$proxy if ($text.Length -gt 5 -and $text.Substring(0, 5) -ieq 'smtp:' -and $text -like '*@*') { $found.Add($text.Substring(5).Trim().ToLowerInvariant()) } } $found.ToArray() | Where-Object { $_ } | Sort-Object -Unique } function Get-NSPInvestigationPageUri { # Only Graph global-cloud v1.0/beta links are accepted; no forwarding credentials elsewhere. param([Parameter(Mandatory)][string]$Uri) if ($Uri -match '^https://') { $parsed = [uri]$Uri if ($parsed.Host -ne 'graph.microsoft.com' -or $parsed.Port -ne 443 -or $parsed.UserInfo -or $parsed.AbsolutePath -notmatch '^/(v1\.0|beta)/') { throw 'Unexpected Graph pagination host or path.' } } elseif ($Uri -notmatch '^(v1\.0/|beta/)?[a-zA-Z]') { throw 'Unexpected Graph pagination URI.' } $Uri } function Export-NSPInvestigationCollection { # Each response page is preserved before the next request. Partial acquisition survives errors. param( [Parameter(Mandatory)][string]$Uri, [Parameter(Mandatory)][string]$OutputPath, [Parameter(Mandatory)][string]$Name, [ValidateSet('v1.0', 'beta')][string]$ApiVersion = 'v1.0', [ValidateRange(1, 10000)][int]$MaxPages = 1000 ) $items = New-Object System.Collections.Generic.List[object] $pages = New-Object System.Collections.Generic.List[object] $seen = @{} $version = $ApiVersion $next = $Uri $errorText = $null $elapsed = [Diagnostics.Stopwatch]::StartNew() Write-NSPInvestigationProgress -Activity $Name -Status 'Starting collection.' try { while ($next) { if ($pages.Count -ge $MaxPages) { throw 'Page limit reached; collection is incomplete.' } if ($seen.ContainsKey($next)) { throw 'Repeated nextLink; collection is incomplete.' } $seen[$next] = $true $next = Get-NSPInvestigationPageUri -Uri $next $requestUri = $next if ($requestUri -match '^(v1\.0|beta)/') { $requestUri = 'https://graph.microsoft.com/' + $requestUri } $pageNumber = $pages.Count + 1 $status = 'Requesting page {0}; {1} records already preserved; elapsed {2:n0}s.' -f $pageNumber, $items.Count, $elapsed.Elapsed.TotalSeconds Write-NSPInvestigationProgress -Activity $Name -Status $status Save-NSPInvestigationJson -Path (Join-Path $OutputPath ('Progress-' + $Name + '.json')) -Data @{ Status='WaitingForGraph';Page=$pageNumber;Records=$items.Count;UpdatedUtc=[datetime]::UtcNow.ToString('o');ElapsedSeconds=$elapsed.Elapsed.TotalSeconds } $response = Invoke-NSPGraphRetry { Invoke-NSPGraphRequest -Uri $requestUri -ApiVersion $version } -Activity ($Name + ' page ' + $pageNumber) if (-not $response.ContainsKey('value')) { throw 'Graph collection response has no value array.' } $pageName = '{0}-{1:D5}.json' -f $Name, ($pages.Count + 1) $file = Join-Path $OutputPath $pageName Save-NSPInvestigationJson -Path $file -Data $response $pages.Add([pscustomobject]@{ File = $pageName; Sha256 = (Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash; Count = @($response['value'] | Where-Object { $null -ne $_ }).Count }) foreach ($item in @($response['value'])) { if ($null -ne $item) { $items.Add($item) } } Write-NSPInvestigationProgress -Activity $Name -Status ('Saved page {0}; {1} records total; elapsed {2:n0}s.' -f $pages.Count, $items.Count, $elapsed.Elapsed.TotalSeconds) $next = [string]$response['@odata.nextLink'] } } catch { $errorText = $_.Exception.Message } $status = if ($errorText) { 'Incomplete' } else { 'Completed' } Save-NSPInvestigationJson -Path (Join-Path $OutputPath ('Progress-' + $Name + '.json')) -Data @{ Status=$status;Page=$pages.Count;Records=$items.Count;UpdatedUtc=[datetime]::UtcNow.ToString('o');ElapsedSeconds=$elapsed.Elapsed.TotalSeconds } Write-NSPInvestigationProgress -Activity $Name -Status ('{0}; {1} pages, {2} records; elapsed {3:n0}s.' -f $status, $pages.Count, $items.Count, $elapsed.Elapsed.TotalSeconds) [pscustomobject]@{ Name = $Name; Complete = -not [bool]$errorText; Error = $errorText; NextLink = $next; Count = $items.Count; Pages = $pages.ToArray(); Items = $items.ToArray() } } function Read-NSPTriageInput { param([Parameter(Mandatory)][string[]]$Path, [switch]$Deduplicate) Import-NSPLogParse $records = New-Object System.Collections.Generic.List[object] $inputs = New-Object System.Collections.Generic.List[object] $identities = @{} $duplicates = 0 $conflicts = 0 foreach ($file in $Path) { $resolved = (Resolve-Path -LiteralPath $file -ErrorAction Stop).ProviderPath $hash = (Get-FileHash -LiteralPath $resolved -Algorithm SHA256).Hash $rawRows = @(Import-Csv -LiteralPath $resolved -Encoding (Get-NSPFileEncoding $resolved) -ErrorAction Stop) if (-not $rawRows.Count) { throw "No data rows in $resolved. Preserve the export and verify its headers and search coverage." } if (-not $rawRows[0].PSObject.Properties['AuditData']) { throw "AuditData column is missing in $resolved." } # Normalize through the existing LogParse parser, including BOM-aware input encoding. # ConvertFrom-NSPM365AuditCsv does not specify encoding; parse UTF-16 rows here instead. $parsed = if ((Get-NSPFileEncoding $resolved) -eq 'UTF8') { @(ConvertFrom-NSPM365AuditCsv -Path $resolved) } else { @(foreach ($r in $rawRows) { [pscustomobject]@{ RecordId=$r.RecordId; CreationDate=$r.CreationDate; RecordType=$r.RecordType; Operation=$r.Operation; UserId=$r.UserId; AuditData=ConvertFrom-Json -InputObject $r.AuditData -ErrorAction Stop; _Raw=$r } }) } $rowNumber = 1 foreach ($record in $parsed) { $rowNumber++ $raw = $rawRows[$rowNumber - 2] $audit = $record.AuditData if ($null -eq $audit -or $audit -is [array] -or $audit -is [string] -or $audit -is [ValueType]) { throw "AuditData must be a JSON object at row $rowNumber in $resolved." } if (-not $record.Operation) { $record.Operation = if ($audit.Operation) { [string]$audit.Operation } else { [string]$raw.Operations } } if (-not $record.UserId) { $record.UserId = if ($audit.UserId) { [string]$audit.UserId } else { [string]$raw.UserIds } } if (-not $record.RecordId) { $record.RecordId = [string]$audit.Id } if (-not $record.Operation) { throw "Operation is missing at row $rowNumber in $resolved." } $time = if ($audit.CreationTime) { $audit.CreationTime } else { $record.CreationDate } try { $null = ConvertTo-NSPHuntUtc $time } catch { throw "Invalid audit timestamp at row $rowNumber in $resolved." } Add-Member -InputObject $record -NotePropertyName SourceFile -NotePropertyValue $resolved Add-Member -InputObject $record -NotePropertyName SourceRow -NotePropertyValue $rowNumber $fingerprint = ConvertTo-Json -InputObject @($record.Operation, $record.UserId, $time, $audit) -Depth 100 -Compress $key = [string]$record.RecordId if ($key) { if (-not $identities.ContainsKey($key)) { $identities[$key] = New-Object System.Collections.Generic.List[string] } if ($identities[$key].Contains($fingerprint)) { $duplicates++; if ($Deduplicate) { continue } } else { if ($identities[$key].Count) { $conflicts++ }; $identities[$key].Add($fingerprint) } } $records.Add($record) } if ((Get-FileHash -LiteralPath $resolved -Algorithm SHA256).Hash -ne $hash) { throw 'Input changed during parsing. Stop and preserve both versions.' } $inputs.Add([pscustomobject]@{ Path = $resolved; Sha256 = $hash; Bytes = (Get-Item -LiteralPath $resolved).Length; Rows = $rawRows.Count }) } [pscustomobject]@{ Records = @($records.ToArray() | Sort-Object { Get-NSPAuditTime $_ }); Inputs = $inputs.ToArray(); DuplicateRecords = $duplicates; ConflictingRecords = $conflicts } } |