Invoke-FabricLogCollector.ps1

<#PSScriptInfo

.VERSION 1.0.0

.GUID bd6817c7-f6c0-42ba-a430-102f9bed1eef

.AUTHOR Patrick Gallucci

.COMPANYNAME

.COPYRIGHT (c) Patrick Gallucci. Licensed under the MIT License.

.TAGS Microsoft-Fabric PowerBI Diagnostics Logging RCA Troubleshooting Azure Telemetry KQL Remediation

.LICENSEURI https://github.com/PatrickGallucci/Fabric-Log-Collector/blob/main/LICENSE

.PROJECTURI https://github.com/PatrickGallucci/Fabric-Log-Collector

.ICONURI

.EXTERNALMODULEDEPENDENCIES

.REQUIREDSCRIPTS

.EXTERNALSCRIPTDEPENDENCIES

.RELEASENOTES
1.0.0 - Initial public release. Fault-isolated collection across the full Microsoft Fabric surface with a consolidated RCA report (Markdown/HTML) and remediation catalog. Requires the Az.Accounts module. See CHANGELOG.md for details.

.PRIVATEDATA

#>


<#
.SYNOPSIS
    Microsoft Fabric diagnostic log & telemetry collector for troubleshooting, RCA, and remediation.

.DESCRIPTION
    Extracts logs, errors, warnings, admin settings, and environment metadata across the full
    Microsoft Fabric surface. Every capability runs inside a fault-isolated wrapper: if a single
    service, workspace, or API call fails, the failure is logged and the run continues.

    Coverage (one collector per Fabric logging surface):
      Data Factory (Pipelines) Dataflow Gen2 Lakehouse
      Warehouse SQL Database in Fabric Eventhouse
      Real-Time Intelligence Semantic Models Power BI Reports
      Notebooks Spark Job Definitions Mirrored Databases
      GraphQL APIs Data Agents Copilot in Fabric
      OneLake Tenant Operations (Audit) Admin settings + metadata

    Outputs (timestamped run folder):
      raw\ Per-service JSON/CSV dumps (source of truth)
      logs\ run.log (every step), errors.log (failures only)
      findings.json Machine-readable RCA findings
      RCA-Report.md / RCA-Report.html Consolidated findings + remediation

.PARAMETER TenantId
    Entra tenant GUID. Required for service-principal auth; optional for interactive.

.PARAMETER ClientId
    App (client) ID for service-principal auth. If supplied with a secret/cert, non-interactive.

.PARAMETER ClientSecret
    App client secret (SecureString or plain). Triggers service-principal auth.

.PARAMETER CertificateThumbprint
    Certificate thumbprint for service-principal cert auth (alternative to ClientSecret).

.PARAMETER WorkspaceId
    One or more workspace GUIDs to scope the run. Omit for tenant-wide (admin) enumeration.

.PARAMETER CapacityId
    One or more capacity GUIDs to scope the run.

.PARAMETER DaysBack
    Look-back window in days for time-bounded logs (audit, monitoring KQL, activity). Default 7.

.PARAMETER OutputRoot
    Root folder for run outputs. Defaults to .\FabricDiagnostics.

.PARAMETER SkipAdminApis
    Skip tenant-level admin APIs (use when the identity is not a Fabric admin).

.PARAMETER IncludeRawExports
    Emit per-service raw dumps. Default $true.

.PARAMETER DeviceCode
    Use device-code interactive login (headless-friendly) instead of the default browser flow.

.EXAMPLE
    .\Invoke-FabricLogCollector.ps1
    Interactive login, tenant-wide, last 7 days, full RCA report.

.EXAMPLE
    .\Invoke-FabricLogCollector.ps1 -TenantId <guid> -ClientId <guid> -ClientSecret $s -DaysBack 14
    Unattended service-principal run over a 14-day window.

.EXAMPLE
    .\Invoke-FabricLogCollector.ps1 -WorkspaceId <guid> -SkipAdminApis
    Single-workspace run for a non-admin user.

.NOTES
    Author : Unified Data Platform toolkit
    Requires: Az.Accounts module (Install-Module Az.Accounts -Scope CurrentUser)
    Docs : Endpoints and KQL table names verified against Microsoft Learn (2026-07).
#>


#Requires -Version 7.0

[CmdletBinding()]
param(
    [string]$TenantId,
    [string]$ClientId,
    [object]$ClientSecret,
    [string]$CertificateThumbprint,
    [string[]]$WorkspaceId,
    [string[]]$CapacityId,
    [int]$DaysBack = 7,
    [string]$OutputRoot = (Join-Path (Get-Location) 'FabricDiagnostics'),
    [switch]$SkipAdminApis,
    [bool]$IncludeRawExports = $true,
    [switch]$DeviceCode
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'   # local try/catch converts to continue-on-error

# ---------------------------------------------------------------------------
# Endpoints (verified against Microsoft Learn)
# ---------------------------------------------------------------------------
$Script:Endpoints = @{
    FabricApi   = 'https://api.fabric.microsoft.com/v1'
    PowerBIApi  = 'https://api.powerbi.com/v1.0/myorg'
    FabricRes   = 'https://api.fabric.microsoft.com'          # token resource for Fabric REST
    PowerBIRes  = 'https://analysis.windows.net/powerbi/api'  # token resource for Power BI REST
    GraphApi    = 'https://graph.microsoft.com'               # Purview/M365 audit search
    StorageRes  = 'https://storage.azure.com'                 # OneLake (ADLS Gen2 API)
    OneLakeDfs  = 'https://onelake.dfs.fabric.microsoft.com'
}

# ---------------------------------------------------------------------------
# Run context / shared state
# ---------------------------------------------------------------------------
$Script:Ctx = [ordered]@{
    StartUtc     = (Get-Date).ToUniversalTime()
    RunId        = (Get-Date -Format 'yyyyMMdd-HHmmss')
    Root         = $null
    RawDir       = $null
    LogDir       = $null
    RunLog       = $null
    ErrLog       = $null
    Tokens       = @{}                       # resource -> @{ Token; Expires }
    Findings     = [System.Collections.Generic.List[object]]::new()
    StepResults  = [System.Collections.Generic.List[object]]::new()
    Workspaces   = @()
    Capacities   = @()
    IsAdmin      = $false
    AuthMode     = 'None'
    Identity     = $null
}

# ===========================================================================
# CORE: logging, resiliency, auth, HTTP, Kusto
# ===========================================================================

function Write-Log {
    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$Message,
        [ValidateSet('INFO','WARN','ERROR','OK','STEP')][string]$Level = 'INFO'
    )
    $ts   = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
    $line = "$ts [$Level] $Message"
    switch ($Level) {
        'ERROR' { Write-Host $line -ForegroundColor Red }
        'WARN'  { Write-Host $line -ForegroundColor Yellow }
        'OK'    { Write-Host $line -ForegroundColor Green }
        'STEP'  { Write-Host $line -ForegroundColor Cyan }
        default { Write-Host $line }
    }
    if ($Script:Ctx.RunLog) { Add-Content -Path $Script:Ctx.RunLog -Value $line -Encoding utf8 }
    if ($Level -eq 'ERROR' -and $Script:Ctx.ErrLog) { Add-Content -Path $Script:Ctx.ErrLog -Value $line -Encoding utf8 }
}

function Invoke-Safe {
    <#
      Fault-isolated step runner. Runs $Action; on failure logs and CONTINUES.
      Returns the action's result on success, or $null on failure. Records a
      structured StepResult either way so the report can show partial coverage.
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$Name,
        [Parameter(Mandatory)][scriptblock]$Action,
        [string]$Category = 'General',
        [string]$Scope    = ''
    )
    $sw = [System.Diagnostics.Stopwatch]::StartNew()
    Write-Log -Level STEP -Message "START $Name$(if($Scope){" [$Scope]"})"
    $result = [ordered]@{
        Name = $Name; Category = $Category; Scope = $Scope
        Status = 'Success'; Error = $null; DurationMs = 0; Items = $null
    }
    try {
        $out = & $Action
        $result.Status = 'Success'
        if ($null -ne $out -and ($out -is [System.Collections.IEnumerable]) -and ($out -isnot [string])) {
            $result.Items = @($out).Count
        }
        Write-Log -Level OK -Message "DONE $Name ($([int]$sw.Elapsed.TotalMilliseconds) ms)$(if($null -ne $result.Items){" - $($result.Items) item(s)"})"
        return $out
    }
    catch {
        $result.Status = 'Failed'
        $result.Error  = $_.Exception.Message
        Write-Log -Level ERROR -Message "FAIL $Name : $($_.Exception.Message)"
        # Every failed capability becomes a finding so nothing is silently lost.
        Add-Finding -Severity 'Warning' -Service $Category -Scope $Scope `
            -Title "Collector '$Name' failed to run" `
            -Detail $_.Exception.Message `
            -Remediation "Verify permissions/feature enablement for this surface. Failure is isolated; other collectors still ran. See logs\errors.log."
        return $null
    }
    finally {
        $sw.Stop()
        $result.DurationMs = [int]$sw.Elapsed.TotalMilliseconds
        $Script:Ctx.StepResults.Add([pscustomobject]$result) | Out-Null
    }
}

function Get-AccessTokenFor {
    [CmdletBinding()]
    param([Parameter(Mandatory)][string]$Resource)

    $cached = $Script:Ctx.Tokens[$Resource]
    if ($cached -and $cached.Expires -gt (Get-Date).AddMinutes(5)) { return $cached.Token }

    $tok = Get-AzAccessToken -ResourceUrl $Resource -ErrorAction Stop
    # PS/Az may return SecureString in newer versions
    $plain = if ($tok.Token -is [System.Security.SecureString]) {
        [System.Net.NetworkCredential]::new('', $tok.Token).Password
    } else { $tok.Token }
    $Script:Ctx.Tokens[$Resource] = @{ Token = $plain; Expires = $tok.ExpiresOn.LocalDateTime }
    return $plain
}

function Connect-FabricContext {
    # Plaintext->SecureString conversion is only used to accept an optional -ClientSecret string
    # for unattended SP auth. Prefer passing a SecureString or -CertificateThumbprint.
    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText','')]
    [CmdletBinding()] param()

    if (-not (Get-Module -ListAvailable -Name Az.Accounts)) {
        throw "Az.Accounts module not found. Run: Install-Module Az.Accounts -Scope CurrentUser"
    }
    Import-Module Az.Accounts -ErrorAction Stop | Out-Null

    $hasSecret = $ClientSecret -and ("$ClientSecret").Length -gt 0
    $useSp     = $ClientId -and ($hasSecret -or $CertificateThumbprint)

    if ($useSp) {
        if (-not $TenantId) { throw "TenantId is required for service-principal authentication." }
        $secure = if ($ClientSecret -is [System.Security.SecureString]) { $ClientSecret }
                  elseif ($hasSecret) { ConvertTo-SecureString "$ClientSecret" -AsPlainText -Force }
                  else { $null }
        if ($secure) {
            $cred = [pscredential]::new($ClientId, $secure)
            Connect-AzAccount -ServicePrincipal -TenantId $TenantId -Credential $cred -WarningAction SilentlyContinue | Out-Null
        }
        else {
            Connect-AzAccount -ServicePrincipal -TenantId $TenantId -ApplicationId $ClientId `
                -CertificateThumbprint $CertificateThumbprint -WarningAction SilentlyContinue | Out-Null
        }
        $Script:Ctx.AuthMode = 'ServicePrincipal'
    }
    else {
        $splat = @{ WarningAction = 'SilentlyContinue' }
        if ($TenantId)   { $splat.TenantId = $TenantId }
        if ($DeviceCode) { $splat.UseDeviceAuthentication = $true }
        Connect-AzAccount @splat | Out-Null
        $Script:Ctx.AuthMode = if ($DeviceCode) { 'Interactive-DeviceCode' } else { 'Interactive-Browser' }
    }

    $ctxAz = Get-AzContext
    $Script:Ctx.Identity = if ($ctxAz.Account) { $ctxAz.Account.Id } else { 'unknown' }
    if (-not $TenantId -and $ctxAz.Tenant) { $script:TenantId = $ctxAz.Tenant.Id }
    Write-Log -Level OK -Message "Authenticated as '$($Script:Ctx.Identity)' via $($Script:Ctx.AuthMode) (tenant $TenantId)"
}

function Invoke-RestSafe {
    <#
      HTTP GET/POST with:
        - bearer token for the given resource
        - 429/5xx retry honoring Retry-After (exponential fallback)
        - continuation-token + @odata.nextLink paging (returns flattened 'value' array)
      Throws on unrecoverable error so the caller's Invoke-Safe records the failure.
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$Uri,
        [Parameter(Mandatory)][string]$Resource,
        [ValidateSet('GET','POST')][string]$Method = 'GET',
        [object]$Body,
        [switch]$Paged,
        [int]$MaxRetry = 5
    )
    $all = [System.Collections.Generic.List[object]]::new()
    $next = $Uri
    do {
        $attempt = 0
        while ($true) {
            $attempt++
            try {
                $headers = @{ Authorization = "Bearer $(Get-AccessTokenFor -Resource $Resource)" }
                $params = @{ Uri = $next; Method = $Method; Headers = $headers; ErrorAction = 'Stop' }
                if ($Body) { $params.Body = ($Body | ConvertTo-Json -Depth 20); $params.ContentType = 'application/json' }
                $resp = Invoke-RestMethod @params
                break
            }
            catch {
                $status = $null
                try { $status = [int]$_.Exception.Response.StatusCode } catch {}
                $retryAfter = 0
                try { $retryAfter = [int]$_.Exception.Response.Headers['Retry-After'] } catch {}
                if (($status -eq 429 -or ($status -ge 500 -and $status -lt 600)) -and $attempt -le $MaxRetry) {
                    $wait = if ($retryAfter -gt 0) { $retryAfter } else { [math]::Min(60, [math]::Pow(2, $attempt)) }
                    Write-Log -Level WARN -Message "HTTP $status on $next - retry $attempt/$MaxRetry after ${wait}s"
                    Start-Sleep -Seconds $wait
                    continue
                }
                throw
            }
        }

        if (-not $Paged) { return $resp }

        # Flatten this page into $all. Envelope shape varies across Fabric/Power BI:
        # { value: [...] } Power BI + many Fabric APIs
        # { workspaces: [...] } admin/workspaces
        # { itemEntities: [...] } admin/items
        # { activityEventEntities: [...] } Power BI admin/activityevents
        # { domains: [...] } etc., or a bare JSON array.
        # Strategy: bare array -> add each element; else flatten 'value' or the first
        # array-valued property (ignoring continuation tokens); else add the object itself.
        if (($resp -is [System.Collections.IEnumerable]) -and ($resp -isnot [string])) {
            foreach ($it in $resp) { $all.Add($it) }
        }
        else {
            $props   = $resp.PSObject.Properties
            $collVal = $null
            if ($props['value'] -and ($props['value'].Value -is [System.Collections.IEnumerable]) -and ($props['value'].Value -isnot [string])) {
                $collVal = $props['value'].Value
            }
            else {
                $cand = $props | Where-Object {
                    $_.Name -notin @('continuationUri','continuationToken','@odata.nextLink','odata.nextLink') -and
                    ($_.Value -is [System.Collections.IEnumerable]) -and ($_.Value -isnot [string])
                } | Select-Object -First 1
                if ($cand) { $collVal = $cand.Value }
            }
            if ($null -ne $collVal) { foreach ($it in $collVal) { $all.Add($it) } }
            else { $all.Add($resp) }
        }

        $next = $null
        if ($resp.PSObject.Properties.Name -contains 'continuationUri' -and $resp.continuationUri) { $next = $resp.continuationUri }
        elseif ($resp.PSObject.Properties.Name -contains '@odata.nextLink' -and $resp.'@odata.nextLink') { $next = $resp.'@odata.nextLink' }
        elseif ($resp.PSObject.Properties.Name -contains 'continuationToken' -and $resp.continuationToken) {
            $sep = if ($Uri -match '\?') { '&' } else { '?' }
            $next = "$Uri$sep" + 'continuationToken=' + [uri]::EscapeDataString($resp.continuationToken)
        }
    } while ($next)

    return $all
}

function Invoke-FabricApi   { param([string]$Path,[switch]$Paged) Invoke-RestSafe -Uri "$($Script:Endpoints.FabricApi)/$Path"  -Resource $Script:Endpoints.FabricRes  -Paged:$Paged }
function Invoke-PowerBIApi  { param([string]$Path,[switch]$Paged) Invoke-RestSafe -Uri "$($Script:Endpoints.PowerBIApi)/$Path" -Resource $Script:Endpoints.PowerBIRes -Paged:$Paged }

function Invoke-KustoQuery {
    <#
      Query a Fabric Workspace-Monitoring Eventhouse (KQL) database via the Kusto REST endpoint.
      $QueryUri = eventhouse queryServiceUri (https://<cluster>.kusto.fabric.microsoft.com)
    #>

    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$QueryUri,
        [Parameter(Mandatory)][string]$Database,
        [Parameter(Mandatory)][string]$Query
    )
    $token = Get-AccessTokenFor -Resource $QueryUri
    $body  = @{ db = $Database; csl = $Query } | ConvertTo-Json
    $resp  = Invoke-RestMethod -Uri "$QueryUri/v1/rest/query" -Method Post -ContentType 'application/json' `
                -Headers @{ Authorization = "Bearer $token" } -Body $body -ErrorAction Stop
    # Primary result = first table with rows mapped to column names
    $primary = $resp.Tables | Where-Object { $_.TableName -eq 'Table_0' -or $_.TableName -eq 'PrimaryResult' } | Select-Object -First 1
    if (-not $primary) { $primary = $resp.Tables | Select-Object -First 1 }
    if (-not $primary) { return @() }
    $cols = $primary.Columns.ColumnName
    $rows = foreach ($r in $primary.Rows) {
        $o = [ordered]@{}
        for ($i = 0; $i -lt $cols.Count; $i++) { $o[$cols[$i]] = $r[$i] }
        [pscustomobject]$o
    }
    return $rows
}

function Save-Raw {
    param([Parameter(Mandatory)][string]$Service,[Parameter(Mandatory)][string]$Name,[object]$Data)
    if (-not $IncludeRawExports -or $null -eq $Data) { return }
    $dir = Join-Path $Script:Ctx.RawDir $Service
    if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }
    $json = Join-Path $dir "$Name.json"
    $Data | ConvertTo-Json -Depth 25 | Set-Content -Path $json -Encoding utf8
    try {
        if (($Data -is [System.Collections.IEnumerable]) -and ($Data -isnot [string])) {
            $arr = @($Data)
            if ($arr.Count -gt 0 -and ($arr[0] -is [pscustomobject] -or $arr[0] -is [hashtable])) {
                $arr | Export-Csv -Path (Join-Path $dir "$Name.csv") -NoTypeInformation -Encoding utf8
            }
        }
    } catch { Write-Log -Level WARN -Message "CSV export skipped for $Service/$Name : $($_.Exception.Message)" }
}

# ===========================================================================
# RCA: findings + remediation catalog
# ===========================================================================

function Add-Finding {
    [CmdletBinding()]
    param(
        [ValidateSet('Critical','Error','Warning','Info')][string]$Severity = 'Info',
        [Parameter(Mandatory)][string]$Service,
        [Parameter(Mandatory)][string]$Title,
        [string]$Detail = '',
        [string]$Scope = '',
        [string]$Remediation = '',
        [int]$Count = 1,
        [object]$Evidence
    )
    $Script:Ctx.Findings.Add([pscustomobject]@{
        Severity = $Severity; Service = $Service; Scope = $Scope
        Title = $Title; Detail = $Detail; Count = $Count
        Remediation = $Remediation
        Evidence = if ($Evidence) { ($Evidence | Select-Object -First 5) } else { $null }
        DetectedUtc = (Get-Date).ToUniversalTime().ToString('s')
    }) | Out-Null
}

# Remediation catalog: message-pattern -> guidance. Extend freely.
$Script:RemediationRules = @(
    @{ Pattern = 'throttl|429|CapacityThrottling|TooManyRequests'; Service='Capacity';   Sev='Error';    Advice='Capacity throttling/interactive delay. Review Capacity Metrics app, reduce concurrent load, enable smoothing headroom, or scale the SKU up. Stagger refreshes/pipelines.' },
    @{ Pattern = 'OutOfMemory|OOM|memory limit|exceeded.*memory';  Service='Spark';      Sev='Error';    Advice='Out-of-memory. Increase executor/driver memory or node size, repartition to reduce skew, cache selectively, and avoid wide collects. Consider a larger Spark pool or resource profile.' },
    @{ Pattern = 'timeout|timed out|deadline';                     Service='General';    Sev='Error';    Advice='Operation timeout. Check source connectivity/gateway, increase command timeout, and inspect long-running queries. For pipelines, add retry policy on the activity.' },
    @{ Pattern = 'Unauthorized|401|Forbidden|403|AccessDenied|permission'; Service='Security'; Sev='Error'; Advice='AuthZ failure. Verify workspace role, item permissions, OneLake data-access roles, and (for SPs) tenant admin-API / service-principal settings. Confirm token audience/resource.' },
    @{ Pattern = 'gateway|on-premises|OPDG|data source.*not found|cannot connect'; Service='Connectivity'; Sev='Error'; Advice='Source connectivity/gateway issue. Validate gateway cluster health & credentials, firewall/private-endpoint rules, and connection binding. Test the connection in Manage Connections & Gateways.' },
    @{ Pattern = 'refresh.*fail|DataSource.*error|processing error'; Service='SemanticModel'; Sev='Error'; Advice='Semantic model refresh failure. Inspect the specific partition/table error, credentials, and source availability. Check refresh history and Large Semantic Model / incremental refresh settings.' },
    @{ Pattern = 'ingestion.*fail|BadRequest.*ingest|mapping';     Service='Eventhouse'; Sev='Error';    Advice='Eventhouse ingestion failure. Validate schema/ingestion mapping, source format, and batching policy. Review IngestionResults/command logs for the failing extent.' },
    @{ Pattern = 'replication|mirror.*fail|CDC|LSN';               Service='MirroredDB';  Sev='Error';   Advice='Mirroring/replication failure. Confirm source CDC/permissions, network path, and table compatibility. Restart mirroring for the affected table and monitor replication logs.' },
    @{ Pattern = 'private endpoint|DNS|name resolution|NSG';        Service='Networking'; Sev='Error';   Advice='Private-networking/DNS issue. Verify private endpoint approval, DNS zone records, and NSG/firewall rules between the workspace capacity and the data source.' }
)

function Resolve-Remediation {
    param([string]$Message,[string]$DefaultService='General')
    foreach ($rule in $Script:RemediationRules) {
        if ($Message -match $rule.Pattern) { return $rule }
    }
    return @{ Service=$DefaultService; Sev='Warning'; Advice='Review the raw evidence for this item. No specific remediation rule matched; inspect the error text and correlate with Monitoring Hub / Capacity Metrics.' }
}

function Add-ErrorFindingsFromRows {
    <#
      Generic helper: scan tabular rows for a status/error signal, group by message,
      map to remediation, and emit findings.
    #>

    param(
        [object[]]$Rows,
        [string]$Service,
        [string]$Scope,
        [string[]]$StatusFields = @('Status','JobStatus','ResultStatus','State'),
        [string[]]$FailValues   = @('Failed','Failure','Error','Cancelled','Deduped','Blocked'),
        [string[]]$MessageFields = @('ErrorMessage','Message','FailureType','StatusMessage','EventText','Error','FailureReason')
    )
    if (-not $Rows) { return }
    $failed = foreach ($row in $Rows) {
        $isFail = $false
        foreach ($sf in $StatusFields) {
            $val = $row.PSObject.Properties[$sf]
            if ($val -and $FailValues -contains "$($val.Value)") { $isFail = $true; break }
        }
        if ($isFail) { $row }
    }
    if (-not $failed) { return }

    $grouped = $failed | Group-Object {
        $m = ''
        foreach ($mf in $MessageFields) {
            $p = $_.PSObject.Properties[$mf]
            if ($p -and $p.Value) { $m = "$($p.Value)"; break }
        }
        if (-not $m) { 'Unclassified failure' } else { $m.Substring(0, [math]::Min(160, $m.Length)) }
    }
    foreach ($g in $grouped) {
        $rule = Resolve-Remediation -Message $g.Name -DefaultService $Service
        $sev  = if ($g.Count -ge 10) { 'Critical' } else { $rule.Sev }
        Add-Finding -Severity $sev -Service $Service -Scope $Scope `
            -Title "$($g.Count) failed operation(s): $($g.Name)" `
            -Detail "Grouped failures detected in $Service telemetry." `
            -Remediation $rule.Advice -Count $g.Count -Evidence $g.Group
    }
}

# ===========================================================================
# OBJECT SHAPE NORMALIZATION (StrictMode-safe)
# ===========================================================================

function Resolve-FabricDisplayName {
    <#
      StrictMode-safe display-name resolver. Fabric object shape varies by API:
      admin APIs (e.g. /admin/workspaces) return 'name', while core/item APIs
      return 'displayName'. Resolves displayName -> name -> id -> '' and never
      throws on a missing property (unlike a bare $obj.displayName under
      Set-StrictMode -Version Latest).
    #>

    [CmdletBinding()]
    param([Parameter(ValueFromPipeline)]$InputObject)
    process {
        if ($null -eq $InputObject) { return '' }
        if ($InputObject -is [string]) { return $InputObject }
        foreach ($prop in 'displayName','name') {
            $p = $InputObject.PSObject.Properties[$prop]
            if ($p -and $null -ne $p.Value -and "$($p.Value)".Length -gt 0) { return "$($p.Value)" }
        }
        $idp = $InputObject.PSObject.Properties['id']
        if ($idp -and $idp.Value) { return "$($idp.Value)" }
        return ''
    }
}

function Set-FabricDisplayName {
    <#
      Guarantees every object in the collection exposes a non-empty 'displayName'
      NoteProperty so downstream '$obj.displayName' access is StrictMode-safe
      regardless of the originating API surface. Mutates in place (PSCustomObjects
      are reference types) and returns the same collection.
    #>

    [CmdletBinding()]
    param([Parameter(Mandatory)][AllowNull()]$InputObject)
    foreach ($o in @($InputObject)) {
        if ($null -eq $o -or $o -is [string] -or $o -is [ValueType]) { continue }
        $existing = $o.PSObject.Properties['displayName']
        if (-not $existing -or [string]::IsNullOrEmpty("$($existing.Value)")) {
            $o | Add-Member -NotePropertyName displayName `
                -NotePropertyValue (Resolve-FabricDisplayName -InputObject $o) -Force
        }
    }
    return $InputObject
}

# ===========================================================================
# DISCOVERY: workspaces, capacities, admin settings, metadata
# ===========================================================================

function Get-TargetWorkspaces {
    [CmdletBinding()] param()
    $ws = @()
    if (-not $SkipAdminApis) {
        $adminWs = Invoke-Safe -Name 'Discover workspaces (Admin API)' -Category 'Metadata' -Action {
            Invoke-FabricApi -Path 'admin/workspaces?type=Workspace' -Paged
        }
        if ($adminWs) { $ws = $adminWs; $Script:Ctx.IsAdmin = $true }
    }
    if (-not $ws) {
        $ws = Invoke-Safe -Name 'Discover workspaces (Core API)' -Category 'Metadata' -Action {
            Invoke-FabricApi -Path 'workspaces' -Paged
        }
    }
    $ws = @($ws)
    if ($WorkspaceId) { $ws = $ws | Where-Object { $WorkspaceId -contains $_.id } }
    if ($CapacityId)  { $ws = $ws | Where-Object { $_.PSObject.Properties['capacityId'] -and $CapacityId -contains $_.capacityId } }
    # Only keep real workspace objects (must carry an 'id'). Guards against an API
    # envelope/wrapper ever leaking into the per-workspace loop under StrictMode.
    $ws = @($ws | Where-Object { $_ -and $_.PSObject.Properties['id'] -and $_.id })
    # Admin API returns 'name'; core API returns 'displayName'. Normalize so every
    # downstream '$ws.displayName' access is StrictMode-safe and never terminates the run.
    $ws = @(Set-FabricDisplayName $ws)
    Save-Raw -Service 'Metadata' -Name 'workspaces' -Data $ws
    $Script:Ctx.Workspaces = $ws
    Write-Log -Level INFO -Message "In-scope workspaces: $(@($ws).Count)"
    return $ws
}

function Get-EnvironmentMetadata {
    [CmdletBinding()] param()

    Invoke-Safe -Name 'List capacities' -Category 'Metadata' -Action {
        $caps = if (-not $SkipAdminApis) { Invoke-PowerBIApi -Path 'admin/capacities' -Paged } else { Invoke-FabricApi -Path 'capacities' -Paged }
        if ($CapacityId) { $caps = $caps | Where-Object { $CapacityId -contains $_.id } }
        $caps = @(Set-FabricDisplayName @($caps))
        $Script:Ctx.Capacities = @($caps)
        Save-Raw -Service 'Metadata' -Name 'capacities' -Data $caps
        # Flag paused / non-active capacities as findings
        foreach ($c in @($caps)) {
            $state = if ($c.PSObject.Properties['state']) { "$($c.state)" } else { '' }
            if ($state -and $state -notmatch 'Active') {
                Add-Finding -Severity 'Warning' -Service 'Capacity' -Scope "$($c.displayName) ($($c.id))" `
                    -Title "Capacity not Active (state=$state)" `
                    -Detail 'Workspaces on a paused/suspended capacity cannot run jobs.' `
                    -Remediation 'Resume the capacity or reassign affected workspaces to an active capacity.'
            }
        }
        $caps
    } | Out-Null

    if (-not $SkipAdminApis) {
        Invoke-Safe -Name 'Tenant settings' -Category 'AdminSettings' -Action {
            $ts = Invoke-FabricApi -Path 'admin/tenantsettings' -Paged
            Save-Raw -Service 'AdminSettings' -Name 'tenant-settings' -Data $ts
            $ts
        } | Out-Null

        Invoke-Safe -Name 'Domains' -Category 'Metadata' -Action {
            $d = Invoke-FabricApi -Path 'admin/domains' -Paged
            Save-Raw -Service 'Metadata' -Name 'domains' -Data $d
            $d
        } | Out-Null

        Invoke-Safe -Name 'Git connections (Admin)' -Category 'Metadata' -Action {
            $g = Invoke-FabricApi -Path 'admin/workspaces/discoverGitConnections' -Paged
            Save-Raw -Service 'Metadata' -Name 'git-connections' -Data $g
            $g
        } | Out-Null

        Invoke-Safe -Name 'All items (Admin inventory)' -Category 'Metadata' -Action {
            $items = Invoke-FabricApi -Path 'admin/items?state=Active' -Paged
            Save-Raw -Service 'Metadata' -Name 'items-inventory' -Data $items
            $items
        } | Out-Null
    }
}

function Get-WorkspaceItems {
    param([Parameter(Mandatory)]$Workspace)
    $wsId = $Workspace.id
    $items = Invoke-Safe -Name 'List items' -Category 'Metadata' -Scope (Resolve-FabricDisplayName $Workspace) -Action {
        Invoke-FabricApi -Path "workspaces/$wsId/items" -Paged
    }
    # Defensive: guarantee displayName on every item so type/name filters below never throw under StrictMode.
    return @(Set-FabricDisplayName @($items))
}

# ===========================================================================
# WORKSPACE-MONITORING (KQL) collectors
# ===========================================================================

function Get-MonitoringEventhouse {
    <#
      Locate the Workspace-Monitoring eventhouse for a workspace and return
      @{ QueryUri; Database } or $null if monitoring is not enabled.
    #>

    param([Parameter(Mandatory)]$Workspace,[object[]]$Items)
    $wsId = $Workspace.id
    $eh = $Items | Where-Object { $_.type -eq 'Eventhouse' -and ($_.displayName -match 'Monitor') }
    if (-not $eh) { $eh = $Items | Where-Object { $_.type -eq 'Eventhouse' } }
    foreach ($e in @($eh)) {
        $full = Invoke-Safe -Name "Get eventhouse '$($e.displayName)'" -Category 'Monitoring' -Scope $Workspace.displayName -Action {
            Invoke-FabricApi -Path "workspaces/$wsId/eventhouses/$($e.id)"
        }
        $uri = $null
        try { $uri = $full.properties.queryServiceUri } catch {}
        if ($uri) {
            return @{ QueryUri = $uri; Database = 'Monitoring Eventhouse'; RawName = $e.displayName; ItemId = $e.id }
        }
    }
    return $null
}

# KQL table -> service mapping for the Workspace Monitoring DB
$Script:MonitoringQueries = @(
    @{ Service='DataFactory';    Table='ItemJobEventLogs';                 Filter="| where JobStatus == 'Failed'" },
    @{ Service='DataFactory';    Table='FabricDataPipelineActivityRunsLogs'; Filter="| where Status == 'Failed'" },
    @{ Service='DataFactory';    Table='CopyJobActivityRunDetailsLogs';    Filter="| where Status == 'Failed'" },
    @{ Service='SemanticModel';  Table='SemanticModelLogs';                Filter="| where Status != 'Success' and isnotempty(Status)" },
    @{ Service='Eventhouse';     Table='EventhouseQueryLogs';              Filter="| where isnotempty(FailureReason) or Success == false" },
    @{ Service='Eventhouse';     Table='EventhouseIngestionResultsLogs';   Filter="| where Result != 'Success'" },
    @{ Service='Eventhouse';     Table='EventhouseCommandLogs';            Filter="| where State != 'Completed' and isnotempty(State)" },
    @{ Service='GraphQL';        Table='GraphQLLogs';                      Filter="| where isnotempty(ErrorType) or Success == false" },
    @{ Service='MirroredDB';     Table='MirroredDatabaseTableLogs';        Filter="| where Status == 'Failed' or isnotempty(ErrorMessage)" }
)

function Invoke-WorkspaceMonitoringCollectors {
    param([Parameter(Mandatory)]$Workspace,[object[]]$Items)

    $mon = Get-MonitoringEventhouse -Workspace $Workspace -Items $Items
    if (-not $mon) {
        Add-Finding -Severity 'Info' -Service 'Monitoring' -Scope $Workspace.displayName `
            -Title 'Workspace Monitoring not enabled' `
            -Detail 'No monitoring eventhouse found; KQL-based logs (pipelines L2, semantic model ops, eventhouse, GraphQL, mirroring) are unavailable for this workspace.' `
            -Remediation 'Enable it: Workspace settings > Monitoring > Log workspace activity. Provisions a read-only KQL DB with ItemJobEventLogs, SemanticModelLogs, Eventhouse*, GraphQLLogs, etc.'
        return
    }
    Write-Log -Level INFO -Message "Monitoring eventhouse: $($mon.QueryUri) [$($Workspace.displayName)]"
    $since = "| where Timestamp > ago($($DaysBack)d)"

    foreach ($q in $Script:MonitoringQueries) {
        Invoke-Safe -Name "KQL $($q.Table)" -Category $q.Service -Scope $Workspace.displayName -Action {
            # Guard: only query tables that exist
            $exists = Invoke-KustoQuery -QueryUri $mon.QueryUri -Database $mon.Database -Query ".show database schema | where TableName == '$($q.Table)' | summarize c=count()"
            if (-not $exists -or [int]($exists[0].c) -eq 0) {
                Write-Log -Level WARN -Message "Table $($q.Table) absent in $($Workspace.displayName) monitoring DB (feature not emitting yet)"
                return
            }
            $csl  = "$($q.Table) $since $($q.Filter) | take 5000"
            $rows = Invoke-KustoQuery -QueryUri $mon.QueryUri -Database $mon.Database -Query $csl
            Save-Raw -Service $q.Service -Name "$($Workspace.displayName)_$($q.Table)" -Data $rows
            Add-ErrorFindingsFromRows -Rows $rows -Service $q.Service -Scope $Workspace.displayName
            $rows
        } | Out-Null
    }
}

# ===========================================================================
# BUILT-IN / REST collectors (do not depend on Workspace Monitoring)
# ===========================================================================

function Invoke-DataFactoryBuiltInCollector {
    param($Workspace,[object[]]$Items)
    $wsId = $Workspace.id
    $pipes = $Items | Where-Object { $_.type -eq 'DataPipeline' }
    foreach ($p in @($pipes)) {
        Invoke-Safe -Name "Pipeline job history '$($p.displayName)'" -Category 'DataFactory' -Scope $Workspace.displayName -Action {
            $runs = Invoke-FabricApi -Path "workspaces/$wsId/items/$($p.id)/jobs/instances" -Paged
            $recent = $runs | Where-Object {
                $_.PSObject.Properties['startTimeUtc'] -and $_.startTimeUtc -and
                ([datetime]$_.startTimeUtc) -gt (Get-Date).AddDays(-$DaysBack)
            }
            Save-Raw -Service 'DataFactory' -Name "$($Workspace.displayName)_$($p.displayName)_jobs" -Data $recent
            $failed = $recent | Where-Object { $_.status -in @('Failed','Cancelled') }
            foreach ($f in @($failed)) {
                $msg = if ($f.PSObject.Properties['failureReason'] -and $f.failureReason) { "$($f.failureReason.message)" } else { 'Pipeline run failed' }
                $rule = Resolve-Remediation -Message $msg -DefaultService 'DataFactory'
                Add-Finding -Severity $rule.Sev -Service 'DataFactory' -Scope "$($Workspace.displayName)/$($p.displayName)" `
                    -Title "Pipeline run $($f.status): $msg" -Remediation $rule.Advice -Evidence $f
            }
            $recent
        } | Out-Null
    }
}

function Invoke-SemanticModelRefreshCollector {
    param($Workspace,[object[]]$Items)
    $wsId = $Workspace.id
    $models = $Items | Where-Object { $_.type -eq 'SemanticModel' }
    foreach ($m in @($models)) {
        Invoke-Safe -Name "Refresh history '$($m.displayName)'" -Category 'SemanticModel' -Scope $Workspace.displayName -Action {
            $hist = Invoke-PowerBIApi -Path "groups/$wsId/datasets/$($m.id)/refreshes?`$top=50"
            $rows = if ($hist.PSObject.Properties['value']) { $hist.value } else { $hist }
            Save-Raw -Service 'SemanticModel' -Name "$($Workspace.displayName)_$($m.displayName)_refreshes" -Data $rows
            $failed = $rows | Where-Object { $_.status -in @('Failed','Disabled') }
            foreach ($f in @($failed)) {
                $msg = if ($f.PSObject.Properties['serviceExceptionJson'] -and $f.serviceExceptionJson) { "$($f.serviceExceptionJson)" } else { 'Refresh failed' }
                $rule = Resolve-Remediation -Message $msg -DefaultService 'SemanticModel'
                Add-Finding -Severity $rule.Sev -Service 'SemanticModel' -Scope "$($Workspace.displayName)/$($m.displayName)" `
                    -Title "Refresh $($f.status)" -Detail $msg -Remediation $rule.Advice -Evidence $f
            }
            $rows
        } | Out-Null
    }
}

function Invoke-WarehouseSqlCollector {
    param($Workspace,[object[]]$Items)
    # Presence + config capture. Query Insights (queryinsights.exec_requests_history) requires a SQL
    # connection; captured here as metadata + guidance so RCA can point the operator to it.
    $wh = $Items | Where-Object { $_.type -in @('Warehouse','SQLDatabase','MirroredWarehouse','DataWarehouse') }
    foreach ($w in @($wh)) {
        Invoke-Safe -Name "Warehouse/SQL metadata '$($w.displayName)'" -Category 'Warehouse' -Scope $Workspace.displayName -Action {
            Save-Raw -Service 'Warehouse' -Name "$($Workspace.displayName)_$($w.displayName)" -Data $w
            Add-Finding -Severity 'Info' -Service 'Warehouse' -Scope "$($Workspace.displayName)/$($w.displayName)" `
                -Title "Warehouse/SQL present: query failure telemetry available via Query Insights" `
                -Detail 'Rich query-failure detail lives in queryinsights.exec_requests_history (T-SQL).' `
                -Remediation "Run against the SQL endpoint: SELECT * FROM queryinsights.exec_requests_history WHERE command_type IS NOT NULL AND status <> 'Succeeded' ORDER BY submit_time DESC; -- also sys.dm_exec_requests for live sessions."
            $w
        } | Out-Null
    }
}

function Invoke-NotebookSparkCollector {
    param($Workspace,[object[]]$Items)
    $wsId = $Workspace.id
    $sparkItems = $Items | Where-Object { $_.type -in @('Notebook','SparkJobDefinition') }
    if (-not $sparkItems) { return }
    Invoke-Safe -Name 'Spark livy sessions/applications' -Category 'Spark' -Scope $Workspace.displayName -Action {
        $apps = Invoke-FabricApi -Path "workspaces/$wsId/spark/livySessions" -Paged
        $recent = $apps | Where-Object {
            $_.PSObject.Properties['submittedDateTime'] -and $_.submittedDateTime -and
            ([datetime]$_.submittedDateTime) -gt (Get-Date).AddDays(-$DaysBack)
        }
        Save-Raw -Service 'Spark' -Name "$($Workspace.displayName)_livySessions" -Data $recent
        $failed = $recent | Where-Object { $_.state -in @('Error','Dead','Killed','Failed') -or $_.PSObject.Properties['jobStatus'].Value -eq 'Failed' }
        foreach ($f in @($failed)) {
            $rule = Resolve-Remediation -Message "$($f.state) spark" -DefaultService 'Spark'
            Add-Finding -Severity $rule.Sev -Service 'Spark' -Scope "$($Workspace.displayName)/$($f.itemName)" `
                -Title "Spark application $($f.state)" -Remediation $rule.Advice -Evidence $f
        }
        $recent
    } | Out-Null
}

function Invoke-OneLakeDiagnosticsCollector {
    param($Workspace,[object[]]$Items)
    # OneLake diagnostics land as JSON in a lakehouse Files path. Detect enablement + point the way.
    $lakehouses = $Items | Where-Object { $_.type -eq 'Lakehouse' }
    Invoke-Safe -Name 'OneLake diagnostics detection' -Category 'OneLake' -Scope $Workspace.displayName -Action {
        $found = $false
        foreach ($lh in @($lakehouses)) {
            $path = "$($Script:Endpoints.OneLakeDfs)/$($Workspace.id)/$($lh.id)/Files/DiagnosticLogs?resource=filesystem&recursive=false"
            try {
                $listing = Invoke-RestSafe -Uri $path -Resource $Script:Endpoints.StorageRes -Method GET
                if ($listing) { $found = $true; Save-Raw -Service 'OneLake' -Name "$($Workspace.displayName)_$($lh.displayName)_diag-listing" -Data $listing }
            } catch { }
        }
        if (-not $found) {
            Add-Finding -Severity 'Info' -Service 'OneLake' -Scope $Workspace.displayName `
                -Title 'OneLake diagnostics not detected' `
                -Detail 'No DiagnosticLogs path found in workspace lakehouses.' `
                -Remediation 'Enable Workspace settings > OneLake > Add diagnostic events to a lakehouse. Events land under Files/DiagnosticLogs/OneLake/Workspaces/<id>/... (up to 1h latency).'
        }
        $found
    } | Out-Null
}

function Invoke-DataAgentCollector {
    param($Workspace,[object[]]$Items)
    $agents = $Items | Where-Object { $_.type -in @('DataAgent','AISkill') }
    foreach ($a in @($agents)) {
        Invoke-Safe -Name "Data Agent metadata '$($a.displayName)'" -Category 'DataAgent' -Scope $Workspace.displayName -Action {
            Save-Raw -Service 'DataAgent' -Name "$($Workspace.displayName)_$($a.displayName)" -Data $a
            Add-Finding -Severity 'Info' -Service 'DataAgent' -Scope "$($Workspace.displayName)/$($a.displayName)" `
                -Title 'Data Agent present: diagnostics/evaluations are UI/SDK-exported' `
                -Detail 'Agent diagnostics, evaluations, and usage are not in the REST inventory.' `
                -Remediation 'Export diagnostics from the Data Agent UI or fabric SDK; review token/usage in the Capacity Metrics app. Use the fabric-data-agent-remediate playbook for failures.'
            $a
        } | Out-Null
    }
}

function Invoke-CopilotCapacityCollector {
    Invoke-Safe -Name 'Copilot / Capacity consumption pointer' -Category 'Copilot' -Action {
        Add-Finding -Severity 'Info' -Service 'Copilot' `
            -Title 'Copilot token consumption is reported via Capacity Metrics' `
            -Detail 'Copilot/Fabric AI usage and token consumption are not exposed as discrete REST logs.' `
            -Remediation 'Open the Microsoft Fabric Capacity Metrics app > Compute, filter to Copilot operations, and correlate CU spikes with the run window.'
        $true
    } | Out-Null
}

function Invoke-TenantAuditCollector {
    if ($SkipAdminApis) { return }
    Invoke-Safe -Name 'Tenant audit activity events' -Category 'TenantOps' -Action {
        $all = [System.Collections.Generic.List[object]]::new()
        for ($d = 0; $d -lt $DaysBack; $d++) {
            $day    = (Get-Date).ToUniversalTime().Date.AddDays(-$d)
            $start  = $day.ToString('yyyy-MM-ddT00:00:00')
            $end    = $day.ToString('yyyy-MM-ddT23:59:59')
            try {
                $resp = Invoke-PowerBIApi -Path "admin/activityevents?startDateTime='$start'&endDateTime='$end'" -Paged
                foreach ($e in @($resp)) { $all.Add($e) }
            } catch { Write-Log -Level WARN -Message "Activity events for $($day.ToShortDateString()) failed: $($_.Exception.Message)" }
        }
        Save-Raw -Service 'TenantOps' -Name 'activity-events' -Data $all
        $failures = $all | Where-Object { $_.PSObject.Properties['Activity'] -and ("$($_.Activity)" -match 'Fail|Error|Delete|Unauthorized') }
        $grouped  = $failures | Group-Object Activity | Sort-Object Count -Descending | Select-Object -First 20
        foreach ($g in $grouped) {
            Add-Finding -Severity 'Warning' -Service 'TenantOps' `
                -Title "$($g.Count) audit event(s): $($g.Name)" `
                -Detail 'High-signal tenant operation from Power BI/Fabric audit log.' `
                -Remediation 'Correlate the actor/object in Purview Audit. Investigate spikes of deletes or unauthorized access.' -Count $g.Count -Evidence $g.Group
        }
        $all
    } | Out-Null
}

# ===========================================================================
# REPORT GENERATION
# ===========================================================================

function New-RcaReport {
    [CmdletBinding()] param()

    $findings = @($Script:Ctx.Findings | Sort-Object @{
        Expression = { switch ($_.Severity) { 'Critical'{0} 'Error'{1} 'Warning'{2} 'Info'{3} default{4} } }
    }, Service)
    $sevCounts = @($findings | Group-Object Severity | ForEach-Object { "$($_.Name)=$($_.Count)" })
    $steps     = $Script:Ctx.StepResults
    $okSteps   = @($steps | Where-Object Status -eq 'Success').Count
    $failSteps = @($steps | Where-Object Status -eq 'Failed').Count
    $durMin    = [math]::Round(((Get-Date).ToUniversalTime() - $Script:Ctx.StartUtc).TotalMinutes, 1)

    # --- findings.json ---
    $findings | ConvertTo-Json -Depth 8 | Set-Content -Path (Join-Path $Script:Ctx.Root 'findings.json') -Encoding utf8

    # --- manifest.json (run-level record so downstream trending can join runs) ---
    $sevMap = @{}
    foreach ($g in ($findings | Group-Object Severity)) { $sevMap[$g.Name] = $g.Count }
    [ordered]@{
        RunId          = $Script:Ctx.RunId
        StartUtc       = $Script:Ctx.StartUtc.ToString('o')
        EndUtc         = (Get-Date).ToUniversalTime().ToString('o')
        DurationMin    = $durMin
        Identity       = $Script:Ctx.Identity
        AuthMode       = $Script:Ctx.AuthMode
        IsAdmin        = $Script:Ctx.IsAdmin
        DaysBack       = $DaysBack
        WorkspaceCount = @($Script:Ctx.Workspaces).Count
        CapacityCount  = @($Script:Ctx.Capacities).Count
        CollectorsOk   = $okSteps
        CollectorsFail = $failSteps
        FindingCount   = @($findings).Count
        Critical       = [int]($sevMap['Critical'])
        Error          = [int]($sevMap['Error'])
        Warning        = [int]($sevMap['Warning'])
        Info           = [int]($sevMap['Info'])
    } | ConvertTo-Json -Depth 4 | Set-Content -Path (Join-Path $Script:Ctx.Root 'manifest.json') -Encoding utf8

    # --- Markdown ---
    $md = [System.Text.StringBuilder]::new()
    [void]$md.AppendLine("# Microsoft Fabric Diagnostic RCA Report")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("Run ID: **$($Script:Ctx.RunId)** | Generated: $((Get-Date).ToUniversalTime().ToString('u'))")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("Identity: ``$($Script:Ctx.Identity)`` via $($Script:Ctx.AuthMode) | Admin APIs: $($Script:Ctx.IsAdmin) | Look-back: $DaysBack day(s)")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("## Run summary")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("| Metric | Value |")
    [void]$md.AppendLine("|---|---|")
    [void]$md.AppendLine("| Workspaces in scope | $(@($Script:Ctx.Workspaces).Count) |")
    [void]$md.AppendLine("| Capacities in scope | $(@($Script:Ctx.Capacities).Count) |")
    [void]$md.AppendLine("| Collectors succeeded | $okSteps |")
    [void]$md.AppendLine("| Collectors failed (isolated) | $failSteps |")
    [void]$md.AppendLine("| Total findings | $(@($findings).Count) |")
    [void]$md.AppendLine("| Severity breakdown | $([string]::Join(', ', @($sevCounts))) |")
    [void]$md.AppendLine("| Duration (min) | $durMin |")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("## Findings & remediation")
    [void]$md.AppendLine("")
    if (@($findings).Count -eq 0) {
        [void]$md.AppendLine("_No errors or warnings detected in the collected telemetry._")
    }
    foreach ($f in $findings) {
        [void]$md.AppendLine("### [$($f.Severity)] $($f.Service) - $($f.Title)")
        [void]$md.AppendLine("")
        if ($f.Scope)       { [void]$md.AppendLine("- **Scope:** $($f.Scope)") }
        if ($f.Count -gt 1) { [void]$md.AppendLine("- **Occurrences:** $($f.Count)") }
        if ($f.Detail)      { [void]$md.AppendLine("- **Detail:** $($f.Detail)") }
        if ($f.Remediation) { [void]$md.AppendLine("- **Remediation:** $($f.Remediation)") }
        [void]$md.AppendLine("")
    }
    [void]$md.AppendLine("## Collector coverage")
    [void]$md.AppendLine("")
    [void]$md.AppendLine("| Collector | Scope | Status | Items | ms |")
    [void]$md.AppendLine("|---|---|---|---|---|")
    foreach ($s in $steps) {
        [void]$md.AppendLine("| $($s.Name) | $($s.Scope) | $($s.Status) | $($s.Items) | $($s.DurationMs) |")
    }
    $md.ToString() | Set-Content -Path (Join-Path $Script:Ctx.Root 'RCA-Report.md') -Encoding utf8

    # --- HTML ---
    New-RcaHtml -Findings $findings -Steps $steps -SevCounts $sevCounts -DurMin $durMin
    Write-Log -Level OK -Message "Reports written: RCA-Report.md / RCA-Report.html / findings.json"
}

function New-RcaHtml {
    param($Findings,$Steps,$SevCounts,$DurMin)
    $sevColor = @{ Critical='#b30000'; Error='#d9534f'; Warning='#e0a800'; Info='#5bc0de' }
    $rows = foreach ($f in $Findings) {
        $c = $sevColor[$f.Severity]; if (-not $c) { $c = '#777' }
        @"
<tr>
  <td><span style="background:$c;color:#fff;padding:2px 8px;border-radius:10px;font-size:12px;">$($f.Severity)</span></td>
  <td>$([System.Web.HttpUtility]::HtmlEncode($f.Service))</td>
  <td>$([System.Web.HttpUtility]::HtmlEncode($f.Scope))</td>
  <td><strong>$([System.Web.HttpUtility]::HtmlEncode($f.Title))</strong><br><span style="color:#555;font-size:13px;">$([System.Web.HttpUtility]::HtmlEncode($f.Detail))</span></td>
  <td style="font-size:13px;">$([System.Web.HttpUtility]::HtmlEncode($f.Remediation))</td>
</tr>
"@

    }
    $stepRows = foreach ($s in $Steps) {
        $sc = if ($s.Status -eq 'Failed') { '#d9534f' } else { '#28a745' }
        "<tr><td>$([System.Web.HttpUtility]::HtmlEncode($s.Name))</td><td>$([System.Web.HttpUtility]::HtmlEncode($s.Scope))</td><td style='color:$sc;'>$($s.Status)</td><td>$($s.Items)</td><td>$($s.DurationMs)</td></tr>"
    }
    Add-Type -AssemblyName System.Web -ErrorAction SilentlyContinue
    $html = @"
<!DOCTYPE html><html><head><meta charset="utf-8"><title>Fabric RCA Report $($Script:Ctx.RunId)</title>
<style>
 body{font-family:Segoe UI,Arial,sans-serif;margin:24px;color:#222;}
 h1{font-size:22px;} h2{font-size:17px;border-bottom:1px solid #ddd;padding-bottom:4px;margin-top:28px;}
 table{border-collapse:collapse;width:100%;margin-top:10px;font-size:14px;}
 th,td{border:1px solid #e2e2e2;padding:8px;text-align:left;vertical-align:top;}
 th{background:#f4f6f8;} .kpi{display:inline-block;background:#f4f6f8;border-radius:8px;padding:10px 16px;margin:4px;}
 .kpi b{display:block;font-size:20px;}
</style></head><body>
<h1>Microsoft Fabric Diagnostic RCA Report</h1>
<p>Run <code>$($Script:Ctx.RunId)</code> &middot; $((Get-Date).ToUniversalTime().ToString('u')) &middot; identity <code>$($Script:Ctx.Identity)</code> ($($Script:Ctx.AuthMode)) &middot; look-back $DaysBack day(s)</p>
<div>
 <span class="kpi"><b>$(@($Script:Ctx.Workspaces).Count)</b>workspaces</span>
 <span class="kpi"><b>$(@($Script:Ctx.Capacities).Count)</b>capacities</span>
 <span class="kpi"><b>$(@($Findings).Count)</b>findings</span>
 <span class="kpi"><b>$(@($Steps | Where-Object Status -eq 'Failed').Count)</b>collector failures</span>
 <span class="kpi"><b>$DurMin</b>minutes</span>
</div>
<p>Severity: $([string]::Join(' &middot; ', @($SevCounts)))</p>
<h2>Findings &amp; remediation</h2>
<table><tr><th>Severity</th><th>Service</th><th>Scope</th><th>Finding</th><th>Recommended remediation</th></tr>
$([string]::Join("`n", @($rows)))
</table>
<h2>Collector coverage</h2>
<table><tr><th>Collector</th><th>Scope</th><th>Status</th><th>Items</th><th>ms</th></tr>
$([string]::Join("`n", @($stepRows)))
</table>
</body></html>
"@

    $html | Set-Content -Path (Join-Path $Script:Ctx.Root 'RCA-Report.html') -Encoding utf8
}

# ===========================================================================
# MAIN
# ===========================================================================

function Initialize-Run {
    $Script:Ctx.Root   = Join-Path $OutputRoot "FabricDiag_$($Script:Ctx.RunId)"
    $Script:Ctx.RawDir = Join-Path $Script:Ctx.Root 'raw'
    $Script:Ctx.LogDir = Join-Path $Script:Ctx.Root 'logs'
    New-Item -ItemType Directory -Path $Script:Ctx.RawDir -Force | Out-Null
    New-Item -ItemType Directory -Path $Script:Ctx.LogDir -Force | Out-Null
    $Script:Ctx.RunLog = Join-Path $Script:Ctx.LogDir 'run.log'
    $Script:Ctx.ErrLog = Join-Path $Script:Ctx.LogDir 'errors.log'
    Write-Log -Level INFO -Message "Fabric Log Collector run $($Script:Ctx.RunId) starting. Output: $($Script:Ctx.Root)"
}

function Invoke-Main {
    Initialize-Run
    if (-not (Invoke-Safe -Name 'Authenticate' -Category 'Auth' -Action { Connect-FabricContext; $true })) {
        Write-Log -Level ERROR -Message "Authentication failed - cannot continue. See logs\errors.log."
        New-RcaReport
        return
    }

    Get-EnvironmentMetadata
    $workspaces = Get-TargetWorkspaces
    Invoke-TenantAuditCollector
    Invoke-CopilotCapacityCollector

    $i = 0; $n = @($workspaces).Count
    foreach ($ws in @($workspaces)) {
        $i++
        $wsName = Resolve-FabricDisplayName -InputObject $ws
        Write-Log -Level STEP -Message "=== Workspace $i/$n : $wsName ($($ws.id)) ==="
        # Fault-isolate the whole per-workspace pass: one workspace can never terminate the run.
        Invoke-Safe -Name 'Process workspace' -Category 'Workspace' -Scope $wsName -Action {
            $items = Get-WorkspaceItems -Workspace $ws
            Save-Raw -Service 'Metadata' -Name "items_$wsName" -Data $items

            Invoke-WorkspaceMonitoringCollectors  -Workspace $ws -Items $items
            Invoke-DataFactoryBuiltInCollector    -Workspace $ws -Items $items
            Invoke-SemanticModelRefreshCollector  -Workspace $ws -Items $items
            Invoke-WarehouseSqlCollector          -Workspace $ws -Items $items
            Invoke-NotebookSparkCollector         -Workspace $ws -Items $items
            Invoke-OneLakeDiagnosticsCollector    -Workspace $ws -Items $items
            Invoke-DataAgentCollector             -Workspace $ws -Items $items
            $true
        } | Out-Null
    }

    New-RcaReport
    Write-Log -Level OK -Message "Complete. Findings: $(@($Script:Ctx.Findings).Count). Open: $($Script:Ctx.Root)\RCA-Report.html"
}

# Auto-run only when invoked directly (.\Invoke-FabricLogCollector.ps1).
# When dot-sourced for testing (. .\Invoke-FabricLogCollector.ps1) the entry point
# is skipped so Pester can exercise individual functions without side effects.
if ($MyInvocation.InvocationName -ne '.') {
    Invoke-Main
}