Private/AzStackHci.ConnectivityCluster.Helpers.ps1
|
# //////////////////////////////////////////////////////////////////////////// # Strict Mode v1 (PS 5.1 safe) - surfaces reads of uninitialised variables at runtime. Set-StrictMode -Version 1.0 # AzStackHci.ConnectivityCluster.Helpers.ps1 # # Cluster fan-out for Test-AzureLocalConnectivity -Scope Cluster (added v0.6.7). # # Owner constraints (see .github/copilot-instructions.md, "Owner Constraints"): # * PowerShell 5.1 target — no PS 7 features. # * NO CredSSP / WinRM auth-mode / TrustedHosts changes. # Cluster fan-out runs over standard Kerberos / Invoke-Command remoting and # assumes the operator has valid credentials when running from a node. # * NO silent shared-host security state mutation. # # Architecture: # The public cmdlet `Test-AzureLocalConnectivity -Scope Cluster` short-circuits # into `Invoke-AzureLocalConnectivityClusterFanOut` at the very top of its # `begin` block. The orchestrator: # 1. Enumerates Get-ClusterNode (UP-state nodes only). # 2. Verifies the module is installed on every node. # 3. Defaults per-node -Parallelism to 8 (each node runs its own local # Layer-7 worker sweep) unless the caller passed an explicit value. The # worker jobs run locally on each node, so the orchestrator only holds one # lightweight remoting job per node and the worker count stays bounded # per-machine — exactly like a single-node -Parallelism run. # 4. Fans out via `Invoke-Command -ComputerName $nodes` calling the same # public function with -Scope Node -PassThru -NoOutput. # 5. Collects per-node ArrayLists into a Cluster pscustomobject. # 6. Generates a tabbed HTML report (Phase C: New-ConnectivityClusterReport) # and (optionally) wires Send-DiagnosticData upload (Phase D). # # All functions in this file are PRIVATE — not exported via FunctionsToExport. # //////////////////////////////////////////////////////////////////////////// Function Invoke-AzureLocalConnectivityClusterFanOut { <# .SYNOPSIS Cluster orchestrator for Test-AzureLocalConnectivity -Scope Cluster. .DESCRIPTION Enumerates cluster nodes, fans out the connectivity test to each node via `Invoke-Command`, collects per-node ArrayList results, and produces a cluster-shaped pscustomobject for downstream reporting. Returns a `[pscustomobject]` with these properties: ClusterName : [string] name from Get-Cluster OrchestratorMachine : [string] name of the node running the orchestrator RunGuid : [guid] unique run identifier StartTime : [datetime] orchestrator start EndTime : [datetime] orchestrator end (after all nodes return) Nodes : [hashtable] {<node-name> = $perNodeResultArrayList} Errors : [array] per-node error messages (empty on success) .PARAMETER ForwardedParameters A hashtable of the parameters originally passed to Test-AzureLocalConnectivity. Cluster-mode-specific parameters (-Scope, -Parallelism, -PassThru, -NoOutput, -ExcludeUploadResults) are stripped/forced before fanning out. .PARAMETER ExportPath The top-level path to write the cluster report HTML/JSON to. If not provided, defaults to `$env:USERPROFILE\AzStackHci.DiagnosticSettings\<ClusterName>\<timestamp>`. .PARAMETER NoOutput When set, suppresses orchestrator console output (per-node calls already use -NoOutput). .PARAMETER ExcludeUploadResults When set, skips the Send-DiagnosticData upload step at the end. .PARAMETER InstallMissingModuleOnNodes When set, side-loads the orchestrator's exact module version onto any node that is missing the module or has a different version (drift), copying over a PSSession from the orchestrator's installed module folder. Opt-in — never mutates nodes silently. Without it, missing/drifted nodes cause a graceful pre-flight failure. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [hashtable]$ForwardedParameters, [Parameter(Mandatory=$false)] [string]$ExportPath, [Parameter(Mandatory=$false)] [switch]$NoOutput, [Parameter(Mandatory=$false)] [switch]$ExcludeUploadResults, [Parameter(Mandatory=$false)] [switch]$InstallMissingModuleOnNodes ) # ── Verify cluster cmdlets are available ───────────────────────────────── # NOTE: Test-CommandExists declares Param ($command) — it has NO -CommandName # parameter. Calling with -CommandName silently leaves $command empty (no # CmdletBinding strict-binding), Get-Command '' returns nothing, and the # precheck wrongly throws even when the cmdlet exists. Use positional binding. if (-not (Test-CommandExists 'Get-Cluster')) { Throw "Get-Cluster cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster. Install RSAT-Clustering or run from a cluster node." } if (-not (Test-CommandExists 'Get-ClusterNode')) { Throw "Get-ClusterNode cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster." } # ── Enumerate the cluster ──────────────────────────────────────────────── [string]$clusterName = '' [array]$nodes = @() try { $clusterObj = Get-Cluster -ErrorAction Stop $clusterName = $clusterObj.Name # Only fan out to UP nodes — DOWN/PAUSED nodes would just produce error rows. $nodes = @(Get-ClusterNode -ErrorAction Stop | Where-Object { $_.State -eq 'Up' } | Select-Object -ExpandProperty Name) } catch { Throw "Cluster enumeration failed: $($_.Exception.Message). -Scope Cluster requires the current host to be a member of an active failover cluster (or to have RSAT remote access to one)." } if ($nodes.Count -eq 0) { Throw "Get-ClusterNode returned 0 UP nodes for cluster '$clusterName'. Cannot fan out." } if (-not $NoOutput.IsPresent) { Write-HostAzS "════════════════════════════════════════════════════════════════════" Write-HostAzS " Cluster connectivity test — Scope: Cluster" Write-HostAzS " Cluster: $clusterName Orchestrator: $env:COMPUTERNAME" Write-HostAzS " Nodes ($($nodes.Count)): $($nodes -join ', ')" Write-HostAzS "════════════════════════════════════════════════════════════════════" } # ── Verify the module is installed on every node ───────────────────────── # Categorise failures so the user gets actionable guidance: # * Remoting failures (WinRM access denied, name resolution, firewall) # usually mean the orchestrator isn't an admin on the remote node — NOT # a missing module. Surface those separately to avoid the confusing # "install the module" prompt when the real fix is admin / network. # * Truly missing modules need a different remediation (Install-Module). # # On any blocking failure (missing module and/or remoting failure) we DO NOT # Throw a raw exception — that surfaces a confusing red script stack trace to # the operator. Instead we print a clean, formatted summary that names the # exact node(s) at fault with remediation guidance, also emit a single # Write-Error so scripted / -NoOutput callers can still detect the failure, # then return $null to stop the cluster fan-out gracefully. if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Verifying 'AzStackHci.DiagnosticSettings' module (exact version) is installed on each cluster node..." -ForegroundColor Cyan } $orchestratorVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$orchestratorVersionString = if ($orchestratorVersion) { $orchestratorVersion.ToString() } else { $null } # Probe every node for the FULL list of installed versions so we can enforce an # EXACT version match against the orchestrator. This prevents a silent mixed-version # cluster report (different nodes running different module code, merged into one # report). Classification per node: # OK — the orchestrator's exact version is present # Missing — no version of the module at all # Drifted — has the module, but NOT the orchestrator's exact version # Remoting failures (WinRM / admin / firewall) are tracked separately — side-loading # cannot fix those, so the remediation hint must not point at -InstallMissingModuleOnNodes. $okNodes = @() $missingNodes = @() $driftedNodes = @() # display strings: "node (has X, Y; orchestrator has Z)" $driftedNodeNames = @() # bare names for side-load targeting $remotingFailures = @() foreach ($node in $nodes) { try { $remoteVersions = @(Invoke-Command -ComputerName $node -ScriptBlock { Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Version | ForEach-Object { $_.ToString() } } -ErrorAction Stop) $remoteVersions = @($remoteVersions | Where-Object { $_ }) # drop any $null/empty if ($remoteVersions.Count -eq 0) { $missingNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module NOT installed" -ForegroundColor Red } } elseif (-not $orchestratorVersionString) { # Orchestrator version is indeterminate — fall back to "any version present is OK" # rather than falsely reporting drift on every node. $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module present (orchestrator version unknown — exact-match check skipped)" -ForegroundColor Yellow } } elseif ($remoteVersions -contains $orchestratorVersionString) { $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module v$orchestratorVersionString present" -ForegroundColor Green } } else { $driftedNodes += "$node (has $($remoteVersions -join ', '); orchestrator has $orchestratorVersionString)" $driftedNodeNames += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Version drift — has $($remoteVersions -join ', '), needs $orchestratorVersionString" -ForegroundColor Yellow } } } catch { $remotingFailures += "$node ($($_.Exception.Message))" if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Remoting failed: $($_.Exception.Message)" -ForegroundColor Red } } } # ── Optional remediation: side-load the orchestrator's exact version ───── # When -InstallMissingModuleOnNodes is specified, copy the orchestrator's installed # module folder (exact version) to every Missing / Drifted node over a PSSession, # then reclassify successes as OK. This is a deliberate, OPT-IN host mutation (see # Owner Constraints) — never performed silently. We side-load rather than # Install-Module from PSGallery because Azure Local nodes are frequently air-gapped, # and copying guarantees a byte-identical version match with the orchestrator (no # PSGallery drift, no NuGet provider, no internet dependency). Remoting-failed nodes # are NOT targeted — the PSSession would fail for the same reason the probe did. if ($InstallMissingModuleOnNodes.IsPresent -and ($missingNodes.Count -gt 0 -or $driftedNodeNames.Count -gt 0)) { if (-not $orchestratorVersionString) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module version is unknown — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { # Resolve the orchestrator's on-disk module folder for the exact version. # Prefer the -ListAvailable entry matching the version; fall back to the # loaded module's ModuleBase (covers dev / non-PSModulePath installs). $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' -ListAvailable -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $orchestratorVersionString } | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase } if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module folder could not be located on disk — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { $targets = @(@($missingNodes + $driftedNodeNames) | Select-Object -Unique) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS " -InstallMissingModuleOnNodes: side-loading AzStackHci.DiagnosticSettings v$orchestratorVersionString to $($targets.Count) node(s) from:" -ForegroundColor Cyan Write-HostAzS " $sourceModuleBase" -ForegroundColor DarkGray } foreach ($targetNode in $targets) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Copying module v$orchestratorVersionString..." -ForegroundColor Cyan } $copied = Copy-AzStackHciModuleToNode -NodeName $targetNode -SourceModuleBase $sourceModuleBase -Version $orchestratorVersion if ($copied) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load OK — module v$orchestratorVersionString now present" -ForegroundColor Green } $okNodes += $targetNode $missingNodes = @($missingNodes | Where-Object { $_ -ne $targetNode }) $driftedNodeNames = @($driftedNodeNames | Where-Object { $_ -ne $targetNode }) $driftedNodes = @($driftedNodes | Where-Object { $_ -notlike "$targetNode (*" }) } else { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load FAILED — re-run with -Verbose for the copy error" -ForegroundColor Red } } } } } } # ── Graceful failure when one or more nodes cannot run the test ────────── if ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0 -or $remotingFailures.Count -gt 0) { $moduleProblem = ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red Write-HostAzS " Cluster connectivity test cannot continue — pre-flight checks failed" -ForegroundColor Red Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red if ($missingNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The 'AzStackHci.DiagnosticSettings' module is NOT installed on $($missingNodes.Count) node(s):" -ForegroundColor Red foreach ($mn in $missingNodes) { Write-HostAzS " - $mn" -ForegroundColor Yellow } } if ($driftedNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The module version does NOT match the orchestrator (v$orchestratorVersionString) on $($driftedNodes.Count) node(s):" -ForegroundColor Red foreach ($dn in $driftedNodes) { Write-HostAzS " - $dn" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " All nodes must run the SAME module version, otherwise the cluster report can" -ForegroundColor Yellow Write-HostAzS " silently blend results collected by different module versions." -ForegroundColor Yellow } if ($moduleProblem) { Write-HostAzS "" if ($InstallMissingModuleOnNodes.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes was specified but the side-load did not succeed on the" -ForegroundColor Yellow Write-HostAzS " node(s) above. Re-run with -Verbose for the copy error, or install the exact version" -ForegroundColor Yellow Write-HostAzS " manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } else { Write-HostAzS " TIP: re-run with -InstallMissingModuleOnNodes to automatically copy the orchestrator's" -ForegroundColor Green Write-HostAzS " exact version (v$orchestratorVersionString) to the affected node(s), e.g.:" -ForegroundColor Green Write-HostAzS " Test-AzureLocalConnectivity -Scope Cluster -InstallMissingModuleOnNodes ..." -ForegroundColor Green Write-HostAzS "" Write-HostAzS " Or install the exact version manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } } if ($remotingFailures.Count -gt 0) { Write-HostAzS "" Write-HostAzS " PowerShell remoting (Invoke-Command) failed on $($remotingFailures.Count) node(s):" -ForegroundColor Red foreach ($rf in $remotingFailures) { Write-HostAzS " - $rf" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " -Scope Cluster requires the orchestrator session to be Administrator on every" -ForegroundColor Yellow Write-HostAzS " cluster node, with WinRM reachable (default port 5985 TCP). Verify with:" -ForegroundColor Yellow Write-HostAzS " Test-WSMan -ComputerName <nodeName>" -ForegroundColor Yellow } Write-HostAzS "" } # Single error-stream message so scripted / -NoOutput callers detect the failure. $errorParts = @() if ($missingNodes.Count -gt 0) { $errorParts += "module not installed on: $($missingNodes -join ', ')" } if ($driftedNodes.Count -gt 0) { $errorParts += "module version drift on: $($driftedNodeNames -join ', ')" } if ($remotingFailures.Count -gt 0) { $errorParts += "remoting failed on: $($remotingFailures -join '; ')" } $hint = if ($moduleProblem -and -not $InstallMissingModuleOnNodes.IsPresent) { " Re-run with -InstallMissingModuleOnNodes to copy the orchestrator's exact version (v$orchestratorVersionString) to the affected node(s)." } else { '' } Write-Error "Test-AzureLocalConnectivity -Scope Cluster aborted — $($errorParts -join ' | ').$hint" return $null } # ── Build the per-node parameter set (strip cluster-specific switches) ── # NOTE: 'Parallelism' is intentionally NOT stripped — each node runs its own # Layer-7 sweep in parallel. The worker Start-Jobs run LOCALLY on each node # (children of that node's wsmprovhost), so the orchestrator only holds one # lightweight remoting job per node; the worker count is bounded per-machine # exactly like a normal single-node -Parallelism run. Version parity (enforced # by the pre-flight) guarantees every node's workers import identical code. $perNodeParams = @{} foreach ($k in $ForwardedParameters.Keys) { if ($k -notin @('Scope','PassThru','NoOutput','ExcludeUploadResults','InstallMissingModuleOnNodes')) { $perNodeParams[$k] = $ForwardedParameters[$k] } } $perNodeParams['Scope'] = 'Node' if (-not $perNodeParams.ContainsKey('Parallelism')) { # Cluster default: 8 local workers per node (only applied when the caller # did NOT pass an explicit -Parallelism). A single-node run still defaults # to 1; the higher default here just speeds up the per-node sweep. $perNodeParams['Parallelism'] = 8 } $perNodeParams['PassThru'] = $true $perNodeParams['NoOutput'] = $true $perNodeParams['ExcludeUploadResults'] = $true # orchestrator handles upload (Phase D) # ── Resolve the export path on the orchestrator ────────────────────────── # Default to the SAME ProgramData location single-node runs use, so cluster # reports land alongside single-node reports (C:\ProgramData\...) instead of # under the operator's roaming profile (C:\Users\<admin>\...). if (-not $ExportPath) { $ExportPath = 'C:\ProgramData\AzStackHci.DiagnosticSettings' } if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # ── Fan out: run every node IN PARALLEL via Start-Job ─────────────────── # Each node gets its own background job (process-isolated, PS 5.1 safe — no # ForEach-Object -Parallel). The job wraps an Invoke-Command -ComputerName so # the actual test runs on the node; Start-Job isolation keeps each node's # remote Write-Progress / host output out of the orchestrator console, and # lets us show a single cluster-level progress bar instead of per-endpoint # noise. All nodes launch at once (unbounded) — each node runs its own local # Layer-7 worker sweep (per-node -Parallelism, default 8), so the worker # count stays bounded per-machine. $nodeJobScript = { param($NodeName, $Params, $RequiredVersion) $jobStart = Get-Date try { $remote = Invoke-Command -ComputerName $NodeName -ScriptBlock { param($P, $Ver) # Pin the import to the orchestrator's EXACT version so every node runs # byte-identical code even if a node also has other versions installed # (the pre-flight has already guaranteed this version is present here). if ($Ver) { Import-Module 'AzStackHci.DiagnosticSettings' -RequiredVersion $Ver -Force } else { Import-Module 'AzStackHci.DiagnosticSettings' -Force } $raw = Test-AzureLocalConnectivity @P # v0.6.8: Test-AzureLocalConnectivity -PassThru now returns a structured # PSCustomObject — per-URL rows live under $raw.Results and the run-level # telemetry are REAL properties on $raw (no longer fragile NoteProperties on # an ArrayList). Capture them explicitly so the remoting CliXml boundary # preserves them (Invoke-Command still enumerates a returned collection, so we # forward $raw.Results as the Rows payload rather than the container object). [pscustomobject]@{ Rows = $raw.Results DownloadSpeed = $raw.DownloadSpeed RequestMethod = $raw.RequestMethod Parallelism = $raw.Parallelism TotalDurationSeconds = $raw.TotalDurationSeconds Layer7TotalDurationSeconds = $raw.Layer7TotalDurationSeconds Layer7WallClockSeconds = $raw.Layer7WallClockSeconds Layer7TestedEndpoints = $raw.Layer7TestedEndpoints # v0.6.8: per-node report paths — these files live on THIS remote node's # local ProgramData, not on the orchestrator. ReportPath = $raw.ReportPath JSONReportPath = $raw.JSONReportPath # v0.6.8 §7.3: forward the per-node run-level detections (flat scalars + # string arrays survive the double CliXml boundary) so the unified cluster # -PassThru can carry the SAME Detections fields each node-scope run exposes. SSLInspectionDetected = $raw.SSLInspectionDetected SSLInspectedURLs = $raw.SSLInspectedURLs PrivateLinkDetected = $raw.PrivateLinkDetected PrivateLinkCriticalArray = $raw.PrivateLinkCriticalArray PrivateLinkProxyBypassArray = $raw.PrivateLinkProxyBypassArray # v0.6.8: full detection set + verified proxy-bypass breakdown (flat arrays # survive the double CliXml boundary). PrivateLinkDetectedArray = $raw.PrivateLinkDetectedArray OtherRfc1918Count = $raw.OtherRfc1918Count PrivateLinkBypassConfirmedArray = $raw.PrivateLinkBypassConfirmedArray PrivateLinkBypassMissingArray = $raw.PrivateLinkBypassMissingArray ProxyBypassList = $raw.ProxyBypassList NoProxyList = $raw.NoProxyList CRLOfflineDetected = $raw.CRLOfflineDetected CRLOfflineURLs = $raw.CRLOfflineURLs } } -ArgumentList $Params, $RequiredVersion -ErrorAction Stop # Return a FLAT wrapper (Rows one level deep, same nesting as the working # single-hop design) so the extra Start-Job CliXml boundary preserves the # already-deserialised flat row property bags. [pscustomobject]@{ Node = $NodeName Error = $null Duration = ((Get-Date) - $jobStart).ToString() Rows = $remote.Rows DownloadSpeed = $remote.DownloadSpeed RequestMethod = $remote.RequestMethod Parallelism = $remote.Parallelism TotalDurationSeconds = $remote.TotalDurationSeconds Layer7TotalDurationSeconds = $remote.Layer7TotalDurationSeconds Layer7WallClockSeconds = $remote.Layer7WallClockSeconds Layer7TestedEndpoints = $remote.Layer7TestedEndpoints ReportPath = $remote.ReportPath JSONReportPath = $remote.JSONReportPath SSLInspectionDetected = $remote.SSLInspectionDetected SSLInspectedURLs = $remote.SSLInspectedURLs PrivateLinkDetected = $remote.PrivateLinkDetected PrivateLinkCriticalArray = $remote.PrivateLinkCriticalArray PrivateLinkProxyBypassArray = $remote.PrivateLinkProxyBypassArray PrivateLinkDetectedArray = $remote.PrivateLinkDetectedArray OtherRfc1918Count = $remote.OtherRfc1918Count PrivateLinkBypassConfirmedArray = $remote.PrivateLinkBypassConfirmedArray PrivateLinkBypassMissingArray = $remote.PrivateLinkBypassMissingArray ProxyBypassList = $remote.ProxyBypassList NoProxyList = $remote.NoProxyList CRLOfflineDetected = $remote.CRLOfflineDetected CRLOfflineURLs = $remote.CRLOfflineURLs } } catch { [pscustomobject]@{ Node = $NodeName Error = $_.Exception.Message Duration = ((Get-Date) - $jobStart).ToString() Rows = $null } } } $clusterStart = Get-Date $perNodeResults = @{} $perNodeErrors = @{} $perNodeDurations = @{} $perNodeDownloadSpeed = @{} # per-node download-speed result (string, e.g. '123.4 Mbps') $perNodeTelemetry = @{} # per-node run-level telemetry (timing + RequestMethod) $perNodeDetections = @{} # per-node run-level detections (SSL / PrivateLink / CRL) # Launch all node jobs at once. $jobMap = @{} foreach ($node in $nodes) { $jobMap[$node] = Start-Job -Name "AzSHciConn_$node" -ScriptBlock $nodeJobScript ` -ArgumentList $node, $perNodeParams, $orchestratorVersionString } if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Fanned out to $($nodes.Count) node(s) in parallel: $($nodes -join ', ')" -ForegroundColor Cyan } # ── Poll for completion + show a node-level progress bar ───────────────── $jobList = @($jobMap.Values) $totalNodes = $nodes.Count do { $pending = @($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }) $doneCount = $totalNodes - $pending.Count if (-not $NoOutput.IsPresent) { $pendingNodes = @($nodes | Where-Object { $jobMap[$_].State -eq 'Running' -or $jobMap[$_].State -eq 'NotStarted' }) $pct = if ($totalNodes -gt 0) { [int](($doneCount / $totalNodes) * 100) } else { 100 } $statusText = if ($pendingNodes.Count -gt 0) { "Waiting on $($pendingNodes.Count) of $totalNodes node(s): $($pendingNodes -join ', ')" } else { 'Finalising...' } Write-Progress -Activity "Cluster connectivity test ($doneCount of $totalNodes nodes complete)" -Status $statusText -PercentComplete $pct } if ($pending.Count -gt 0) { $null = Wait-Job -Job $jobList -Any -Timeout 2 } } while (@($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }).Count -gt 0) if (-not $NoOutput.IsPresent) { Write-Progress -Activity "Cluster connectivity test" -Completed } # ── Receive each job + attribute results back to its node ──────────────── foreach ($node in $nodes) { $job = $jobMap[$node] $jobOutput = $null try { $jobOutput = Receive-Job -Job $job -ErrorAction Stop } catch { $jobOutput = $null } Remove-Job -Job $job -Force -ErrorAction SilentlyContinue # Start-Job can emit multiple objects; pick our wrapper (it carries a 'Node' prop). $nodeWrapper = $null if ($jobOutput) { $nodeWrapper = @($jobOutput | Where-Object { $_ -and $_.PSObject.Properties['Node'] }) | Select-Object -First 1 } if ($nodeWrapper -and (-not $nodeWrapper.Error) -and $nodeWrapper.PSObject.Properties['Rows'] -and $nodeWrapper.Rows) { $perNodeResults[$node] = @($nodeWrapper.Rows) $perNodeDownloadSpeed[$node] = $nodeWrapper.DownloadSpeed $perNodeTelemetry[$node] = [pscustomobject]@{ DownloadSpeed = $nodeWrapper.DownloadSpeed RequestMethod = $nodeWrapper.RequestMethod Parallelism = $nodeWrapper.Parallelism TotalDurationSeconds = $nodeWrapper.TotalDurationSeconds Layer7TotalDurationSeconds = $nodeWrapper.Layer7TotalDurationSeconds Layer7WallClockSeconds = $nodeWrapper.Layer7WallClockSeconds Layer7TestedEndpoints = $nodeWrapper.Layer7TestedEndpoints # v0.6.8: report paths on the REMOTE node (this node's local ProgramData). ReportPath = $nodeWrapper.ReportPath JSONReportPath = $nodeWrapper.JSONReportPath } # v0.6.8 §7.3: per-node run-level detections for the unified cluster -PassThru. $perNodeDetections[$node] = [pscustomobject]@{ SSLInspectionDetected = [bool]$nodeWrapper.SSLInspectionDetected SSLInspectedURLs = @($nodeWrapper.SSLInspectedURLs) PrivateLinkDetected = [bool]$nodeWrapper.PrivateLinkDetected PrivateLinkCriticalArray = @($nodeWrapper.PrivateLinkCriticalArray) PrivateLinkProxyBypassArray = @($nodeWrapper.PrivateLinkProxyBypassArray) PrivateLinkDetectedArray = @($nodeWrapper.PrivateLinkDetectedArray) OtherRfc1918Count = [int]$nodeWrapper.OtherRfc1918Count PrivateLinkBypassConfirmedArray = @($nodeWrapper.PrivateLinkBypassConfirmedArray) PrivateLinkBypassMissingArray = @($nodeWrapper.PrivateLinkBypassMissingArray) ProxyBypassList = @($nodeWrapper.ProxyBypassList) NoProxyList = @($nodeWrapper.NoProxyList) CRLOfflineDetected = [bool]$nodeWrapper.CRLOfflineDetected CRLOfflineURLs = @($nodeWrapper.CRLOfflineURLs) } $perNodeErrors[$node] = @() if (-not $NoOutput.IsPresent) { $rowCount = @($perNodeResults[$node]).Count $dlNote = if ($perNodeDownloadSpeed[$node]) { " (download speed: $($perNodeDownloadSpeed[$node]))" } else { '' } Write-HostAzS " [$node] Returned $rowCount result row(s).$dlNote" -ForegroundColor Green } } else { $perNodeResults[$node] = $null $perNodeDownloadSpeed[$node] = $null $perNodeTelemetry[$node] = $null $perNodeDetections[$node] = $null $errMsg = if ($nodeWrapper -and $nodeWrapper.Error) { $nodeWrapper.Error } else { "No result returned from node '$node' (job produced no output)." } $perNodeErrors[$node] = @($errMsg) if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] FAILED: $errMsg" -ForegroundColor Red } } if ($nodeWrapper -and $nodeWrapper.PSObject.Properties['Duration'] -and $nodeWrapper.Duration) { $perNodeDurations[$node] = try { [TimeSpan]::Parse($nodeWrapper.Duration) } catch { [TimeSpan]::Zero } } else { $perNodeDurations[$node] = [TimeSpan]::Zero } } $clusterEnd = Get-Date # ── Console summary: per-node connectivity FAILURES only ───────────────── # -Scope Node prints every endpoint result inline; -Scope Cluster suppresses the # per-node console (Start-Job isolation keeps each node's endpoint sweep out of the # orchestrator). To restore the at-a-glance failure visibility of a Node-scope run, # surface ONLY the failing endpoints here, attributed to the node they came from. # (The full per-endpoint detail still lives in the tabbed HTML / JSON report.) if (-not $NoOutput.IsPresent) { $failureLines = [System.Collections.Generic.List[string]]::new() foreach ($node in $nodes) { $rows = @($perNodeResults[$node]) if ($rows.Count -eq 0) { continue } $nodeFailures = @($rows | Where-Object { $_ -and ( ($_.PSObject.Properties['TCPStatus'] -and $_.TCPStatus -eq 'Failed') -or ($_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status -eq 'Failed') ) }) foreach ($f in $nodeFailures) { $u = if ($f.PSObject.Properties['URL']) { $f.URL } else { '<unknown>' } $p = if ($f.PSObject.Properties['Port']) { $f.Port } else { '' } $tcp = if ($f.PSObject.Properties['TCPStatus']) { $f.TCPStatus } else { '' } $l7 = if ($f.PSObject.Properties['Layer7Status']) { $f.Layer7Status } else { '' } # NOTE: wrap the -f in EXTRA parens — commas inside method () are arg # separators, so '... -f a, b' would pass b as a 2nd .Add() argument. # Leading tab indents the row under the banner so it stands out in console. $failureLines.Add(("`t [{0}] {1}:{2} TCP={3} L7={4}" -f $node, $u, $p, $tcp, $l7)) } } # Header banner (matches the module's //// section style) plus a tab indent so the # cluster failure summary is easy to spot in a busy console — mirroring the # at-a-glance "Test results summary:" block that -Scope Node prints. Write-HostAzS "`n`t//////////////// Connectivity Test Results ////////////////`n" -ForegroundColor Cyan if ($failureLines.Count -gt 0) { Write-HostAzS "`tConnectivity FAILURES ($($failureLines.Count) across $($nodes.Count) node(s)):" -ForegroundColor Red foreach ($line in $failureLines) { Write-HostAzS $line -ForegroundColor Red } } else { Write-HostAzS "`tNo connectivity failures detected on any node." -ForegroundColor Green } Write-HostAzS "" } # ── Build cluster-shaped result object ─────────────────────────────────── $clusterResult = [pscustomobject]@{ ClusterName = $clusterName OrchestratorMachine = $env:COMPUTERNAME RunGuid = [guid]::NewGuid() StartTime = $clusterStart EndTime = $clusterEnd Duration = ($clusterEnd - $clusterStart) Nodes = $perNodeResults NodeDurations = $perNodeDurations NodeDownloadSpeeds = $perNodeDownloadSpeed NodeTelemetry = $perNodeTelemetry NodeDetections = $perNodeDetections Errors = $perNodeErrors ExportPath = $ExportPath # v0.6.8: merged orchestrator report paths (set in Phase C after the report is written). ReportPath = $null JSONReportPath = $null } # ── Phase C — Generate cluster report (tabbed HTML, or merged CSV) ─────── # Honour the caller's -OutputFormat (HTML default; CSV writes a merged CSV). # JSON is always emitted alongside. AzureRegion isn't needed for cluster file # naming (cluster name only), so we don't depend on it here. $clusterOutputFormat = if ($ForwardedParameters.ContainsKey('OutputFormat') -and $ForwardedParameters['OutputFormat']) { [string]$ForwardedParameters['OutputFormat'] } else { 'HTML' } try { $reportPath = New-ConnectivityClusterReport -ClusterResult $clusterResult -ExportPath $ExportPath -OutputFormat $clusterOutputFormat if ($reportPath) { # Surface the MERGED orchestrator report paths (local to the orchestrator) at the top # level of the -Scope Cluster -PassThru object. JSON is written alongside with the same # base name, so swap the extension to derive its path. $clusterResult.ReportPath = $reportPath $clusterResult.JSONReportPath = $reportPath -replace '\.(html|csv)$', '.json' } if ($reportPath -and -not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Cluster connectivity report: $reportPath" -ForegroundColor Cyan } } catch { Write-Warning "New-ConnectivityClusterReport failed: $($_.Exception.Message)" } # ── Phase D — Upload bundle (Send-DiagnosticData integration) ─────────── # Test-CommandExists takes the command positionally (Param ($command)); -CommandName # is NOT a valid parameter and would silently bind to nothing. if (-not $ExcludeUploadResults.IsPresent -and (Test-CommandExists 'Send-DiagnosticData')) { try { Invoke-UploadDiagnosticResults ` -FolderToUpload $ExportPath ` -MessageToDisplay "Send the cluster connectivity results bundle to Microsoft? [Y/N]" ` -Context 'Cluster Connectivity Test results' ` -ErrorAction Stop } catch { Write-Warning "Cluster upload via Send-DiagnosticData failed: $($_.Exception.Message). Bundle remains at: $ExportPath" } } return $clusterResult } Function ConvertTo-ConnectivityClusterPassThru { <# .SYNOPSIS Transforms the internal cluster orchestrator result into the unified v0.6.8 (§7.3) structured -PassThru contract. .DESCRIPTION -Scope Node -PassThru returns a flat PSCustomObject (run-level fields on real properties, per-URL rows under .Results). For -Scope Cluster -PassThru we return the SAME structured shape but with per-node results nested under .Nodes[], where each .Nodes[] entry carries the identical flat run-level fields + Detections + .Results a node-scope run exposes. This gives consumers ONE contract regardless of scope. The internal $ClusterResult shape (used by the HTML/CSV report generator) is left untouched — this is a pure projection for callers. .PARAMETER ClusterResult The pscustomobject returned by Invoke-AzureLocalConnectivityClusterFanOut. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] $ClusterResult ) $nodeObjects = @() foreach ($nodeName in @($ClusterResult.Nodes.Keys)) { $rows = @($ClusterResult.Nodes[$nodeName]) $tel = $ClusterResult.NodeTelemetry[$nodeName] $det = $ClusterResult.NodeDetections[$nodeName] $errs = @($ClusterResult.Errors[$nodeName]) $nodeObjects += [pscustomobject]@{ SchemaVersion = '1.1' Hostname = $nodeName Collected = ($rows.Count -gt 0) Error = if ($errs.Count -gt 0) { $errs[0] } else { $null } DownloadSpeed = if ($tel) { $tel.DownloadSpeed } else { $null } RequestMethod = if ($tel) { $tel.RequestMethod } else { $null } Parallelism = if ($tel) { $tel.Parallelism } else { $null } TotalDurationSeconds = if ($tel) { $tel.TotalDurationSeconds } else { $null } Layer7TotalDurationSeconds = if ($tel) { $tel.Layer7TotalDurationSeconds } else { $null } Layer7WallClockSeconds = if ($tel) { $tel.Layer7WallClockSeconds } else { $null } Layer7TestedEndpoints = if ($tel) { $tel.Layer7TestedEndpoints } else { $null } # v0.6.8: per-node report paths — these files live on the REMOTE node ($nodeName), # not on the orchestrator. The merged cluster report is the top-level ReportPath. ReportPath = if ($tel) { $tel.ReportPath } else { $null } JSONReportPath = if ($tel) { $tel.JSONReportPath } else { $null } SSLInspectionDetected = if ($det) { [bool]$det.SSLInspectionDetected } else { $false } SSLInspectedURLs = if ($det) { @($det.SSLInspectedURLs) } else { @() } PrivateLinkDetected = if ($det) { [bool]$det.PrivateLinkDetected } else { $false } PrivateLinkCriticalArray = if ($det) { @($det.PrivateLinkCriticalArray) } else { @() } PrivateLinkProxyBypassArray = if ($det) { @($det.PrivateLinkProxyBypassArray) } else { @() } PrivateLinkDetectedArray = if ($det) { @($det.PrivateLinkDetectedArray) } else { @() } OtherRfc1918Count = if ($det) { [int]$det.OtherRfc1918Count } else { 0 } PrivateLinkBypassConfirmedArray = if ($det) { @($det.PrivateLinkBypassConfirmedArray) } else { @() } PrivateLinkBypassMissingArray = if ($det) { @($det.PrivateLinkBypassMissingArray) } else { @() } ProxyBypassList = if ($det) { @($det.ProxyBypassList) } else { @() } NoProxyList = if ($det) { @($det.NoProxyList) } else { @() } CRLOfflineDetected = if ($det) { [bool]$det.CRLOfflineDetected } else { $false } CRLOfflineURLs = if ($det) { @($det.CRLOfflineURLs) } else { @() } Results = $rows } } [pscustomobject]@{ SchemaVersion = '1.1' Scope = 'Cluster' ClusterName = $ClusterResult.ClusterName OrchestratorMachine = $ClusterResult.OrchestratorMachine Timestamp = if ($ClusterResult.StartTime) { $ClusterResult.StartTime.ToString('yyyy-MM-dd HH:mm:ss') } else { (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') } RunGuid = $ClusterResult.RunGuid StartTime = $ClusterResult.StartTime EndTime = $ClusterResult.EndTime Duration = $ClusterResult.Duration ExportPath = $ClusterResult.ExportPath # v0.6.8: the MERGED cluster report on the orchestrator (local to the caller, openable # directly — unlike the per-node ReportPath values, which are on the remote nodes). ReportPath = $ClusterResult.ReportPath JSONReportPath = $ClusterResult.JSONReportPath Nodes = @($nodeObjects) Errors = $ClusterResult.Errors } } Function Copy-AzStackHciModuleToNode { <# .SYNOPSIS Side-loads (copies) the orchestrator's installed AzStackHci.DiagnosticSettings module folder onto a remote cluster node over a PSSession. .DESCRIPTION Used by Invoke-AzureLocalConnectivityClusterFanOut when -InstallMissingModuleOnNodes is specified. Copies the orchestrator's exact versioned module folder (`...\Modules\AzStackHci.DiagnosticSettings\<version>`) to the same AllUsers module path on the target node, guaranteeing a byte-identical version match without any PowerShell Gallery / internet dependency (Azure Local nodes are frequently air-gapped). The copy is performed over a `New-PSSession` (NOT the `C$` admin share) so it depends only on the same WinRM remoting already required by -Scope Cluster — no extra SMB / firewall surface. Any existing copy of the SAME version on the node is removed first to avoid a partial-merge. Copied files are Unblock-File'd, then the version is re-verified via `Get-Module -ListAvailable` on the node. Owner constraints: this is an OPT-IN host mutation only; it does not change CredSSP, WinRM auth mode, TrustedHosts, or any shared security state. Returns $true if the exact version is present on the node after the copy, else $false. .PARAMETER NodeName The remote node to copy the module to. .PARAMETER SourceModuleBase The orchestrator's on-disk versioned module folder, e.g. `C:\Program Files\WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings\0.6.7`. .PARAMETER Version The exact module version being side-loaded (used for the destination subfolder and post-copy verification). .NOTES SECURITY / TRUST BOUNDARY: this copies the orchestrator's local module folder ($SourceModuleBase) to every node's %ProgramFiles%\WindowsPowerShell\Modules and imports it. Whatever is in the orchestrator's module install is what gets pushed to all nodes — run the side-load (-InstallMissingModuleOnNodes) only from a trusted, clean module install. Uses default Kerberos remoting (no CredSSP) and fails closed (returns $false) on any error. #> [CmdletBinding()] [OutputType([bool])] param( [Parameter(Mandatory)] [string]$NodeName, [Parameter(Mandatory)] [string]$SourceModuleBase, [Parameter(Mandatory)] [version]$Version ) $session = $null try { $session = New-PSSession -ComputerName $NodeName -ErrorAction Stop # Destination module-name root on the node (PS 5.1 AllUsers path). Resolved on the # node itself so a non-default %ProgramFiles% is honoured. $destModuleNameRoot = Invoke-Command -Session $session -ScriptBlock { Join-Path $env:ProgramFiles 'WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings' } $verString = $Version.ToString() # Ensure the module-name root exists and clear any partial copy of this exact version. Invoke-Command -Session $session -ScriptBlock { param($root, $ver) if (-not (Test-Path -Path $root)) { New-Item -Path $root -ItemType Directory -Force | Out-Null } $verPath = Join-Path $root $ver # Defence-in-depth: only delete when the leaf is a well-formed version string # (e.g. 0.6.7 / 0.6.7.0) AND the resolved path is still under the module-name # root, so a malformed $ver can never escalate this into deleting an # unintended path on the remote node. if (($ver -match '^\d+\.\d+\.\d+(\.\d+)?$') -and ($verPath -like (Join-Path $root '*')) -and (Test-Path -Path $verPath)) { Remove-Item -Path $verPath -Recurse -Force -ErrorAction SilentlyContinue } } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop # Copy the versioned folder INTO the module-name root → ...\<ModuleName>\<version>. Copy-Item -Path $SourceModuleBase -Destination $destModuleNameRoot -ToSession $session -Recurse -Force -ErrorAction Stop # Unblock copied files and confirm the exact version is now discoverable on the node. $ok = Invoke-Command -Session $session -ScriptBlock { param($root, $ver) $verPath = Join-Path $root $ver Get-ChildItem -Path $verPath -Recurse -ErrorAction SilentlyContinue | Unblock-File -ErrorAction SilentlyContinue $found = Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $ver } [bool]$found } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop return [bool]$ok } catch { Write-Verbose "Copy-AzStackHciModuleToNode: side-load to '$NodeName' failed: $($_.Exception.Message)" return $false } finally { if ($session) { Remove-PSSession -Session $session -ErrorAction SilentlyContinue } } } Function New-ConnectivityClusterReport { <# .SYNOPSIS Builds the tabbed HTML cluster report from a Cluster-shaped PassThru object. .DESCRIPTION Mirrors the structure of `New-OsConfigReport` (Private/AzStackHci.OsConfigReport.Helpers.ps1) but specialised for connectivity-test data. Sections: * Status Overview (top of Summary tab) — Node | Tested | Passed | Failed | SSL-Inspected | Private-Link | Status badge. * Per-node tabs — same tabular result layout the existing CSV/HTML emits today, scoped to that node's PassThru ArrayList. Returns the full HTML report file path on success, or $null on failure. .PARAMETER ClusterResult The pscustomobject produced by Invoke-AzureLocalConnectivityClusterFanOut. .PARAMETER ExportPath The directory to write the HTML/JSON files to. #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory)] [pscustomobject]$ClusterResult, [Parameter(Mandatory)] [string]$ExportPath, [Parameter(Mandatory=$false)] [ValidateSet('HTML','CSV')] [string]$OutputFormat = 'HTML' ) if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # Nested HTML-encode helper. The ConvertTo-HtmlSafe in OsConfigReport.Helpers.ps1 # is nested inside New-OsConfigReport, so it isn't visible from this scope; we # define our own with the same name + signature so all the call sites below read # cleanly. PowerShell function lookup resolves nested functions before sibling # function names, so this does NOT shadow the OsConfigReport version. function ConvertTo-HtmlSafe { param([string]$Text) if (-not $Text) { return '' } return [System.Net.WebUtility]::HtmlEncode($Text) } # File naming: cluster name only (no orchestrator host name), matching the # single-node AzureLocal_ConnectivityTest_* convention so all reports group # together in C:\ProgramData\AzStackHci.DiagnosticSettings. $reportStamp = Get-Date -Format 'yyyy-MM-dd-HH-mm-ss' $safeClusterName = ([string]$ClusterResult.ClusterName) -replace '[^A-Za-z0-9._-]', '_' $baseName = "AzureLocal_ConnectivityTest_Cluster_${safeClusterName}_${reportStamp}" $reportExt = if ($OutputFormat -eq 'CSV') { '.csv' } else { '.html' } $reportFile = Join-Path $ExportPath ($baseName + $reportExt) $jsonFile = Join-Path $ExportPath ($baseName + '.json') $sb = [System.Text.StringBuilder]::new() # ── HTML head + CSS ────────────────────────────────────────────────────── $cssBlock = @' <style> body { font-family: 'Segoe UI', Tahoma, Geneva, sans-serif; margin: 16px; color: #222; background: #f7f7f9; } h1 { color: #0078d4; margin-bottom: 4px; } h2 { color: #333; border-bottom: 2px solid #e1e1e1; padding-bottom: 4px; margin-top: 20px; } h3 { color: #444; margin-top: 14px; } table { border-collapse: collapse; width: 100%; background: #fff; margin-bottom: 12px; } th, td { border: 1px solid #d8d8d8; padding: 6px 10px; text-align: left; vertical-align: top; font-size: 13px; } th { background: #f0f4f8; color: #003366; } tr:nth-child(even) { background: #fafbfc; } .summary-card { background: #fff; border: 1px solid #d8d8d8; padding: 12px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .tab-bar { display: flex; flex-wrap: wrap; gap: 4px; border-bottom: 2px solid #0078d4; margin-bottom: 0; } .tab-btn { background: #e1e1e1; border: 1px solid #c1c1c1; border-bottom: none; padding: 6px 14px; cursor: pointer; font-size: 13px; border-radius: 4px 4px 0 0; } .tab-btn.active { background: #fff; border-color: #0078d4; font-weight: bold; color: #0078d4; } .tab-panel { display: none; padding: 14px; background: #fff; border: 1px solid #c1c1c1; border-top: none; } .tab-panel.active { display: block; } .badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: bold; } .badge-ok { background: #d4edda; color: #155724; border: 1px solid #c3e6cb; } .badge-warn { background: #fff3cd; color: #856404; border: 1px solid #ffeeba; } .badge-error { background: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; } .badge-na { background: #e2e3e5; color: #383d41; border: 1px solid #d6d8db; } .footer { margin-top: 24px; color: #888; font-size: 11px; text-align: center; } .row-failed { background: #fdecea !important; } .row-warn { background: #fff8e1 !important; } /* Per-node result-row colouring — identical palette to the single-node report. */ tr.status-failed { background-color: #fde7e9 !important; } tr.status-success { background-color: #e6f4ea !important; } tr.status-skipped { background-color: #fff4ce !important; } .totals-card { background: #fff; border: 1px solid #d8d8d8; padding: 10px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .totals-card .num { font-size: 18px; font-weight: bold; } .totals-card .ok { color: #155724; } .totals-card .fail { color: #721c24; } .totals-card .skip { color: #856404; } </style> '@ [void]$sb.AppendLine("<!DOCTYPE html>") [void]$sb.AppendLine("<html lang='en'><head><meta charset='utf-8'>") [void]$sb.AppendLine("<title>Cluster Connectivity Report — $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)</title>") [void]$sb.AppendLine($cssBlock) [void]$sb.AppendLine("</head><body>") [void]$sb.AppendLine("<h1>Cluster Connectivity Report</h1>") [void]$sb.AppendLine("<div class='summary-card'>") [void]$sb.AppendLine("<strong>Cluster:</strong> $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)<br>") [void]$sb.AppendLine("<strong>Orchestrator:</strong> $(ConvertTo-HtmlSafe $ClusterResult.OrchestratorMachine)<br>") [void]$sb.AppendLine("<strong>Run GUID:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.RunGuid.ToString()))<br>") [void]$sb.AppendLine("<strong>Start:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.StartTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>End:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.EndTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>Duration:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.Duration.ToString()))<br>") [void]$sb.AppendLine("</div>") # ── Build per-node summary stats ───────────────────────────────────────── $nodeNames = @($ClusterResult.Nodes.Keys | Sort-Object) $summary = @{} foreach ($node in $nodeNames) { $rows = @($ClusterResult.Nodes[$node] | Where-Object { $_ -and $_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status }) $tested = $rows.Count $passed = @($rows | Where-Object { $_.Layer7Status -eq 'Success' }).Count $failed = @($rows | Where-Object { $_.Layer7Status -eq 'Failed' }).Count $skipped = @($rows | Where-Object { $_.Layer7Status -eq 'Skipped' }).Count $sslInspected = @($rows | Where-Object { $_.Layer7Response -match 'SSL Inspection' }).Count $privateLink = @($rows | Where-Object { $_.Layer7Response -match 'Private Link|PrivateLink' }).Count $hasErrors = @($ClusterResult.Errors[$node]).Count -gt 0 $dlSpeed = if ($ClusterResult.PSObject.Properties['NodeDownloadSpeeds'] -and $ClusterResult.NodeDownloadSpeeds) { $ClusterResult.NodeDownloadSpeeds[$node] } else { $null } $status = if ($failed -gt 0 -or $hasErrors) { 'error' } elseif ($sslInspected -gt 0 -or $privateLink -gt 0) { 'warn' } else { 'ok' } $summary[$node] = [pscustomobject]@{ Tested = $tested Passed = $passed Failed = $failed Skipped = $skipped SSLInspected = $sslInspected PrivateLink = $privateLink DownloadSpeed = $dlSpeed Status = $status } } # ── Build the MERGED, flat result set (ComputerName as the FIRST column) ── # Used for the always-on JSON sidecar and the CSV output. ComputerName is set # to the node we fanned out to. The per-node HTML tabs intentionally OMIT this # column (the tab name + heading already identify the node). $mergedColumns = @( 'RowID','URL','Port','ArcGateway','IsWildcard','Source','IPAddress', 'Layer7Status','Layer7Response','Layer7ResponseTime','Note','TCPStatus', 'CertificateIssuer','CertificateSubject','CertificateThumbprint', 'IntermediateCertificateIssuer','IntermediateCertificateSubject','IntermediateCertificateThumbprint', 'RootCertificateIssuer','RootCertificateSubject','RootCertificateThumbprint' ) $mergedRows = [System.Collections.ArrayList]::new() foreach ($node in $nodeNames) { $nrows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) foreach ($r in $nrows) { $ordered = [ordered]@{ ComputerName = $node } foreach ($c in $mergedColumns) { $ordered[$c] = if ($r.PSObject.Properties[$c]) { $r.$c } else { $null } } [void]$mergedRows.Add([pscustomobject]$ordered) } } # ── Rolled-up cluster totals (for the Summary tab + JSON) ───────────────── $totalTested = ($summary.Values | ForEach-Object { $_.Tested } | Measure-Object -Sum).Sum $totalPassed = ($summary.Values | ForEach-Object { $_.Passed } | Measure-Object -Sum).Sum $totalFailed = ($summary.Values | ForEach-Object { $_.Failed } | Measure-Object -Sum).Sum $totalSkipped = ($summary.Values | ForEach-Object { $_.Skipped } | Measure-Object -Sum).Sum if (-not $totalTested) { $totalTested = 0 } if (-not $totalPassed) { $totalPassed = 0 } if (-not $totalFailed) { $totalFailed = 0 } if (-not $totalSkipped) { $totalSkipped = 0 } # ── Tab bar (Summary + per-node) with status indicators ────────────────── [void]$sb.AppendLine('<div class="tab-bar">') [void]$sb.AppendLine('<button class="tab-btn active" onclick="showTab(event, ''tab-summary'')">Summary</button>') foreach ($node in $nodeNames) { $st = $summary[$node].Status $badge = switch ($st) { 'ok' { '<span class="badge badge-ok" style="font-size:9px;padding:1px 6px;">✓</span>' } 'warn' { '<span class="badge badge-warn" style="font-size:9px;padding:1px 6px;">⚠</span>' } 'error' { '<span class="badge badge-error" style="font-size:9px;padding:1px 6px;">✗</span>' } default { '' } } $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<button class=`"tab-btn`" onclick=`"showTab(event, 'tab-$safeNode')`">$safeNode $badge</button>") } [void]$sb.AppendLine('</div>') # ── Summary tab — rolled-up totals + per-node Status Overview ──────────── [void]$sb.AppendLine('<div id="tab-summary" class="tab-panel active">') [void]$sb.AppendLine('<h2>Cluster Totals</h2>') [void]$sb.AppendLine('<div class="totals-card">') [void]$sb.AppendLine("<table style='width:auto;'><thead><tr><th>Nodes</th><th>Endpoints Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th></tr></thead><tbody>") [void]$sb.AppendLine("<tr><td class='num'>$($nodeNames.Count)</td><td class='num'>$totalTested</td><td class='num ok'>$totalPassed</td><td class='num fail'>$totalFailed</td><td class='num skip'>$totalSkipped</td></tr>") [void]$sb.AppendLine('</tbody></table>') [void]$sb.AppendLine('</div>') [void]$sb.AppendLine('<h2>Per-Node Status Overview</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th><th>SSL-Inspected</th><th>Private-Link</th><th>Download Speed</th><th>Status</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $s = $summary[$node] $badgeClass = 'badge-' + $s.Status $badgeText = switch ($s.Status) { 'ok' { 'OK' } 'warn' { 'WARN' } 'error' { 'FAIL' } default { '?' } } $dlText = if ($s.PSObject.Properties['DownloadSpeed'] -and $s.DownloadSpeed) { ConvertTo-HtmlSafe ([string]$s.DownloadSpeed) } else { 'N/A' } [void]$sb.AppendLine("<tr><td>$(ConvertTo-HtmlSafe $node)</td><td>$($s.Tested)</td><td>$($s.Passed)</td><td>$($s.Failed)</td><td>$($s.Skipped)</td><td>$($s.SSLInspected)</td><td>$($s.PrivateLink)</td><td>$dlText</td><td><span class=`"badge $badgeClass`">$badgeText</span></td></tr>") } [void]$sb.AppendLine('</tbody></table>') # Errors block (if any) $totalErrors = ($ClusterResult.Errors.Values | ForEach-Object { @($_).Count } | Measure-Object -Sum).Sum if ($totalErrors -gt 0) { [void]$sb.AppendLine('<h2>Per-Node Errors</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Error</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $errs = @($ClusterResult.Errors[$node]) foreach ($e in $errs) { [void]$sb.AppendLine("<tr class='row-failed'><td>$(ConvertTo-HtmlSafe $node)</td><td>$(ConvertTo-HtmlSafe $e)</td></tr>") } } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') # ── Per-node tabs ──────────────────────────────────────────────────────── foreach ($node in $nodeNames) { $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<div id=`"tab-$safeNode`" class=`"tab-panel`">") [void]$sb.AppendLine("<h2>Node: $safeNode</h2>") # Filter $null entries — a failed Invoke-Command leaves Nodes[<name>] = $null # and `@($null)` is a 1-element array containing $null (not @()), which would # otherwise blow up `$r.PSObject.Properties[...]` below. $rows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) if ($rows.Count -eq 0) { [void]$sb.AppendLine("<p><em>No results returned from this node. See Errors on the Summary tab.</em></p>") } else { [void]$sb.AppendLine('<table><thead><tr>') # Per-node tab columns — full single-node column set, WITHOUT ComputerName # (the tab name + "Node:" heading already identify the host). $columns = $mergedColumns foreach ($c in $columns) { [void]$sb.AppendLine("<th>$c</th>") } [void]$sb.AppendLine('</tr></thead><tbody>') foreach ($r in $rows) { # Colour rows by Layer7Status using the SAME palette as the single-node # report: Success=green, Failed=red, Skipped=yellow. $statusVal = if ($r.PSObject.Properties['Layer7Status']) { [string]$r.Layer7Status } else { '' } $rowClass = switch ($statusVal) { 'Success' { 'status-success' } 'Failed' { 'status-failed' } 'Skipped' { 'status-skipped' } default { '' } } $cls = if ($rowClass) { " class=`"$rowClass`"" } else { '' } [void]$sb.AppendLine("<tr$cls>") foreach ($c in $columns) { $val = if ($r.PSObject.Properties[$c]) { $r.$c } else { '' } [void]$sb.AppendLine("<td>$(ConvertTo-HtmlSafe ([string]$val))</td>") } [void]$sb.AppendLine('</tr>') } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') } # ── Footer + JS ───────────────────────────────────────────────────────── # Stamp the running module version into the footer so portable HTML artefacts # always record which build produced them (best-effort; 'version unknown' if unresolved). $footerModuleVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$footerModuleVersionLabel = if ($footerModuleVersion) { "v$($footerModuleVersion.ToString())" } else { 'version unknown' } [void]$sb.AppendLine(@" <div class="footer"> Generated by <strong>AzStackHci.DiagnosticSettings</strong> $footerModuleVersionLabel — Test-AzureLocalConnectivity -Scope Cluster — $(Get-Date -Format 'u') </div> <script> function showTab(evt, tabId) { var panels = document.querySelectorAll('.tab-panel'); for (var i = 0; i < panels.length; i++) panels[i].classList.remove('active'); var btns = document.querySelectorAll('.tab-bar > .tab-btn'); for (var i = 0; i < btns.length; i++) btns[i].classList.remove('active'); document.getElementById(tabId).classList.add('active'); evt.currentTarget.classList.add('active'); } </script> </body></html> "@) # ── Write the primary output file ──────────────────────────────────────── # HTML -> tabbed cluster report ($sb). CSV -> merged flat rows (ComputerName # first). JSON is ALWAYS written below in addition. (BOM-less UTF-8; PS 5.1 # Set-Content -Encoding UTF8 would add a BOM.) if ($OutputFormat -eq 'CSV') { try { # ConvertTo-Csv + Write-Utf8NoBom instead of Export-Csv -Encoding UTF8: the # latter prepends a UTF-8 BOM on PS 5.1 (the same BOM the comment above warns # about for the HTML path). @() guards the empty-rows case so an empty # collection binds cleanly to Write-Utf8NoBom's [string[]] parameter. $csvLines = @($mergedRows | ConvertTo-Csv -NoTypeInformation) Write-Utf8NoBom -Path $reportFile -Content $csvLines } catch { Write-Warning "Failed to write merged cluster CSV '$reportFile': $($_.Exception.Message)" } } else { Write-Utf8NoBom -Path $reportFile -Content $sb.ToString() } # ── JSON sidecar — full-fidelity cluster bundle (always written) ───────── # Schema v2: the per-node `Nodes` map is replaced by a single MERGED `Results` # array where every row carries ComputerName as its FIRST property, plus rolled # up `Totals`. Per-node summary/telemetry/errors are retained for drill-down. $nodeDurStrings = [ordered]@{} foreach ($node in $nodeNames) { $d = $ClusterResult.NodeDurations[$node] $nodeDurStrings[$node] = if ($d) { $d.ToString() } else { '' } } $orchestratorVer = (Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings') $jsonObj = [ordered]@{ Schema = 'AzStackHciClusterConnectivity/v2' ModuleVersion = if ($orchestratorVer) { $orchestratorVer.ToString() } else { 'unknown' } ClusterName = $ClusterResult.ClusterName OrchestratorMachine = $ClusterResult.OrchestratorMachine RunGuid = $ClusterResult.RunGuid.ToString() StartTime = $ClusterResult.StartTime.ToString('u') EndTime = $ClusterResult.EndTime.ToString('u') Duration = $ClusterResult.Duration.ToString() Totals = [ordered]@{ Nodes = $nodeNames.Count Tested = $totalTested Successful = $totalPassed Failed = $totalFailed Skipped = $totalSkipped } NodeSummary = $summary NodeDurations = $nodeDurStrings NodeTelemetry = if ($ClusterResult.PSObject.Properties['NodeTelemetry']) { $ClusterResult.NodeTelemetry } else { @{} } Errors = $ClusterResult.Errors Results = $mergedRows.ToArray() } # Depth 8 is enough for: { Results: [ { flat row props } ] }. Write-Utf8NoBom -Path $jsonFile -Content ($jsonObj | ConvertTo-Json -Depth 8) return $reportFile } # SIG # Begin signature block # MIInRgYJKoZIhvcNAQcCoIInNzCCJzMCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDl0jo4Gq//n86x # aegUXe2EVQO/V/vNHVqfGpccuug/AqCCDLowggX1MIID3aADAgECAhMzAAACHU0Z # yE7XD1dIAAAAAAIdMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNVBAYTAlVTMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBD # b2RlIFNpZ25pbmcgUENBIDIwMjQwHhcNMjYwNDE2MTg1OTQzWhcNMjcwNDE1MTg1 # OTQzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYD # VQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IB # DwAwggEKAoIBAQDQvewXxx9gZZFC6Ys1WBay8BJ8kGA4JQnH5CMafqOASlTpK9H8 # o5ZXTXt0caVQTNMUPt445wXYD+dFtaKWTwDn1I52oUSrC9vJin1Gsqt+zyKJL5Dg # 3eQXbQNR61DmMy20GLTIO3SFed9Rfi/ophgCLGFLDR3r0KvHjwMb/jYWS0celV/4 # Lz27LfAekm8v9E5IXaeiXbAUYZKK090n4CVl3JBtbN+9DtI9SNu/yjvozW52/u7R # X/Ttpa/KDlpuokZ+Zcbvmtd9ur9gFLvZzh41o9MsE/clQtdaFWGvuo6Jua/ntpgk # ey3E5/vBFe+MJPG6phdnuo6r57ZudCudiI1bAgMBAAGjggGbMIIBlzAOBgNVHQ8B # Af8EBAMCB4AwHwYDVR0lBBgwFgYKKwYBBAGCN0wIAQYIKwYBBQUHAwMwHQYDVR0O # BBYEFH6QuMwqcPG0hQlQ6c5jCtTTLrVeMEUGA1UdEQQ+MDykOjA4MR4wHAYDVQQL # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xFjAUBgNVBAUTDTIzMDAxMis1MDc1NTkw # HwYDVR0jBBgwFoAUf1k/VCHarU/vBeXmo9ctBpQSCDEwYAYDVR0fBFkwVzBVoFOg # UYZPaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0 # JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNybDBtBggrBgEFBQcBAQRh # MF8wXQYIKwYBBQUHMAKGUWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y2VydHMvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNy # dDAMBgNVHRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4ICAQBKTbYOjzwTG/DXGaz9 # s6+fQeaTtDcFmMY+5UyVFCyj7Pv+5i37qfX8lSL/tBIfYQfWsMuBQlfZurJD6r4H # VJ2CeH+1fgiq8dcHdVKoZ3Sa2qXoX3cq9iS8cVb06B7+5/XJ7I0OxHH9fDsvJ3T3 # w5V/ZtAIFmLrl+P0CtG+92uzRsn0nTbdFjOkLMLWPLAU3THohKRlSEMgFJpPkm5n # 5UAZ35xX6FWCrDLsSKb555bTifwa8mJBwdlof0bmfYidH+dxZ1FdDxvLnNl9zeKs # A4kejaaIqqIPguhwAti5Ql7BlTNoJNwxCvBmqW2MQLnCkYN/VVUsR3V2x/rcTNzo # Bf/Z/SpROvdaA2ZOOd1uioXJt3tdLQ7vHpqpib0KfWr/FWXW10q38VxfCnRQBqzb # SuztR7nEMuzX7Ck+B/XaPDXd1qh72+QYyB0Z2VzWmO9zsnb9Uq/dwu8LGeQqnyu6 # 7SDGACvnXii2fb9+US492VTnXSnFKyqwgzUyFMtZK1/sHYTv6bG4TtQUygQxTN+Z # V+aJIlKO2MqZ7bKrAnOzS9m6NgoTdWOq11bTOZwKlIEV/EhV9SWkDmdpR/hPPT2v # 6TEj4F8PT/zHjRezIU5c/DGlt/VhY/pK0XkJtEyMmmS1BMtjU/rqBZVMIm3dnxQs # /TBByr+Cf8Z1r7aifQVQ+WSqzjCCBr0wggSloAMCAQICEzMAAAA5O7Y3Gb8GHWcA # AAAAADkwDQYJKoZIhvcNAQEMBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDExMB4XDTI0MDgwODIwNTQxOFoXDTM2MDMyMjIyMTMwNFow # VzELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEo # MCYGA1UEAxMfTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAyNDCCAiIwDQYJ # KoZIhvcNAQEBBQADggIPADCCAgoCggIBANgBnB7jOMeqlRYHNa265v4IY9fH8TKh # emHfPINe1gpLaV3dhg324WwH06LcHbpnsBukCDNitryo0dtS/EW6I/yEL/bLSY8h # KpbfQuWusBPr9qazYcDxCW/qnjb5JsI1s8bNOg3bVATvQVL4tcf03aTycsz8QeCd # M0l/yHRObJ9QqazM1r6VPEOJ7LL+uEEb73w6QCuhs89a1uv1zerOYMnsneRRwCbp # yW11IcggU0cRKDDq1pjVJzIbIF6+oiXXbReOsgeI8zu1FyQfK0fVkaya8SmVHQ/t # Of23mZ4W9k0Ri22QW9p3UgSC5OUDktKxxcCmGL6tXLfOGSWHIIV4YrTJTT6PNty5 # REojHJuZHArkF9VnHTERWoTjAzfI3kP+5b4alUdhgAZ7ttOu1bVnXfHaqPYl2rPs # 20ji03LOVWsh/radgE17es5hL+t6lV0eVHrVhsssROWJuz2MXMCt7iw7lFPG9LXK # Gjsmonn2gotGdHIuEg5JnJMJVmixd5LRlkmgYRZKzhxSCwyoGIq0PhaA7Y+VPct5 # pCHkijcIIDm0nlkK+0KyepolcqGm0T/GYQRMhHJlGOOmVQop36wUVUYklUy++vDW # eEgEo4s7hxN6mIbf2MSIQ/iIfMZgJxC69oukMUXCrOC3SkE/xIkgpfl22MM1itkZ # 35nNXkMolU1lAgMBAAGjggFOMIIBSjAOBgNVHQ8BAf8EBAMCAYYwEAYJKwYBBAGC # NxUBBAMCAQAwHQYDVR0OBBYEFH9ZP1Qh2q1P7wXl5qPXLQaUEggxMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # ci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0fBFMwUTBPoE2gS4ZJaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcBAQRSMFAwTgYIKwYBBQUHMAKGQmh0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNydDANBgkqhkiG9w0BAQwFAAOCAgEAFJQfOChP7onn6fLI # MKrSlN1WYKwDFgAddymOUO3FrM8d7B/W/iQ6DxXsDn7D5W4wMwYeLystcEqfkjz4 # NURRgazyMu5yRzQh4LqjA4tStTcJh1opExo7nn5PuPBYnbu0+THSuVHTe0VTTPVh # ily/piFrDo3axQ9P4C+Ol5yet+2gTfekICS5xS+cYfSIvgn0JksVBVMYVI5QFu/q # hnLhsEFEUzG8fvv0hjgkO+lkpV9ty6GkN4vdnd7ya6Q6aR9y34aiM1qmxaxBi6OU # nyNl6fkuun/diTFnYDLTppOkr/mg5WSfCiDVMNCxtj4wPKC5OmHm1DQIt/MNokbb # H3UGsFP1QbzsLocuSqLCvH09Io3fDPTmscR9Y75G4qX7RTX8AdBPo0I6OEojf39z # uFZt0qOHm65YWQE69cZM2ueE1MB05dNNgHK9gTE7zKvK/fg8B2qjW88MT/WF5V5u # vZGtqa9FSL2RazArA+rDPuf6JGYz4HpgMZHB4S6szWSKYBv0VisCzfxgeU+dquXW # 9bd0auYlOB58DPcOYKdc3Se94g+xL4pcEhbB54JOgAkwYTu/9dLeH2pDqeJZAABV # DWRQCaXfO5LgyKwKCLYXpigrZYCjUSBcr+Ve8PFWMhVTQl0v4q8J/AUmQN5W4n10 # 1cY2L4A7GTQG1h32HHAvfQESWP0xghniMIIZ3gIBATBuMFcxCzAJBgNVBAYTAlVT # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jv # c29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMjQCEzMAAAIdTRnITtcPV0gAAAAAAh0w # DQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYK # KwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIC36EK3j # bP3Nw5XnAbvVVWeALhs9icFA4EQUcYcpbVs5MEIGCisGAQQBgjcCAQwxNDAyoBSA # EgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20w # DQYJKoZIhvcNAQEBBQAEggEARfC6QYJGUFj6iHi/zBkPF7TmtfEpGeQZgCSGzRzy # lNkgWanC/jVa5+AF2NDY1ScsphYEr+Wrm5qtXUk4IelhjaLLSnzoVPcrUjnHhn7o # vNRuhjW+LLcwIHUOgKhrSiiHXREvme8X+UW9xCjKLAi7U0HWHgQtFTYySkEqENYQ # Tgt8Tb97dtLD+La52G1g2h6QDnG1UIfcvH+RhckTZ+gnhOfeYlIYrk2924Oc0AaN # cmoDd2D0RfGUGbNGpTxg3Ke3JKaOGstWAKo8bB0fWD04iqEUv+m283UPwhc8+0nz # QaLmQqP1ljbeymd1keP4EoutBwFeQRGRLResezdyKRMUuaGCF5QwgheQBgorBgEE # AYI3AwMBMYIXgDCCF3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUD # BAIBBQAwggFSBgsqhkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoD # ATAxMA0GCWCGSAFlAwQCAQUABCDgKEbjdIuVinA77+euRVj4bA1AJyUH3O/8Z29t # gEfNAgIGakfvQ3H0GBMyMDI2MDcxNTE4MjU1OS4zMjRaMASAAgH0oIHRpIHOMIHL # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxN # aWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRT # UyBFU046OTIwMC0wNUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0 # YW1wIFNlcnZpY2WgghHqMIIHIDCCBQigAwIBAgITMwAAAiNP2WAkU8/+KwABAAAC # IzANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAe # Fw0yNjAyMTkxOTM5NTdaFw0yNzA1MTcxOTM5NTdaMIHLMQswCQYDVQQGEwJVUzET # MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV # TWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmlj # YSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046OTIwMC0wNUUw # LUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIi # MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCK6Q2nk5WUdKzSCSafp+UjUARs # WxHKS63rJhFC/zSabFumTBuaJ0QNrmqevub5Db7fSj5qtwwKnjIO92+HXF67192f # ujL7DFot5WEj/AtEZ/XrzFHimKlN1h6gEQwP5I67wizaPW5ZzSBNpaLBg5oHvASP # OZtwdNUoZ+DQKF3hJl1KZuoIlVK+qi7cLjgak6s5oOZcRCMrKnuC3aoVa6wRDbYv # KUuj7rkFx9KO0PsHJ/k+LnZMggRheh4AVdawyh+oOzKPjlQGUNfSeWUgym2U9CLa # 8tt0mQX4DxDz6+ram50gj1oAfyQ6TQ7r96PADFOKBgaU7+cpHnaZG89dTegQ6ydB # RGIycOw1dRX2eKDRRzziK3cn0WaIm/7OeGsyQKjIzEQuUTDv0Jj/9zQ7truLOOpJ # D98BJVOK7je84Sz2hb3HvUST7j1j2N8peD6olkpFHR/1Z8Jz4F+mkrUF7MmPAirY # HRzunbIg3HrDMNwFYN7yBkDA4/VMo9CY0y9oGUoq2yjbCwTibz9VYl93nB3QQiTC # T9nW3M+TOWB+PMrZpExq1BSHmKPzIqehKqrUDoM33PK+dEKwpYLET6uXq4HuQRMX # WT//sPubUnQAaaUMfQhAZSy23HtxwtN3eK9+T4wCav2wQFt57eUOwUW5/DCzMF9t # ua5He1hNvgcAXaiG1wIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFNbAh89v29nPY9bw # Qb1QYCzxVgeXMB8GA1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1Ud # HwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3Js # L01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggr # BgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNv # bS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIw # MTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgw # DgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4ICAQCHQwe7z5tp4NZwAf1c # B+4c9J4svw3P6WqGBMxtqznS6DdzUzStXHCaPZhM41g1iKHNnmcnLjwLujOEaNjh # SnUDiAZqQjW5ZapOBxgc7Egghh9k+r78qWAe3rJ4QohBbhSGdZtKivTRaeRqmnhy # 8+ThrKhzCeEwaarXJimZwSpdQQUDbheWHeyAxASqultd5KO0m/UFvO03tfepqGXA # 4tCg/WGECwKqOjJzpRAfPIB6y1HyVrk+vmL5rpEbTwwLOtX7WxFGG8+cYLk9HjaD # kxraA/HYlKQRx1sdza+w/gulLwgOnByRJKF2rr8M7FNIlwoi6ywFpaNc8A7HewaG # jgw/tfcE260I1XekGluANI9HnONOYWlI7BKBQbWE2teo6vsQ1Vg8B8rTZSePVdmX # L1PPqqs3KVdFKM5kYocPCDM+6VL32IV96sESf2T7DjxanpCg2D2UYj4Z1i7cy8U1 # LLDGg55KWs4af2RRBjH2MulHgAmW5obKxiZCDQjRaroJ2XElXUhigE9BzvhCFbT/ # HDY2vpVpl5HnSpcCSxmL5i5lIT/xbAQMI7Luh75Xrm+IslfFWOGOGMlCp+24qEJE # glXEP7xwsolNdBNndXihhyIefVGlI1DR7xGELiJrk8ifVWYo9XEbEXv/lbvp6F2R # 2UsnweWckvq0y1HWnLHDqH6dPjCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkA # AAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVow # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUA # A4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX # 9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1q # UoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8d # q6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byN # pOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2k # rnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4d # Pf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgS # Uei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8 # QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6Cm # gyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzF # ER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQID # AQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQU # KqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1 # GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0 # bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMA # QTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbL # j+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1p # Y3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0w # Ni0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIz # LmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwU # tj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN # 3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU # 5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5 # KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGy # qVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB6 # 2FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltE # AY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFp # AUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcd # FYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRb # atGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQd # VTNYs6FwZvKhggNNMIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzAR # BgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1p # Y3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2Eg # T3BlcmF0aW9uczEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjkyMDAtMDVFMC1E # OTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEw # BwYFKw4DAhoDFQA4RWFs+kTiZnoZiAj1BtYj8zCNaqCBgzCBgKR+MHwxCzAJBgNV # BAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4w # HAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29m # dCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA7gI+ODAiGA8y # MDI2MDcxNTE3MTMyOFoYDzIwMjYwNzE2MTcxMzI4WjB0MDoGCisGAQQBhFkKBAEx # LDAqMAoCBQDuAj44AgEAMAcCAQACAg70MAcCAQACAhOWMAoCBQDuA4+4AgEAMDYG # CisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEA # AgMBhqAwDQYJKoZIhvcNAQELBQADggEBAFI4r6+6JND5eQziOdH2yl0pa3cU6mBx # DzrnGiEhjn4kHq2e5BT2X78Oosv9RFO7ufPQaAfY36U83uAAhMqtce4EUb0xsd9z # mV8+RKxMpNp0eMH5juF12Yjo2rGb1qKcG8K1nfTPSWX1iQyocpZiahoRlqHQRXzJ # hMbbJWO3VfD4xEC9lrsOZYjLP5OTcmrOhgj99bSLpXiTFR90nuwRxlki4ej5DCEM # isMtWXZXfwoKZPvKA5rTJZSuwPRZS3+f1pUO4fdZ5s2EHygYf8ZFFYILgBSuet4I # SZoYQqFIO39abARBsqiPZnR6/HcU2FsFgCHeKIL9prabY9iGrGtrJxkxggQNMIIE # CQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYw # JAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAiNP2WAk # U8/+KwABAAACIzANBglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqG # SIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCCTpyPJl7hR8TP3+Q6ytoSljS1sdYIz # V1X8keRuNSAa7jCB+gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIJbwMywRbvcG # iynjnwjAqcaD47yYvebKZRAvtEAR5u6zMIGYMIGApH4wfDELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUt # U3RhbXAgUENBIDIwMTACEzMAAAIjT9lgJFPP/isAAQAAAiMwIgQgLwMQMMBWRlo0 # sR6jV/GLz7liTPYR30d9iURZ7BPOmnYwDQYJKoZIhvcNAQELBQAEggIAXPE1M4m7 # yaS2TyC5AI3n1sJHRWySHARXgAnGqIucDHVMHWMUXKpCMNb00oszvmXxgcsm5/Wv # QeORXhoaqmrP6Y3ANjOJwKm9iDkKvPGAbIchV4rTavb7W+TVZ46FAen6KTIfVsVs # bA6186ZYeO5cMBxU0mCegMD9owrkNgVXHx6vhA8kP0Mxrc9JhBxLEV11R8Uno5Bi # qpSPD4YcYRonbWMhpcWhZ4glsZ/DpLKf3NJzGVkllcr0ge6g6Xqi/XdobXqWkfD6 # qjdzpHrJA/PeX16+yO3e3+w3xoYezVqJuPsqeYotXky5YMVh3nDowc1vpA9vub3/ # r/mDCzdAG2KdaUxIRMVFQ5uC0L9EcwaA3oWIlP7JPjWY/Se+McJHuLOYQr5ApNHx # spZkvkhvdCO/kFP39i6OFjrOBDqzBVx0ciX8EaRVu4/jOVdMmIQdPrE9xL1zZ0yZ # 4K9v6x77TNWM2XiG5Wla/wTP0nrIXXsAnalFkfs6pZAhO7O/G3Op85ppx2TbNSpq # 0XWhl+7MhOoCpIO2J0KDeL6a+9TICuT3t9EESoZrqPUNZZRGX+3XRQtqzrV4Yq14 # f7fH2/1+dwp+aqfJeglCPBuQm39Z+SaRE0HXtcvwevotCb6m3///+PowxjVcKQmj # We+mQ/gXiD8BVIOYx4+fUcgcHkEXfuinGI0= # SIG # End signature block |