Private/AzStackHci.ConnectivityCluster.Helpers.ps1
|
# //////////////////////////////////////////////////////////////////////////// # Strict Mode v1 (PS 5.1 safe) - surfaces reads of uninitialised variables at runtime. Set-StrictMode -Version 1.0 # AzStackHci.ConnectivityCluster.Helpers.ps1 # # Cluster fan-out for Test-AzureLocalConnectivity -Scope Cluster (added v0.6.7). # # Owner constraints (see .github/copilot-instructions.md, "Owner Constraints"): # * PowerShell 5.1 target — no PS 7 features. # * NO CredSSP / WinRM auth-mode / TrustedHosts changes. # Cluster fan-out runs over standard Kerberos / Invoke-Command remoting and # assumes the operator has valid credentials when running from a node. # * NO silent shared-host security state mutation. # # Architecture: # The public cmdlet `Test-AzureLocalConnectivity -Scope Cluster` short-circuits # into `Invoke-AzureLocalConnectivityClusterFanOut` at the very top of its # `begin` block. The orchestrator: # 1. Enumerates Get-ClusterNode (UP-state nodes only). # 2. Verifies the module is installed on every node. # 3. Defaults per-node -Parallelism to 8 (each node runs its own local # Layer-7 worker sweep) unless the caller passed an explicit value. The # worker jobs run locally on each node, so the orchestrator only holds one # lightweight remoting job per node and the worker count stays bounded # per-machine — exactly like a single-node -Parallelism run. # 4. Fans out via `Invoke-Command -ComputerName $nodes` calling the same # public function with -Scope Node -PassThru -NoOutput. # 5. Collects per-node ArrayLists into a Cluster pscustomobject. # 6. Generates a tabbed HTML report (Phase C: New-ConnectivityClusterReport) # and (optionally) wires Send-DiagnosticData upload (Phase D). # # All functions in this file are PRIVATE — not exported via FunctionsToExport. # //////////////////////////////////////////////////////////////////////////// Function Invoke-AzureLocalConnectivityClusterFanOut { <# .SYNOPSIS Cluster orchestrator for Test-AzureLocalConnectivity -Scope Cluster. .DESCRIPTION Enumerates cluster nodes, fans out the connectivity test to each node via `Invoke-Command`, collects per-node ArrayList results, and produces a cluster-shaped pscustomobject for downstream reporting. Returns a `[pscustomobject]` with these properties: ClusterName : [string] name from Get-Cluster OrchestratorMachine : [string] name of the node running the orchestrator RunGuid : [guid] unique run identifier StartTime : [datetime] orchestrator start EndTime : [datetime] orchestrator end (after all nodes return) Nodes : [hashtable] {<node-name> = $perNodeResultArrayList} Errors : [array] per-node error messages (empty on success) .PARAMETER ForwardedParameters A hashtable of the parameters originally passed to Test-AzureLocalConnectivity. Cluster-mode-specific parameters (-Scope, -Parallelism, -PassThru, -NoOutput, -ExcludeUploadResults) are stripped/forced before fanning out. .PARAMETER ExportPath The top-level path to write the cluster report HTML/JSON to. If not provided, defaults to `$env:USERPROFILE\AzStackHci.DiagnosticSettings\<ClusterName>\<timestamp>`. .PARAMETER NoOutput When set, suppresses orchestrator console output (per-node calls already use -NoOutput). .PARAMETER ExcludeUploadResults When set, skips the Send-DiagnosticData upload step at the end. .PARAMETER InstallMissingModuleOnNodes When set, side-loads the orchestrator's exact module version onto any node that is missing the module or has a different version (drift), copying over a PSSession from the orchestrator's installed module folder. Opt-in — never mutates nodes silently. Without it, missing/drifted nodes cause a graceful pre-flight failure. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [hashtable]$ForwardedParameters, [Parameter(Mandatory=$false)] [string]$ExportPath, [Parameter(Mandatory=$false)] [switch]$NoOutput, [Parameter(Mandatory=$false)] [switch]$ExcludeUploadResults, [Parameter(Mandatory=$false)] [switch]$InstallMissingModuleOnNodes ) # ── Verify cluster cmdlets are available ───────────────────────────────── # NOTE: Test-CommandExists declares Param ($command) — it has NO -CommandName # parameter. Calling with -CommandName silently leaves $command empty (no # CmdletBinding strict-binding), Get-Command '' returns nothing, and the # precheck wrongly throws even when the cmdlet exists. Use positional binding. if (-not (Test-CommandExists 'Get-Cluster')) { Throw "Get-Cluster cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster. Install RSAT-Clustering or run from a cluster node." } if (-not (Test-CommandExists 'Get-ClusterNode')) { Throw "Get-ClusterNode cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster." } # ── Enumerate the cluster ──────────────────────────────────────────────── [string]$clusterName = '' [array]$nodes = @() try { $clusterObj = Get-Cluster -ErrorAction Stop $clusterName = $clusterObj.Name # Only fan out to UP nodes — DOWN/PAUSED nodes would just produce error rows. $nodes = @(Get-ClusterNode -ErrorAction Stop | Where-Object { $_.State -eq 'Up' } | Select-Object -ExpandProperty Name) } catch { Throw "Cluster enumeration failed: $($_.Exception.Message). -Scope Cluster requires the current host to be a member of an active failover cluster (or to have RSAT remote access to one)." } if ($nodes.Count -eq 0) { Throw "Get-ClusterNode returned 0 UP nodes for cluster '$clusterName'. Cannot fan out." } if (-not $NoOutput.IsPresent) { Write-HostAzS "════════════════════════════════════════════════════════════════════" Write-HostAzS " Cluster connectivity test — Scope: Cluster" Write-HostAzS " Cluster: $clusterName Orchestrator: $env:COMPUTERNAME" Write-HostAzS " Nodes ($($nodes.Count)): $($nodes -join ', ')" Write-HostAzS "════════════════════════════════════════════════════════════════════" } # ── Verify the module is installed on every node ───────────────────────── # Categorise failures so the user gets actionable guidance: # * Remoting failures (WinRM access denied, name resolution, firewall) # usually mean the orchestrator isn't an admin on the remote node — NOT # a missing module. Surface those separately to avoid the confusing # "install the module" prompt when the real fix is admin / network. # * Truly missing modules need a different remediation (Install-Module). # # On any blocking failure (missing module and/or remoting failure) we DO NOT # Throw a raw exception — that surfaces a confusing red script stack trace to # the operator. Instead we print a clean, formatted summary that names the # exact node(s) at fault with remediation guidance, also emit a single # Write-Error so scripted / -NoOutput callers can still detect the failure, # then return $null to stop the cluster fan-out gracefully. if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Verifying 'AzStackHci.DiagnosticSettings' module (exact version) is installed on each cluster node..." -ForegroundColor Cyan } $orchestratorVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$orchestratorVersionString = if ($orchestratorVersion) { $orchestratorVersion.ToString() } else { $null } # Probe every node for the FULL list of installed versions so we can enforce an # EXACT version match against the orchestrator. This prevents a silent mixed-version # cluster report (different nodes running different module code, merged into one # report). Classification per node: # OK — the orchestrator's exact version is present # Missing — no version of the module at all # Drifted — has the module, but NOT the orchestrator's exact version # Remoting failures (WinRM / admin / firewall) are tracked separately — side-loading # cannot fix those, so the remediation hint must not point at -InstallMissingModuleOnNodes. $okNodes = @() $missingNodes = @() $driftedNodes = @() # display strings: "node (has X, Y; orchestrator has Z)" $driftedNodeNames = @() # bare names for side-load targeting $remotingFailures = @() foreach ($node in $nodes) { try { $remoteVersions = @(Invoke-Command -ComputerName $node -ScriptBlock { Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Version | ForEach-Object { $_.ToString() } } -ErrorAction Stop) $remoteVersions = @($remoteVersions | Where-Object { $_ }) # drop any $null/empty if ($remoteVersions.Count -eq 0) { $missingNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module NOT installed" -ForegroundColor Red } } elseif (-not $orchestratorVersionString) { # Orchestrator version is indeterminate — fall back to "any version present is OK" # rather than falsely reporting drift on every node. $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module present (orchestrator version unknown — exact-match check skipped)" -ForegroundColor Yellow } } elseif ($remoteVersions -contains $orchestratorVersionString) { $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module v$orchestratorVersionString present" -ForegroundColor Green } } else { $driftedNodes += "$node (has $($remoteVersions -join ', '); orchestrator has $orchestratorVersionString)" $driftedNodeNames += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Version drift — has $($remoteVersions -join ', '), needs $orchestratorVersionString" -ForegroundColor Yellow } } } catch { $remotingFailures += "$node ($($_.Exception.Message))" if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Remoting failed: $($_.Exception.Message)" -ForegroundColor Red } } } # ── Optional remediation: side-load the orchestrator's exact version ───── # When -InstallMissingModuleOnNodes is specified, copy the orchestrator's installed # module folder (exact version) to every Missing / Drifted node over a PSSession, # then reclassify successes as OK. This is a deliberate, OPT-IN host mutation (see # Owner Constraints) — never performed silently. We side-load rather than # Install-Module from PSGallery because Azure Local nodes are frequently air-gapped, # and copying guarantees a byte-identical version match with the orchestrator (no # PSGallery drift, no NuGet provider, no internet dependency). Remoting-failed nodes # are NOT targeted — the PSSession would fail for the same reason the probe did. if ($InstallMissingModuleOnNodes.IsPresent -and ($missingNodes.Count -gt 0 -or $driftedNodeNames.Count -gt 0)) { if (-not $orchestratorVersionString) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module version is unknown — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { # Resolve the orchestrator's on-disk module folder for the exact version. # Prefer the -ListAvailable entry matching the version; fall back to the # loaded module's ModuleBase (covers dev / non-PSModulePath installs). $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' -ListAvailable -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $orchestratorVersionString } | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase } if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module folder could not be located on disk — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { $targets = @(@($missingNodes + $driftedNodeNames) | Select-Object -Unique) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS " -InstallMissingModuleOnNodes: side-loading AzStackHci.DiagnosticSettings v$orchestratorVersionString to $($targets.Count) node(s) from:" -ForegroundColor Cyan Write-HostAzS " $sourceModuleBase" -ForegroundColor DarkGray } foreach ($targetNode in $targets) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Copying module v$orchestratorVersionString..." -ForegroundColor Cyan } $copied = Copy-AzStackHciModuleToNode -NodeName $targetNode -SourceModuleBase $sourceModuleBase -Version $orchestratorVersion if ($copied) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load OK — module v$orchestratorVersionString now present" -ForegroundColor Green } $okNodes += $targetNode $missingNodes = @($missingNodes | Where-Object { $_ -ne $targetNode }) $driftedNodeNames = @($driftedNodeNames | Where-Object { $_ -ne $targetNode }) $driftedNodes = @($driftedNodes | Where-Object { $_ -notlike "$targetNode (*" }) } else { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load FAILED — re-run with -Verbose for the copy error" -ForegroundColor Red } } } } } } # ── Graceful failure when one or more nodes cannot run the test ────────── if ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0 -or $remotingFailures.Count -gt 0) { $moduleProblem = ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red Write-HostAzS " Cluster connectivity test cannot continue — pre-flight checks failed" -ForegroundColor Red Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red if ($missingNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The 'AzStackHci.DiagnosticSettings' module is NOT installed on $($missingNodes.Count) node(s):" -ForegroundColor Red foreach ($mn in $missingNodes) { Write-HostAzS " - $mn" -ForegroundColor Yellow } } if ($driftedNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The module version does NOT match the orchestrator (v$orchestratorVersionString) on $($driftedNodes.Count) node(s):" -ForegroundColor Red foreach ($dn in $driftedNodes) { Write-HostAzS " - $dn" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " All nodes must run the SAME module version, otherwise the cluster report can" -ForegroundColor Yellow Write-HostAzS " silently blend results collected by different module versions." -ForegroundColor Yellow } if ($moduleProblem) { Write-HostAzS "" if ($InstallMissingModuleOnNodes.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes was specified but the side-load did not succeed on the" -ForegroundColor Yellow Write-HostAzS " node(s) above. Re-run with -Verbose for the copy error, or install the exact version" -ForegroundColor Yellow Write-HostAzS " manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } else { Write-HostAzS " TIP: re-run with -InstallMissingModuleOnNodes to automatically copy the orchestrator's" -ForegroundColor Green Write-HostAzS " exact version (v$orchestratorVersionString) to the affected node(s), e.g.:" -ForegroundColor Green Write-HostAzS " Test-AzureLocalConnectivity -Scope Cluster -InstallMissingModuleOnNodes ..." -ForegroundColor Green Write-HostAzS "" Write-HostAzS " Or install the exact version manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } } if ($remotingFailures.Count -gt 0) { Write-HostAzS "" Write-HostAzS " PowerShell remoting (Invoke-Command) failed on $($remotingFailures.Count) node(s):" -ForegroundColor Red foreach ($rf in $remotingFailures) { Write-HostAzS " - $rf" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " -Scope Cluster requires the orchestrator session to be Administrator on every" -ForegroundColor Yellow Write-HostAzS " cluster node, with WinRM reachable (default port 5985 TCP). Verify with:" -ForegroundColor Yellow Write-HostAzS " Test-WSMan -ComputerName <nodeName>" -ForegroundColor Yellow } Write-HostAzS "" } # Single error-stream message so scripted / -NoOutput callers detect the failure. $errorParts = @() if ($missingNodes.Count -gt 0) { $errorParts += "module not installed on: $($missingNodes -join ', ')" } if ($driftedNodes.Count -gt 0) { $errorParts += "module version drift on: $($driftedNodeNames -join ', ')" } if ($remotingFailures.Count -gt 0) { $errorParts += "remoting failed on: $($remotingFailures -join '; ')" } $hint = if ($moduleProblem -and -not $InstallMissingModuleOnNodes.IsPresent) { " Re-run with -InstallMissingModuleOnNodes to copy the orchestrator's exact version (v$orchestratorVersionString) to the affected node(s)." } else { '' } Write-Error "Test-AzureLocalConnectivity -Scope Cluster aborted — $($errorParts -join ' | ').$hint" return $null } # ── Build the per-node parameter set (strip cluster-specific switches) ── # NOTE: 'Parallelism' is intentionally NOT stripped — each node runs its own # Layer-7 sweep in parallel. The worker Start-Jobs run LOCALLY on each node # (children of that node's wsmprovhost), so the orchestrator only holds one # lightweight remoting job per node; the worker count is bounded per-machine # exactly like a normal single-node -Parallelism run. Version parity (enforced # by the pre-flight) guarantees every node's workers import identical code. $perNodeParams = @{} foreach ($k in $ForwardedParameters.Keys) { if ($k -notin @('Scope','PassThru','NoOutput','ExcludeUploadResults','InstallMissingModuleOnNodes')) { $perNodeParams[$k] = $ForwardedParameters[$k] } } $perNodeParams['Scope'] = 'Node' if (-not $perNodeParams.ContainsKey('Parallelism')) { # Cluster default: 8 local workers per node (only applied when the caller # did NOT pass an explicit -Parallelism). A single-node run still defaults # to 1; the higher default here just speeds up the per-node sweep. $perNodeParams['Parallelism'] = 8 } $perNodeParams['PassThru'] = $true $perNodeParams['NoOutput'] = $true $perNodeParams['ExcludeUploadResults'] = $true # orchestrator handles upload (Phase D) # ── Resolve the export path on the orchestrator ────────────────────────── # Default to the SAME ProgramData location single-node runs use, so cluster # reports land alongside single-node reports (C:\ProgramData\...) instead of # under the operator's roaming profile (C:\Users\<admin>\...). if (-not $ExportPath) { $ExportPath = Get-AzStackHciArtifactDirectory -ArtifactType Connectivity -Create } if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # ── Fan out: run every node IN PARALLEL via Start-Job ─────────────────── # Each node gets its own background job (process-isolated, PS 5.1 safe — no # ForEach-Object -Parallel). The job wraps an Invoke-Command -ComputerName so # the actual test runs on the node; Start-Job isolation keeps each node's # remote Write-Progress / host output out of the orchestrator console, and # lets us show a single cluster-level progress bar instead of per-endpoint # noise. All nodes launch at once (unbounded) — each node runs its own local # Layer-7 worker sweep (per-node -Parallelism, default 8), so the worker # count stays bounded per-machine. $nodeJobScript = { param($NodeName, $Params, $RequiredVersion) $jobStart = Get-Date try { $remote = Invoke-Command -ComputerName $NodeName -ScriptBlock { param($P, $Ver) # Pin the import to the orchestrator's EXACT version so every node runs # byte-identical code even if a node also has other versions installed # (the pre-flight has already guaranteed this version is present here). if ($Ver) { Import-Module 'AzStackHci.DiagnosticSettings' -RequiredVersion $Ver -Force } else { Import-Module 'AzStackHci.DiagnosticSettings' -Force } $raw = Test-AzureLocalConnectivity @P # v0.6.8: Test-AzureLocalConnectivity -PassThru now returns a structured # PSCustomObject — per-URL rows live under $raw.Results and the run-level # telemetry are REAL properties on $raw (no longer fragile NoteProperties on # an ArrayList). Capture them explicitly so the remoting CliXml boundary # preserves them (Invoke-Command still enumerates a returned collection, so we # forward $raw.Results as the Rows payload rather than the container object). [pscustomobject]@{ Rows = $raw.Results DownloadSpeed = $raw.DownloadSpeed RequestMethod = $raw.RequestMethod Parallelism = $raw.Parallelism TotalDurationSeconds = $raw.TotalDurationSeconds Layer7TotalDurationSeconds = $raw.Layer7TotalDurationSeconds Layer7WallClockSeconds = $raw.Layer7WallClockSeconds Layer7TestedEndpoints = $raw.Layer7TestedEndpoints # v0.6.8: per-node report paths — these files live on THIS remote node's # local ProgramData, not on the orchestrator. ReportPath = $raw.ReportPath JSONReportPath = $raw.JSONReportPath # v0.6.8 §7.3: forward the per-node run-level detections (flat scalars + # string arrays survive the double CliXml boundary) so the unified cluster # -PassThru can carry the SAME Detections fields each node-scope run exposes. SSLInspectionDetected = $raw.SSLInspectionDetected SSLInspectedURLs = $raw.SSLInspectedURLs PrivateLinkDetected = $raw.PrivateLinkDetected PrivateLinkCriticalArray = $raw.PrivateLinkCriticalArray PrivateLinkProxyBypassArray = $raw.PrivateLinkProxyBypassArray # v0.6.8: full detection set + verified proxy-bypass breakdown (flat arrays # survive the double CliXml boundary). PrivateLinkDetectedArray = $raw.PrivateLinkDetectedArray OtherRfc1918Count = $raw.OtherRfc1918Count PrivateLinkBypassConfirmedArray = $raw.PrivateLinkBypassConfirmedArray PrivateLinkBypassMissingArray = $raw.PrivateLinkBypassMissingArray ProxyBypassList = $raw.ProxyBypassList NoProxyList = $raw.NoProxyList CRLOfflineDetected = $raw.CRLOfflineDetected CRLOfflineURLs = $raw.CRLOfflineURLs } } -ArgumentList $Params, $RequiredVersion -ErrorAction Stop # Return a FLAT wrapper (Rows one level deep, same nesting as the working # single-hop design) so the extra Start-Job CliXml boundary preserves the # already-deserialised flat row property bags. [pscustomobject]@{ Node = $NodeName Error = $null Duration = ((Get-Date) - $jobStart).ToString() Rows = $remote.Rows DownloadSpeed = $remote.DownloadSpeed RequestMethod = $remote.RequestMethod Parallelism = $remote.Parallelism TotalDurationSeconds = $remote.TotalDurationSeconds Layer7TotalDurationSeconds = $remote.Layer7TotalDurationSeconds Layer7WallClockSeconds = $remote.Layer7WallClockSeconds Layer7TestedEndpoints = $remote.Layer7TestedEndpoints ReportPath = $remote.ReportPath JSONReportPath = $remote.JSONReportPath SSLInspectionDetected = $remote.SSLInspectionDetected SSLInspectedURLs = $remote.SSLInspectedURLs PrivateLinkDetected = $remote.PrivateLinkDetected PrivateLinkCriticalArray = $remote.PrivateLinkCriticalArray PrivateLinkProxyBypassArray = $remote.PrivateLinkProxyBypassArray PrivateLinkDetectedArray = $remote.PrivateLinkDetectedArray OtherRfc1918Count = $remote.OtherRfc1918Count PrivateLinkBypassConfirmedArray = $remote.PrivateLinkBypassConfirmedArray PrivateLinkBypassMissingArray = $remote.PrivateLinkBypassMissingArray ProxyBypassList = $remote.ProxyBypassList NoProxyList = $remote.NoProxyList CRLOfflineDetected = $remote.CRLOfflineDetected CRLOfflineURLs = $remote.CRLOfflineURLs } } catch { [pscustomobject]@{ Node = $NodeName Error = $_.Exception.Message Duration = ((Get-Date) - $jobStart).ToString() Rows = $null } } } $clusterStart = Get-Date $perNodeResults = @{} $perNodeErrors = @{} $perNodeDurations = @{} $perNodeDownloadSpeed = @{} # per-node download-speed result (string, e.g. '123.4 Mbps') $perNodeTelemetry = @{} # per-node run-level telemetry (timing + RequestMethod) $perNodeDetections = @{} # per-node run-level detections (SSL / PrivateLink / CRL) # Launch all node jobs at once. $jobMap = @{} foreach ($node in $nodes) { $jobMap[$node] = Start-Job -Name "AzSHciConn_$node" -ScriptBlock $nodeJobScript ` -ArgumentList $node, $perNodeParams, $orchestratorVersionString } if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Fanned out to $($nodes.Count) node(s) in parallel: $($nodes -join ', ')" -ForegroundColor Cyan } # ── Poll for completion + show a node-level progress bar ───────────────── $jobList = @($jobMap.Values) $totalNodes = $nodes.Count $deadlineStopwatch = [System.Diagnostics.Stopwatch]::StartNew() $timedOutNodes = @{} do { $pending = @($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }) $doneCount = $totalNodes - $pending.Count if (-not $NoOutput.IsPresent) { $pendingNodes = @($nodes | Where-Object { $jobMap[$_].State -eq 'Running' -or $jobMap[$_].State -eq 'NotStarted' }) $pct = if ($totalNodes -gt 0) { [int](($doneCount / $totalNodes) * 100) } else { 100 } $statusText = if ($pendingNodes.Count -gt 0) { "Waiting on $($pendingNodes.Count) of $totalNodes node(s): $($pendingNodes -join ', ')" } else { 'Finalising...' } Write-Progress -Activity "Cluster connectivity test ($doneCount of $totalNodes nodes complete)" -Status $statusText -PercentComplete $pct } if ($pending.Count -gt 0) { $null = Wait-Job -Job $jobList -Any -Timeout $script:JOB_POLL_INTERVAL_SEC } } while ( @($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }).Count -gt 0 -and $deadlineStopwatch.Elapsed.TotalSeconds -lt $script:OPERATION_DEADLINE_SEC ) $deadlineStopwatch.Stop() $remainingJobs = @($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }) foreach ($node in $nodes) { if ($remainingJobs -contains $jobMap[$node]) { $timedOutNodes[$node] = $true Stop-Job -Job $jobMap[$node] -ErrorAction SilentlyContinue } } if (-not $NoOutput.IsPresent) { Write-Progress -Activity "Cluster connectivity test" -Completed } # ── Receive each job + attribute results back to its node ──────────────── foreach ($node in $nodes) { $job = $jobMap[$node] $jobOutput = $null try { $jobOutput = Receive-Job -Job $job -ErrorAction Stop } catch { $jobOutput = $null } Remove-Job -Job $job -Force -ErrorAction SilentlyContinue # Start-Job can emit multiple objects; pick our wrapper (it carries a 'Node' prop). $nodeWrapper = $null if ($jobOutput) { $nodeWrapper = @($jobOutput | Where-Object { $_ -and $_.PSObject.Properties['Node'] }) | Select-Object -First 1 } if ($nodeWrapper -and (-not $nodeWrapper.Error) -and $nodeWrapper.PSObject.Properties['Rows'] -and $nodeWrapper.Rows) { $perNodeResults[$node] = @($nodeWrapper.Rows) $perNodeDownloadSpeed[$node] = $nodeWrapper.DownloadSpeed $perNodeTelemetry[$node] = [pscustomobject]@{ DownloadSpeed = $nodeWrapper.DownloadSpeed RequestMethod = $nodeWrapper.RequestMethod Parallelism = $nodeWrapper.Parallelism TotalDurationSeconds = $nodeWrapper.TotalDurationSeconds Layer7TotalDurationSeconds = $nodeWrapper.Layer7TotalDurationSeconds Layer7WallClockSeconds = $nodeWrapper.Layer7WallClockSeconds Layer7TestedEndpoints = $nodeWrapper.Layer7TestedEndpoints # v0.6.8: report paths on the REMOTE node (this node's local ProgramData). ReportPath = $nodeWrapper.ReportPath JSONReportPath = $nodeWrapper.JSONReportPath } # v0.6.8 §7.3: per-node run-level detections for the unified cluster -PassThru. $perNodeDetections[$node] = [pscustomobject]@{ SSLInspectionDetected = [bool]$nodeWrapper.SSLInspectionDetected SSLInspectedURLs = @($nodeWrapper.SSLInspectedURLs) PrivateLinkDetected = [bool]$nodeWrapper.PrivateLinkDetected PrivateLinkCriticalArray = @($nodeWrapper.PrivateLinkCriticalArray) PrivateLinkProxyBypassArray = @($nodeWrapper.PrivateLinkProxyBypassArray) PrivateLinkDetectedArray = @($nodeWrapper.PrivateLinkDetectedArray) OtherRfc1918Count = [int]$nodeWrapper.OtherRfc1918Count PrivateLinkBypassConfirmedArray = @($nodeWrapper.PrivateLinkBypassConfirmedArray) PrivateLinkBypassMissingArray = @($nodeWrapper.PrivateLinkBypassMissingArray) ProxyBypassList = @($nodeWrapper.ProxyBypassList) NoProxyList = @($nodeWrapper.NoProxyList) CRLOfflineDetected = [bool]$nodeWrapper.CRLOfflineDetected CRLOfflineURLs = @($nodeWrapper.CRLOfflineURLs) } $perNodeErrors[$node] = @() if (-not $NoOutput.IsPresent) { $rowCount = @($perNodeResults[$node]).Count $dlNote = if ($perNodeDownloadSpeed[$node]) { " (download speed: $($perNodeDownloadSpeed[$node]))" } else { '' } Write-HostAzS " [$node] Returned $rowCount result row(s).$dlNote" -ForegroundColor Green } } else { $perNodeResults[$node] = $null $perNodeDownloadSpeed[$node] = $null $perNodeTelemetry[$node] = $null $perNodeDetections[$node] = $null $errMsg = if ($timedOutNodes.ContainsKey($node)) { "Connectivity collection timed out after $($script:OPERATION_DEADLINE_SEC) seconds on node '$node'." } elseif ($nodeWrapper -and $nodeWrapper.Error) { $nodeWrapper.Error } else { "No result returned from node '$node' (job produced no output)." } $perNodeErrors[$node] = @($errMsg) if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] FAILED: $errMsg" -ForegroundColor Red } } if ($nodeWrapper -and $nodeWrapper.PSObject.Properties['Duration'] -and $nodeWrapper.Duration) { $perNodeDurations[$node] = try { [TimeSpan]::Parse($nodeWrapper.Duration) } catch { [TimeSpan]::Zero } } else { $perNodeDurations[$node] = [TimeSpan]::Zero } } $clusterEnd = Get-Date # ── Console summary: per-node connectivity FAILURES only ───────────────── # -Scope Node prints every endpoint result inline; -Scope Cluster suppresses the # per-node console (Start-Job isolation keeps each node's endpoint sweep out of the # orchestrator). To restore the at-a-glance failure visibility of a Node-scope run, # surface ONLY the failing endpoints here, attributed to the node they came from. # (The full per-endpoint detail still lives in the tabbed HTML / JSON report.) if (-not $NoOutput.IsPresent) { $failureLines = [System.Collections.Generic.List[string]]::new() foreach ($node in $nodes) { $rows = @($perNodeResults[$node]) if ($rows.Count -eq 0) { continue } $nodeFailures = @($rows | Where-Object { $_ -and ( ($_.PSObject.Properties['TCPStatus'] -and $_.TCPStatus -eq 'Failed') -or ($_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status -eq 'Failed') ) }) foreach ($f in $nodeFailures) { $u = if ($f.PSObject.Properties['URL']) { $f.URL } else { '<unknown>' } $p = if ($f.PSObject.Properties['Port']) { $f.Port } else { '' } $tcp = if ($f.PSObject.Properties['TCPStatus']) { $f.TCPStatus } else { '' } $l7 = if ($f.PSObject.Properties['Layer7Status']) { $f.Layer7Status } else { '' } # NOTE: wrap the -f in EXTRA parens — commas inside method () are arg # separators, so '... -f a, b' would pass b as a 2nd .Add() argument. # Leading tab indents the row under the banner so it stands out in console. $failureLines.Add(("`t [{0}] {1}:{2} TCP={3} L7={4}" -f $node, $u, $p, $tcp, $l7)) } } # Header banner (matches the module's //// section style) plus a tab indent so the # cluster failure summary is easy to spot in a busy console — mirroring the # at-a-glance "Test results summary:" block that -Scope Node prints. Write-HostAzS "`n`t//////////////// Connectivity Test Results ////////////////`n" -ForegroundColor Cyan if ($failureLines.Count -gt 0) { Write-HostAzS "`tConnectivity FAILURES ($($failureLines.Count) across $($nodes.Count) node(s)):" -ForegroundColor Red foreach ($line in $failureLines) { Write-HostAzS $line -ForegroundColor Red } } else { Write-HostAzS "`tNo connectivity failures detected on any node." -ForegroundColor Green } Write-HostAzS "" } # ── Build cluster-shaped result object ─────────────────────────────────── $clusterResult = [pscustomobject]@{ ClusterName = $clusterName OrchestratorMachine = $env:COMPUTERNAME RunGuid = [guid]::NewGuid() StartTime = $clusterStart EndTime = $clusterEnd Duration = ($clusterEnd - $clusterStart) Nodes = $perNodeResults NodeDurations = $perNodeDurations NodeDownloadSpeeds = $perNodeDownloadSpeed NodeTelemetry = $perNodeTelemetry NodeDetections = $perNodeDetections Errors = $perNodeErrors ExportPath = $ExportPath # v0.6.8: merged orchestrator report paths (set in Phase C after the report is written). ReportPath = $null JSONReportPath = $null } # ── Phase C — Generate cluster report (tabbed HTML, or merged CSV) ─────── # Honour the caller's -OutputFormat (HTML default; CSV writes a merged CSV). # JSON is always emitted alongside. AzureRegion isn't needed for cluster file # naming (cluster name only), so we don't depend on it here. $clusterOutputFormat = if ($ForwardedParameters.ContainsKey('OutputFormat') -and $ForwardedParameters['OutputFormat']) { [string]$ForwardedParameters['OutputFormat'] } else { 'HTML' } try { $reportPath = New-ConnectivityClusterReport -ClusterResult $clusterResult -ExportPath $ExportPath -OutputFormat $clusterOutputFormat if ($reportPath) { # Surface the MERGED orchestrator report paths (local to the orchestrator) at the top # level of the -Scope Cluster -PassThru object. JSON is written alongside with the same # base name, so swap the extension to derive its path. $clusterResult.ReportPath = $reportPath # Derive the JSON path from the report path. ChangeExtension always yields a .json # extension regardless of the source extension (html/csv or none), so the JSON path # can never accidentally equal the report path. A bare extension -replace would be a # no-op (leaving JSONReportPath == ReportPath) if the report path ever lacked a # recognised report extension. $clusterResult.JSONReportPath = [System.IO.Path]::ChangeExtension($reportPath, '.json') } if ($reportPath -and -not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Cluster connectivity report: $reportPath" -ForegroundColor Cyan } } catch { Write-Warning "New-ConnectivityClusterReport failed: $($_.Exception.Message)" } # ── Phase D — Upload bundle (Send-DiagnosticData integration) ─────────── # Test-CommandExists takes the command positionally (Param ($command)); -CommandName # is NOT a valid parameter and would silently bind to nothing. if (-not $ExcludeUploadResults.IsPresent -and (Test-CommandExists 'Send-DiagnosticData')) { try { Invoke-UploadDiagnosticResults ` -FolderToUpload $ExportPath ` -MessageToDisplay "Send the cluster connectivity results bundle to Microsoft? [Y/N]" ` -Context 'Cluster Connectivity Test results' ` -ErrorAction Stop } catch { Write-Warning "Cluster upload via Send-DiagnosticData failed: $($_.Exception.Message). Bundle remains at: $ExportPath" } } return $clusterResult } Function ConvertTo-ConnectivityClusterPassThru { <# .SYNOPSIS Transforms the internal cluster orchestrator result into the unified v0.6.8 (§7.3) structured -PassThru contract. .DESCRIPTION -Scope Node -PassThru returns a flat PSCustomObject (run-level fields on real properties, per-URL rows under .Results). For -Scope Cluster -PassThru we return the SAME structured shape but with per-node results nested under .Nodes[], where each .Nodes[] entry carries the identical flat run-level fields + Detections + .Results a node-scope run exposes. This gives consumers ONE contract regardless of scope. The internal $ClusterResult shape (used by the HTML/CSV report generator) is left untouched — this is a pure projection for callers. .PARAMETER ClusterResult The pscustomobject returned by Invoke-AzureLocalConnectivityClusterFanOut. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] $ClusterResult ) $nodeObjects = @() foreach ($nodeName in @($ClusterResult.Nodes.Keys)) { $rows = @($ClusterResult.Nodes[$nodeName]) $tel = $ClusterResult.NodeTelemetry[$nodeName] $det = $ClusterResult.NodeDetections[$nodeName] $errs = @($ClusterResult.Errors[$nodeName]) $nodeObjects += [pscustomobject]@{ SchemaVersion = '1.1' Hostname = $nodeName Collected = ($rows.Count -gt 0) Error = if ($errs.Count -gt 0) { $errs[0] } else { $null } DownloadSpeed = if ($tel) { $tel.DownloadSpeed } else { $null } RequestMethod = if ($tel) { $tel.RequestMethod } else { $null } Parallelism = if ($tel) { $tel.Parallelism } else { $null } TotalDurationSeconds = if ($tel) { $tel.TotalDurationSeconds } else { $null } Layer7TotalDurationSeconds = if ($tel) { $tel.Layer7TotalDurationSeconds } else { $null } Layer7WallClockSeconds = if ($tel) { $tel.Layer7WallClockSeconds } else { $null } Layer7TestedEndpoints = if ($tel) { $tel.Layer7TestedEndpoints } else { $null } # v0.6.8: per-node report paths — these files live on the REMOTE node ($nodeName), # not on the orchestrator. The merged cluster report is the top-level ReportPath. ReportPath = if ($tel) { $tel.ReportPath } else { $null } JSONReportPath = if ($tel) { $tel.JSONReportPath } else { $null } SSLInspectionDetected = if ($det) { [bool]$det.SSLInspectionDetected } else { $false } SSLInspectedURLs = if ($det) { @($det.SSLInspectedURLs) } else { @() } PrivateLinkDetected = if ($det) { [bool]$det.PrivateLinkDetected } else { $false } PrivateLinkCriticalArray = if ($det) { @($det.PrivateLinkCriticalArray) } else { @() } PrivateLinkProxyBypassArray = if ($det) { @($det.PrivateLinkProxyBypassArray) } else { @() } PrivateLinkDetectedArray = if ($det) { @($det.PrivateLinkDetectedArray) } else { @() } OtherRfc1918Count = if ($det) { [int]$det.OtherRfc1918Count } else { 0 } PrivateLinkBypassConfirmedArray = if ($det) { @($det.PrivateLinkBypassConfirmedArray) } else { @() } PrivateLinkBypassMissingArray = if ($det) { @($det.PrivateLinkBypassMissingArray) } else { @() } ProxyBypassList = if ($det) { @($det.ProxyBypassList) } else { @() } NoProxyList = if ($det) { @($det.NoProxyList) } else { @() } CRLOfflineDetected = if ($det) { [bool]$det.CRLOfflineDetected } else { $false } CRLOfflineURLs = if ($det) { @($det.CRLOfflineURLs) } else { @() } Results = $rows } } [pscustomobject]@{ SchemaVersion = '1.1' Scope = 'Cluster' ClusterName = $ClusterResult.ClusterName OrchestratorMachine = $ClusterResult.OrchestratorMachine Timestamp = if ($ClusterResult.StartTime) { $ClusterResult.StartTime.ToString('yyyy-MM-dd HH:mm:ss') } else { (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') } RunGuid = $ClusterResult.RunGuid StartTime = $ClusterResult.StartTime EndTime = $ClusterResult.EndTime Duration = $ClusterResult.Duration ExportPath = $ClusterResult.ExportPath # v0.6.8: the MERGED cluster report on the orchestrator (local to the caller, openable # directly — unlike the per-node ReportPath values, which are on the remote nodes). ReportPath = $ClusterResult.ReportPath JSONReportPath = $ClusterResult.JSONReportPath Nodes = @($nodeObjects) Errors = $ClusterResult.Errors } } Function Copy-AzStackHciModuleToNode { <# .SYNOPSIS Side-loads (copies) the orchestrator's installed AzStackHci.DiagnosticSettings module folder onto a remote cluster node over a PSSession. .DESCRIPTION Used by Invoke-AzureLocalConnectivityClusterFanOut when -InstallMissingModuleOnNodes is specified. Copies the orchestrator's exact versioned module folder (`...\Modules\AzStackHci.DiagnosticSettings\<version>`) to the same AllUsers module path on the target node, guaranteeing a byte-identical version match without any PowerShell Gallery / internet dependency (Azure Local nodes are frequently air-gapped). The copy is performed over a `New-PSSession` (NOT the `C$` admin share) so it depends only on the same WinRM remoting already required by -Scope Cluster — no extra SMB / firewall surface. Any existing copy of the SAME version on the node is removed first to avoid a partial-merge. Copied files are Unblock-File'd, then the version is re-verified via `Get-Module -ListAvailable` on the node. Owner constraints: this is an OPT-IN host mutation only; it does not change CredSSP, WinRM auth mode, TrustedHosts, or any shared security state. Returns $true if the exact version is present on the node after the copy, else $false. .PARAMETER NodeName The remote node to copy the module to. .PARAMETER SourceModuleBase The orchestrator's on-disk versioned module folder, e.g. `C:\Program Files\WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings\0.6.7`. .PARAMETER Version The exact module version being side-loaded (used for the destination subfolder and post-copy verification). .NOTES SECURITY / TRUST BOUNDARY: this copies the orchestrator's local module folder ($SourceModuleBase) to every node's %ProgramFiles%\WindowsPowerShell\Modules and imports it. Whatever is in the orchestrator's module install is what gets pushed to all nodes — run the side-load (-InstallMissingModuleOnNodes) only from a trusted, clean module install. Uses default Kerberos remoting (no CredSSP) and fails closed (returns $false) on any error. #> [CmdletBinding()] [OutputType([bool])] param( [Parameter(Mandatory)] [string]$NodeName, [Parameter(Mandatory)] [string]$SourceModuleBase, [Parameter(Mandatory)] [version]$Version ) $session = $null try { $session = New-PSSession -ComputerName $NodeName -ErrorAction Stop # Destination module-name root on the node (PS 5.1 AllUsers path). Resolved on the # node itself so a non-default %ProgramFiles% is honoured. $destModuleNameRoot = Invoke-Command -Session $session -ScriptBlock { Join-Path $env:ProgramFiles 'WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings' } $verString = $Version.ToString() # Ensure the module-name root exists and clear any partial copy of this exact version. Invoke-Command -Session $session -ScriptBlock { param($root, $ver) if (-not (Test-Path -Path $root)) { New-Item -Path $root -ItemType Directory -Force | Out-Null } $verPath = Join-Path $root $ver # Defence-in-depth: only delete when the leaf is a well-formed version string # (e.g. 0.6.7 / 0.6.7.0) AND the resolved path is still under the module-name # root, so a malformed $ver can never escalate this into deleting an # unintended path on the remote node. if (($ver -match '^\d+\.\d+\.\d+(\.\d+)?$') -and ($verPath -like (Join-Path $root '*')) -and (Test-Path -Path $verPath)) { Remove-Item -Path $verPath -Recurse -Force -ErrorAction SilentlyContinue } } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop # Copy the versioned folder INTO the module-name root → ...\<ModuleName>\<version>. Copy-Item -Path $SourceModuleBase -Destination $destModuleNameRoot -ToSession $session -Recurse -Force -ErrorAction Stop # Unblock copied files and confirm the exact version is now discoverable on the node. $ok = Invoke-Command -Session $session -ScriptBlock { param($root, $ver) $verPath = Join-Path $root $ver Get-ChildItem -Path $verPath -Recurse -ErrorAction SilentlyContinue | Unblock-File -ErrorAction SilentlyContinue $found = Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $ver } [bool]$found } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop return [bool]$ok } catch { Write-Verbose "Copy-AzStackHciModuleToNode: side-load to '$NodeName' failed: $($_.Exception.Message)" return $false } finally { if ($session) { Remove-PSSession -Session $session -ErrorAction SilentlyContinue } } } Function New-ConnectivityClusterReport { <# .SYNOPSIS Builds the tabbed HTML cluster report from a Cluster-shaped PassThru object. .DESCRIPTION Mirrors the structure of `New-OsConfigReport` (Private/AzStackHci.OsConfigReport.Helpers.ps1) but specialised for connectivity-test data. Sections: * Status Overview (top of Summary tab) — Node | Tested | Passed | Failed | SSL-Inspected | Private-Link | Status badge. * Per-node tabs — same tabular result layout the existing CSV/HTML emits today, scoped to that node's PassThru ArrayList. Returns the full HTML report file path on success, or $null on failure. .PARAMETER ClusterResult The pscustomobject produced by Invoke-AzureLocalConnectivityClusterFanOut. .PARAMETER ExportPath The directory to write the HTML/JSON files to. #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory)] [pscustomobject]$ClusterResult, [Parameter(Mandatory)] [string]$ExportPath, [Parameter(Mandatory=$false)] [ValidateSet('HTML','CSV')] [string]$OutputFormat = 'HTML' ) if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # Nested HTML-encode helper. The ConvertTo-HtmlSafe in OsConfigReport.Helpers.ps1 # is nested inside New-OsConfigReport, so it isn't visible from this scope; we # define our own with the same name + signature so all the call sites below read # cleanly. PowerShell function lookup resolves nested functions before sibling # function names, so this does NOT shadow the OsConfigReport version. function ConvertTo-HtmlSafe { param([string]$Text) if (-not $Text) { return '' } return [System.Net.WebUtility]::HtmlEncode($Text) } # File naming: cluster name only (no orchestrator host name), matching the # single-node AzureLocal_ConnectivityTest_* convention so all reports group # together in C:\ProgramData\AzStackHci.DiagnosticSettings. $reportStamp = Get-Date -Format 'yyyy-MM-dd-HH-mm-ss' $safeClusterName = ([string]$ClusterResult.ClusterName) -replace '[^A-Za-z0-9._-]', '_' $baseName = "AzureLocal_ConnectivityTest_Cluster_${safeClusterName}_${reportStamp}" $reportExt = if ($OutputFormat -eq 'CSV') { '.csv' } else { '.html' } $reportFile = Join-Path $ExportPath ($baseName + $reportExt) $jsonFile = Join-Path $ExportPath ($baseName + '.json') $sb = [System.Text.StringBuilder]::new() # ── HTML head + CSS ────────────────────────────────────────────────────── $cssBlock = @' <style> body { font-family: 'Segoe UI', Tahoma, Geneva, sans-serif; margin: 16px; color: #222; background: #f7f7f9; } h1 { color: #0078d4; margin-bottom: 4px; } h2 { color: #333; border-bottom: 2px solid #e1e1e1; padding-bottom: 4px; margin-top: 20px; } h3 { color: #444; margin-top: 14px; } table { border-collapse: collapse; width: 100%; background: #fff; margin-bottom: 12px; } th, td { border: 1px solid #d8d8d8; padding: 6px 10px; text-align: left; vertical-align: top; font-size: 13px; } th { background: #f0f4f8; color: #003366; } tr:nth-child(even) { background: #fafbfc; } .summary-card { background: #fff; border: 1px solid #d8d8d8; padding: 12px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .tab-bar { display: flex; flex-wrap: wrap; gap: 4px; border-bottom: 2px solid #0078d4; margin-bottom: 0; } .tab-btn { background: #e1e1e1; border: 1px solid #c1c1c1; border-bottom: none; padding: 6px 14px; cursor: pointer; font-size: 13px; border-radius: 4px 4px 0 0; } .tab-btn.active { background: #fff; border-color: #0078d4; font-weight: bold; color: #0078d4; } .tab-panel { display: none; padding: 14px; background: #fff; border: 1px solid #c1c1c1; border-top: none; } .tab-panel.active { display: block; } .badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: bold; } .badge-ok { background: #d4edda; color: #155724; border: 1px solid #c3e6cb; } .badge-warn { background: #fff3cd; color: #856404; border: 1px solid #ffeeba; } .badge-error { background: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; } .badge-na { background: #e2e3e5; color: #383d41; border: 1px solid #d6d8db; } .footer { margin-top: 24px; color: #888; font-size: 11px; text-align: center; } .row-failed { background: #fdecea !important; } .row-warn { background: #fff8e1 !important; } /* Per-node result-row colouring — identical palette to the single-node report. */ tr.status-failed { background-color: #fde7e9 !important; } tr.status-success { background-color: #e6f4ea !important; } tr.status-skipped { background-color: #fff4ce !important; } .totals-card { background: #fff; border: 1px solid #d8d8d8; padding: 10px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .totals-card .num { font-size: 18px; font-weight: bold; } .totals-card .ok { color: #155724; } .totals-card .fail { color: #721c24; } .totals-card .skip { color: #856404; } </style> '@ [void]$sb.AppendLine("<!DOCTYPE html>") [void]$sb.AppendLine("<html lang='en'><head><meta charset='utf-8'>") [void]$sb.AppendLine("<title>Cluster Connectivity Report — $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)</title>") [void]$sb.AppendLine($cssBlock) [void]$sb.AppendLine("</head><body>") [void]$sb.AppendLine("<h1>Cluster Connectivity Report</h1>") [void]$sb.AppendLine("<div class='summary-card'>") [void]$sb.AppendLine("<strong>Cluster:</strong> $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)<br>") [void]$sb.AppendLine("<strong>Orchestrator:</strong> $(ConvertTo-HtmlSafe $ClusterResult.OrchestratorMachine)<br>") [void]$sb.AppendLine("<strong>Run GUID:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.RunGuid.ToString()))<br>") [void]$sb.AppendLine("<strong>Start:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.StartTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>End:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.EndTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>Duration:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.Duration.ToString()))<br>") [void]$sb.AppendLine("</div>") # ── Build per-node summary stats ───────────────────────────────────────── $nodeNames = @($ClusterResult.Nodes.Keys | Sort-Object) $summary = @{} foreach ($node in $nodeNames) { $rows = @($ClusterResult.Nodes[$node] | Where-Object { $_ -and $_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status }) $tested = $rows.Count $passed = @($rows | Where-Object { $_.Layer7Status -eq 'Success' }).Count $failed = @($rows | Where-Object { $_.Layer7Status -eq 'Failed' }).Count $skipped = @($rows | Where-Object { $_.Layer7Status -eq 'Skipped' }).Count $sslInspected = @($rows | Where-Object { $_.Layer7Response -match 'SSL Inspection' }).Count $privateLink = @($rows | Where-Object { $_.Layer7Response -match 'Private Link|PrivateLink' }).Count $hasErrors = @($ClusterResult.Errors[$node]).Count -gt 0 $dlSpeed = if ($ClusterResult.PSObject.Properties['NodeDownloadSpeeds'] -and $ClusterResult.NodeDownloadSpeeds) { $ClusterResult.NodeDownloadSpeeds[$node] } else { $null } $status = if ($failed -gt 0 -or $hasErrors) { 'error' } elseif ($sslInspected -gt 0 -or $privateLink -gt 0) { 'warn' } else { 'ok' } $summary[$node] = [pscustomobject]@{ Tested = $tested Passed = $passed Failed = $failed Skipped = $skipped SSLInspected = $sslInspected PrivateLink = $privateLink DownloadSpeed = $dlSpeed Status = $status } } # ── Build the MERGED, flat result set (ComputerName as the FIRST column) ── # Used for the always-on JSON sidecar and the CSV output. ComputerName is set # to the node we fanned out to. The per-node HTML tabs intentionally OMIT this # column (the tab name + heading already identify the node). $mergedColumns = @( 'RowID','URL','Port','ArcGateway','IsWildcard','Source','IPAddress', 'Layer7Status','Layer7Response','Layer7ResponseTime','Note','TCPStatus', 'CertificateIssuer','CertificateSubject','CertificateThumbprint', 'IntermediateCertificateIssuer','IntermediateCertificateSubject','IntermediateCertificateThumbprint', 'RootCertificateIssuer','RootCertificateSubject','RootCertificateThumbprint' ) $mergedRows = [System.Collections.ArrayList]::new() foreach ($node in $nodeNames) { $nrows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) foreach ($r in $nrows) { $ordered = [ordered]@{ ComputerName = $node } foreach ($c in $mergedColumns) { $ordered[$c] = if ($r.PSObject.Properties[$c]) { $r.$c } else { $null } } [void]$mergedRows.Add([pscustomobject]$ordered) } } # ── Rolled-up cluster totals (for the Summary tab + JSON) ───────────────── $totalTested = ($summary.Values | ForEach-Object { $_.Tested } | Measure-Object -Sum).Sum $totalPassed = ($summary.Values | ForEach-Object { $_.Passed } | Measure-Object -Sum).Sum $totalFailed = ($summary.Values | ForEach-Object { $_.Failed } | Measure-Object -Sum).Sum $totalSkipped = ($summary.Values | ForEach-Object { $_.Skipped } | Measure-Object -Sum).Sum if (-not $totalTested) { $totalTested = 0 } if (-not $totalPassed) { $totalPassed = 0 } if (-not $totalFailed) { $totalFailed = 0 } if (-not $totalSkipped) { $totalSkipped = 0 } # ── Tab bar (Summary + per-node) with status indicators ────────────────── [void]$sb.AppendLine('<div class="tab-bar">') [void]$sb.AppendLine('<button class="tab-btn active" onclick="showTab(event, ''tab-summary'')">Summary</button>') foreach ($node in $nodeNames) { $st = $summary[$node].Status $badge = switch ($st) { 'ok' { '<span class="badge badge-ok" style="font-size:9px;padding:1px 6px;">✓</span>' } 'warn' { '<span class="badge badge-warn" style="font-size:9px;padding:1px 6px;">⚠</span>' } 'error' { '<span class="badge badge-error" style="font-size:9px;padding:1px 6px;">✗</span>' } default { '' } } $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<button class=`"tab-btn`" onclick=`"showTab(event, 'tab-$safeNode')`">$safeNode $badge</button>") } [void]$sb.AppendLine('</div>') # ── Summary tab — rolled-up totals + per-node Status Overview ──────────── [void]$sb.AppendLine('<div id="tab-summary" class="tab-panel active">') [void]$sb.AppendLine('<h2>Cluster Totals</h2>') [void]$sb.AppendLine('<div class="totals-card">') [void]$sb.AppendLine("<table style='width:auto;'><thead><tr><th>Nodes</th><th>Endpoints Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th></tr></thead><tbody>") [void]$sb.AppendLine("<tr><td class='num'>$($nodeNames.Count)</td><td class='num'>$totalTested</td><td class='num ok'>$totalPassed</td><td class='num fail'>$totalFailed</td><td class='num skip'>$totalSkipped</td></tr>") [void]$sb.AppendLine('</tbody></table>') [void]$sb.AppendLine('</div>') [void]$sb.AppendLine('<h2>Per-Node Status Overview</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th><th>SSL-Inspected</th><th>Private-Link</th><th>Download Speed</th><th>Status</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $s = $summary[$node] $badgeClass = 'badge-' + $s.Status $badgeText = switch ($s.Status) { 'ok' { 'OK' } 'warn' { 'WARN' } 'error' { 'FAIL' } default { '?' } } $dlText = if ($s.PSObject.Properties['DownloadSpeed'] -and $s.DownloadSpeed) { ConvertTo-HtmlSafe ([string]$s.DownloadSpeed) } else { 'N/A' } [void]$sb.AppendLine("<tr><td>$(ConvertTo-HtmlSafe $node)</td><td>$($s.Tested)</td><td>$($s.Passed)</td><td>$($s.Failed)</td><td>$($s.Skipped)</td><td>$($s.SSLInspected)</td><td>$($s.PrivateLink)</td><td>$dlText</td><td><span class=`"badge $badgeClass`">$badgeText</span></td></tr>") } [void]$sb.AppendLine('</tbody></table>') # Errors block (if any) $totalErrors = ($ClusterResult.Errors.Values | ForEach-Object { @($_).Count } | Measure-Object -Sum).Sum if ($totalErrors -gt 0) { [void]$sb.AppendLine('<h2>Per-Node Errors</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Error</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $errs = @($ClusterResult.Errors[$node]) foreach ($e in $errs) { [void]$sb.AppendLine("<tr class='row-failed'><td>$(ConvertTo-HtmlSafe $node)</td><td>$(ConvertTo-HtmlSafe $e)</td></tr>") } } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') # ── Per-node tabs ──────────────────────────────────────────────────────── foreach ($node in $nodeNames) { $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<div id=`"tab-$safeNode`" class=`"tab-panel`">") [void]$sb.AppendLine("<h2>Node: $safeNode</h2>") # Filter $null entries — a failed Invoke-Command leaves Nodes[<name>] = $null # and `@($null)` is a 1-element array containing $null (not @()), which would # otherwise blow up `$r.PSObject.Properties[...]` below. $rows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) if ($rows.Count -eq 0) { [void]$sb.AppendLine("<p><em>No results returned from this node. See Errors on the Summary tab.</em></p>") } else { [void]$sb.AppendLine('<table><thead><tr>') # Per-node tab columns — full single-node column set, WITHOUT ComputerName # (the tab name + "Node:" heading already identify the host). $columns = $mergedColumns foreach ($c in $columns) { [void]$sb.AppendLine("<th>$c</th>") } [void]$sb.AppendLine('</tr></thead><tbody>') foreach ($r in $rows) { # Colour rows by Layer7Status using the SAME palette as the single-node # report: Success=green, Failed=red, Skipped=yellow. $statusVal = if ($r.PSObject.Properties['Layer7Status']) { [string]$r.Layer7Status } else { '' } $rowClass = switch ($statusVal) { 'Success' { 'status-success' } 'Failed' { 'status-failed' } 'Skipped' { 'status-skipped' } default { '' } } $cls = if ($rowClass) { " class=`"$rowClass`"" } else { '' } [void]$sb.AppendLine("<tr$cls>") foreach ($c in $columns) { $val = if ($r.PSObject.Properties[$c]) { $r.$c } else { '' } [void]$sb.AppendLine("<td>$(ConvertTo-HtmlSafe ([string]$val))</td>") } [void]$sb.AppendLine('</tr>') } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') } # ── Footer + JS ───────────────────────────────────────────────────────── # Stamp the running module version into the footer so portable HTML artefacts # always record which build produced them (best-effort; 'version unknown' if unresolved). $footerModuleVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$footerModuleVersionLabel = if ($footerModuleVersion) { "v$($footerModuleVersion.ToString())" } else { 'version unknown' } [void]$sb.AppendLine(@" <div class="footer"> Generated by <strong>AzStackHci.DiagnosticSettings</strong> $footerModuleVersionLabel — Test-AzureLocalConnectivity -Scope Cluster — $(Get-Date -Format 'u') </div> <script> function showTab(evt, tabId) { var panels = document.querySelectorAll('.tab-panel'); for (var i = 0; i < panels.length; i++) panels[i].classList.remove('active'); var btns = document.querySelectorAll('.tab-bar > .tab-btn'); for (var i = 0; i < btns.length; i++) btns[i].classList.remove('active'); document.getElementById(tabId).classList.add('active'); evt.currentTarget.classList.add('active'); } </script> </body></html> "@) # ── Write the primary output file ──────────────────────────────────────── # HTML -> tabbed cluster report ($sb). CSV -> merged flat rows (ComputerName # first). JSON is ALWAYS written below in addition. (BOM-less UTF-8; PS 5.1 # Set-Content -Encoding UTF8 would add a BOM.) if ($OutputFormat -eq 'CSV') { try { # ConvertTo-Csv + Write-Utf8NoBom instead of Export-Csv -Encoding UTF8: the # latter prepends a UTF-8 BOM on PS 5.1 (the same BOM the comment above warns # about for the HTML path). @() guards the empty-rows case so an empty # collection binds cleanly to Write-Utf8NoBom's [string[]] parameter. $csvLines = @($mergedRows | ConvertTo-Csv -NoTypeInformation) Write-Utf8NoBom -Path $reportFile -Content $csvLines } catch { Write-Warning "Failed to write merged cluster CSV '$reportFile': $($_.Exception.Message)" } } else { Write-Utf8NoBom -Path $reportFile -Content $sb.ToString() } # ── JSON sidecar — full-fidelity cluster bundle (always written) ───────── # Schema v2: the per-node `Nodes` map is replaced by a single MERGED `Results` # array where every row carries ComputerName as its FIRST property, plus rolled # up `Totals`. Per-node summary/telemetry/errors are retained for drill-down. $nodeDurStrings = [ordered]@{} foreach ($node in $nodeNames) { $d = $ClusterResult.NodeDurations[$node] $nodeDurStrings[$node] = if ($d) { $d.ToString() } else { '' } } $orchestratorVer = (Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings') $jsonObj = [ordered]@{ Schema = 'AzStackHciClusterConnectivity/v2' ModuleVersion = if ($orchestratorVer) { $orchestratorVer.ToString() } else { 'unknown' } ClusterName = $ClusterResult.ClusterName OrchestratorMachine = $ClusterResult.OrchestratorMachine RunGuid = $ClusterResult.RunGuid.ToString() StartTime = $ClusterResult.StartTime.ToString('u') EndTime = $ClusterResult.EndTime.ToString('u') Duration = $ClusterResult.Duration.ToString() Totals = [ordered]@{ Nodes = $nodeNames.Count Tested = $totalTested Successful = $totalPassed Failed = $totalFailed Skipped = $totalSkipped } NodeSummary = $summary NodeDurations = $nodeDurStrings NodeTelemetry = if ($ClusterResult.PSObject.Properties['NodeTelemetry']) { $ClusterResult.NodeTelemetry } else { @{} } Errors = $ClusterResult.Errors Results = $mergedRows.ToArray() } # Depth 8 is enough for: { Results: [ { flat row props } ] }. Write-Utf8NoBom -Path $jsonFile -Content ($jsonObj | ConvertTo-Json -Depth 8) return $reportFile } # SIG # Begin signature block # MIInUAYJKoZIhvcNAQcCoIInQTCCJz0CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAzHl4v81ca1dnS # Em2hsyHhyc1eIbsA0d5lJ3BsjtRHfaCCDMkwggYEMIID7KADAgECAhMzAAACHPrN # xZvoL37EAAAAAAIcMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNVBAYTAlVTMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBD # b2RlIFNpZ25pbmcgUENBIDIwMjQwHhcNMjYwNDE2MTg1OTQxWhcNMjcwNDE1MTg1 # OTQxWjB0MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYD # VQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IB # DwAwggEKAoIBAQDVsZfgOKmM31HPfoWOoNEiw0SlCiIxUMC0I9NMWbucKOw/e9lP # oAoehQVu6SG65V4EPzrYsnBnFPNoi4/HoOdjhz1qkrEt4I6tEcxXU6oOeY9zGveC # /3iBeuhLYxM3M/PkcUoebF+Nednm8OkdSPoDu8imViHPQq/8CQUu0WRR4rE+dMRf # rpVqfmNi2qWCX94T4MsepijGVkwE//tJg0ryAiYdHT34LSnlG/RSBZmQRGWZ5g8j # qnKjRParSqMft1gvjuUTVgtWNZfgcLFSK5Wa0myrq8OPcgTGGsRgun+tnSS+IxDT # xVsAPH1OzvPjwomguByhUe/OcvUN0D5Wmp7xAgMBAAGjggGqMIIBpjAOBgNVHQ8B # Af8EBAMCB4AwHwYDVR0lBBgwFgYKKwYBBAGCN0wIAQYIKwYBBQUHAwMwHQYDVR0O # BBYEFNoH7a2YDjOSwpkp6DHcmUS7J+0yMFQGA1UdEQRNMEukSTBHMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxFjAUBgNVBAUT # DTIzMDAxMis1MDc1NjkwHwYDVR0jBBgwFoAUf1k/VCHarU/vBeXmo9ctBpQSCDEw # YAYDVR0fBFkwVzBVoFOgUYZPaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9w # cy9jcmwvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNy # bDBtBggrBgEFBQcBAQRhMF8wXQYIKwYBBQUHMAKGUWh0dHA6Ly93d3cubWljcm9z # b2Z0LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmcl # MjBQQ0ElMjAyMDI0LmNydDAMBgNVHRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4IC # AQAUnEqhaRXe0T3hIJjvdQErEkrA/7bByjn6t5IArODkkRjzkYwtKMc2yYj2quaN # rLutWw2YZcngKPy1b71YyDJQTy4NDRwaSh9Tw5thrk3NmcPrAHia5vtcBJ1CgtKK # 7mQbIcQ22d/N3813ayCDDFewu1+jsZmX+r/aTEqaOM4TVxVtRSkuCy8nAXKuChOK # Li/zA4XuH8iEYqIsj2YoNaeSxVmeGiERXpKdo3dDmYi0kO5w2D8VS4c3+9h6gElY # BaAAg/dYErBg27qT3vv0zRDJhJufvCNylA8S7/+8H5E/PV5cng6na9VV/w9OV3qu # uND6zdGa2EX38Glp50F9AIQk3p2xXmcvorDeM4XJ7UlWYBi6g80J1SSOQnInCYFE # msfUNn3+1AaTJKSJL83quKArTac2pKhu0Yzzzrzo6HrsRiQKzpnRBb1/dMa6P3hz # 75XbMRBctNsFhZC07WCmjExdLg2eHW5uV0TY8D5+6wozJf7vF3+WHkYPO85Z+BC6 # U4FkNbYNycZ9cE4j1tXRdyDCfml6c0HWPHjNVDObrv9lKt3qUqFpX38VCqVCyNOO # 1UcXfQiVjJw32U2WUKZjt/neJKHEBsm9kFsLuWzkQ53+qcaSaytmsCnk2gOglrlD # 5d3kKyvvAw+rzm0lT8K38P6PLxfZQHhu4W8dV7Av8N2ZmDCCBr0wggSloAMCAQIC # EzMAAAA5O7Y3Gb8GHWcAAAAAADkwDQYJKoZIhvcNAQEMBQAwgYgxCzAJBgNVBAYT # AlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBS # b290IENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDExMB4XDTI0MDgwODIwNTQxOFoX # DTM2MDMyMjIyMTMwNFowVzELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQ # Q0EgMjAyNDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANgBnB7jOMeq # lRYHNa265v4IY9fH8TKhemHfPINe1gpLaV3dhg324WwH06LcHbpnsBukCDNitryo # 0dtS/EW6I/yEL/bLSY8hKpbfQuWusBPr9qazYcDxCW/qnjb5JsI1s8bNOg3bVATv # QVL4tcf03aTycsz8QeCdM0l/yHRObJ9QqazM1r6VPEOJ7LL+uEEb73w6QCuhs89a # 1uv1zerOYMnsneRRwCbpyW11IcggU0cRKDDq1pjVJzIbIF6+oiXXbReOsgeI8zu1 # FyQfK0fVkaya8SmVHQ/tOf23mZ4W9k0Ri22QW9p3UgSC5OUDktKxxcCmGL6tXLfO # GSWHIIV4YrTJTT6PNty5REojHJuZHArkF9VnHTERWoTjAzfI3kP+5b4alUdhgAZ7 # ttOu1bVnXfHaqPYl2rPs20ji03LOVWsh/radgE17es5hL+t6lV0eVHrVhsssROWJ # uz2MXMCt7iw7lFPG9LXKGjsmonn2gotGdHIuEg5JnJMJVmixd5LRlkmgYRZKzhxS # CwyoGIq0PhaA7Y+VPct5pCHkijcIIDm0nlkK+0KyepolcqGm0T/GYQRMhHJlGOOm # VQop36wUVUYklUy++vDWeEgEo4s7hxN6mIbf2MSIQ/iIfMZgJxC69oukMUXCrOC3 # SkE/xIkgpfl22MM1itkZ35nNXkMolU1lAgMBAAGjggFOMIIBSjAOBgNVHQ8BAf8E # BAMCAYYwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFH9ZP1Qh2q1P7wXl5qPX # LQaUEggxMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMB # Af8wHwYDVR0jBBgwFoAUci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0fBFMwUTBP # oE2gS4ZJaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMv # TWljUm9vQ2VyQXV0MjAxMV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcBAQRSMFAw # TgYIKwYBBQUHMAKGQmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMv # TWljUm9vQ2VyQXV0MjAxMV8yMDExXzAzXzIyLmNydDANBgkqhkiG9w0BAQwFAAOC # AgEAFJQfOChP7onn6fLIMKrSlN1WYKwDFgAddymOUO3FrM8d7B/W/iQ6DxXsDn7D # 5W4wMwYeLystcEqfkjz4NURRgazyMu5yRzQh4LqjA4tStTcJh1opExo7nn5PuPBY # nbu0+THSuVHTe0VTTPVhily/piFrDo3axQ9P4C+Ol5yet+2gTfekICS5xS+cYfSI # vgn0JksVBVMYVI5QFu/qhnLhsEFEUzG8fvv0hjgkO+lkpV9ty6GkN4vdnd7ya6Q6 # aR9y34aiM1qmxaxBi6OUnyNl6fkuun/diTFnYDLTppOkr/mg5WSfCiDVMNCxtj4w # PKC5OmHm1DQIt/MNokbbH3UGsFP1QbzsLocuSqLCvH09Io3fDPTmscR9Y75G4qX7 # RTX8AdBPo0I6OEojf39zuFZt0qOHm65YWQE69cZM2ueE1MB05dNNgHK9gTE7zKvK # /fg8B2qjW88MT/WF5V5uvZGtqa9FSL2RazArA+rDPuf6JGYz4HpgMZHB4S6szWSK # YBv0VisCzfxgeU+dquXW9bd0auYlOB58DPcOYKdc3Se94g+xL4pcEhbB54JOgAkw # YTu/9dLeH2pDqeJZAABVDWRQCaXfO5LgyKwKCLYXpigrZYCjUSBcr+Ve8PFWMhVT # Ql0v4q8J/AUmQN5W4n101cY2L4A7GTQG1h32HHAvfQESWP0xghndMIIZ2QIBATBu # MFcxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # KDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMjQCEzMAAAIc # +s3Fm+gvfsQAAAAAAhwwDQYJYIZIAWUDBAIBBQCggZAwGQYJKoZIhvcNAQkDMQwG # CisGAQQBgjcCAQQwLwYJKoZIhvcNAQkEMSIEIGN9dYttYfamVc2HbIhxQNXC06T1 # 1q96i6JFWykj95+LMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8AcwBv # AGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEBBQAE # ggEAbdkpjeTcy+Lv/xX/YC0qren/F2CdQOfoghKpz8t1/AhhVFyGn0M7mGab2+0Z # YfRd9TVW3FJ0FnqJ7RAuiwpYdHFuGwlgvewxmKoGAAgkFZKeT4w+MH+PdmQuhfK4 # V5fZLTyk0V+EQn/rLzKB3AEplcVUldtG+toqhyX7mlWtJjBQYSSwZADdyo/i8EuC # l7yEacYjpGVTehtb5lZGdCjR1cO7JpKkvmXmubBih8i6kd2KTA0i0Hbr6b4xfY80 # zbI3hxIAP+CDdw23qq2M0v/cYzNYwOKzJtU4Uuf6SGY4bIbHtsXElnYE/KWNPcAb # 9RfTBIugf0pfl0zc+pun1xpMGqGCF60wghepBgorBgEEAYI3AwMBMYIXmTCCF5UG # CSqGSIb3DQEHAqCCF4YwgheCAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsqhkiG # 9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFlAwQC # AQUABCBf60bE2kSf2VKgaa0AlHTFHeLCKtLOnC7wgSQxUnWb1AIGaohOpXWqGBMy # MDI2MDgyODE0NTUzMy44OTdaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJVUzET # MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV # TWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJlbGFu # ZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjoz # NjA1LTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaCCEfswggcoMIIFEKADAgECAhMzAAACE7BDNWbPr5XoAAEAAAITMA0GCSqG # SIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI1MDgx # NDE4NDgxN1oXDTI2MTExMzE4NDgxN1owgdMxCzAJBgNVBAYTAlVTMRMwEQYDVQQI # EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv # ZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjM2MDUtMDVF # MC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNlMIIC # IjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9Jl64LoZxDINSFgz+9KS5Ozv # 5m548ePVzc9RXWe4T4/Mplfga4eq12RGdp5cVvnjde5vxfq2ax/jnu7vUW4rZN4m # OUm5vh+kcYsQlYQ53FwgIB3nEjcQHomrG3mZe/ozjFSAr6JbglKtIeAySPzAcFzy # Aer5lLNUHBEvQMM8BOjMyapCvh0xsg4xKFcVEJQLKEfCGBffMZI/amutHFb3CUTZ # 7aVpG2KHEFUNlZ1vwMKvxXTPRDnbwPGzyyqJJznfsLNHQ4vXt2ttS1PeCoGI0hN1 # Peq8yGsIXM9oocwC06DGNSM/4LAx2uKvwmUn6NwLc0+tmvny6w28rZLejskRfnVW # ofEv1mWY0jHUnHrwSGBS8gVP9gcBs6P5g0OpJPMfxdUkHXRkcMPPW0hIP8NbW8W5 # Sup8HuwnSKbjpyAlGBUdM/V5rZb0sZmkn714r6ULGK+cLLAN6R3FhX6N0nj64F27 # LTK2BbS0pJZaXjo0eDNz1QcxeIFLUgF+RBsLYDn8E8cCkexK8Nlt3Gi9zJf55w6U # fTZ+kwTMxMqFxh7+Tfx7+aBObZ+nx961AtiqAy7zVV69o/LWRdKPZdvZn9ESyGbT # nPfjkBERv22prSlETlRwzP6bmEVOKWLWVwxuwh7bUWUuUb1cj93zvttQYGQat5E9 # ALLJNmlvLKCskB7raLsCAwEAAaOCAUkwggFFMB0GA1UdDgQWBBQTnhBKx+FryphQ # WMRipH49sMFAOjAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBfBgNV # HR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2Ny # bC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmwwbAYI # KwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAy # MDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMI # MA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAgEAgmxaJrGqQ2D6UJhZ # 6Ql2SZFOaNuGbW3LzB+ES+l2BB1MJtBRSFdi/hVY33NpxsJQhQ5TLVp0DXYOkIoP # Qc17rH+IVhemO8jCt+U6I1TIw6cR7c+tEo/Jjp6EqEU1c4/mraMjgHhQ+raC/OUA # m98A1r4bIPHtsBmLROGmeE5XLIFaBIZWHvh2COXITKObXVd5wGtJ1dZZdwaHACXF # 506jta+uoUdyzAeuNlTPLTrZ8nyhxGwk9Vh6eiDQ7CQMWSSa8DJS9PUXjeoi9vTd # S7ZMXqu+tv6Qz3xtoBF5+YFK4uE+miGs90Fxm0VK2lWrmFhjkRl5zyoHOdwG7spN # YkDomCPNWIudUQmQYKpt/Hsspfcb+xpnWIDQdMzgE8pj1vpwLgWEnH7LtT4dZCeo # Do9PK40RxBD8kKJ769ngkEwfwCD2EX/MQk79eIvOhpnH12GuVByvaKZk5XZvqtPO # NNwr8q/qA3877IuWwWgnaeX+prpw0dZ/QLtbGGVrgP+TRQjt+2dcZA5P3X4LwANh # iPsy0Ol4XCdj7OxBLFvOzsCPDPaVnkp+dfDFG+NOBir7aqTJ68622pymg1V+6gc/ # 1RvxC/wgvYyG033ecJqv0On0ZRNYr+i/OkwgA3HP1aLD0aHrEpw6lt0263iRkCvr # cdcOW8w3jC8TJuaGWyC2S9jEjzgwggdxMIIFWaADAgECAhMzAAAAFcXna54Cm0mZ # AAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMyMjVa # MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMT # HU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0BAQEF # AAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51yMo1 # V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY6GB9 # alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9cmmv # Haus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN7928 # jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDuaRr3t # pK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74kpEe # HT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2K26o # ElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5TI4C # vEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZki1ug # poMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9QBXps # xREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3PmriLq0C # AwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUCBBYE # FCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJlpxtT # NRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIBFjNo # dHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9yeS5o # dG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBD # AEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZW # y4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5t # aWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAt # MDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0y # My5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/ypb+pc # FLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulmZzpT # Td2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM9W0j # VOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECWOKz3 # +SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4FOmR # sqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3UwxTSw # ethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPXfx5b # RAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVXVAmx # aQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGConsX # HRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU5nR0 # W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEGahC0 # HVUzWLOhcGbyoYIDVjCCAj4CAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJVUzET # MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV # TWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJlbGFu # ZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjoz # NjA1LTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUAmBE8SCjxgjacmy8/VEdk7NxpR6aggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsFAAIF # AO48BuMwIhgPMjAyNjA4MjgxMzA4NTFaGA8yMDI2MDgyOTEzMDg1MVowdDA6Bgor # BgEEAYRZCgQBMSwwKjAKAgUA7jwG4wIBADAHAgEAAgIEmzAHAgEAAgISmTAKAgUA # 7j1YYwIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAID # B6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBCwUAA4IBAQA33M4fRehS2ouMtJsa # +FcBshAC1whmYaxqqzrPjEyIaaT2DhuNF5AfEQNOJcXsAtX8pBQf9JXxN76N7DLU # 93njgMYBBM9aq6Qt7z4ITwAH10h8kBz3WuMW6fDy+rhYVI93X8IN/6vigD7IeYW7 # ek/8dUOAnngeAxfQ8AD5uG05ysX/BP+9tkWHub1NgAp7kcHPaGvI/w3BuZnYqxDm # KtMXnz0dA55+0eD8OzQYDa87ScPt2p0pSvYfxjJX8tP2VD8Ki3Po7drRPQtuPsvx # L+z+Z6JnpEIQKk+NEr4Wrmdaj0XeYV6TaI//uKzXDN2JV6BNVH7HyGRcmCf+oQWU # rXBcMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAC # EzMAAAITsEM1Zs+vlegAAQAAAhMwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3 # DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQxIgQgpi6x/uEnt9wDEKyE # X52ukwXuMZGMI1hprnSQRm22iigwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9 # BCDM4QltFIUz8J4DjAzP4nVodZvQxYGleUIfp86Oa5xYaDCBmDCBgKR+MHwxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jv # c29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAACE7BDNWbPr5XoAAEAAAITMCIE # IKaiHLCllo8wZMklh+yQTNhSt4maXwyRI1ofDVz9S1QQMA0GCSqGSIb3DQEBCwUA # BIICAAm4Wk+RTr4mT1la4e6rFOM0ZwBwZnF6Qhs1ENfsWbZZ8nuiqlL0lCWRh22T # x973QwIZF5RlOJ6DQniNwddBjwzY4GYNisjY44itg0C8ReDODjbAhD2ftRXE0BjH # 0E9oJ39ArSIV0Rr6zsq0pFMiWXscZvbAvnxxGlPtMH1XMuYltIia3k4RfO0RqgAE # m1fOKI4byw4UrnsJF/NGOIySRKiVo0x8pnEsugtug6/s/dlplPkAB76CUCShRnqD # 3Eosvj9IDGjIMuNtqrrBMwwgIVP71C830qQqOI3aw3092nWb0I3ezNI/Zplto0m8 # ltS0MbHsExjiFlErM7QH3Bw/MgW8yEXJQksZkuVWB8M7vqwNr6Kdu84mpuokXDqM # K9ZaXbuBGbZK9CYlv2e1d8DtXphO3kWpV3qSKjkDYddfLZ6QoDVuTsgFbsOxJPPQ # JWloI9R0w3pc8Hbz1RWpsErpIwWn3cadQ+hnrZforEtloC+JPVLBoOhJqUdaxlFU # OAvRuynF+BcMph0NFP3pOJv8N3ggtPjvyGAB9F+h1+pi9hz6VoFRbxzJltleIFGf # LdFuA8Y3yMBssZeYJ1vm4FkkUzd3cPm+9BIKmPQUB4WB8+I9/h3EVnXTAOeD6283 # R/ZF05Z8nT6XSPe++tQRYOUxW26TRBLofZOasCkfLLIEFBhF # SIG # End signature block |