Private/NutanixPrismCentral.ps1
|
<# Prism Central v4 collection cycle. clustermgmt supplies cluster/host inventory and per-host history; vmm supplies VM inventory and VM history through one paginated list endpoint. Host history is fetched with bounded concurrency (Private/Nutanix.ps1), VM history is aggregated per host, and both are joined by sample timestamp into the platform hypervisor payload (Classes/HypervisorPayload.ps1). Reference: https://developers.nutanix.com/api-reference?namespace=clustermgmt&version=v4.0 https://developers.nutanix.com/api-reference?namespace=vmm&version=v4.0 #> # ---------------------------- # Unit conversions # ---------------------------- function ConvertFrom-NutanixPpmToHundredthsOfPercent { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # The Nexthink hypervisor-processor divides usage_average by 100 before storing it, because # vSphere's cpu.usage.average / mem.usage.average counters arrive as hundredths of a percent # ("2938" = 29.38%). Sending a plain percentage here shows up 100x too small in the product. # 1 ppm = 0.0001%, so ppm / 100 is already the hundredths-of-a-percent figure. return [Math]::Round(([double]$Value / 100), 2) } function ConvertFrom-NutanixBytesToMegabytes { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # memory.committed is defined in megabytes (the vSphere connector sums Config.Hardware.MemoryMB) # and the hypervisor-processor multiplies it by 1024^2 on the way to the data model. return [Math]::Round(([double]$Value / 1048576), 0) } function ConvertFrom-NutanixBytesToKilobytes { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # mem.swapused.average and mem.vmmemctl.average are KB on the vSphere wire; the processor applies # kbToBytes() to ballooning on the way to the data model. return ConvertTo-NutanixWholeNumber ([Math]::Round(([double]$Value / 1024), 0)) } function ConvertFrom-NutanixMicrosecondsToMilliseconds { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # Nexthink's Hypervisor API deserializes disk.max_total_latency_latest as a Long; a fractional # value here fails with a 400 (java.lang.Long from a decimal string). return [Math]::Round(([double]$Value / 1000), 0) } function ConvertTo-NutanixWholeNumber { <# .SYNOPSIS Rounds a counter to a whole number for the payload fields the Hypervisor API deserializes as Long (byte counts and kBps rates). Averaging two 30s points otherwise yields '690.5', which the API rejects with a bare 400. #> param([AllowNull()][object]$Value) if ($null -eq $Value -or [string]::IsNullOrWhiteSpace([string]$Value)) { return $null } $parsed = 0.0 if (-not [double]::TryParse([string]$Value, [System.Globalization.NumberStyles]::Float, [System.Globalization.CultureInfo]::InvariantCulture, [ref]$parsed)) { return $null } return [Math]::Round($parsed, 0) } function Get-NutanixHostSwapRateKbps { param( [AllowNull()][object]$HostSample, [Parameter(Mandatory = $true)][string]$HostFieldName, [Parameter(Mandatory = $true)][pscustomobject]$VmMetricTotals, [Parameter(Mandatory = $true)][string]$VmTotalName ) $hostRate = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName $HostFieldName if ($null -ne $hostRate) { return ConvertTo-NutanixWholeNumber $hostRate } $availableFlag = $VmMetricTotals."${VmTotalName}Available" if ($availableFlag) { return ConvertTo-NutanixWholeNumber $VmMetricTotals.$VmTotalName } return '0' } function Get-NutanixNestedValue { <# .SYNOPSIS Null-safe read of a nested property path such as 'host.extId' on an API record. #> param( [AllowNull()][object]$Record, [Parameter(Mandatory = $true)][string]$Path ) $current = $Record foreach ($segment in $Path -split '\.') { if ($null -eq $current) { return $null } $property = $current.PSObject.Properties[$segment] if ($null -eq $property) { return $null } $current = $property.Value } return $current } # ---------------------------- # Cluster filtering # ---------------------------- function Test-NutanixClusterHasHosts { <# .SYNOPSIS False for the Prism Central self-registration, which the host list API rejects (CLU-10008). Clusters without a readable clusterFunction are assumed to be AOS clusters. #> param([AllowNull()][object]$Cluster) $functions = @(Get-NutanixNestedValue -Record $Cluster -Path 'config.clusterFunction') foreach ($function in $functions) { if ([string]$function -in $script:NUTANIX_CLUSTER_FUNCTION_SKIP) { return $false } } return $true } # ---------------------------- # VM inventory and history (vmm namespace) # ---------------------------- function Get-NutanixVms { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context ) Write-CustomLog -Message "Retrieving VMs from Prism Central" -Severity 'DEBUG' $extraQuery = @{} if (-not [string]::IsNullOrWhiteSpace($script:NUTANIX_VM_CONFIG_SELECT)) { $extraQuery['$select'] = $script:NUTANIX_VM_CONFIG_SELECT } $vms = @(Get-NutanixPaginatedResults -Context $Context -Path 'vmm/v4.0/ahv/config/vms' -ExtraQuery $extraQuery) Write-CustomLog -Message "Found $($vms.Count) VMs" -Severity 'DEBUG' return $vms } function Get-NutanixVmsByHostExtId { param([AllowEmptyCollection()][object[]]$Vms) $vmsByHostExtId = @{} foreach ($vm in @($Vms)) { # number_of_vms / number_of_vcpus / committed are defined over powered-on VMs only (the vSphere # connector filters Runtime.PowerState = poweredOn). A powered-off VM normally carries no host # reference, but filter explicitly on powerState too; a VM without powerState is kept. $powerState = [string]$vm.powerState if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -ne 'ON') { continue } $hostExtId = [string](Get-NutanixNestedValue -Record $vm -Path 'host.extId') if ([string]::IsNullOrWhiteSpace($hostExtId)) { continue } if (-not $vmsByHostExtId.ContainsKey($hostExtId)) { $vmsByHostExtId[$hostExtId] = [System.Collections.Generic.List[object]]::new() } $vmsByHostExtId[$hostExtId].Add($vm) } return $vmsByHostExtId } function Get-NutanixVmStatsByExtId { <# .SYNOPSIS Fetches one window of VM history for every VM through the paginated list endpoint and returns a hashtable of VM extId -> stats tuples. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context, [Parameter(Mandatory = $true)] [datetime]$StartUtc, [Parameter(Mandatory = $true)] [datetime]$EndUtc ) $extraQuery = @{ '$startTime' = $StartUtc.ToString('o') '$endTime' = $EndUtc.ToString('o') '$samplingInterval' = $script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS # Mandatory on the list endpoint (Prism Central answers 400 VMM-30102 without it), unlike the # per-host stats endpoint where it is optional. '$statType' = $script:NUTANIX_VM_STATS_TYPE } if (-not [string]::IsNullOrWhiteSpace($script:NUTANIX_VM_STATS_SELECT)) { $extraQuery['$select'] = $script:NUTANIX_VM_STATS_SELECT } $results = @(Get-NutanixPaginatedResults -Context $Context -Path 'vmm/v4.0/ahv/stats/vms' -ExtraQuery $extraQuery) $statsByExtId = @{} foreach ($item in $results) { $extId = [string]$item.extId if ([string]::IsNullOrWhiteSpace($extId)) { continue } $statsByExtId[$extId] = @($item.stats) } Write-CustomLog -Message "Retrieved VM history for $($statsByExtId.Count) VM(s) between $($StartUtc.ToString('o')) and $($EndUtc.ToString('o'))" -Severity 'DEBUG' return $statsByExtId } function New-NutanixVmMetricTotals { return [pscustomobject]@{ Ready = 0.0; ReadyAvailable = $false SwapIn = 0.0; SwapInAvailable = $false SwapOut = 0.0; SwapOutAvailable = $false Balloon = 0.0; BalloonAvailable = $false } } function Add-NutanixVmMetricTotal { param( [Parameter(Mandatory = $true)][pscustomobject]$Totals, [Parameter(Mandatory = $true)][object]$Tuple, [Parameter(Mandatory = $true)][string]$FieldName, [Parameter(Mandatory = $true)][string]$TotalName ) $property = $Tuple.PSObject.Properties[$FieldName] if ($null -eq $property -or $null -eq $property.Value) { return } $parsed = 0.0 if (-not [double]::TryParse([string]$property.Value, [ref]$parsed)) { return } $Totals.$TotalName += $parsed $Totals."${TotalName}Available" = $true } function Get-NutanixVmMetricTotalsByTimestamp { <# .SYNOPSIS Sums the VM-only counters of the given VMs per sample timestamp, so each host event can carry the aggregate of its VMs at the same instant. Keys are the normalized timestamp strings Get-NutanixSampleTimestampKey produces. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [AllowEmptyCollection()] [object[]]$Vms, [Parameter(Mandatory = $true)] [hashtable]$VmStatsByExtId ) $totalsByTimestamp = @{} foreach ($vm in @($Vms)) { $extId = [string]$vm.extId if (-not $VmStatsByExtId.ContainsKey($extId)) { continue } foreach ($tuple in @($VmStatsByExtId[$extId])) { $timestampKey = Get-NutanixSampleTimestampKey -Sample $tuple if ([string]::IsNullOrWhiteSpace($timestampKey)) { continue } if (-not $totalsByTimestamp.ContainsKey($timestampKey)) { $totalsByTimestamp[$timestampKey] = New-NutanixVmMetricTotals } $totals = $totalsByTimestamp[$timestampKey] Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorCpuReadyTimePpm' -TotalName 'Ready' Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorSwapInRateKbps' -TotalName 'SwapIn' Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorSwapOutRateKbps' -TotalName 'SwapOut' Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorMemoryBalloonReclaimedBytes' -TotalName 'Balloon' } } return $totalsByTimestamp } # ---------------------------- # Payload mapping # ---------------------------- function Get-NutanixVmVcpuCount { param([AllowNull()][object]$Vm) if ($null -eq $Vm -or $null -eq $Vm.numSockets) { return 0 } $coresPerSocket = if ($null -ne $Vm.numCoresPerSocket) { [int]$Vm.numCoresPerSocket } else { 1 } $threadsPerCore = if ($null -ne $Vm.numThreadsPerCore) { [int]$Vm.numThreadsPerCore } else { 1 } return [int]$Vm.numSockets * $coresPerSocket * $threadsPerCore } function ConvertTo-NutanixPrismCentralVmInfo { param([object]$Vm) $vmInfo = [NutanixHypervisorVMInfo]::new() $vmInfo.name = [string]$Vm.name $vmInfo.guest_tools_version = Get-NutanixNestedValue -Record $Vm -Path 'guestTools.version' $vmInfo.resource_pool = $null $vmInfo.cpu_limit = $null $vmInfo.cpu_shares = $null $throttledIops = Get-NutanixNestedValue -Record $Vm -Path 'storageConfig.qosConfig.throttledIops' $vmInfo.disk_io_limit = if ($null -ne $throttledIops) { [long]$throttledIops } else { $null } return $vmInfo } function Test-NutanixHostSampleHasCounters { <# .SYNOPSIS True when the joined host sample carries at least one time-series counter. Prism Central returns memoryCapacityBytes as a single point stamped with the last capacity change, so a sample carrying only that field is not a performance bucket. #> param([AllowNull()][object]$Sample) if ($null -eq $Sample) { return $false } foreach ($fieldName in $script:NUTANIX_HOST_EVENT_SERIES) { $property = $Sample.PSObject.Properties[$fieldName] if ($null -ne $property -and $null -ne $property.Value) { return $true } } return $false } function Get-NutanixLatestHostSampleValue { <# .SYNOPSIS The value of the given field from the most recent sample that carries it, or null. #> param( [AllowEmptyCollection()][object[]]$Samples, [Parameter(Mandatory = $true)][string]$FieldName ) $latest = $null $latestKey = $null foreach ($sample in @($Samples)) { $value = Get-NutanixStatValueOrNull -Sample $sample -FieldName $FieldName if ($null -eq $value) { continue } $key = Get-NutanixSampleTimestampKey -Sample $sample if ($null -eq $latestKey -or [string]::CompareOrdinal($key, $latestKey) -gt 0) { $latest = $value $latestKey = $key } } return $latest } # ---------------------------- # Complete minutes of 30s Prism Central points # ---------------------------- function Get-NutanixCompleteMinuteBuckets { <# .SYNOPSIS Selects the 30s host samples and VM aggregates of complete minutes, one bucket per sample. .DESCRIPTION Starting at StartUtc (minute-aligned), a minute is complete when the host has a point at :00 and :30. Leading incomplete minutes are skipped (a gap Prism Central will never fill); the first incomplete minute after a complete one ends the run, because the trailing minute is the one Prism Central is still filling and it is re-read next cycle. .OUTPUTS Buckets: ordered list of { TimestampKey; Sample; VmTotals }. CoveredEndUtc: end of the last emitted minute, or StartUtc when nothing was emitted. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$HostSamples, [Parameter(Mandatory = $true)][hashtable]$VmTotalsByTimestamp, [Parameter(Mandatory = $true)][datetime]$StartUtc, [Parameter(Mandatory = $true)][datetime]$EndUtc ) $samplesByKey = @{} foreach ($sample in @($HostSamples)) { $key = Get-NutanixSampleTimestampKey -Sample $sample if (-not [string]::IsNullOrWhiteSpace($key)) { $samplesByKey[$key] = $sample } } $bucketSeconds = [int]$script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS $buckets = [System.Collections.Generic.List[object]]::new() $coveredEnd = $StartUtc $runStarted = $false for ($minuteStart = $StartUtc; $minuteStart.AddSeconds(60) -le $EndUtc; $minuteStart = $minuteStart.AddSeconds(60)) { $keys = @(for ($offset = 0; $offset -lt 60; $offset += $bucketSeconds) { $minuteStart.AddSeconds($offset).ToString('o') }) $complete = @($keys | Where-Object { -not $samplesByKey.ContainsKey($_) }).Count -eq 0 if (-not $complete) { if ($runStarted) { break } continue } $runStarted = $true foreach ($key in $keys) { $buckets.Add([pscustomobject]@{ TimestampKey = $key Sample = $samplesByKey[$key] VmTotals = if ($VmTotalsByTimestamp.ContainsKey($key)) { $VmTotalsByTimestamp[$key] } else { New-NutanixVmMetricTotals } }) } $coveredEnd = $minuteStart.AddSeconds(60) } return [pscustomobject]@{ Buckets = @($buckets); CoveredEndUtc = $coveredEnd } } function New-NutanixPrismCentralHostEvent { param( [Parameter(Mandatory = $true)][object]$HostItem, [Parameter(Mandatory = $true)][object]$HostSample, [Parameter(Mandatory = $true)][int]$NumberOfVcpus, [Parameter(Mandatory = $true)][pscustomobject]$VmMetricTotals, [Parameter(Mandatory = $false)][AllowNull()][object]$MemoryCapacityBytes = $null ) $intervalSeconds = $script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS $cpuPpm = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'hypervisorCpuUsagePpm' $memoryPpm = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'aggregateHypervisorMemoryUsagePpm' $hostEvent = [NutanixHypervisorEvent]::new() # A v4 stats tuple is stamped with the start of its sampling bucket, like a v2 series point. $hostEvent.start_time = ConvertTo-Rfc3339UtcZ -Timestamp (ConvertFrom-RfcUtcTimestamp -Value (Get-NutanixSampleTimestampKey -Sample $HostSample)) $hostEvent.duration = $intervalSeconds $cpu = [NutanixHypervisorCpuMetrics]::new() $cpu.number_of_threads = [int]$HostItem.numberOfCpuThreads $cpu.number_of_packages = [int]$HostItem.numberOfCpuSockets $cpu.number_of_vcpus = $NumberOfVcpus # Raw sum of the host's VM hypervisorCpuReadyTimePpm: the hypervisor-processor turns it into the # per-VM readiness percentage (sum ppm / 10 000 / number of VMs) for source=nutanix-connector. $cpu.ready_summation = if ($VmMetricTotals.ReadyAvailable) { ConvertTo-NutanixWholeNumber $VmMetricTotals.Ready } else { $null } $cpu.usage_average = ConvertFrom-NutanixPpmToHundredthsOfPercent $cpuPpm $cpu.used_summation = if ($null -ne $cpuPpm) { # Nexthink's Hypervisor API deserializes cpu.used_summation as a Long, so it must be a whole # number of milliseconds. [Math]::Round(([double]$cpuPpm / 1000000) * [int]$HostItem.numberOfCpuThreads * $intervalSeconds * 1000, 0) } else { $null } $hostEvent.cpu = $cpu $disk = [NutanixHypervisorDiskMetrics]::new() $disk.read_average = ConvertTo-NutanixWholeNumber (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerReadIoBandwidthKbps') $disk.write_average = ConvertTo-NutanixWholeNumber (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerWriteIoBandwidthKbps') $disk.max_total_latency_latest = ConvertFrom-NutanixMicrosecondsToMilliseconds (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerAvgIoLatencyUsecs') $hostEvent.disk = $disk $memory = [NutanixHypervisorMemoryMetrics]::new() $swapIn = Get-NutanixHostSwapRateKbps -HostSample $HostSample -HostFieldName 'hostMemorySwapInRateKbps' ` -VmMetricTotals $VmMetricTotals -VmTotalName 'SwapIn' $swapOut = Get-NutanixHostSwapRateKbps -HostSample $HostSample -HostFieldName 'hostMemorySwapOutRateKbps' ` -VmMetricTotals $VmMetricTotals -VmTotalName 'SwapOut' $memory.swap_in_rate_average = $swapIn $memory.swap_out_rate_average = $swapOut # PM decision: swap_used, ballooning and committed are sent as null until validated on a lab cluster. $memory.swap_used_average = $null $memory.vm_mem_ctl_average = $null $memory.usage_average = ConvertFrom-NutanixPpmToHundredthsOfPercent $memoryPpm $installedMemory = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'memoryCapacityBytes' if ($null -eq $installedMemory -and $null -ne $MemoryCapacityBytes) { $installedMemory = [string]$MemoryCapacityBytes } $memory.installed = if ($null -ne $installedMemory) { ConvertTo-NutanixWholeNumber $installedMemory } else { ConvertTo-NutanixWholeNumber $HostItem.memorySizeBytes } $memory.committed = $null $hostEvent.memory = $memory return $hostEvent } function ConvertTo-NutanixPrismCentralDataItem { <# .OUTPUTS { DataItem; CoveredEndUtc }. CoveredEndUtc is the end of the last complete minute emitted, which is how far the host's watermark may advance; StartUtc when no minute was complete. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)][object]$HostItem, [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$HostSamples, [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Vms, [Parameter(Mandatory = $true)][hashtable]$VmStatsByExtId, [Parameter(Mandatory = $true)][datetime]$StartUtc, [Parameter(Mandatory = $true)][datetime]$EndUtc ) $numberOfVcpus = 0 foreach ($vm in @($Vms)) { $numberOfVcpus += Get-NutanixVmVcpuCount -Vm $vm } $vmTotalsByTimestamp = Get-NutanixVmMetricTotalsByTimestamp -Vms @($Vms) -VmStatsByExtId $VmStatsByExtId $memoryCapacityBytes = Get-NutanixLatestHostSampleValue -Samples @($HostSamples) -FieldName 'memoryCapacityBytes' $counterSamples = @($HostSamples | Where-Object { Test-NutanixHostSampleHasCounters -Sample $_ }) $minutes = Get-NutanixCompleteMinuteBuckets -HostSamples $counterSamples -VmTotalsByTimestamp $vmTotalsByTimestamp -StartUtc $StartUtc -EndUtc $EndUtc $events = [System.Collections.Generic.List[NutanixHypervisorEvent]]::new() foreach ($bucket in $minutes.Buckets) { $hostEvent = New-NutanixPrismCentralHostEvent -HostItem $HostItem -HostSample $bucket.Sample -NumberOfVcpus $numberOfVcpus ` -VmMetricTotals $bucket.VmTotals -MemoryCapacityBytes $memoryCapacityBytes $events.Add($hostEvent) } if ($events.Count -eq 0 -and $counterSamples.Count -gt 0) { Write-CustomLog -Message "Host '$($HostItem.hostName)' returned $($counterSamples.Count) sample(s) but no complete minute between $($StartUtc.ToString('o')) and $($EndUtc.ToString('o')); nothing emitted, window retried next cycle." -Severity 'WARNING' } $hostInfo = [NutanixHypervisorHostInfo]::new() $hostInfo.name = [string]$HostItem.hostName $hostInfo.cluster = [string]$HostItem.ClusterName $hostInfo.number_of_vms = @($Vms).Count $hostInfo.power_policy = $null # No AHV equivalent. $hostInfo.hyperthreading = ([int]$HostItem.numberOfCpuThreads -gt [int]$HostItem.numberOfCpuCores) $virtualMachines = [System.Collections.Generic.List[NutanixHypervisorVMInfo]]::new() foreach ($vm in @($Vms)) { $virtualMachines.Add((ConvertTo-NutanixPrismCentralVmInfo -Vm $vm)) } $dataItem = [NutanixHypervisorDataItem]::new() $dataItem.host = $hostInfo $dataItem.events = @($events) $dataItem.virtual_machines = @($virtualMachines) return [pscustomobject]@{ DataItem = $dataItem; CoveredEndUtc = $minutes.CoveredEndUtc } } # ---------------------------- # Cycle orchestrator # ---------------------------- function Invoke-NutanixPrismCentralCollection { [CmdletBinding()] param( [Parameter(Mandatory = $true)][NutanixConnectorConfiguration]$Config, [Parameter(Mandatory = $true)][NutanixConnectorState]$State ) $cycleStopwatch = [Diagnostics.Stopwatch]::StartNew() $context = Get-NutanixApiContext -EnvironmentConfig $Config.EnvironmentConfig $now = [datetime]::UtcNow $summary = [ordered]@{ Clusters = 0; ClusterFailures = 0; Hosts = 0; HostFailures = 0; Vms = 0; VmStatsFailures = 0 } # Inventory $clusters = @(Get-NutanixClusters -Context $context) $summary.Clusters = $clusters.Count $allHosts = [System.Collections.Generic.List[object]]::new() foreach ($cluster in $clusters) { if (-not (Test-NutanixClusterHasHosts -Cluster $cluster)) { Write-CustomLog -Message "Skipping cluster '$($cluster.name)' ($($cluster.extId)): clusterFunction $((@(Get-NutanixNestedValue -Record $cluster -Path 'config.clusterFunction')) -join ',') has no AHV hosts." -Severity 'DEBUG' continue } try { $clusterHosts = @(Get-NutanixHosts -Context $context -ClusterExtId $cluster.extId -ClusterName $cluster.name) } catch { # Prism Central registers itself as a cluster without AHV hosts and rejects the host # listing; one unreadable cluster must not cost the others their cycle. $summary.ClusterFailures++ Write-CustomLog -Message "Skipping cluster '$($cluster.name)' ($($cluster.extId)): host listing failed. Error=$($_.Exception.Message)" -Severity 'WARNING' continue } foreach ($hostItem in $clusterHosts) { $allHosts.Add($hostItem) } } $summary.Hosts = $allHosts.Count $vms = @() if ($allHosts.Count -gt 0) { $vms = @(Get-NutanixVms -Context $context) } $summary.Vms = $vms.Count $vmsByHostExtId = Get-NutanixVmsByHostExtId -Vms $vms # Per-host history windows $fallbackWatermark = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.last_received_utc $workItems = [System.Collections.Generic.List[object]]::new() foreach ($hostItem in $allHosts) { $hostExtId = [string]$hostItem.extId $lastReceivedUtc = $fallbackWatermark if ($State.watermarks.host_last_received_utc.ContainsKey($hostExtId)) { $lastReceivedUtc = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.host_last_received_utc[$hostExtId] } $range = Get-HypervisorMetricsTimeRange -Now $now -LastReceivedUtc $lastReceivedUtc -MaxMinutesRead $script:MAX_MINUTES_READ if ($range.End -le $range.Start) { continue } $workItems.Add((New-NutanixHostStatsWorkItem -Host $hostItem -StartUtc $range.Start -EndUtc $range.End)) } $rangeContext = $null if ($workItems.Count -gt 0) { $rangeContext = [pscustomobject]@{ Start = @($workItems | ForEach-Object { $_.StartUtc }) | Sort-Object | Select-Object -First 1 End = @($workItems | ForEach-Object { $_.EndUtc }) | Sort-Object -Descending | Select-Object -First 1 } } # History $hostStats = Get-NutanixHostStatsConcurrent -Context $context -WorkItems @($workItems) $summary.HostFailures = @($hostStats.FailedWorkItems).Count $vmStatsByExtId = @{} if ($null -ne $rangeContext -and $vms.Count -gt 0) { try { $vmStatsByExtId = Get-NutanixVmStatsByExtId -Context $context -StartUtc $rangeContext.Start -EndUtc $rangeContext.End } catch { $summary.VmStatsFailures = 1 Write-CustomLog -Message "VM history collection failed; host events carry no VM-derived counters this cycle. Error=$($_.Exception.Message)" -Severity 'ERROR' } } # Assembly $dataItems = [System.Collections.Generic.List[NutanixHypervisorDataItem]]::new() $coveredEnds = [System.Collections.Generic.List[datetime]]::new() foreach ($workItem in $workItems) { $hostExtId = [string]$workItem.Host.extId if (-not $hostStats.SamplesByHostExtId.ContainsKey($hostExtId)) { # Failed host: keep its previous watermark so the window is retried next cycle. continue } $hostVms = @() if ($vmsByHostExtId.ContainsKey($hostExtId)) { $hostVms = @($vmsByHostExtId[$hostExtId]) } $assembled = ConvertTo-NutanixPrismCentralDataItem -HostItem $workItem.Host ` -HostSamples @($hostStats.SamplesByHostExtId[$hostExtId]) -Vms $hostVms -VmStatsByExtId $vmStatsByExtId ` -StartUtc $workItem.StartUtc -EndUtc $workItem.EndUtc $dataItems.Add($assembled.DataItem) # Only the complete minutes actually emitted count as received; the trailing minute Prism # Central had not finished stays after the watermark and is read again next cycle. if ($assembled.CoveredEndUtc -gt $workItem.StartUtc) { $coveredEnds.Add($assembled.CoveredEndUtc) $State.watermarks.host_last_received_utc[$hostExtId] = $assembled.CoveredEndUtc.ToString('o') } } if ($coveredEnds.Count -gt 0) { # The spool timestamp and reported range end reflect what was actually emitted. $rangeContext.End = @($coveredEnds) | Sort-Object -Descending | Select-Object -First 1 } $payload = [NutanixHypervisorPayload]::new() $payload.schema_version = '1.1' $payload.source = 'nutanix-connector' $payload.customer_environment = $Config.EnvironmentConfig.Name $payload.version = Get-ModuleVersion $payload.data = @($dataItems) if ($dataItems.Count -gt 0 -and $null -ne $rangeContext) { $null = Write-SpoolReceived -Config $Config -Timestamp ([DateTimeOffset]$rangeContext.End) -Content $payload } $hostWatermarks = foreach ($hostItem in $allHosts) { $hostExtId = [string]$hostItem.extId if ($State.watermarks.host_last_received_utc.ContainsKey($hostExtId)) { ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.host_last_received_utc[$hostExtId] } else { $fallbackWatermark } } if (@($hostWatermarks).Count -gt 0) { $State.watermarks.last_received_utc = (@($hostWatermarks) | Sort-Object | Select-Object -First 1).ToString('o') } Save-State -State $State -Config $Config $cycleStopwatch.Stop() $summary.StartUtc = if ($null -ne $rangeContext) { $rangeContext.Start.ToString('o') } else { $null } $summary.EndUtc = if ($null -ne $rangeContext) { $rangeContext.End.ToString('o') } else { $null } $summary.DurationSeconds = [Math]::Round($cycleStopwatch.Elapsed.TotalSeconds, 2) return [pscustomobject]@{ Payload = $payload Range = $rangeContext Summary = [pscustomobject]$summary } } |