Private/VSphere.ps1
|
# ---------------------------- # Helper: Find the main disk I/O allocation limit # ---------------------------- function Get-MainDiskIOLimit { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [AllowNull()] [array]$Devices ) if ($null -eq $Devices -or $Devices.Count -eq 0) { return $null } foreach ($device in $Devices) { if ($device -is [VMware.Vim.VirtualDisk]) { $controller = @($Devices | Where-Object { $_.Key -eq $device.ControllerKey })[0] if ($null -ne $controller -and $controller.BusNumber -eq 0 -and $device.UnitNumber -eq 0) { if ($null -ne $device.StorageIOAllocation) { return $device.StorageIOAllocation.Limit } return $null } } } return $null } # ---------------------------- # Phase 1: Retrieve ESXi hosts, clusters, resource pools, and VM data # ---------------------------- function Get-VSphereHostsData { [CmdletBinding()] param() Write-CustomLog -Message "Retrieving ESXi hosts from vSphere using Get-View" -Severity 'DEBUG' $hostViews = @(Get-View -ViewType HostSystem -Property Name, Parent, Hardware.SystemInfo.Uuid, Hardware.CpuInfo.NumCpuPackages, Summary.Hardware.NumCpuThreads, Summary.Hardware.MemorySize, Config.PowerSystemInfo.CurrentPolicy.Key, Config.HyperThread.Active, Vm -ErrorAction Stop) Write-CustomLog -Message "Found $($hostViews.Count) ESXi hosts" -Severity 'DEBUG' Write-CustomLog -Message "Retrieving cluster information" -Severity 'DEBUG' $clusterNameByMoRef = @{} $clusterViews = @(Get-View -ViewType ClusterComputeResource -Property Name -ErrorAction SilentlyContinue) foreach ($clusterView in $clusterViews) { if ($null -eq $clusterView -or $null -eq $clusterView.MoRef -or $null -eq $clusterView.MoRef.Value) { continue } $clusterNameByMoRef[$clusterView.MoRef.Value] = $clusterView.Name } Write-CustomLog -Message "Retrieving resource pool information" -Severity 'DEBUG' $resourcePoolNameByMoRef = @{} $resourcePoolViews = @(Get-View -ViewType ResourcePool -Property Name -ErrorAction SilentlyContinue) foreach ($rpView in $resourcePoolViews) { if ($null -eq $rpView -or $null -eq $rpView.MoRef -or $null -eq $rpView.MoRef.Value) { continue } $resourcePoolNameByMoRef[$rpView.MoRef.Value] = $rpView.Name } Write-CustomLog -Message "Retrieving VM data per host" -Severity 'DEBUG' $vmCountByHostMoRef = @{} $vcpuCountByHostMoRef = @{} $memoryCommittedByHostMoRef = @{} $vmDataByHostMoRef = @{} $vmViews = @(Get-View -ViewType VirtualMachine -Property Runtime.Host, Name, Config.Hardware.NumCPU, Config.Hardware.MemoryMB, Guest.ToolsVersion, ResourcePool, Config.CpuAllocation, Config.Hardware.Device -Filter @{ 'Runtime.PowerState' = 'poweredOn' } -ErrorAction SilentlyContinue) foreach ($vm in $vmViews) { if ($null -eq $vm.Runtime -or $null -eq $vm.Runtime.Host) { continue } $hostMoRef = $vm.Runtime.Host.Value if (-not $vmCountByHostMoRef.ContainsKey($hostMoRef)) { $vmCountByHostMoRef[$hostMoRef] = 0 } $vmCountByHostMoRef[$hostMoRef]++ $numCpu = if ($null -ne $vm.Config -and $null -ne $vm.Config.Hardware) { [int]$vm.Config.Hardware.NumCPU } else { 0 } if (-not $vcpuCountByHostMoRef.ContainsKey($hostMoRef)) { $vcpuCountByHostMoRef[$hostMoRef] = 0 } $vcpuCountByHostMoRef[$hostMoRef] += $numCpu $memoryMB = if ($null -ne $vm.Config -and $null -ne $vm.Config.Hardware) { [long]$vm.Config.Hardware.MemoryMB } else { 0 } if (-not $memoryCommittedByHostMoRef.ContainsKey($hostMoRef)) { $memoryCommittedByHostMoRef[$hostMoRef] = 0 } $memoryCommittedByHostMoRef[$hostMoRef] += $memoryMB $rpName = $null if ($null -ne $vm.ResourcePool) { $rpName = $resourcePoolNameByMoRef[$vm.ResourcePool.Value] } $cpuLimit = $null $cpuShares = $null if ($null -ne $vm.Config -and $null -ne $vm.Config.CpuAllocation) { $cpuLimit = $vm.Config.CpuAllocation.Limit if ($null -ne $vm.Config.CpuAllocation.Shares) { $cpuShares = [string]$vm.Config.CpuAllocation.Shares.Level } } $devices = $null if ($null -ne $vm.Config -and $null -ne $vm.Config.Hardware) { $devices = $vm.Config.Hardware.Device } $diskIOLimit = Get-MainDiskIOLimit -Devices $devices $guestToolsVersion = $null if ($null -ne $vm.Guest) { $guestToolsVersion = $vm.Guest.ToolsVersion } if (-not $vmDataByHostMoRef.ContainsKey($hostMoRef)) { $vmDataByHostMoRef[$hostMoRef] = [System.Collections.Generic.List[pscustomobject]]::new() } $vmDataByHostMoRef[$hostMoRef].Add([pscustomobject]@{ Name = $vm.Name GuestToolsVersion = $guestToolsVersion ResourcePool = $rpName CpuLimit = $cpuLimit CpuShares = $cpuShares DiskIOLimit = $diskIOLimit }) } return [pscustomobject]@{ HostViews = $hostViews ClusterNameByMoRef = $clusterNameByMoRef VMCountByHostMoRef = $vmCountByHostMoRef VCPUCountByHostMoRef = $vcpuCountByHostMoRef MemoryCommittedByHostMoRef = $memoryCommittedByHostMoRef VMDataByHostMoRef = $vmDataByHostMoRef } } # ---------------------------- # Phase 2a: Set up PerfManager and counter mappings # ---------------------------- function Initialize-PerfCounterMappings { [CmdletBinding()] param( [Parameter(Mandatory = $true)] $PerfManager ) $counterIdByName = @{} if ($null -ne $PerfManager.PerfCounter) { foreach ($counter in $PerfManager.PerfCounter) { if ($null -eq $counter -or $null -eq $counter.Key) { continue } $groupKey = if ($null -ne $counter.GroupInfo) { $counter.GroupInfo.Key } else { '' } $nameKey = if ($null -ne $counter.NameInfo) { $counter.NameInfo.Key } else { '' } $rollupType = if ($null -ne $counter.RollupType) { $counter.RollupType } else { '' } $metricName = "{0}.{1}.{2}" -f $groupKey, $nameKey, $rollupType if (-not $counterIdByName.ContainsKey($metricName)) { $counterIdByName[$metricName] = $counter.Key } } } $requestedCounterIds = @() foreach ($metricName in $script:VSPHERE_HYPERVISOR_METRIC_IDS) { if (-not $counterIdByName.ContainsKey($metricName)) { Write-CustomLog -Message "Counter not found in vCenter: $metricName" -Severity 'WARNING' continue } $requestedCounterIds += [pscustomobject]@{ Name = $metricName Id = [int]$counterIdByName[$metricName] } } $counterNameById = @{} foreach ($counterInfo in $requestedCounterIds) { $counterNameById[$counterInfo.Id] = $counterInfo.Name } return [pscustomobject]@{ RequestedCounterIds = $requestedCounterIds CounterNameById = $counterNameById } } # ---------------------------- # Phase 2b: Build host data structures and query metrics in batches # ---------------------------- function Get-HostPerformanceMetrics { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [array]$HostViews, [Parameter(Mandatory = $true)] $PerfManager, [Parameter(Mandatory = $true)] [pscustomobject]$CounterMappings, [Parameter(Mandatory = $true)] [hashtable]$ClusterNameByMoRef, [Parameter(Mandatory = $true)] [hashtable]$VMCountByHostMoRef, [Parameter(Mandatory = $true)] [hashtable]$VCPUCountByHostMoRef, [Parameter(Mandatory = $true)] [hashtable]$MemoryCommittedByHostMoRef, [Parameter(Mandatory = $true)] [hashtable]$VMDataByHostMoRef, [Parameter(Mandatory = $true)] [datetime]$StartUtc, [Parameter(Mandatory = $true)] [datetime]$EndUtc ) # Helper: Split array into batches function Split-ArrayIntoBatches { param( [Parameter(Mandatory = $true)] [array]$InputArray, [Parameter(Mandatory = $true)] [int]$Size ) for ($batchStartIndex = 0; $batchStartIndex -lt $InputArray.Count; $batchStartIndex += $Size) { , $InputArray[$batchStartIndex..([Math]::Min($batchStartIndex + $Size - 1, $InputArray.Count - 1))] } } $hostDataByMoRef = @{} foreach ($hostView in $HostViews) { if ($null -eq $hostView -or $null -eq $hostView.MoRef) { continue } $hostMoRef = $hostView.MoRef.Value if ($null -eq $hostMoRef) { continue } $clusterName = $null if ($null -ne $hostView.Parent -and $hostView.Parent.Type -eq 'ClusterComputeResource') { $clusterName = $ClusterNameByMoRef[$hostView.Parent.Value] } $vmCount = 0 if ($VMCountByHostMoRef.ContainsKey($hostMoRef)) { $vmCount = $VMCountByHostMoRef[$hostMoRef] } $vcpuCount = 0 if ($VCPUCountByHostMoRef.ContainsKey($hostMoRef)) { $vcpuCount = $VCPUCountByHostMoRef[$hostMoRef] } $memoryCommitted = $null if ($MemoryCommittedByHostMoRef.ContainsKey($hostMoRef)) { $memoryCommitted = $MemoryCommittedByHostMoRef[$hostMoRef] } $vmData = @() if ($VMDataByHostMoRef.ContainsKey($hostMoRef)) { $vmData = $VMDataByHostMoRef[$hostMoRef] } $hostDataByMoRef[$hostMoRef] = [pscustomobject]@{ HostView = $hostView Name = $hostView.Name Cluster = $clusterName VMCount = $vmCount VCPUCount = $vcpuCount MemoryCommitted = $memoryCommitted VMData = $vmData EventsByTimestamp = @{} } } Write-CustomLog -Message "Querying performance metrics for $($HostViews.Count) hosts in batches of $($script:PERF_QUERY_BATCH_SIZE)" -Severity 'DEBUG' $intervalId = $script:METRICS_INTERVAL_SECONDS $requestedCounterIds = $CounterMappings.RequestedCounterIds $counterNameById = $CounterMappings.CounterNameById $metricIds = foreach ($counterInfo in $requestedCounterIds) { $perfMetricId = New-Object VMware.Vim.PerfMetricId $perfMetricId.CounterId = $counterInfo.Id $perfMetricId.Instance = "" $perfMetricId } foreach ($batch in (Split-ArrayIntoBatches -InputArray $HostViews -Size $script:PERF_QUERY_BATCH_SIZE)) { $specs = foreach ($hostView in $batch) { $querySpec = New-Object VMware.Vim.PerfQuerySpec $querySpec.Entity = $hostView.MoRef $querySpec.IntervalId = $intervalId $querySpec.StartTime = $StartUtc $querySpec.EndTime = $EndUtc $querySpec.MetricId = $metricIds $querySpec } $results = $PerfManager.QueryPerf($specs) foreach ($perfResult in $results) { $hostMoRef = $perfResult.Entity.Value if (-not $hostDataByMoRef.ContainsKey($hostMoRef)) { continue } $hostData = $hostDataByMoRef[$hostMoRef] if ($null -eq $perfResult.SampleInfo) { continue } for ($sampleIndex = 0; $sampleIndex -lt $perfResult.SampleInfo.Count; $sampleIndex++) { $timestamp = $perfResult.SampleInfo[$sampleIndex].Timestamp $timestampKey = $timestamp.ToString('o') if (-not $hostData.EventsByTimestamp.ContainsKey($timestampKey)) { $hostData.EventsByTimestamp[$timestampKey] = @{ Timestamp = $timestamp Metrics = @{} } } foreach ($series in $perfResult.Value) { $counterId = [int]$series.Id.CounterId if (-not $counterNameById.ContainsKey($counterId)) { continue } $metricName = $counterNameById[$counterId] $value = $series.Value[$sampleIndex] $hostData.EventsByTimestamp[$timestampKey].Metrics[$metricName] = $value } } } } return $hostDataByMoRef } # ---------------------------- # Phase 3: Build output data items # ---------------------------- function ConvertTo-HypervisorDataItems { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [hashtable]$HostDataByMoRef ) # Helper: Get metric value as string or null from stats hashtable function Get-MetricValueOrNull { param( [Parameter(Mandatory = $true)] [hashtable]$StatsByMetricName, [Parameter(Mandatory = $true)] [string]$MetricName ) if ($StatsByMetricName.ContainsKey($MetricName)) { return [string]$StatsByMetricName[$MetricName] } return $null } Write-CustomLog -Message "Building output data items" -Severity 'DEBUG' $dataItems = foreach ($hostMoRef in $HostDataByMoRef.Keys) { $hostData = $HostDataByMoRef[$hostMoRef] $hostInfo = [VSphereHypervisorHostInfo]::new() $hostInfo.name = [string]$hostData.Name $hostInfo.cluster = $hostData.Cluster $hostInfo.number_of_vms = [int]$hostData.VMCount $hostInfo.power_policy = $null if ($null -ne $hostData.HostView.Config -and $null -ne $hostData.HostView.Config.PowerSystemInfo -and $null -ne $hostData.HostView.Config.PowerSystemInfo.CurrentPolicy) { $hostInfo.power_policy = [int]$hostData.HostView.Config.PowerSystemInfo.CurrentPolicy.Key } $hostInfo.hyperthreading = $false if ($null -ne $hostData.HostView.Config -and $null -ne $hostData.HostView.Config.HyperThread) { $hostInfo.hyperthreading = [bool]$hostData.HostView.Config.HyperThread.Active } $events = [System.Collections.Generic.List[VSphereHypervisorEvent]]::new() $sortedTimestamps = $hostData.EventsByTimestamp.Keys | Sort-Object foreach ($timestampKey in $sortedTimestamps) { $eventData = $hostData.EventsByTimestamp[$timestampKey] $eventTimestamp = $eventData.Timestamp $metrics = $eventData.Metrics $hypervisorEvent = [VSphereHypervisorEvent]::new() $hypervisorEvent.start_time = ConvertTo-Rfc3339UtcZ -Timestamp $eventTimestamp.AddSeconds(-$script:METRICS_INTERVAL_SECONDS) $hypervisorEvent.duration = [int]$script:METRICS_INTERVAL_SECONDS $hypervisorEvent.cpu = [VSphereHypervisorCpuMetrics]::new() $hypervisorEvent.disk = [VSphereHypervisorDiskMetrics]::new() $hypervisorEvent.memory = [VSphereHypervisorMemoryMetrics]::new() # CPU $hypervisorEvent.cpu.number_of_threads = [int]$hostData.HostView.Summary.Hardware.NumCpuThreads $hypervisorEvent.cpu.number_of_packages = [int]$hostData.HostView.Hardware.CpuInfo.NumCpuPackages $hypervisorEvent.cpu.number_of_vcpus = [int]$hostData.VCPUCount $hypervisorEvent.cpu.ready_summation = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuReadySummation $hypervisorEvent.cpu.usage_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuUsageAverage $hypervisorEvent.cpu.used_summation = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuUsedSummation # Disk $hypervisorEvent.disk.read_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskReadAverage $hypervisorEvent.disk.write_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskWriteAverage $hypervisorEvent.disk.max_total_latency_latest = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskMaxTotalLatencyLatest # Memory $hypervisorEvent.memory.swap_in_rate_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapInRateAverage $hypervisorEvent.memory.swap_out_rate_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapOutRateAverage $hypervisorEvent.memory.swap_used_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapUsedAverage $hypervisorEvent.memory.state_latest = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemStateLatest $hypervisorEvent.memory.vm_mem_ctl_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemVmmemctlAverage $hypervisorEvent.memory.usage_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemUsageAverage $hypervisorEvent.memory.installed = [string]$hostData.HostView.Summary.Hardware.MemorySize if ($null -ne $hostData.MemoryCommitted) { $hypervisorEvent.memory.committed = [string]$hostData.MemoryCommitted } $events.Add($hypervisorEvent) } $virtualMachines = [System.Collections.Generic.List[VSphereHypervisorVMInfo]]::new() if ($null -ne $hostData.VMData) { foreach ($vmEntry in @($hostData.VMData)) { if ($null -eq $vmEntry) { continue } $vmInfo = [VSphereHypervisorVMInfo]::new() $vmInfo.name = $vmEntry.Name $vmInfo.guest_tools_version = $vmEntry.GuestToolsVersion $vmInfo.resource_pool = $vmEntry.ResourcePool $vmInfo.cpu_limit = $vmEntry.CpuLimit $vmInfo.cpu_shares = $vmEntry.CpuShares $vmInfo.disk_io_limit = $vmEntry.DiskIOLimit $virtualMachines.Add($vmInfo) } } $dataItem = [VSphereHypervisorDataItem]::new() $dataItem.host = $hostInfo $dataItem.events = $events $dataItem.virtual_machines = $virtualMachines $dataItem } return $dataItems } # ---------------------------- # Main orchestrator function # ---------------------------- function Get-VSphereHypervisorMetrics { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [VSphereEnvironmentConfiguration]$EnvironmentConfig, [Parameter(Mandatory = $true)] [datetime]$Start, [Parameter(Mandatory = $true)] [datetime]$End ) $connection = $null try { Write-CustomLog -Message "Connecting to vSphere server: $($EnvironmentConfig.vCenterFQDN)" -Severity 'DEBUG' $connection = Connect-VSphereServer -Server $EnvironmentConfig.vCenterFQDN -Target $EnvironmentConfig.WindowsCredentialEntry $startUtc = $Start.ToUniversalTime() $endUtc = $End.ToUniversalTime() if ($endUtc -le $startUtc) { throw "Invalid time window: End must be after Start. Start='$startUtc' End='$endUtc'" } # Phase 1: Retrieve hosts, clusters, and VM data $hostsData = Get-VSphereHostsData if ($hostsData.HostViews.Count -eq 0) { return @() } # Phase 2: Setup PerfManager and counter mappings Write-CustomLog -Message "Setting up Performance Manager" -Severity 'DEBUG' $serviceInstance = Get-View ServiceInstance -ErrorAction Stop $perfMgr = Get-View $serviceInstance.Content.PerfManager -ErrorAction Stop $counterMappings = Initialize-PerfCounterMappings -PerfManager $perfMgr # Phase 2: Query performance metrics $hostDataByMoRef = Get-HostPerformanceMetrics ` -HostViews $hostsData.HostViews ` -PerfManager $perfMgr ` -CounterMappings $counterMappings ` -ClusterNameByMoRef $hostsData.ClusterNameByMoRef ` -VMCountByHostMoRef $hostsData.VMCountByHostMoRef ` -VCPUCountByHostMoRef $hostsData.VCPUCountByHostMoRef ` -MemoryCommittedByHostMoRef $hostsData.MemoryCommittedByHostMoRef ` -VMDataByHostMoRef $hostsData.VMDataByHostMoRef ` -StartUtc $startUtc ` -EndUtc $endUtc # Phase 3: Build output data items return ConvertTo-HypervisorDataItems -HostDataByMoRef $hostDataByMoRef } catch { throw "Failed to retrieve vSphere hypervisor metrics: $_" } finally { Disconnect-VSphereServer -Connection $connection } } function Get-HypervisorPayloadWithContext { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [VSphereConnectorState]$State, [Parameter(Mandatory = $true)] [VSphereConnectorConfiguration]$Config ) $now = [DateTime]::UtcNow # Parse last_received_utc from state $lastReceivedUtc = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.last_received_utc $range = Get-HypervisorMetricsTimeRange -Now $now -LastReceivedUtc $lastReceivedUtc -MaxMinutesRead $script:MAX_MINUTES_READ $start = $range.Start $end = $range.End Write-CustomLog -Message "Fetching hypervisor metrics from $start to $end" -Severity 'INFO' # Fetch metrics $dataItems = @(Get-VSphereHypervisorMetrics -EnvironmentConfig $Config.EnvironmentConfig -Start $start -End $end) # Build payload $payload = [VSphereHypervisorPayload]::new() $payload.schema_version = '1.1' $payload.source = 'vsphere-connector' $payload.customer_environment = $Config.EnvironmentConfig.Name $payload.version = Get-ModuleVersion $payload.data = $dataItems # Save payload to spool (timestamp aligned with fetched time window) # In PowerShell, any uncaptured function return values automatically become part of the calling function's output stream. $null = Write-SpoolReceived -Config $Config -Timestamp ([DateTimeOffset]$end) -Content $payload # Update state watermark $State.watermarks.last_received_utc = $end.ToString('o') Save-State -State $State -Config $Config Write-CustomLog -Message "Retrieved metrics for $($dataItems.Count) hosts. Updated last_received_utc to $($end.ToString('o'))" -Severity 'INFO' return @{Payload = $payload ; Range = $range} } function Get-HypervisorMetricsTimeRange { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [datetime]$Now, [Parameter(Mandatory = $true)] [datetime]$LastReceivedUtc, [Parameter(Mandatory = $true)] [int]$MaxMinutesRead ) # If last_received is more than 1 hour in the past, move start to 55 minutes ago $oneHourAgo = $Now.AddHours(-1) if ($LastReceivedUtc -lt $oneHourAgo) { Write-CustomLog -Message "Last received timestamp ($LastReceivedUtc) is more than 1 hour ago. Moving start to 55 minutes ago." -Severity 'INFO' $start = $Now.AddMinutes(-55) } else { $start = $LastReceivedUtc } # Round start to minute (floor/truncate) $start = [datetime]::new($start.Year, $start.Month, $start.Day, $start.Hour, $start.Minute, 0, [System.DateTimeKind]::Utc) # End = min(now - 1 minute, start + MaxMinutesRead) $nowMinus1 = $Now.AddMinutes(-1) $startPlusMax = $start.AddMinutes($MaxMinutesRead) $end = if ($nowMinus1 -lt $startPlusMax) { $nowMinus1 } else { $startPlusMax } # Round end to minute (floor/truncate) $end = [datetime]::new($end.Year, $end.Month, $end.Day, $end.Hour, $end.Minute, 0, [System.DateTimeKind]::Utc) return [pscustomobject]@{ Start = $start End = $end } } |