Noveris.VMwareReport

0.2.1

Noveris.VMwareReport.psm1


################
# Global settings
$ErrorActionPreference = "Stop"
$InformationPreference = "Continue"
Set-StrictMode -Version 2

enum VMwareReportStatus
{
    None = 0
    Ok
    Warning
    Error
}

Class VMwareReportSectionDef
{
    VMwareReportSectionDef()
    {
    }

    [string]$Report
    [string]$Section
}

Class VMwareReportConfig
{
    [VMwareReportSectionDef[]]$Include
    [VMwareReportSectionDef[]]$Exclude
    [string]$Target
    [string]$Username
    [string]$Password
    [string]$SmtpServer
    [string]$SmtpSender
    [string[]]$Recipients
    [string[]]$IssueRecipients
    [string]$Site
    [PSCustomObject]$Settings

    VMwareReportConfig()
    {
        $this.Include = $null
        $this.Exclude = $null
        $this.Target = ""
        $this.Username = ""
        $this.Password = ""
        $this.SmtpServer = ""
        $this.SmtpSender = ""
        $this.Recipients = [string[]]@()
        $this.IssueRecipients = [string[]]@()
        $this.Site = ""
        $this.Settings = [PSCustomObject]@{}
    }
}

Class VMwareReportSummaryNotice
{
    [VMwareReportStatus]$Status
    [string]$Report
    [string]$Description

    VMwareReportSummaryNotice()
    {
        $this.Status = [VMwareReportStatus]::None
        $this.Report = ""
        $this.Description = ""
    }
}

<#
#>
Function Set-VMwareReportConfig
{
    [CmdletBinding()]
    param(
        [Parameter(mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string]$ConfigPath,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string[]]$Reports,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string]$Target,

        [Parameter(mandatory=$false)]
        [ValidateNotNull()]
        [PSCredential]$Credential,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string]$SmtpServer,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string]$SmtpSender,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string[]]$Recipients,

        [Parameter(mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string[]]$IssueRecipients,

        [Parameter(mandatory=$false)]
        [AllowEmptyString()]
        [ValidateNotNull()]
        [string]$Site = ""
    )

    process
    {
        [VMwareReportConfig]$config = New-Object VMwareReportConfig
        if (Test-Path -PathType Leaf $ConfigPath)
        {
            $config = [VMwareReportConfig](Get-Content $ConfigPath -Encoding UTF8 | ConvertFrom-Json)
        }

        if ($PSBoundParameters.Keys -contains "Reports")
        {
            $config.Reports = $Reports
        }

        if ($PSBoundParameters.Keys -contains "Target")
        {
            $config.Target = $Target
        }

        if ($PSBoundParameters.Keys -contains "Credential")
        {
            $config.Username = $Credential.Username
            $config.Password = $Credential.Password | ConvertFrom-SecureString
        }

        if ($PSBoundParameters.Keys -contains "SmtpServer")
        {
            $config.SmtpServer = $SmtpServer
        }

        if ($PSBoundParameters.Keys -contains "SmtpSender")
        {
            $config.SmtpSender = $SmtpSender
        }

        if ($PSBoundParameters.Keys -contains "Recipients")
        {
            $config.Recipients = $Recipients
        }

        if ($PSBoundParameters.Keys -contains "IssueRecipients")
        {
            $config.IssueRecipients = $IssueRecipients
        }

        if ($PSBoundParameters.Keys -contains "Site")
        {
            $config.Site = $Site
        }

        $config | ConvertTo-Json | Out-File -Encoding UTF8 $ConfigPath
    }
}

Function New-VMwareReportSummaryNotice
{
    [CmdletBinding()]
    param(
        [Parameter(mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string]$Report,

        [Parameter(mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string]$Description,

        [Parameter(mandatory=$false)]
        [ValidateNotNull()]
        [VMwareReportStatus]$Status = [VMwareReportStatus]::Ok
    )

    process
    {
        $notice = New-Object VMwareReportSummaryNotice
        $notice.Status = $Status
        $notice.Report = $Report
        $notice.Description = $Description

        $notice
    }
}

Function New-VMwareReportCell
{
    [CmdletBinding()]
    param(
        [Parameter(mandatory=$false)]
        [AllowNull()]
        [AllowEmptyString()]
        [string]$Content = "",

        [Parameter(mandatory=$false)]
        [ValidateNotNull()]
        [VMwareReportStatus]$Status,

        [Parameter(mandatory=$false)]
        [ValidateNotNull()]
        [int]$ColumnSpan
    )

    process
    {
        $header = "<td "
        if ($PSBoundParameters.Keys -contains "Status")
        {
            if ($Status -eq [VMwareReportStatus]::Error)
            {
                $header += " bgcolor=`"#FF0000`" "
            } elseif ($Status -eq [VMwareReportStatus]::Warning)
            {
                $header += " bgcolor=`"#FFCC00`" "
            } elseif ($Status -eq [VMwareReportStatus]::Ok)
            {
                $header += " bgcolor=`"#009900`" "
            }
        }

        if ($PSBoundParameters.Keys -contains "ColumnSpan")
        {
            $header += " colspan=`"$ColumnSpan`" "
        }

        $header + ">" + $Content + "</td>"
    }
}

$script:Reports = [ordered]@{}

$script:Reports["Snapshot Summary"] = [PSCustomObject]@{
    "Begin" = {
        "<table><tr><th>VM</th><th>Consolidation Required</th><th>Snapshots</th></tr>"
    }
    "End" = {
        "</table>"
    }
    "Sections" = [ordered]@{
        "Snapshots" = {
            $runtime = $_
            $config = $runtime.Config

            $vmCount = 0
            foreach ($vm in Get-VM -Server $config.Target)
            {
                Write-Information ("Getting snapshot information for: " + $vm.Name)

                $snapshots = $vm | Get-Snapshot
                $consolidate = $vm.ExtensionData.Runtime.consolidationNeeded

                if ($consolidate -eq $false -and ($snapshots | Measure-Object).Count -lt 1)
                {
                    continue
                }
                
                $vmCount++

                "<tr>"

                # Display VM name
                New-VMwareReportCell -Content $vm.Name

                # Consolidate status
                $cellStatus = [VMwareReportStatus]::None
                if ($consolidate -eq $true)
                {
                    $cellStatus = [VMwareReportStatus]::Warning
                }

                New-VMwareReportCell -Status $cellStatus -Content $consolidate

                # Snapshot status
                $cellStatus = [VMwareReportStatus]::None
                $content = "None"
                if (($snapshots | Measure-Object).Count -gt 0)
                {
                    $cellStatus = [VMwareReportStatus]::Warning
                    $content = $snapshots | ForEach-Object {
                        $snap = $_
                        ("<b>Snapshot: </b>" + $snap.Description + "<br>")
                        ("<b>Size (MB): </b>" + ([int] $snap.SizeMB) + "<br>")
                        ("<b>Created: </b>" + $snap.Created + "<br>")
                        "<br>"
                    } | Out-String
                }

                New-VMwareReportCell -Status $cellStatus -Content $content

                "</tr>"
            }

            if ($vmCount -eq 0)
            {
                "<tr>"
                New-VMwareReportCell -ColumnSpan 3 -Content "None"
                "</tr>"
            } else {
                New-VMwareReportSummaryNotice -Status Warning -Report "Snapshot Summary" -Description "Some VMs require consolidation or have snapshots"
            }
        }
    }
}

$script:Reports["Host Health"] = [PSCustomObject]@{
    "Begin" = {
        $runtime = $_
        $config = $runtime.Config
        
        "<table><tr><th>Condition</th>"
        $runtime.VMHosts | ForEach-Object {
            $name = $_.Name
            if (($config.Settings | Get-Member).Name -contains "StripHostSuffix" -and $config.Settings.StripHostSuffix -ne $null)
            {
                [string]$strip = $config.Settings.StripHostSuffix.ToString()
                $name = $name -replace $strip, ""
            }

            $name
        } | ForEach-Object { ("<th>{0}</th>" -f $_) }
        "</tr>"
    }
    "End" = {
        # End table
        "</table>"
    }
    "Sections" = [ordered]@{
        "ConnectionState" = {
            $runtime = $_
            $config = $runtime.Config
            
            $disconnected = 0
            $maintenance = 0
            "<tr>"
            New-VMwareReportCell -Content "<b>Connection State</b>"

            foreach ($vmhost in $runtime.VMHosts)
            {
                $status = [VMwareReportStatus]::Error
                $content = "Unknown"
                switch ($vmhost.ConnectionState)
                {
                    "Connected" {
                        $status = [VMwareReportStatus]::Ok
                        $content = "Connected"
                        break
                    }

                    "Maintenance" {
                        $maintenance++
                        $status = [VMwareReportStatus]::Warning
                        $content = "Maintenance"
                        break
                    }

                    default {
                        $disconnected++
                        $status = [VMwareReportStatus]::Error
                        $content = ("Not connected: " + $vmhost.ConnectionState.ToString())
                        break
                    }
                }

                New-VMwareReportCell -Status $status -Content $content
            }

            "</tr>"

            if ($maintenance -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts are in maintenance mode"
            }

            if ($disconnected -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts are not connected"
            }
        }

        "OverallStatus" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Overall Status</b>"

            $errorhosts = 0
            $warningHosts = 0
            foreach ($vmhost in $vmhosts)
            {
                $overallStatus = $vmhost.ExtensionData.OverallStatus.ToString()
                $status = [VMwareReportStatus]::Ok
                if ($overallStatus -eq "red")
                {
                    $status = [VMwareReportStatus]::Error
                    $errorHosts++
                } elseif ($overallStatus -eq "yellow")
                {
                    $status = [VMwareReportStatus]::Warning
                    $warningHosts++
                }

                New-VMwareReportCell -Status $status -Content $overallStatus
            }

            if ($errorHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts have alerts"
            } elseif ($warningHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts have alerts"
            }

            "</tr>"
        }
        
        "ConfigStatus" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Config Status</b>"

            $errorhosts = 0
            $warningHosts = 0
            foreach ($vmhost in $vmhosts)
            {
                $configStatus = $vmhost.ExtensionData.ConfigStatus.ToString()
                $status = [VMwareReportStatus]::Ok
                if ($configStatus -eq "red")
                {
                    $status = [VMwareReportStatus]::Error
                    $errorHosts++
                } elseif ($configStatus -eq "yellow")
                {
                    $status = [VMwareReportStatus]::Warning
                    $warningHosts++
                }

                New-VMwareReportCell -Status $status -Content $configStatus
            }

            if ($errorHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts have config issues"
            } elseif ($warningHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts have config issues"
            }

            "</tr>"
        }

        "VMCount" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"
            $allVMs = Get-VM -Server $Config.Target
            $vmCount = ($allVMs | Measure-Object).Count
            $vmhostCount = ($vmhosts | Measure-Object).Count

            $vmPerHost = 0
            $avgPerHost = 0
            if ($vmhostCount -gt 0 -and $vmCount -gt 0)
            {
                $avgPerHost = 1 / $vmhostCount * 100
                $vmPerHost = $vmCount / $vmhostCount
            }

            $content = "<b>Virtual Machines</b>"
            $content += "<br>VMs Per Host Count (Balanced): " + $vmPerHost.ToString("0.00")
            $content += "<br>VM Per Host % (Balanced): " + $avgPerHost.ToString("0.00")
            $content += "<br>Total VMs: " + $vmCount
            New-VMwareReportCell -Content $content

            foreach ($vmhost in $vmhosts)
            {
                $hostVMCount = ($vmhost | Get-VM | Measure-Object).Count
                $hostAvg = $hostVMCount / $vmCount * 100
                New-VMwareReportCell -Content ("VMs: {0}<br>Avg: {1}" -f $hostVMCount, $hostAvg.ToString("0.00"))
            }

            "</tr>"
        }

        "CPUUtilisation" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            $cpuTotalMhz = ($vmhosts | Measure-Object -sum -property CpuTotalMhz).Sum
            $cpuTotalGhz = $cpuTotalMhz / 1024
            $cpuUsageAvgMhz = ($vmhosts | Measure-Object -average -property CpuUsageMhz).Average
            $cpuUsageAvgGhz = $cpuUsageAvgMhz / 1024
            $cpuUsageMhz = ($vmhosts | Measure-Object -sum -property CpuUsageMhz).Sum
            $cpuUsageGhz = $cpuUsageMhz / 1024

            "<tr>"
            $content = ("<b>CPU</b><br>CPU Usage (Ghz): [{0}/{1}]" -f $cpuUsageGhz.ToString("0.00"), $cpuTotalGhz.ToString("0.00"))
            $usagePct = 0
            if ($cpuTotalGhz -gt 0)
            {
                $usagePct = (($cpuUsageGhz / $cpuTotalGhz) * 100)
            }

            $content += ("<br>CPU Usage %: {0}" -f $usagePct.ToString("0.00"))
            New-VMwareReportCell -Content $content

            foreach ($vmhost in $vmhosts)
            {
                $hostUsageMhz = $vmhost.CpuUsageMhz
                $hostTotalMhz = $vmhost.CpuTotalMhz
                $hostUsageGhz = $hostUsageMhz / 1024
                $hostTotalGhz = $hostTotalMhz / 1024

                # Calculate utilisation pct
                $utl = $hostUsageMhz / $hostTotalMhz * 100
                $status = [VMwareReportStatus]::Ok
                if ($utl -gt 90) {
                    $status = [VMwareReportStatus]::Error
                } elseif ($utl -gt 80) {
                    $status = [VMwareReportStatus]::Warn
                }

                $content = ("Host Utilisation (Ghz): [{0}/{1}] = {2}%" -f $hostUsageGhz.ToString("0.00"), $hostTotalGhz.ToString("0.00"), $utl.ToString("0.00"))

                $loadContribution = 0
                if ($cpuUsageGhz -gt 0)
                {
                    $loadContribution = (($hostUsageGhz / $cpuUsageGhz) * 100)
                }

                $content += ("<br>Load Contribution: [{0}/{1}] = {2}%" -f $hostUsageGhz.ToString("0.00"), $cpuUsageGhz.ToString("0.00"), $loadContribution.ToString("0.00"))

                New-VMwareReportCell -Status $status -Content $content
            }

            "</tr>"
        }

        "MemoryUtilisation" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            $warningHosts = 0
            $errorHosts = 0

            $memTotalGB = ((Get-VMHost).MemoryTotalGB | Measure-Object -Sum).Sum
            $memUsageGB = ((Get-VMHost).MemoryUsageGB | Measure-Object -Sum).Sum
            $memAvgGB = ((Get-VMHost).MemoryUsageGB | Measure-Object -Average).Average
            $memUsagePct = 0
            if ($memTotalGB -gt 0)
            {
                $memUsagePct = $memUsageGB / $memTotalGB * 100
            }

            $content = ("<b>Memory</b><br>Memory Usage (GB): [{0}/{1}]" -f $memUsageGB.ToString("0.00"), $memTotalGB.ToString("0.00"))
            $content += ("<br>Memory Usage %: {0}" -f $memUsagePct.ToString("0.00"))
            New-VMwareReportCell -Content $content

            foreach ($vmhost in $vmhosts)
            {
                $hostUsageGB = $vmhost.MemoryUsageGB
                $hostTotalGB = $vmhost.MemoryTotalGB
                $hostUsagePct = 0
                if ($hostTotalGB -gt 0)
                {
                    $hostUsagePct = $hostUsageGB / $hostTotalGB * 100
                }

                # Check memory usage threshold
                $status = [VMwareReportStatus]::Ok
                if ($hostUsagePct -gt 90)
                {
                    $status = [VMwareReportStatus]::Error
                    $errorHosts++
                } elseif ($hostUsagePct -gt 80)
                {
                    $status = [VMwareReportStatus]::Warning
                    $warningHosts++
                }

                $content = ("Host Utilisation (GB): [{0}/{1}] = {2}%" -f $hostUsageGB.ToString("0.00"), $hostTotalGB.ToString("0.00"), $hostUsagePct.ToString("0.00"))

                $loadContribution = 0
                if ($memUsageGB -gt 0)
                {
                    $loadContribution = (($hostUsageGB / $memUsageGB) * 100)
                }

                $content += ("<br>Load Contribution: [{0}/{1}] = {2}%" -f $hostUsageGB.ToString("0.00"), $memUsageGB.ToString("0.00"), $loadContribution.ToString("0.00"))

                New-VMwareReportCell -Status $status -Content $content
            }

            if ($warningHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts have high memory utilisation"
            }

            if ($errorHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts have very high memory utilisation"
            }

            "</tr>"
        }

        "StoragePaths" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Path Check</b>"

            $inactivePaths = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error

                try {
                    $esxcli = $vmhost | Get-EsxCli -V2
                    $paths = $esxcli.storage.core.path.list.Invoke()
                    $inactive = ($paths | Where-Object {$_.State -ne "active"} | Measure-Object).Count
                    $active = ($paths | Where-Object {$_.State -eq "active"} | Measure-Object).Count

                    $status = [VMwareReportStatus]::Ok
                    if ($inactive -gt 0)
                    {
                        $inactivePaths = $true
                        $status = [VMwareReportStatus]::Error
                    }

                    $content = ("Active Paths: {0}<br>Inactive Paths: {1}" -f $active, $inactive)

                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($inactivePaths)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts have inactive paths"
            }

            "</tr>"
        }

        "TimeConsistency" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content ("<b>Time Consistency<br></b>Reference System: {0}" -f [System.Net.DNS]::GetHostname())

            $timeErrorHosts = 0
            $timeWarnHosts = 0
            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $esxcli = $vmhost | Get-EsxCli -V2
                    $hostTime = [DateTime]::Parse($esxcli.system.time.get.Invoke()).ToUniversalTime()
                    $sysTime = [DateTime]::UtcNow

                    $diffMins = ($hostTime - $sysTime).TotalMinutes
                    $status = [VMwareReportStatus]::Ok
                    if ([Math]::Abs($diffMins) -gt 4)
                    {
                        $status = [VMwareReportStatus]::Error
                        $timeErrorHosts++
                    } elseif ([Math]::Abs($diffMins) -gt 1) {
                        $status = [VMwareReportStatus]::Warning
                        $timeWarnHosts++
                    }

                    $mod = "+"
                    if ($diffMins -lt 0)
                    {
                        $mod = ""
                    }

                    $content = ("{0}{1} min" -f $mod, $diffMins.ToString("0.00"))
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($timeErrorHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Error -Report "Host Health" -Description "Some hosts have significant time drift"
            }

            if ($timeWarnHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts have minor time drift"
            }

            "</tr>"
        }

        "BuildConsistency" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Host Build<br>(+ Consistency)</b>"

            $versions = New-Object 'System.Collections.Generic.Hashset[string]'

            $mismatch = $false
            $reference = $null

            foreach ($vmhost in $vmhosts)
            {
                $versionStr = ("{0}-{1}" -f $vmhost.Version, $vmhost.Build)

                $status = [VMwareReportStatus]::None
                if ($reference -eq $null)
                {
                    $reference = $versionStr
                } elseif ($reference -ne $versionStr)
                {
                    $status = [VMwareReportStatus]::Warning
                    $mismatch = $true
                }

                New-VMwareReportCell -Status $status -Content $versionStr
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in host version and build"
            }

            "</tr>"
        }

        "TimezoneConsistency" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Timezone<br>(+ Consistency)</b>"

            $timezones = New-Object 'System.Collections.Generic.Hashset[string]'

            # Collect version information first
            foreach ($vmhost in $vmhosts)
            {
                $tz = $vmhost.Timezone.Description
                $timezones.Add($tz) | Out-Null
            }

            # Report on each host version
            $tzCount = $timezones.Count
            $status = [VMwareReportStatus]::Ok
            if ($tzCount -gt 1)
            {
                $status = [VMwareReportStatus]::Warning
            }

            foreach ($vmhost in $vmhosts)
            {
                $tz = $vmhost.Timezone.Description
                New-VMwareReportCell -Status $status -Content $tz
            }

            if ($tzCount -gt 1)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in timezone configuration"
            }

            "</tr>"
        }

        "BootTime" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Last boot</b>"

            $matchHosts = 0
            foreach ($vmhost in $vmhosts)
            {
                $bootTime = $vmhost.ExtensionData.Runtime.BootTime

                $status = [VMwareReportStatus]::None
                if ($bootTime -gt [DateTime]::Now.AddDays(-2))
                {
                    $status = [VMwareReportStatus]::Warning
                    $matchHosts++
                }

                $content = ("{0} ({1} days ago)" -f $bootTime.ToString("yyyy/MM/dd HH:mm"), ([DateTime]::Now - $bootTime).TotalDays.ToString("0.00"))
                New-VMwareReportCell -Status $status -Content $content
            }

            if ($matchHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts have restarted recently"
            }

            "</tr>"
        }

        "RebootRequired" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Reboot Required?</b>"

            $matchHosts = 0

            foreach ($vmhost in $vmhosts)
            {
                $status = [VMwareReportStatus]::Ok
                $rebootRequired = $vmhost.ExtensionData.Summary.RebootRequired
                if ($rebootRequired)
                {
                    $status = [VMwareReportStatus]::Warning
                    $matchHosts++
                }

                New-VMwareReportCell -Status $status -Content $rebootRequired.ToString()
            }

            if ($matchHosts -gt 0)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts require a restart"
            }

            "</tr>"
        }

        "NTPSource" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>NTP Time Source<br>(+ Consistency)</b>"

            $reference = $null

            $mismatch = $false
            $missing = $false

            foreach ($vmhost in $vmhosts)
            {
                $status = [VMwareReportStatus]::None
                $targets = ($vmhost | Get-VMHostNtpServer) -join "<br>"

                # Check if NTP configuration is different than the reference
                if ($reference -eq $null)
                {
                    $reference = $targets
                } elseif ($reference -ne $targets)
                {
                    $mismatch = $true
                    $status = [VMwareReportStatus]::Warning
                }

                # Check if ntp configuration is missing
                if ([string]::IsNullOrEmpty($targets))
                {
                    $missing = $true
                    $status = [VMwareReportStatus]::Warning
                }

                $content = $targets
                if ([string]::IsNullOrEmpty($content))
                {
                    $content = "(missing)"
                }

                New-VMwareReportCell -Status $status -Content $content
            }

            if ($missing)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts are missing NTP sync configuration"
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts vary in NTP configuration"
            }

            "</tr>"
        }

        "ImageConsistency" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>ESXi Image<br>(+ Consistency)</b>"

            $mismatch = $false
            $reference = $null

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $imageProfile = $cli.software.profile.get.Invoke()
                    $content = ("{0}<br>{1}" -f $imageProfile.Name, $imageProfile.Vendor)

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $content
                    } elseif ($reference -ne $content)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in ESXi image"
            }

            "</tr>"
        }

        "SNMPConfiguration" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            $reference = $null
            $mismatch = $false

            New-VMwareReportCell -Content "<b>SNMP Configuration<br>(+ Consistency)</b>"

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $snmpconfig = $cli.system.snmp.get.Invoke()

                    $content = ($snmpconfig.communities | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $content
                    } elseif ($reference -ne $content)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in SNMP configuration"
            }

            "</tr>"
        }

        "WBEMConfig" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            $reference = $null
            $mismatch = $false

            New-VMwareReportCell -Content "<b>WBEM Configuration<br>(+ Consistency)</b>"

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $config = $cli.system.wbem.get.Invoke()

                    $content = ("Enabled: {0}<br>Port: {1}" -f $config.Enabled, $config.Port)

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $content
                    } elseif ($reference -ne $content)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in WBEM configuration"
            }

            "</tr>"
        }

        "SyslogConfig" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>Syslog Targets<br>(+ Consistency)</b>"

            $reference = $null

            $mismatch = $false
            $missing = $false

            foreach ($vmhost in $vmhosts)
            {
                $status = [VMwareReportStatus]::None
                $targets = ($vmhost | Get-VMHostSyslogServer) -join "<br>"

                # Check if syslog target configuration is different than the reference
                if ($reference -eq $null)
                {
                    $reference = $targets
                } elseif ($reference -ne $targets)
                {
                    $mismatch = $true
                    $status = [VMwareReportStatus]::Warning
                }

                # Check if syslog target configuration is missing
                if ([string]::IsNullOrEmpty($targets))
                {
                    $missing = $true
                    $status = [VMwareReportStatus]::Warning
                }

                $content = $targets
                if ([string]::IsNullOrEmpty($content))
                {
                    $content = "(missing)"
                }

                New-VMwareReportCell -Status $status -Content $content
            }

            if ($missing)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts are missing syslog target configuration"
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts vary in syslog target configuration"
            }

            "</tr>"
        }

        "iSCSITargets" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>iSCSI Targets<br>(+ Consistency)</b>"

            $reference = $null
            $mismatch = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $targets = ($cli.iscsi.adapter.target.list.Invoke() | ForEach-Object { $_.Target } | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $targets
                    } elseif ($reference -ne $targets)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                    $content = $targets
                    if ([string]::IsNullOrEmpty($content))
                    {
                        $content = "(empty)"
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in active iSCSI targets"
            }

            "</tr>"
        }

        "iSCSISendTargets" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>iSCSI SendTargets<br>(+ Consistency)</b>"

            $reference = $null
            $mismatch = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $targets = (($cli.iscsi.adapter.discovery.sendtarget.list.Invoke() | ForEach-Object { $_.SendTarget } ) | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $targets
                    } elseif ($reference -ne $targets)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                    $content = $targets
                    if ([string]::IsNullOrEmpty($content))
                    {
                        $content = "(empty)"
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in iSCSI SendTargets configuration"
            }

            "</tr>"
        }

        "iSCSIStaticTargets" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>iSCSI StaticTarget<br>(+ Consistency)</b>"

            $reference = $null
            $mismatch = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $targets = (($cli.iscsi.adapter.discovery.statictarget.list.Invoke() | ForEach-Object { $_.SendTarget } ) | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $targets
                    } elseif ($reference -ne $targets)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                    $content = $targets
                    if ([string]::IsNullOrEmpty($content))
                    {
                        $content = "(empty)"
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in iSCSI StaticTarget configuration"
            }

            "</tr>"
        }

        "iSCSISessions" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>iSCSI Sessions<br>(+ Consistency)</b>"

            $reference = $null
            $mismatch = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $sessions = ($cli.iscsi.session.list.Invoke() | ForEach-Object { $_.Target } | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $sessions
                    } elseif ($reference -ne $sessions)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                    $content = $sessions
                    if ([string]::IsNullOrEmpty($content))
                    {
                        $content = "(empty)"
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in active iSCSI sessions"
            }

            "</tr>"
        }

        "iSCSIRemoteSessions" = {
            $runtime = $_
            $config = $runtime.Config
            $vmhosts = $runtime.VMHosts

            "<tr>"

            New-VMwareReportCell -Content "<b>iSCSI Session Connections<br>(+ Consistency)</b>"

            $reference = $null
            $mismatch = $false

            foreach ($vmhost in $vmhosts)
            {
                $content = "Error"
                $status = [VMwareReportStatus]::Error
                
                try {
                    $cli = $vmhost | Get-EsxCli -V2
                    $sessions = ($cli.iscsi.session.connection.list.Invoke() | ForEach-Object { ("{0}-{1}" -f $_.RemoteAddress, $_.State ) } | Sort-Object -Unique) -join "<br>"

                    $status = [VMwareReportStatus]::None
                    if ($reference -eq $null)
                    {
                        $reference = $sessions
                    } elseif ($reference -ne $sessions)
                    {
                        $mismatch = $true
                        $status = [VMwareReportStatus]::Warning
                    }

                    $content = $sessions
                    if ([string]::IsNullOrEmpty($content))
                    {
                        $content = "(empty)"
                    }
                } catch {
                    Write-Information ("Error during section: " + $_.ToString())
                }

                New-VMwareReportCell -Content $content -Status $status
            }

            if ($mismatch)
            {
                New-VMwareReportSummaryNotice -Status Warning -Report "Host Health" -Description "Some hosts differ in active iSCSI session connections"
            }

            "</tr>"
        }
    }
}

<#
#>
Function Invoke-BlockWithRuntime
{
    [CmdletBinding()]
    param(
         [Parameter(mandatory=$true)]
         [ValidateNotNull()]
         $runtime,
         
         [Parameter(mandatory=$true)]
         [ValidateNotNull()]
         [ScriptBlock]$ScriptBlock
    )

    process
    {
        # This is separate to provide a limited variable scope to the executed script block
        ForEach-Object -InputObject $runtime -Process $ScriptBlock
    }
}

<#
#>
Function New-VMwareReport
{
    [CmdletBinding()]
    param(
        [Parameter(mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string]$ConfigPath,

        [Parameter(mandatory=$false)]
        [switch]$DisplayContent = $false
    )

    process
    {
        # Test for existance of config file
        if (!(Test-Path -PathType Leaf $ConfigPath))
        {
            Write-Error "ConfigPath ($ConfigPath) does not exist"
        }

        # Read config as json and convert to VMwareReportConfig object
        Write-Information "Reading configuration"
        [VMwareReportConfig]$config = $null
        try {
            $config = [VMwareReportConfig](Get-Content $ConfigPath -Encoding UTF8 | ConvertFrom-Json)
        } catch {
            Write-Information "Failed to read the VMwareReport configuration file. See error below."
            Write-Information ("Error: " + $_.ToString())
            throw $_
        }

        # decrypt credential string
        Write-Information "Decrypting password"
        $secureString = $config.Password | ConvertTo-SecureString
        $byteStr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($secureString)
        $password = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto($byteStr)

        # Display active configuration
        Write-Information "Current configuration:"
        $config | ConvertTo-Json

        # Import VMware.VimAutomation.Core module
        try {
            Write-Information "Importing VMware.VimAutomation.Core"
            #Install-Module VMware.PowerCli -Scope CurrentUser -Force -Confirm:$false -EA Ignore
            Import-Module VMware.VimAutomation.Core -EA Stop -WA Ignore | Out-Null
        } catch {
            Write-Information "Failed to import VMware.VimAutomation.Core module. This may not be installed. See error below."
            Write-Information ("Error: " + $_.ToString())
            throw $_
        }

        # Set PowerCli session configuration
        try {
            Write-Information "Setting PowerCli session configuration"
            Set-PowerCliConfiguration -Scope Session -DefaultVIServerMode Single -ParticipateInCeip $false -DisplayDeprecationWarnings $false -InvalidCertificateAction Ignore -Confirm:$false | Out-Null
        } catch {
            Write-Information "Error setting session configuration for PowerCli. See error below."
            Write-Information ("Error: " + $_.ToString())
            throw $_
        }

        # Connect to target server
        try {
            Write-Information "Connecting to target server"
            Connect-VIserver -Server $config.Target -User $config.Username -Password $password
        } catch {
            Write-Information "Error connecting to target server. See error below."
            Write-Information ("Error: " + $_.ToString())
            throw $_
        }

        # Create a new report frame
        $warnings = 0
        $errors = 0

        Write-Information "Generating new report frame"
        $content = New-VMwareReportFrame -Script {

            $runtime = [PSCustomObject]@{
                Config = $config
                VMHosts = Get-VMHost
                VMs = Get-VM
            }

            foreach ($reportName in $script:Reports.Keys)
            {
                $report = $script:Reports[$reportName]
                $beginHasRun = $false

                foreach ($sectionName in $report.Sections.Keys)
                {
                    $section = $report.Sections[$sectionName]

                    # Check if the section is included
                    $included = $false
                    foreach ($def in $config.Include)
                    {
                        if ([string]::IsNullOrEmpty($def.Report) -or [string]::IsNullOrEmpty($def.Section))
                        {
                            continue
                        }

                        if ($reportName -match $def.Report -and $sectionName -match $def.Section)
                        {
                            $included = $true
                            break
                        }
                    }

                    if (!$included)
                    {
                        # Didn't find a match to move to next section
                        continue
                    }

                    # Check if the section is excluded
                    $excluded = $false
                    foreach ($def in $config.Exclude)
                    {
                        if ([string]::IsNullOrEmpty($def.Report) -or [string]::IsNullOrEmpty($def.Section))
                        {
                            continue
                        }

                        if ($reportName -match $def.Report -and $sectionName -match $def.Section)
                        {
                            $excluded = $true
                            break
                        }
                    }

                    if ($excluded)
                    {
                        # Section has been excluded, try next section
                        continue
                    }

                    # Run the begin block, if not run already
                    if (!$beginHasRun)
                    {
                        Write-Information "Running report begin for $reportName"
                        try {
                            Invoke-BlockWithRuntime -Runtime $runtime -ScriptBlock $report.begin
                        } catch {
                            Write-Information ("Error beging report ($reportName). Exception: " + $_.ToString())
                            "Error"
                        }

                        $beginHasRun = $true
                    }

                    # Run section
                    Write-Information "Running section $sectionName"
                    try {
                        Invoke-BlockWithRuntime -Runtime $runtime -ScriptBlock $section
                    } catch {
                        Write-Information ("Error generating section ($sectionName). Exception: " + $_.ToString())
                        "Error"
                    }
                }

                # Run the End code block
                if ($beginHasRun)
                {
                    Write-Information "Running report end for $reportName"
                    try {
                        Invoke-BlockWithRuntime -Runtime $runtime -ScriptBlock $report.End
                    } catch {
                        Write-Information ("Error ending report ($reportName). Exception: " + $_.ToString())
                        "Error"
                    }

                    "<br>"
                }
            }

        } | ForEach-Object {
            # Check if it is a string or status object
            if ([VMwareReportSummaryNotice].IsAssignableFrom($_.GetType()))
            {
                [VMwareReportSummaryNotice]$notice = $_
                if ($notice.Status -eq [VMwareReportStatus]::Warning)
                {
                    $warnings++
                } elseif ($notice.Status -eq [VMwareReportStatus]::Error)
                {
                    $errors++
                }
            } else {
                $_

                if ($DisplayContent)
                {
                    Write-Information $_
                }
            }
        } | Out-String

        if ($DisplayContent)
        {
            Write-Information "Complete Report Content:"
            $content
        }

        # Disconnect from target server
        try {
            Write-Information "Disconnecting from target server"
            Disconnect-VIServer -Server $config.Target -Confirm:$false -Force
        } catch {
            Write-Information "Error disconnecting from target server. See error below. Will stil continue/non-terminating"
            Write-Information ("Error: " + $_.ToString())
        }
        
        # Build a list of recipients to email results to
        Write-Information "Building email recipient list"
        $recipients = New-Object 'System.Collections.Generic.Hashset[string]'
        $config.Recipients | ForEach-Object {
            $recipients.Add($_) | Out-Null
        }

        # Add issue recipients, if there are any errors or warnings
        if ($warnings -gt 0 -or $errors -gt 0)
        {
            Write-Information "Warnings or errors within report. Adding issue recipients."
            $config.IssueRecipients | ForEach-Object {
                $recipients.Add($_) | Out-Null
            }
        }

        # Send notification to recipients
        $dateStr = [DateTime]::Now.ToString("yyyyMMdd HHmm")
        $recipients | ForEach-Object {
            Write-Information "Sending notification to $_"
            $subject = "${dateStr}: "
            if (![string]::IsNullOrEmpty($config.Site))
            {
                $subject += ("{0} - " -f $config.Site)                
            }

            $subject += "VMware Status Report"

            $attempts = 3
            while ($attempts -gt 0)
            {
                try
                {
                    Send-MailMessage -To $_ -Subject $subject -Body $content -SmtpServer $config.SmtpServer -From $config.SmtpSender -BodyAsHtml
                    break
                } catch {
                    Write-Information ("Failure to send message. Exception: " + $_.ToString())
                    Write-Information "Retrying in 5 seconds."
                    Start-Sleep 5
                }

                $attempts--
            }

            if ($attempts -lt 1)
            {
                Write-Information "Failed to send message to $_"
            }
        }
    }
}

Function New-VMwareReportFrame
{
    [CmdletBinding()]
    param(
        [Parameter(mandatory=$true)]
        [ValidateNotNull()]
        [ScriptBlock]$Script
    )

    process
    {
        # Write header to output
        "<head>
        <style>
        table {
            font-family: arial, sans-serif;
            border-collapse: collapse;
            width: 100%;
        }
 
        td, th {
            border: 1px solid #dddddd;
            text-align: left;
            padding: 8px;
        }
 
        tr:nth-child(even) {
            background-color: #dddddd;
        }
        </style>
        </head>
        <body>"

        try {
            $summaries = New-Object 'System.Collections.Generic.Hashset[VMwareReportSummaryNotice]'

            $content = & $Script | ForEach-Object {
                # Check if there is a summary status and save that separately to be written shortly
                if ([VMwareReportSummaryNotice].IsAssignableFrom($_.GetType()))
                {
                    $summaries.Add($_) | Out-Null
                }

                $_
            }

            # Write summary table
            "<table><tr><th>Report</th><th>Description</th></tr>"
            if ($summaries.Count -gt 0)
            {
                $summaries | ForEach-Object {
                    "<tr>"
                    New-VMwareReportCell -Content $_.Report
                    New-VMwareReportCell -Status $_.Status -Content $_.Description
                    "</tr>"
                }
            } else {
                New-VMwareReportCell -ColumnSpan 2 -Content "No Summaries"
            }
            "</table><br><p>"

            # Write report content
            $content

        } catch {
            "<b>Error during report processing</b>"
            ("Error: " + $_.ToString())
            ($_.Exception | Format-List | Out-String -Stream | ForEach-Object { "{0}<br>" -f $_})
        }

        "</body></html>"
    }
}