Private/AzStackHci.WildcardAndSubdomain.Helpers.ps1

# ////////////////////////////////////////////////////////////////////////////
# Helper to perform DNS + optional TCP + Layer7 testing for a URL/port and return a result hashtable.
# Used by Expand-WildcardUrlsDynamically and Test-ManuallyDefinedSubdomains to avoid duplicated test logic.
Function Invoke-EndpointConnectivityTest {
    [CmdletBinding()]
    param(
        [Parameter(Mandatory)][string]$Url,
        [Parameter(Mandatory)][int]$Port,
        [switch]$IncludeTCPConnectivityTests
    )

    try {
        $DNSCheck = Get-DnsRecord -url (Get-DomainFromURL -url $Url).Domain
    } catch {
        Write-HostAzS "Error: DNS resolution failed for '$Url': $($_.Exception.Message)" -ForegroundColor Red
        return @{
            TCPStatus                        = "Failed"
            IPAddress                        = "DNS Error"
            Layer7Status                     = "Failed"
            Layer7Response                   = "DNS resolution error: $($_.Exception.Message)"
            Layer7ResponseTime               = "N/A"
            CertificateIssuer                = "N/A"
            CertificateSubject               = "N/A"
            CertificateThumbprint            = "N/A"
            CertificateIntermediateIssuer    = "N/A"
            CertificateIntermediateSubject   = "N/A"
            CertificateIntermediateThumbprint= "N/A"
            CertificateRootIssuer            = "N/A"
            CertificateRootSubject           = "N/A"
            CertificateRootThumbprint        = "N/A"
        }
    }

    if (($DNSCheck.DNSExists) -or ($script:Proxy.Enabled)) {
        if ($IncludeTCPConnectivityTests.IsPresent) {
            try {
                $tcpStatus, $ipAddress = Test-TCPConnectivity -url $Url -port $Port
            } catch {
                Write-HostAzS "Error: TCP connectivity test failed for '$Url': $($_.Exception.Message)" -ForegroundColor Red
                $tcpStatus = "Failed"
                $ipAddress = "TCP Error"
            }
        } else {
            $tcpStatus = "N/A"
            $ipAddress = $DNSCheck.IpAddress
        }
        try {
            $Layer7Results = Test-Layer7Connectivity -url $Url -port $Port
        } catch {
            Write-HostAzS "Error: Layer 7 connectivity test failed for '$Url': $($_.Exception.Message)" -ForegroundColor Red
            $Layer7Results = @{
                Layer7Status = "Failed"
                Layer7Response = "Layer 7 test error: $($_.Exception.Message)"
                Layer7ResponseTime = "N/A"
                CertificateIssuer = "N/A"
                CertificateSubject = "N/A"
                CertificateThumbprint = "N/A"
                CertificateIntermediateIssuer = "N/A"
                CertificateIntermediateSubject = "N/A"
                CertificateIntermediateThumbprint = "N/A"
                CertificateRootIssuer = "N/A"
                CertificateRootSubject = "N/A"
                CertificateRootThumbprint = "N/A"
            }
        }
        return @{
            TCPStatus                        = $tcpStatus
            IPAddress                        = $ipAddress
            Layer7Status                     = $Layer7Results.Layer7Status
            Layer7Response                   = $Layer7Results.Layer7Response
            Layer7ResponseTime               = $Layer7Results.Layer7ResponseTime
            CertificateIssuer                = $Layer7Results.CertificateIssuer
            CertificateSubject               = $Layer7Results.CertificateSubject
            CertificateThumbprint            = $Layer7Results.CertificateThumbprint
            CertificateIntermediateIssuer    = $Layer7Results.CertificateIntermediateIssuer
            CertificateIntermediateSubject   = $Layer7Results.CertificateIntermediateSubject
            CertificateIntermediateThumbprint= $Layer7Results.CertificateIntermediateThumbprint
            CertificateRootIssuer            = $Layer7Results.CertificateRootIssuer
            CertificateRootSubject           = $Layer7Results.CertificateRootSubject
            CertificateRootThumbprint        = $Layer7Results.CertificateRootThumbprint
        }
    } else {
        # DNS name does not exist
        return @{
            TCPStatus                        = if ($IncludeTCPConnectivityTests.IsPresent) { "Failed" } else { "N/A" }
            IPAddress                        = $DNSCheck.IpAddress
            Layer7Status                     = "Failed"
            Layer7Response                   = "N/A"
            Layer7ResponseTime               = "N/A"
            CertificateIssuer                = "N/A"
            CertificateSubject               = "N/A"
            CertificateThumbprint            = "N/A"
            CertificateIntermediateIssuer    = "N/A"
            CertificateIntermediateSubject   = "N/A"
            CertificateIntermediateThumbprint= "N/A"
            CertificateRootIssuer            = "N/A"
            CertificateRootSubject           = "N/A"
            CertificateRootThumbprint        = "N/A"
        }
    }
}


# ////////////////////////////////////////////////////////////////////////////
# Enhanced Function to expand wildcard URLs dynamically using pattern matching *.domain.com
#
# How this works:
# 1. Splits $script:Results into wildcard entries (*.foo.com) and non-wildcard entries (bar.foo.com)
# 2. For each wildcard, converts it to a regex (e.g. *.foo.com -> .*\.foo\.com)
# 3. Finds non-wildcard URLs that match the regex pattern
# 4. For each match: if already tested, adds a cross-reference Note; if new, runs connectivity tests
# 5. Updates the wildcard entry's Note field to show which specific URLs were tested for it
#
# This enables testing of wildcard firewall rules by verifying connectivity to known specific endpoints
# that fall under the wildcard pattern.
Function Expand-WildcardUrlsDynamically {

    begin {
        # Write-Verbose "Starting Expand-WildcardUrlsDynamically function"
    }

    process {
        # Split into wildcard and non-wildcard results
        [array]$wildcardResults = $script:Results | Where-Object { $_.IsWildcard -eq $true }
        [array]$nonWildcardResults = $script:Results | Where-Object { $_.IsWildcard -eq $false }

        [int]$wildcardCount = 0

        ForEach ($wildcard in $wildcardResults) {

            $wildcardCount++
            Write-Progress -Id 1 -ParentId 0 -Activity "Expanding Wildcard URLs" -Status "Wildcard $wildcardCount of $($wildcardResults.Count): $($wildcard.URL)" -PercentComplete (($wildcardCount / [math]::Max($wildcardResults.Count,1)) * 100)

            # Create a regex pattern from the wildcard URL
            # Escape dots first, then replace * with .* for proper regex matching
            $wildcardPattern = [regex]::Escape($wildcard.URL) -replace "\\\*", ".*"

            # Find matching non-wildcard URLs, for each wildcard URL that exists
            [array]$matchingUrls = @()
            $matchingUrls = $nonWildcardResults | Where-Object { $_.URL -match "^$wildcardPattern$" }

            Write-HostAzS "`n$wildcardCount of $($wildcardResults.Count): Processing $($wildcard.Source): $($wildcard.URL)"

            if($matchingUrls.Count -eq 0) {
                Write-HostAzS "`tInfo: No match found for $($wildcard.source), $($wildcard.URL)" -ForegroundColor Yellow
                if($wildcard.Note -notlike "Wildcard URL,*"){
                    # Update the note to include the matching URL
                    Write-Verbose "Updating Note of wildcard URL: $($wildcard.URL)"
                    ($script:Results | Where-Object { $PSItem.URL -eq $wildcard.URL } | Select-Object -First 1).Note = "Wildcard URL, not tested (as no non-wildcard) - $($wildcard.Note)"
                }
                # Continue to the next matching URL
                Continue

            } elseif ($matchingUrls.Count -gt 0) {
                # Matches found, expand the wildcard entry
                # Incremental counter for matching URLs
                [int]$counter = 0
                # Loop for each matching URL
                foreach ($match in $matchingUrls) {
                    $counter++
                    Write-HostAzS "`n`tProcessing match $counter of $($matchingUrls.count): $($match.URL)"
                    # Add URL based on test for either HTTP or HTTPS
                    if($match.Port -is [int]){

                        # If the URL already exists in the results, update the note
                        if($script:Results.URL -contains $match.URL){
                            # URL already exists in the results, skip
                            Write-HostAzS "`tEndpoint matches wildcard '$($wildcard.URL)', adding cross-reference in Notes column" -ForegroundColor Green
                            if(($match.Note -notlike "URL matches *") -and ($match.Note -notlike "Manually defined URL *")){
                                # Update the note to include the matching URL
                                Write-Verbose "Updating Note of matched URL: $($match.URL)"
                            ($script:Results | Where-Object { $PSItem.URL -eq $match.URL } | Select-Object -First 1).Note = "URL matches $($wildcard.Source), to test URL $($wildcard.URL) - $($match.Note)"
                            }
                            # Continue to the next matching URL
                            Continue

                        } elseif ($script:Results.URL -notcontains $match.URL){
                            # Specific URL matched to Wildcard URL does not exist in the results, unexpected, but will add it
                            Write-HostAzS "`tEndpoint matches wildcard '$($wildcard.URL)', adding to results" -ForegroundColor Green
                            $newEntry = $wildcard.PSObject.Copy()
                            $newEntry.URL = $match.URL
                            $newEntry.Port = $match.Port
                            $newEntry.ArcGateway = $match.ArcGateway
                            $newEntry.Source = "$($match.Source)"
                            $newEntry.Note = "URL matches $($match.Source), to test URL $($wildcard.URL) - $($match.Note)"
                            # Set IsWildcard to false, as this is no longer a wildcard URL, it is a specific URL to test a wildcard URL
                            $newEntry.IsWildcard = $false
                            if($ArcGatewayDeployment.IsPresent){
                                # Skip URLs with ArcGateway -eq $True
                                if($match.ArcGateway){
                                    # Skip URLs that support Arc Gateway
                                    Write-HostAzS "Skipped URL: '$($newEntry.URL)' as supported by Arc Gateway" -ForegroundColor Yellow
                                    $newEntry.TCPStatus = "Skipped"
                                    $newEntry.Note = "Skipped, URL is supported by Arc Gateway - $($match.Note)"
                                    $newEntry.IPAddress = "N/A"
                                    $newEntry.Layer7Status = "Skipped"
                                    $newEntry.Layer7Response = "N/A"
                                    $newEntry.Layer7ResponseTime = "N/A"
                                    $newEntry.CertificateIssuer = "N/A"
                                    $newEntry.CertificateSubject = "N/A"
                                    $newEntry.CertificateThumbprint = "N/A"
                                    $newEntry.IntermediateCertificateIssuer = "N/A"
                                    $newEntry.IntermediateCertificateSubject = "N/A"
                                    $newEntry.IntermediateCertificateThumbprint = "N/A"
                                    $newEntry.RootCertificateIssuer = "N/A"
                                    $newEntry.RootCertificateSubject = "N/A"
                                    $newEntry.RootCertificateThumbprint = "N/A"
                                } else {
                                    # Does not support Arc Gateway - run connectivity tests
                                    $testResult = Invoke-EndpointConnectivityTest -Url $match.URL -Port $match.Port -IncludeTCPConnectivityTests:$IncludeTCPConnectivityTests.IsPresent
                                    $newEntry.TCPStatus = $testResult.TCPStatus
                                    $newEntry.IPAddress = $testResult.IPAddress
                                    $newEntry.Layer7Status = $testResult.Layer7Status
                                    $newEntry.Layer7Response = $testResult.Layer7Response
                                    $newEntry.Layer7ResponseTime = $testResult.Layer7ResponseTime
                                    $newEntry.CertificateIssuer = $testResult.CertificateIssuer
                                    $newEntry.CertificateSubject = $testResult.CertificateSubject
                                    $newEntry.CertificateThumbprint = $testResult.CertificateThumbprint
                                    $newEntry.IntermediateCertificateIssuer = $testResult.CertificateIntermediateIssuer
                                    $newEntry.IntermediateCertificateSubject = $testResult.CertificateIntermediateSubject
                                    $newEntry.IntermediateCertificateThumbprint = $testResult.CertificateIntermediateThumbprint
                                    $newEntry.RootCertificateIssuer = $testResult.CertificateRootIssuer
                                    $newEntry.RootCertificateSubject = $testResult.CertificateRootSubject
                                    $newEntry.RootCertificateThumbprint = $testResult.CertificateRootThumbprint
                                } # End of ArcGateway -eq $True check
                            } else {
                                # Else process URLs with ArcGateway -eq $False
                                $testResult = Invoke-EndpointConnectivityTest -Url $match.URL -Port $match.Port -IncludeTCPConnectivityTests:$IncludeTCPConnectivityTests.IsPresent
                                $newEntry.TCPStatus = $testResult.TCPStatus
                                $newEntry.IPAddress = $testResult.IPAddress
                                $newEntry.Layer7Status = $testResult.Layer7Status
                                $newEntry.Layer7Response = $testResult.Layer7Response
                                $newEntry.Layer7ResponseTime = $testResult.Layer7ResponseTime
                                $newEntry.CertificateIssuer = $testResult.CertificateIssuer
                                $newEntry.CertificateSubject = $testResult.CertificateSubject
                                $newEntry.CertificateThumbprint = $testResult.CertificateThumbprint
                                $newEntry.IntermediateCertificateIssuer = $testResult.CertificateIntermediateIssuer
                                $newEntry.IntermediateCertificateSubject = $testResult.CertificateIntermediateSubject
                                $newEntry.IntermediateCertificateThumbprint = $testResult.CertificateIntermediateThumbprint
                                $newEntry.RootCertificateIssuer = $testResult.CertificateRootIssuer
                                $newEntry.RootCertificateSubject = $testResult.CertificateRootSubject
                                $newEntry.RootCertificateThumbprint = $testResult.CertificateRootThumbprint
                            } # End of DNS check
                            # Add the new match to the expanded results
                            $script:Results.Add($newEntry) | Out-Null
                            Write-Verbose "Added: '$($newEntry.URL)' that matches Wildcard '$($wildcard.URL)'"
                        
                        } else {
                            # Unexpected condition, should not occur, as we use the Results array for the list of Wildcards, so the URL should always exist
                            Write-Warning "Unexpected: URL '$($match.URL)' matches Wildcard '$($wildcard.URL)', but conditions not met, skipping"
                            Continue
                        } # End of URL exists check

                    } else {
                        Write-Warning "Port not valid for url: '$($match.URL)', port: '$($match.Port)', note: '$($match.Source)' - skipping"
                    }

                } # End of matching URLs loop
            } else {
                # Unexpected condition, should not occur, as we use the Results array for the list of Wildcards, so the URL should always exist
                Write-Warning "Unexpected: Wildcard URL $($wildcard.URL) should match either zero or more results, but matches status unknown, skipping"
                Continue

            } # End of matching URLs check
        } # End of wildcard loop
        Write-Progress -Id 1 -Activity "Expanding Wildcard URLs" -Completed

    } # End of process block

    end {
        # Write-Debug "Completed Expand-WildcardUrlsDynamically function"
    }

} # End of Expand-WildcardUrlsDynamically function


# ////////////////////////////////////////////////////////////////////////////
# Function to manually test known subdomains for wildcard URLs
Function Test-ManuallyDefinedSubdomains {

    Param (
    )

    begin {
        # Incremental counter for subdomains (separate from the main progress bar)
        [int]$urlCount = 0
        # Total number of manually defined subdomains for sub-progress tracking
        [int]$totalSubdomains = $script:MANUAL_SUBDOMAIN_COUNT
        # Use the constant subdomain list
        $manualSubdomains = $script:MANUAL_SUBDOMAINS
    }

    process {
        # Test each manual subdomain for each wildcard to validate connectivity.
        ForEach ($entry in $manualSubdomains) {
            $wildcard = $entry.Wildcard
            $subdomains = $entry.Subdomains

            # Check if the wildcard URL is supported by Arc Gateway, if so, skip the wildcard URL
            if($ArcGatewayDeployment.IsPresent -and ($PreArcGatewayRemoval | Where-Object { ($_.URL -eq $wildcard) -and ($_.ArcGateway -eq $true) })) {
                # Skip wildcard URLs that have already been tested
                Write-Verbose "Info: Wildcard '$wildcard', supports the Arc Gateway, skipping subdomain tests."
                Continue
            }

            # Check if the wildcard URL exists in the results, if not, skip testing subdomains
            if($script:Results.URL -notcontains $wildcard) {
                # Skip wildcard URLs that have already been tested
                Write-Verbose "Info: Wildcard '$wildcard', is not present in Results array, skipping subdomain tests."
                Continue
            }

            Write-HostAzS "`nTesting manually defined subdomains for wildcard $wildcard"

            # ForEach loop to process each subdomain
            ForEach ($subdomain in $subdomains) {
                # Test each subdomain for TCP and Layer 7 connectivity
                $urlCount++
                if ($totalSubdomains -gt 0) {
                    Write-Progress -Id 1 -ParentId 0 -Activity "Validating Wildcard Endpoints" -Status "Subdomain $urlCount of $($totalSubdomains): $subdomain" -PercentComplete (($urlCount / $totalSubdomains) * 100)
                }
                Write-HostAzS "`nWildcard validation $urlCount of $($totalSubdomains): Subdomain: $subdomain"
                # Test port 80 and 443 for each subdomain
                ForEach ($port in @(80, 443)) {
                    
                    # Exceptions to some URLs, that do not support port 80 or 443
                    # Skip port 443 for ctldl.windowsupdate.com, download.windowsupdate.com, fe2.update.microsoft.com and 1a.au.download.windowsupdate.com
                    if(($subdomain -in ("ctldl.windowsupdate.com","download.windowsupdate.com","fe2.update.microsoft.com","1a.au.download.windowsupdate.com")) -and ($port -eq 443)){
                        # Skip port 443 for ctldl.windowsupdate.com, download.windowsupdate.com, fe2.update.microsoft.com and 1a.au.download.windowsupdate.com
                        Write-HostAzS "Skipping port 443 for $subdomain" -ForegroundColor Yellow
                        Continue
                    }
                    # Skip port 80 for prod5.prod.hot.ingest.monitor.core.windows.net and edr-neu3.eu.endpoint.security.microsoft.com
                    if(($subdomain -in ("prod5.prod.hot.ingest.monitor.core.windows.net","edr-neu3.eu.endpoint.security.microsoft.com")) -and ($port -eq 80)){
                        # Skip port 80 for prod5.prod.hot.ingest.monitor.core.windows.net and edr-neu3.eu.endpoint.security.microsoft.com
                        Write-HostAzS "Skipping port 80 for $subdomain" -ForegroundColor Yellow
                        Continue
                    }
                    
                    # Run connectivity tests using shared helper
                    Write-HostAzS "Testing $subdomain on port $port"
                    $testResult = Invoke-EndpointConnectivityTest -Url $subdomain -Port $port -IncludeTCPConnectivityTests:$IncludeTCPConnectivityTests.IsPresent
                    $TCPstatus = $testResult.TCPStatus
                    $ipAddress = $testResult.IPAddress
                    $Layer7Status = $testResult.Layer7Status
                    $Layer7Response = $testResult.Layer7Response
                    $Layer7ResponseTime = $testResult.Layer7ResponseTime
                    $CertificateIssuer = $testResult.CertificateIssuer
                    $CertificateSubject = $testResult.CertificateSubject
                    $CertificateThumbprint = $testResult.CertificateThumbprint
                    $CertificateIntermediateIssuer = $testResult.CertificateIntermediateIssuer
                    $CertificateIntermediateSubject = $testResult.CertificateIntermediateSubject
                    $CertificateIntermediateThumbprint = $testResult.CertificateIntermediateThumbprint
                    $CertificateRootIssuer = $testResult.CertificateRootIssuer
                    $CertificateRootSubject = $testResult.CertificateRootSubject
                    $CertificateRootThumbprint = $testResult.CertificateRootThumbprint
                    
                    # Check if the URL already exists in the "allTestedUrls" results, only add new ones
                    if($script:Results.URL -notcontains $subdomain) {
                        # ArcGateway = $true, as all wildcards are "microsoft.com" or "windows.net", so should support ArcGateway
                        $subdomainEntry = [PSCustomObject]@{
                            RowID = 0
                            URL = $subdomain
                            Port = $port
                            ArcGateway = $true
                            IsWildcard = $false
                            Source = "Test for $(($script:Results | Where-Object { ($_.URL -eq $wildcard) } | Select-Object -First 1).Source)"
                            Note = "Manually defined URL to test $Source Wildcard URL $wildcard"
                            TCPStatus = $TCPstatus
                            IPAddress = $ipAddress
                            Layer7Status = $Layer7Status
                            Layer7Response = $Layer7Response
                            Layer7ResponseTime = $Layer7ResponseTime
                            CertificateIssuer = $CertificateIssuer
                            CertificateSubject = $CertificateSubject
                            CertificateThumbprint = $CertificateThumbprint
                            IntermediateCertificateIssuer = $CertificateIntermediateIssuer
                            IntermediateCertificateSubject = $CertificateIntermediateSubject
                            IntermediateCertificateThumbprint = $CertificateIntermediateThumbprint
                            RootCertificateIssuer = $CertificateRootIssuer
                            RootCertificateSubject = $CertificateRootSubject
                            RootCertificateThumbprint = $CertificateRootThumbprint
                        }
                        # Add the subdomain entry to the results
                        Write-Debug "Info: Added $($subdomain) to the results array"
                        $script:Results.Add($subdomainEntry) | Out-Null
                        $MatchedWildcardURLs = ($script:Results | Where-Object { $PSItem.URL -eq $wildcard } )
                        ForEach($MatchedWildcardURL in $MatchedWildcardURLs) {
                            if($MatchedWildcardURL.Note -notlike "Wildcard URL,*"){
                                # Update the note to include the matching URL
                                Write-Verbose "Updating Note of wildcard URL: $($wildcard), with cross-reference to $($subdomain)"
                                $MatchedWildcardURL.Note = "Wildcard URL, tested by manually defined URL $($subdomain) - $($MatchedWildcardURL.Note)"
                            }
                        }

                    } else {
                        # URL already exists in the results, skip
                        Write-Verbose "Info: Url $($subdomain) already exists in the results, skipping"
                        $MatchedWildcardURLs = ($script:Results | Where-Object { $PSItem.URL -eq $wildcard } )
                        ForEach($MatchedWildcardURL in $MatchedWildcardURLs) {
                            if($MatchedWildcardURL.Note -notlike "Wildcard URL,*"){
                                # Update the note to include the matching URL
                                $MatchedWildcardURL.Note = "Wildcard URL, tested by manually defined URL $($subdomain) - $($MatchedWildcardURL.Note)"
                                Write-Verbose "Updated Note of wildcard URL: $($wildcard), with cross-reference to $($subdomain)"
                            }
                        }
                    }
                }
            } # End of ForEach per $port

        } # End of ForEach $subdomain

    } # End of Process block

    end {
        # Progress bar is managed by the caller (unified with main endpoint testing)
    }
} # End of Test-ManuallyDefinedSubdomains function


# ////////////////////////////////////////////////////////////////////////////
# Function to parse endpoints from markdown lines
Function Get-EndpointsFromMarkdown {
    <#
    .SYNOPSIS
        Parse endpoints from markdown lines.
    .DESCRIPTION
        This function parses the provided markdown lines to extract endpoint URLs.
    #>

    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [AllowEmptyCollection()]
        [string[]]$InputMarkdown,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string]$Source
    )

    begin {
        # Write-Debug "Get-EndpointsFromMarkdown: Beginning endpoint parsing from markdown lines"
        # Initialize counter to store the number of parsed endpoints
        [int]$parsedEndpoints = 0
        
        # Validate input is not empty
        if (-not $InputMarkdown -or $InputMarkdown.Count -eq 0) {
            Write-Warning "No markdown content provided to parse in function: 'Get-EndpointsFromMarkdown'. Returning 0 endpoints."
            return $parsedEndpoints
        }
    }

    process {
        # Parse markdown table rows to extract endpoint URLs.
        # Expected format: | RowNum | ... | URL | Ports | Note | ArcGateway | ...
        # The regex matches lines starting with | followed by a number (the row ID column).
        # Columns are split on '|' and indexed: [3]=URL, [4]=Ports, [5]=Note, [6]=ArcGateway.
        ForEach ($line in $InputMarkdown) {
            if ($line -match "^\|\s*(\d+)\s*\|") {
                # $rowId = [int]($line -replace "^\|\s*(\d+)\s*\|.*", '$1')
                $columns = $line -split "\|"
                # Validate column count before accessing indices (need at least 7 columns: empty, rowId, desc, URL, Ports, Note, ArcGateway)
                if ($columns.Count -lt 7) {
                    Write-Warning "Skipping malformed markdown row (expected at least 7 columns, found $($columns.Count)): $line"
                    Continue
                }
                $url = $columns[3].Trim()
                # Sanitize URL: only allow FQDN characters (alphanumeric, hyphens, dots, wildcards, forward slashes, colons for port, underscores)
                # This prevents injection of unexpected values from crafted markdown files.
                if ($url -notmatch '^[a-zA-Z0-9\*\.\-\:/_]+$') {
                    Write-Warning "Skipping invalid URL from markdown: '$url' - contains unexpected characters"
                    Continue
                }
                $ports = $columns[4].Trim() -split ','
                $note = $columns[5].Trim()
                $ArcGatewayText = $columns[6].Trim()
                if(($ArcGatewayText.Length -ge 2) -and ($ArcGatewayText.Substring(0,2) -eq "No")){
                    # "No"
                    [bool]$ArcGateway = $false
                } elseif(($ArcGatewayText.Length -ge 3) -and ($ArcGatewayText.Substring(0,3) -eq "Yes")){
                    # "Yes"
                    [bool]$ArcGateway = $true
                } else {
                    # Unknown
                    Write-Warning "Unknown ArcGateway status for url: '$url' : $ArcGatewayText"
                    [bool]$ArcGateway = $false
                }
                # Use only the domain name from the URL, remove any absolute path from the endpoint when checking if it exists
                $urlcheck = (Get-DomainFromURL -url $url).Domain
                foreach ($port in $ports) {
                    $isWildcard = $url.Contains("*")
                    if($script:Results | Where-Object { ((Get-DomainFromURL -url $_.URL).Domain -eq $urlcheck) -and ($_.Port -eq $port) }) {
                        Write-Debug "$($Source): Skipping '$url' with port '$port', as it already exists in the results array."
                        # If the ArcGateway value is different, update it
                        $script:Results | Where-Object { ((Get-DomainFromURL -url $_.URL).Domain -like $urlcheck) -and ($_.Port -eq $port) } | ForEach-Object {
                            $_.ArcGateway = $ArcGateway
                        }
                        Continue
                    } else {
                        Write-Debug "$($Source): Adding '$urlcheck' with port '$port' to results array for processing."
                        $script:Results.Add([PSCustomObject]@{
                            RowID = 0
                            URL = $url
                            Port = [int]$port
                            ArcGateway = $ArcGateway
                            IsWildcard = $isWildcard
                            Source = if ($isWildcard) { "$Source Wildcard" } else { "$Source" }
                            Note = $note
                            TCPStatus = ""
                            IPAddress = ""
                            Layer7Status = ""
                            Layer7Response = ""
                            Layer7ResponseTime = ""
                            CertificateIssuer = ""
                            CertificateSubject = ""
                            CertificateThumbprint = ""
                            IntermediateCertificateIssuer = ""
                            IntermediateCertificateSubject = ""
                            IntermediateCertificateThumbprint = ""
                            RootCertificateIssuer = ""
                            RootCertificateSubject = ""
                            RootCertificateThumbprint = ""
                        }) | Out-Null
                        # Increment the parsed endpoints counter
                        $parsedEndpoints++
                    }
                }
            }
            
        }
    } # End of process block

    end {
        # Write-Debug "Get-EndpointsFromMarkdown: Endpoint parsing completed"
        Return $parsedEndpoints

    }
} # End Function Get-EndpointsFromMarkdown