configmap

1.2.1.0

test/Measure-ConfigMapPerformance.ps1

#requires -version 7.0

[CmdletBinding()]
param(
    [ValidateRange(5, 10000)]
    [int]$Iterations = 50,

    [ValidateNotNullOrEmpty()]
    [int[]]$EntryCount = @(100, 1000),

    [switch]$AsJson
)

$modulePath = Join-Path $PSScriptRoot '..\configmap.psm1'
Import-Module $modulePath -Force

function New-BenchmarkMap {
    param(
        [ValidateRange(1, [int]::MaxValue)]
        [int]$Count,
        [pscustomobject]$State
    )

    $map = [ordered]@{}
    $command = {
        $State.InvocationCount++
    }

    foreach ($number in 1..$Count) {
        $map["task$number"] = $command
    }

    return $map
}

function Measure-BenchmarkAction {
    param(
        [string]$Scenario,
        [int]$Entries,
        [int]$Iterations,
        [scriptblock]$Action
    )

    $null = & $Action
    $samples = [System.Collections.Generic.List[double]]::new()
    $allocatedBytes = [System.Collections.Generic.List[long]]::new()

    foreach ($iteration in 1..$Iterations) {
        $before = [GC]::GetTotalAllocatedBytes($true)
        $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
        $null = & $Action
        $stopwatch.Stop()

        $samples.Add($stopwatch.Elapsed.TotalMilliseconds)
        $allocatedBytes.Add([GC]::GetTotalAllocatedBytes($true) - $before)
    }

    $orderedSamples = @($samples | Sort-Object)
    $orderedAllocations = @($allocatedBytes | Sort-Object)
    $p95Index = [math]::Ceiling($Iterations * 0.95) - 1

    return [pscustomobject]@{
        Scenario             = $Scenario
        Entries              = $Entries
        Iterations           = $Iterations
        MedianMilliseconds   = [math]::Round($orderedSamples[[int]($Iterations / 2)], 3)
        P95Milliseconds      = [math]::Round($orderedSamples[$p95Index], 3)
        MinimumMilliseconds  = [math]::Round($orderedSamples[0], 3)
        MaximumMilliseconds  = [math]::Round($orderedSamples[-1], 3)
        MedianAllocatedBytes = $orderedAllocations[[int]($Iterations / 2)]
        P95AllocatedBytes    = $orderedAllocations[$p95Index]
    }
}

$results = foreach ($count in $EntryCount) {
    $state = [pscustomobject]@{ InvocationCount = 0 }
    $map = New-BenchmarkMap -Count $count -State $state
    $targetKey = "task$([math]::Floor($count / 2))"

    $lookup = {
        $entries = @(Get-MapEntries $map $targetKey -language build)
        if ($entries.Count -ne 1 -or $entries[0].Key -ne $targetKey) {
            throw "Expected one lookup result for '$targetKey'."
        }
    }

    $completion = {
        $completions = @(Get-EntryCompletion -map $map -language build -wordToComplete '')
        if ($completions.Count -ne $count -or $completions -notcontains $targetKey) {
            throw "Completion results did not match the $count-entry fixture."
        }
    }

    $qbuild = {
        qbuild -map $map $targetKey
    }

    Measure-BenchmarkAction -Scenario 'Get-MapEntries' -Entries $count -Iterations $Iterations -Action $lookup
    Measure-BenchmarkAction -Scenario 'Get-EntryCompletion' -Entries $count -Iterations $Iterations -Action $completion
    $qbuildResult = Measure-BenchmarkAction -Scenario 'qbuild no-op' -Entries $count -Iterations $Iterations -Action $qbuild

    if ($state.InvocationCount -ne ($Iterations + 1)) {
        throw "Expected qbuild to invoke '$targetKey' $($Iterations + 1) times; invoked $($state.InvocationCount) times."
    }

    $qbuildResult
}

$includeRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("configmap-include-benchmark-" + [guid]::NewGuid().ToString())
try {
    $includeDirectory = Join-Path $includeRoot 'child'
    New-Item -ItemType Directory -Path $includeDirectory -Force | Out-Null
    Set-Content -Path (Join-Path $includeDirectory '.build.map.ps1') -Value @'
@{
    run = { }
}
'@
    $includeMap = @{
        '#include' = @{
            child = @{ prefix = $true }
        }
        _baseDir = $includeRoot
    }
    $module = Get-Module ConfigMap

    $withoutOperationCache = {
        & $module {
            param($Map)

            $target = @(Get-MapEntries $Map 'child.run' -language build)[0]
            if (!$target) {
                throw 'Unable to resolve the included benchmark entry.'
            }

            Get-MapEntry $Map 'child.run' -language build | Out-Null
            Invoke-WithEntrySettings -Map $Map -EntryKey 'child.run' -Entry $target.Value -ScriptBlock { }
        } $includeMap
    }

    $withOperationCache = {
        & $module {
            param($Map)

            $context = New-ConfigMapOperationContext
            $target = @(Get-MapEntries $Map 'child.run' -language build -OperationContext $context)[0]
            if (!$target) {
                throw 'Unable to resolve the included benchmark entry.'
            }

            Get-MapEntry $Map 'child.run' -language build -OperationContext $context | Out-Null
            Invoke-WithEntrySettings -Map $Map -EntryKey 'child.run' -Entry $target.Value -OperationContext $context -ScriptBlock { }
        } $includeMap
    }

    $results += Measure-BenchmarkAction -Scenario 'included map without operation cache' -Entries 1 -Iterations $Iterations -Action $withoutOperationCache
    $results += Measure-BenchmarkAction -Scenario 'included map with operation cache' -Entries 1 -Iterations $Iterations -Action $withOperationCache
}
finally {
    if (Test-Path $includeRoot) {
        Remove-Item -Path $includeRoot -Recurse -Force
    }
}

if ($AsJson) {
    $results | ConvertTo-Json
}
else {
    $results
}