test/perf/Invoke-PerfSuite.ps1
|
#requires -version 7.0 <# .SYNOPSIS Runs the qbuild performance scenarios against one configmap module and emits JSON. .DESCRIPTION Run in a fresh pwsh process. Public scenarios only use qbuild, `qbuild list` and TabExpansion2, so the same suite can measure an older module checkout (A/B in Compare-Perf.ps1). Internal scenarios call exported helpers and are skipped when the module does not have them. Cache states: warm - repeated call in a loaded session (discovery cache memory hit) edited - a child map was just touched, so the discovery cache is rebuilt new-shell - fresh pwsh: module import + first TAB (discovery cache disk hit) .EXAMPLE ./Invoke-PerfSuite.ps1 .EXAMPLE ./Invoke-PerfSuite.ps1 -FixtureRoot $env:TEMP/perf -OutFile result.json #> [CmdletBinding()] param( [string]$ModulePath = (Join-Path $PSScriptRoot '..\..\configmap.psm1'), # Prepared by New-PerfFixture.ps1. Omitted: a temporary copy of all fixtures. [string]$FixtureRoot, [ValidateRange(3, 10000)] [int]$Iterations = 10, # Slow scenarios stop after this many seconds (but run at least 3 iterations). [double]$ScenarioBudgetSeconds = 8, [ValidateRange(1, 1000)] [int]$NewShellIterations = 8, [string]$Label = 'local', [string]$OutFile ) $ErrorActionPreference = 'Stop' # Ambient QCONF_* settings (cache location, opt-outs) would skew results. Get-ChildItem env: | Where-Object Name -like 'QCONF_*' | ForEach-Object { Remove-Item "env:$($_.Name)" } $ModulePath = (Resolve-Path $ModulePath).Path $tempFixtures = $null if (!$FixtureRoot) { $tempFixtures = Join-Path ([System.IO.Path]::GetTempPath()) "configmap-perf-fixtures-$([guid]::NewGuid())" & (Join-Path $PSScriptRoot 'New-PerfFixture.ps1') -Path $tempFixtures $FixtureRoot = $tempFixtures } $FixtureRoot = (Resolve-Path $FixtureRoot).Path Import-Module $ModulePath -Force function Measure-Scenario { param( [string]$Name, [string]$Fixture, [scriptblock]$Action, [scriptblock]$Setup, [object[]]$ArgumentList = @(), [bool]$Gate = $true ) $times = [System.Collections.Generic.List[double]]::new() $allocs = [System.Collections.Generic.List[long]]::new() try { if ($Setup) { & $Setup } $null = & $Action @ArgumentList $budget = [System.Diagnostics.Stopwatch]::StartNew() while ($times.Count -lt $Iterations -and ($times.Count -lt 3 -or $budget.Elapsed.TotalSeconds -lt $ScenarioBudgetSeconds)) { if ($Setup) { & $Setup } $before = [GC]::GetTotalAllocatedBytes($true) $sw = [System.Diagnostics.Stopwatch]::StartNew() $null = & $Action @ArgumentList $sw.Stop() $allocs.Add([GC]::GetTotalAllocatedBytes($true) - $before) $times.Add($sw.Elapsed.TotalMilliseconds) } } catch { # e.g. a base checkout that does not support a feature used by the fixture Write-Host (" {0,-14} {1,-34} FAILED: {2}" -f $Fixture, $Name, $_) -ForegroundColor Yellow return [pscustomobject]@{ Scenario = $Name; Fixture = $Fixture; Gate = $Gate; Error = "$_" } } $result = New-ScenarioResult -Name $Name -Fixture $Fixture -Gate $Gate -Times $times -Allocs $allocs Write-Host (" {0,-14} {1,-34} median {2,9:n1} ms (n={3})" -f $Fixture, $Name, $result.MedianMs, $result.Iterations) -ForegroundColor DarkGray $result } function New-ScenarioResult { param([string]$Name, [string]$Fixture, [bool]$Gate, $Times, $Allocs) $t = @($Times | Sort-Object) $a = @($Allocs | Sort-Object) $p95 = [math]::Ceiling($t.Count * 0.95) - 1 [pscustomobject]@{ Scenario = $Name Fixture = $Fixture Gate = $Gate Iterations = $t.Count MedianMs = [math]::Round($t[[int]($t.Count / 2)], 3) P95Ms = [math]::Round($t[$p95], 3) MinMs = [math]::Round($t[0], 3) MedianAllocKB = if ($a.Count) { [math]::Round($a[[int]($a.Count / 2)] / 1KB, 1) } else { $null } Error = $null } } function Measure-Tab { param([string]$Line, [int]$Min = 1) $found = (TabExpansion2 $Line $Line.Length).CompletionMatches if ($found.Count -lt $Min) { throw "TabExpansion2 '$Line' returned $($found.Count) matches, expected at least $Min." } } function Measure-NewShell { param([string]$Fixture, [string]$Directory, [string]$Line) $probe = { param($ModulePath, $Directory, $Line) Get-ChildItem env: | Where-Object Name -like 'QCONF_*' | ForEach-Object { Remove-Item "env:$($_.Name)" } $sw = [System.Diagnostics.Stopwatch]::StartNew() Import-Module $ModulePath $importMs = $sw.Elapsed.TotalMilliseconds Set-Location $Directory $before = [GC]::GetTotalAllocatedBytes($true) $sw.Restart() $count = (TabExpansion2 $Line $Line.Length).CompletionMatches.Count $tabMs = $sw.Elapsed.TotalMilliseconds $alloc = [GC]::GetTotalAllocatedBytes($true) - $before @{ importMs = $importMs; tabMs = $tabMs; alloc = $alloc; count = $count } | ConvertTo-Json -Compress } $command = "& { $probe } '$ModulePath' '$Directory' '$Line'" $encoded = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($command)) $import = [System.Collections.Generic.List[double]]::new() $tab = [System.Collections.Generic.List[double]]::new() $allocs = [System.Collections.Generic.List[long]]::new() try { # Writes the disk cache so every timed run is a disk hit. $null = pwsh -NoProfile -NonInteractive -EncodedCommand $encoded foreach ($i in 1..$NewShellIterations) { $json = pwsh -NoProfile -NonInteractive -EncodedCommand $encoded | Select-Object -Last 1 $r = $json | ConvertFrom-Json if ($r.count -lt 1) { throw "new-shell TabExpansion2 '$Line' returned no matches." } $import.Add($r.importMs) $tab.Add($r.tabMs) $allocs.Add($r.alloc) } } catch { Write-Host (" {0,-14} {1,-34} FAILED: {2}" -f $Fixture, 'new-shell', $_) -ForegroundColor Yellow [pscustomobject]@{ Scenario = "new-shell: TAB '$Line'"; Fixture = $Fixture; Gate = $true; Error = "$_" } return } New-ScenarioResult -Name 'new-shell: Import-Module' -Fixture $Fixture -Gate $true -Times $import -Allocs @() New-ScenarioResult -Name "new-shell: TAB '$Line'" -Fixture $Fixture -Gate $true -Times $tab -Allocs $allocs } $results = [System.Collections.Generic.List[object]]::new() $startLocation = Get-Location try { foreach ($fixture in 'workspace-M', 'workspace-XL') { $dir = Join-Path $FixtureRoot $fixture if (!(Test-Path $dir)) { continue } Set-Location $dir $childMap = Join-Path $dir 'p03\.build.map.ps1' $touch = { (Get-Item $childMap).LastWriteTimeUtc = [datetime]::UtcNow }.GetNewClosure() $list = { qbuild list 6>&1 | Out-String } $results.Add((Measure-Scenario -Name 'list (warm)' -Fixture $fixture -Action $list)) $results.Add((Measure-Scenario -Name 'list (edited)' -Fixture $fixture -Action $list -Setup $touch)) $tabs = [ordered]@{ 'TAB entry' = 'qbuild ' 'TAB nested' = 'qbuild p03.tools.db.' 'TAB param name' = 'qbuild p03.build -' 'TAB param value' = 'qbuild p03.build -Configuration ' 'TAB pkg script' = 'qbuild p03.web ' } foreach ($kv in $tabs.GetEnumerator()) { $results.Add((Measure-Scenario -Name "$($kv.Key) (warm)" -Fixture $fixture -Action { param($l) Measure-Tab $l } -ArgumentList $kv.Value)) } $entryTab = { Measure-Tab 'qbuild ' } $results.Add((Measure-Scenario -Name 'TAB entry (edited)' -Fixture $fixture -Action $entryTab -Setup $touch)) foreach ($r in Measure-NewShell -Fixture $fixture -Directory $dir -Line 'qbuild ') { $results.Add($r) } } $samplesDir = Join-Path $FixtureRoot 'samples' if (Test-Path $samplesDir) { Set-Location $samplesDir $rootMap = Join-Path $samplesDir '.build.map.ps1' $touchRoot = { (Get-Item $rootMap).LastWriteTimeUtc = [datetime]::UtcNow }.GetNewClosure() $results.Add((Measure-Scenario -Name 'list (warm)' -Fixture 'samples' -Action { qbuild list 6>&1 | Out-String })) $results.Add((Measure-Scenario -Name 'TAB entry (warm)' -Fixture 'samples' -Action { Measure-Tab 'qbuild ' })) $results.Add((Measure-Scenario -Name 'TAB entry (edited)' -Fixture 'samples' -Action { Measure-Tab 'qbuild ' } -Setup $touchRoot)) foreach ($r in Measure-NewShell -Fixture 'samples' -Directory $samplesDir -Line 'qbuild ') { $results.Add($r) } } $flatDir = Join-Path $FixtureRoot 'flat-1k' if (Test-Path $flatDir) { Set-Location $flatDir $results.Add((Measure-Scenario -Name 'list (warm)' -Fixture 'flat-1k' -Action { qbuild list 6>&1 | Out-String })) $results.Add((Measure-Scenario -Name 'TAB entry (warm)' -Fixture 'flat-1k' -Action { Measure-Tab 'qbuild ' })) } # Internal helpers: report-only, they may legitimately change shape between versions. if ((Get-Command Get-MapEntries, Get-EntryCompletion -Module ConfigMap -ErrorAction Ignore).Count -eq 2) { $map = [ordered]@{} foreach ($n in 1..1000) { $map["task$n"] = { } } $results.Add((Measure-Scenario -Name 'internal: Get-MapEntries' -Fixture 'in-memory-1k' -Gate $false -Action { @(Get-MapEntries $map 'task500' -language build) })) $results.Add((Measure-Scenario -Name 'internal: Get-EntryCompletion' -Fixture 'in-memory-1k' -Gate $false -Action { @(Get-EntryCompletion -map $map -language build -wordToComplete '') })) } $xlMap = Join-Path $FixtureRoot 'workspace-XL\.build.map.ps1' if (Test-Path $xlMap) { $results.Add((Measure-Scenario -Name 'internal: Import-ConfigMap' -Fixture 'workspace-XL' -Gate $false -Action { Import-ConfigMap -Map $xlMap -LookUp:$false })) } } finally { Set-Location $startLocation if ($tempFixtures) { Remove-Item $tempFixtures -Recurse -Force -ErrorAction Ignore } } $report = [ordered]@{ label = $Label module = $ModulePath pwsh = $PSVersionTable.PSVersion.ToString() os = [System.Runtime.InteropServices.RuntimeInformation]::OSDescription iterations = $Iterations results = $results } if ($OutFile) { $report | ConvertTo-Json -Depth 5 | Set-Content $OutFile } else { $results | Format-Table -AutoSize | Out-String | Write-Host } |