Modules/businessdev.ALbuild.Apps/Public/Get-BcTestQuality.ps1
|
function Get-BcTestQuality { <# .SYNOPSIS Heuristically assesses the QUALITY of AL tests (assertions, arrange/act/assert) -- not their coverage. .DESCRIPTION Code coverage is not test quality: a [Test] method can execute every line of the code under test and still assert nothing, so it would show as "covered" while catching no regression. This scans the AL [Test] methods in a workspace and flags quality smells per test: * NoAssertions - the body contains no assertion (no Assert/LibraryAssert call, no TestField, no asserterror, no guard Error()), so it cannot fail on a wrong result. * Empty - the body has no executable statements. * NoAct - the body asserts but never invokes anything outside the assertion library (a test that only checks constants). It returns a per-test breakdown and a suite summary with a 0-100 quality score (share of tests that actually assert, penalised for empty tests). The heuristics are intentionally conservative and documented; treat the score as a smell detector, not a grade. .PARAMETER WorkspaceRoot AL source root to scan for [Test] methods (skips tooling folders, like Get-BcAlObjectMap). .PARAMETER Path A single .al file to assess instead of a whole workspace. .OUTPUTS PSCustomObject: Summary, Tests[]. #> [CmdletBinding(DefaultParameterSetName = 'Workspace')] [OutputType([PSCustomObject])] param( [Parameter(Mandatory, ParameterSetName = 'Workspace')] [ValidateNotNullOrEmpty()] [string] $WorkspaceRoot, [Parameter(Mandatory, ParameterSetName = 'File')] [ValidateNotNullOrEmpty()] [string] $Path ) # Assertion signals: the Library Assert codeunit, TestField/FieldError, the asserterror keyword, and a guard # Error() (the hand-rolled assert pattern `if <unexpected> then Error(...)`). $assertRx = [regex]::new('(?i)(\b(assert|libraryassert|assertion)\s*\.)|(\.testfield\s*\()|(\.fielderror\s*\()|(\basserterror\b)|(\berror\s*\()') $callRx = [regex]::new('[A-Za-z0-9_"]\s*\(') # a procedure/method call on the line $files = if ($PSCmdlet.ParameterSetName -eq 'File') { , (Get-Item -LiteralPath $Path) } else { $root = (Resolve-Path -LiteralPath $WorkspaceRoot).Path Get-ChildItem -LiteralPath $root -Filter '*.al' -File -Recurse -ErrorAction SilentlyContinue | Where-Object { $_.FullName.Substring($root.Length) -notmatch '[\\/](\.alpackages|\.altemplates|\.snapshots|\.output|output|\.git|node_modules)[\\/]' } } $tests = [System.Collections.Generic.List[object]]::new() foreach ($file in $files) { $lines = @(Get-Content -LiteralPath $file.FullName -ErrorAction SilentlyContinue) if ($lines.Count -eq 0) { continue } $cuName = '' $m = [regex]::Match(($lines -join "`n"), '(?im)^\s*codeunit\s+\d+\s+("(?<q>[^"]+)"|(?<b>[A-Za-z0-9_]+))') if ($m.Success) { $cuName = if ($m.Groups['q'].Success) { $m.Groups['q'].Value } else { $m.Groups['b'].Value } } $isTestAttr = $false for ($i = 0; $i -lt $lines.Count; $i++) { $t = $lines[$i].Trim() if ($t -match '^\[Test\b') { $isTestAttr = $true; continue } $pm = [regex]::Match($t, '(?i)^(local\s+|internal\s+)*procedure\s+(?<n>[A-Za-z0-9_]+)') if (-not $pm.Success) { if ($t -ne '' -and $t -notmatch '^\[') { $isTestAttr = $false }; continue } if (-not $isTestAttr) { continue } $isTestAttr = $false $name = $pm.Groups['n'].Value $startLine = $i + 1 # Capture the body: first begin after the header, to its matching end. $depth = 0; $started = $false; $assertCount = 0; $stmtCount = 0; $callCount = 0 for ($j = $i + 1; $j -lt $lines.Count; $j++) { $code = ($lines[$j] -replace '//.*$', '').Trim() if ($code -eq '') { continue } $lower = $code.ToLowerInvariant() $opens = ([regex]::Matches($lower, '(?<![A-Za-z0-9_])begin(?![A-Za-z0-9_])')).Count $closes = ([regex]::Matches($lower, '(?<![A-Za-z0-9_])end(?![A-Za-z0-9_])')).Count if (-not $started) { if ($opens -gt 0) { $started = $true; $depth = 0 } else { continue } } $isKeyword = $lower -match '^(begin|end|end;|else|do|then|var)$' if ($started -and -not $isKeyword -and $lower -notmatch '^[{}();]+$' -and $lower -notmatch '^(begin|end)\b') { $stmtCount++ if ($assertRx.IsMatch($code)) { $assertCount++ } elseif ($callRx.IsMatch($code)) { $callCount++ } } $depth += $opens - $closes if ($started -and $depth -le 0) { break } } $issues = [System.Collections.Generic.List[string]]::new() if ($stmtCount -eq 0) { $issues.Add('Empty') } if ($assertCount -eq 0 -and $stmtCount -gt 0) { $issues.Add('NoAssertions') } if ($assertCount -gt 0 -and $callCount -eq 0 -and $stmtCount -gt 0) { $issues.Add('NoAct') } $rel = $file.FullName $tests.Add([PSCustomObject]@{ codeunit = $cuName testName = $name filePath = $rel line = $startLine statementCount = $stmtCount assertionCount = $assertCount hasAssertions = $assertCount -gt 0 invokesCode = $callCount -gt 0 issues = @($issues) }) } } $testCount = $tests.Count $withAsserts = @($tests | Where-Object { $_.hasAssertions }).Count $empty = @($tests | Where-Object { $_.issues -contains 'Empty' }).Count $noAssert = @($tests | Where-Object { $_.issues -contains 'NoAssertions' }).Count # Score: share asserting, with empty tests counted as worst-case (0). $score = if ($testCount -gt 0) { [math]::Round((($withAsserts) / $testCount) * 100, 2) } else { 100.0 } $summary = [PSCustomObject]@{ testCount = $testCount testCodeunitCount = @($tests | Select-Object -ExpandProperty codeunit -Unique).Count testsWithAssertions = $withAsserts testsWithoutAssertions = $noAssert emptyTests = $empty assertionDensity = if ($testCount -gt 0) { [math]::Round((($tests | Measure-Object assertionCount -Sum).Sum / $testCount), 2) } else { 0.0 } qualityScore = $score } Write-ALbuildLog -Level Success ("Test quality: {0}/{1} test(s) assert ({2}% score); {3} without assertions, {4} empty." -f $withAsserts, $testCount, $score, $noAssert, $empty) return [PSCustomObject]@{ Summary = $summary; Tests = @($tests) } } |