businessdev.ALbuild

2.34.27473

Modules/businessdev.ALbuild.RuntimePackages/Private/Invoke-BcSourceEmit.ps1

                                function Invoke-BcSourceEmit {

    <#

    .SYNOPSIS

        Drives Microsoft's AL compiler: manifest, symbol references, compilation, emit.

    .DESCRIPTION

        Late-bound throughout. The BC assemblies are version-specific and live wherever the artifact

        cache put them, so nothing here may be resolved at parse time.

        Arguments are bound BY PARAMETER NAME, not by position. Compilation.Create takes twelve

        parameters and CompilationOptions twenty-one, and BC majors have added to both. Binding by name

        means an unknown parameter falls back to its default instead of shifting everything after it.

    .PARAMETER Compiler

        An Import-BcAlCompiler result.

    .OUTPUTS

        PSCustomObject with Path, Bytes and Objects.

    #>

    [CmdletBinding()]

    [OutputType([PSCustomObject])]

    param(

        [Parameter(Mandatory)] [PSCustomObject] $Compiler,

        [Parameter(Mandatory)] [string] $ProjectFolder,

        [Parameter(Mandatory)] [string] $OutputFolder,

        [string[]] $SymbolFolder = @(),

        [ValidateSet('Runtime', 'App')] [string] $Kind = 'Runtime',

        [switch] $IncludeAlCode,

        [string] $FileName

    )

    $assembly = $Compiler.Assembly

    $types = @()

    try { $types = $assembly.GetTypes() } catch { $types = @($_.Exception.Types | Where-Object { $_ }) }

    function Get-CompilerType([string] $name) {

        $t = $types | Where-Object { $_ -and $_.Name -eq $name } | Select-Object -First 1

        if (-not $t) { throw "This BC version's AL compiler does not expose '$name'. Report the BC version; the binding needs an additional shape." }

        return $t

    }

    # PowerShell unrolls an empty collection on return, which turns a fresh List into $null. The

    # leading comma is what stops it.

    function New-TypedList([Type] $elementType) {

        return , [Activator]::CreateInstance([System.Collections.Generic.List`1].MakeGenericType(@($elementType)))

    }

    function Get-DefaultFor([Type] $t) {

        if ($t.IsValueType) { return [Activator]::CreateInstance($t) }

        return $null

    }

    # Builds a reflection argument list by PARAMETER NAME from a name->value map.

    #

    # Deliberately not a `switch` expression: switch writes its result to the pipeline, and PowerShell

    # UNROLLS collections on the way. One of these values is an ImmutableArray<Guid> that is often in

    # its default state, and enumerating that throws "This operation cannot be performed on a default

    # instance of ImmutableArray<T>" - from inside the argument list, naming nothing that led there.

    function Build-Arguments([System.Reflection.MethodBase] $method, [hashtable] $map) {

        $result = [object[]]::new($method.GetParameters().Count)

        $i = 0

        foreach ($p in $method.GetParameters()) {

            if ($map.ContainsKey($p.Name)) { $result[$i] = $map[$p.Name] }

            else { $result[$i] = Get-DefaultFor $p.ParameterType }

            $i++

        }

        return , $result

    }

    $projectFull = (Resolve-Path -LiteralPath $ProjectFolder).Path

    $manifestPath = Join-Path $projectFull 'app.json'

    # --- manifest, read by the compiler's own reader ------------------------------------------------

    # Hardcoding these was tried and was wrong twice over: the ID ranges came out as the generic

    # extension range so every object of a real project was reported out of range, and the compilation

    # target ignored what app.json actually says.

    $projectManifestType = Get-CompilerType 'ProjectManifest'

    $readFromString = $projectManifestType.GetMethods('Public,Static') |

        Where-Object { $_.Name -eq 'ReadFromString' -and $_.GetParameters().Count -eq 3 } | Select-Object -First 1

    if (-not $readFromString) { throw "This BC version's ProjectManifest does not expose the expected ReadFromString." }

    $diagnosticType = $readFromString.GetParameters()[2].ParameterType.GetGenericArguments()[0]

    $manifestDiagnostics = New-TypedList $diagnosticType

    $projectManifest = $readFromString.Invoke($null, @(

            [string]$manifestPath, [string][System.IO.File]::ReadAllText($manifestPath), $manifestDiagnostics))

    $manifest = $projectManifestType.GetProperty('AppManifest').GetValue($projectManifest)

    if (-not $manifest) {

        throw "'$manifestPath' could not be read as an AL manifest. Check it against a project that builds - id, publisher, version and idRanges are all required."

    }

    # --- symbol references --------------------------------------------------------------------------

    $packageFolders = New-TypedList ([string])

    $defaultPackages = Join-Path $projectFull '.alpackages'

    if (Test-Path -LiteralPath $defaultPackages) { $packageFolders.Add($defaultPackages) }

    foreach ($folder in $SymbolFolder) {

        if ([string]::IsNullOrWhiteSpace($folder)) { continue }

        if (-not (Test-Path -LiteralPath $folder)) { continue }

        $full = (Resolve-Path -LiteralPath $folder).Path

        if (-not ($packageFolders -contains $full)) { $packageFolders.Add($full) }

    }

    Write-ALbuildLog -Level Verbose "Symbol folders: $($packageFolders -join '; ')"

    # RelativeFileSystem, not FileSystem: relative resource paths (control add-in JS/CSS) otherwise

    # resolve against the PROCESS working directory, which produced 93 phantom AL0572 errors.

    $fileSystem = (Get-CompilerType 'RelativeFileSystem').GetConstructors()[0].Invoke(@($projectFull))

    # The loader takes MANY directories from BC 18 onwards and exactly ONE before that:

    #   BC17: FileSystemSymbolReferenceLoader(IFileSystem, String      directory,   IDocumentationProviderFactory)

    #   BC28: FileSystemSymbolReferenceLoader(IFileSystem, IEnumerable directories, IDocumentationProviderFactory)

    # Handing a list to the older shape fails with 'List`1[System.String] cannot be converted to

    # System.String' from deep inside a reflection call, naming nothing that leads back to here.

    #

    # The older compilers have no chaining or composite loader either - MemoryCached wraps a single

    # next loader, nothing aggregates - so the one directory has to hold everything. Every symbol

    # package is therefore hard-linked into one folder: instant, and it costs no disk because the links

    # share the original file's data. A copy is the fallback when the folders sit on different volumes.

    $symbolLoaderType = Get-CompilerType 'FileSystemSymbolReferenceLoader'

    $symbolLoaderCtor = @($symbolLoaderType.GetConstructors())[0]

    $directoryParameter = $symbolLoaderCtor.GetParameters()[1]

    # Typed on purpose. Reflection does NOT unwrap a PSObject, and PowerShell wraps values freely -

    # '.GetType()' then still reports 'System.String', because PSObject is transparent to it. The

    # failure reads 'PSObject cannot be converted to System.String' while every diagnostic insists the

    # value IS a string. A typed variable settles it.

    [string] $singleFolder = ''

    if ($directoryParameter.ParameterType -eq [string]) {

        $mergedFolder = Join-Path ([System.IO.Path]::GetTempPath()) ("albsym-" + [guid]::NewGuid().ToString('N').Substring(0, 10))

        New-Item -Path $mergedFolder -ItemType Directory -Force | Out-Null

        $linked = 0

        foreach ($folder in $packageFolders) {

            foreach ($package in @(Get-ChildItem -LiteralPath $folder -Filter '*.app' -File -ErrorAction SilentlyContinue)) {

                $target = Join-Path $mergedFolder $package.Name

                if (Test-Path -LiteralPath $target) { continue }

                try { New-Item -ItemType HardLink -Path $target -Value $package.FullName -ErrorAction Stop | Out-Null }

                catch { Copy-Item -LiteralPath $package.FullName -Destination $target -Force }

                $linked++

            }

        }

        Write-ALbuildLog -Level Verbose ("This compiler accepts a single symbol directory; $linked package(s) " +

            "gathered into '$mergedFolder'.")

        $singleFolder = $mergedFolder

    }

    # MemoryCached in front of the file-system loader: each symbol package is read once, not once per

    # reference that mentions it. Written as plain statements rather than a pipeline so the arguments

    # reach reflection as themselves.

    $symbolLoader = if ($directoryParameter.ParameterType -eq [string]) {

        $symbolLoaderCtor.Invoke(@($fileSystem, $singleFolder, $null))

    }

    else {

        $symbolLoaderCtor.Invoke(@($fileSystem, $packageFolders, $null))

    }

    $memoryCachedCtor = @((Get-CompilerType 'MemoryCachedSymbolReferenceLoader').GetConstructors() |

            Where-Object { $_.GetParameters().Count -eq 1 })[0]

    $loader = $memoryCachedCtor.Invoke(@($symbolLoader))

    $specificationType = Get-CompilerType 'SymbolReferenceSpecification'

    $references = New-TypedList $specificationType

    $manifestType = $manifest.GetType()

    # Taking the references FROM the manifest rather than re-deriving them from app.json means

    # dependency shapes we have never seen still resolve.

    $declared = $manifestType.GetProperty('DependencyReferences').GetValue($manifest)

    if ($declared) { foreach ($d in $declared) { if ($d) { $references.Add($d) } } }

    foreach ($implied in 'AppReference', 'PlatformReference', 'TestReference') {

        $value = $manifestType.GetProperty($implied).GetValue($manifest)

        if ($value) { $references.Add($value) }

    }

    Write-ALbuildLog -Level Verbose "Symbol references: $($references.Count)"

    $resolverFactory = [Activator]::CreateInstance((Get-CompilerType 'NullDotNetResolverFactory'))

    # ReferenceManager is the one internal type in this chain; constructing it beats reimplementing

    # reference resolution.

    $referenceManager = (Get-CompilerType 'ReferenceManager').GetConstructors('NonPublic,Public,Instance')[0].Invoke(

        @($references, $loader, $resolverFactory, $true))

    # --- options ------------------------------------------------------------------------------------

    $optionsType = Get-CompilerType 'CompilationOptions'

    $optionsCtor = $optionsType.GetConstructors() | Sort-Object { $_.GetParameters().Count } -Descending | Select-Object -First 1

    $optionArgs = Build-Arguments $optionsCtor @{

        warningLevel          = 4

        concurrentBuild       = $true

        # Without Code the emit writes a package index naming code files it never wrote.

        generateOptions       = [Enum]::Parse((Get-CompilerType 'CompilationGenerationOptions'), 'All')

        codeGenerationOptions = [Enum]::Parse((Get-CompilerType 'CodeGenerationOption'), 'Text')

        idSpaces              = (Get-CompilerType 'IdSpaces').GetProperty('ExtensionDevelopmentIdSpace').GetValue($null)

    }

    $options = $optionsType.GetMethod('WithManifestOptions').Invoke($optionsCtor.Invoke($optionArgs), @($manifest))

    # --- syntax trees -------------------------------------------------------------------------------

    $syntaxTreeType = Get-CompilerType 'SyntaxTree'

    $parse = $syntaxTreeType.GetMethods('Public,Static') |

        Where-Object { $_.Name -eq 'ParseObjectText' -and $_.GetParameters().Count -eq 5 } | Select-Object -First 1

    if (-not $parse) { throw "This BC version's compiler does not expose the expected ParseObjectText overload." }

    # THE PREPROCESSOR SYMBOLS. Without them every '#if BC25' is parsed as if the symbol were undefined,

    # so version-guarded code is included or excluded exactly the wrong way round - and the result is not

    # an error but a package built from the wrong branch.

    #

    # Measured on run 27447: 49 of 60 packages failed that the container route had built from the same

    # source. Print Agent guards a block with '#if not BC25' (PrAPrinterAccessPerm.Codeunit.al:320); it

    # references the 'User Group Member' table Microsoft removed in BC25. With BC25 defined - as the

    # stamped manifest defines it - the block is skipped. With no symbols at all it was compiled, and

    # AL0433/AL0185/AL0171 followed. The 11 that did compile are no better off: they were built from

    # branches meant for a different platform, silently.

    #

    # 'includeRuntimeVersion' is passed so the parse also honours the manifest's runtime version; the

    # default is 17.0 regardless of the target. CompilationOptions already went through

    # WithManifestOptions - the parse side was simply missed.

    $parseOptionsType = Get-CompilerType 'ParseOptions'

    $parseOptions = $parseOptionsType.GetProperty('Default').GetValue($null)

    $withManifestParse = $parseOptionsType.GetMethod('WithManifestOptions')

    if (-not $withManifestParse) { throw "This BC version's ParseOptions does not expose WithManifestOptions." }

    $parseOptions = $withManifestParse.Invoke($parseOptions, @($manifest, $true))

    # Read from the MANIFEST, not from ParseOptions: ParseOptions exposes only RuntimeVersion and

    # DocumentationMode as properties - the symbols sit in a backing field. A first attempt looked for a

    # 'Preprocessor' property on the options and therefore reported '(none)' in every case, including

    # the ones that were working. A diagnostic that always says the same thing is worse than none.

    $symbolNames = @()

    $manifestSymbols = $manifestType.GetProperty('PreprocessorSymbols')

    if ($manifestSymbols) { $symbolNames = @($manifestSymbols.GetValue($manifest)) }

    Write-ALbuildLog "Preprocessor symbols: $(if ($symbolNames.Count) { $symbolNames -join ', ' } else { '(none in the manifest)' })"

    $ignored = @('.alpackages', '.altestrunner', '.snapshots', '.vscode', '.git', 'bin', 'obj')

    $trees = New-TypedList $syntaxTreeType

    $sourceCount = 0

    foreach ($file in (Get-ChildItem -LiteralPath $projectFull -Filter '*.al' -File -Recurse | Sort-Object FullName)) {

        $relative = $file.FullName.Substring($projectFull.Length).TrimStart([System.IO.Path]::DirectorySeparatorChar)

        $segments = $relative.Split([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar)

        # Only FOLDER names disqualify - a file may legitimately be called 'bin.al'.

        $inIgnored = $false

        for ($i = 0; $i -lt $segments.Count - 1; $i++) { if ($ignored -contains $segments[$i]) { $inIgnored = $true; break } }

        if ($inIgnored) { continue }

        $trees.Add($parse.Invoke($null, @(

                    [string][System.IO.File]::ReadAllText($file.FullName), [string]$file.FullName,

                    [System.Text.Encoding]::UTF8, $parseOptions, [System.Threading.CancellationToken]::None)))

        $sourceCount++

    }

    Write-ALbuildLog -Level Verbose "Source files: $sourceCount"

    # --- compilation --------------------------------------------------------------------------------

    # 'AppAlternateIds' exists from BC 18 onwards; on BC 17 the property is simply absent and reading it

    # straight cost a bare 'You cannot call a method on a null-valued expression', pointing at a

    # hashtable that spans a dozen lines.

    #

    # Read with a plain assignment, NOT through a helper function: a function returns through the

    # pipeline, and PowerShell unrolls collections on the way - this value is an ImmutableArray<Guid>

    # that is usually in its default state, and enumerating THAT throws 'This operation cannot be

    # performed on a default instance of ImmutableArray<T>'. The comment on Build-Arguments below warns

    # about exactly this; the first attempt at the BC 17 fix walked into it anyway and took six tests

    # of the modern path down with it.

    $alternateIdsProperty = $manifestType.GetProperty('AppAlternateIds')

    $alternateIds = $null

    if ($alternateIdsProperty) { $alternateIds = $alternateIdsProperty.GetValue($manifest) }

    $compilationType = Get-CompilerType 'Compilation'

    $create = $compilationType.GetMethod('Create')

    $createArgs = Build-Arguments $create @{

        moduleName            = $manifestType.GetProperty('AppName').GetValue($manifest)

        publisher             = $manifestType.GetProperty('AppPublisher').GetValue($manifest)

        version               = $manifestType.GetProperty('AppVersion').GetValue($manifest)

        appId                 = $manifestType.GetProperty('AppId').GetValue($manifest)

        internalsVisibleTo    = New-TypedList $specificationType

        syntaxTrees           = $trees

        options               = $options

        fileSystem            = $fileSystem

        dotNetResolverFactory = $resolverFactory

        referenceManager      = $referenceManager

        skipInvalidModules    = $true

        alternateIds          = $alternateIds

    }

    $compilation = $create.Invoke($null, $createArgs)

    $diagnostics = @($compilationType.GetMethod('GetDiagnostics').Invoke(

            $compilation, @([System.Threading.CancellationToken]::None)))

    $errors = @($diagnostics | Where-Object { "$($_.Severity)" -eq 'Error' })

    foreach ($warning in ($diagnostics | Where-Object { "$($_.Severity)" -eq 'Warning' })) {

        Write-ALbuildLog -Level Verbose "  $($warning.Id): $($warning.GetMessage())"

    }

    if ($errors.Count -gt 0) {

        # The whole list, not a count: a caller reading the log must be able to fix it without rerunning.

        $detail = ($errors | Select-Object -First 20 | ForEach-Object {

                $span = $_.Location.GetLineSpan()

                "  $($_.Id) $($span.Path):$($span.StartLinePosition.Line + 1) - $($_.GetMessage())"

            }) -join [Environment]::NewLine

        $more = if ($errors.Count -gt 20) { "$([Environment]::NewLine)  ... and $($errors.Count - 20) more" } else { '' }

        throw "The project did not compile: $($errors.Count) error(s).$([Environment]::NewLine)$detail$more"

    }

    # --- emit ---------------------------------------------------------------------------------------

    $emitOptionsType = Get-CompilerType 'EmitOptions'

    $extensionValues = (Get-CompilerType 'ExtensionEmitValues').GetConstructors()[0].Invoke(@(

            $manifestType.GetProperty('AppId').GetValue($manifest),

            $manifestType.GetProperty('AppVersion').GetValue($manifest)))

    $emitCtor = $emitOptionsType.GetConstructors() | Sort-Object { $_.GetParameters().Count } -Descending | Select-Object -First 1

    $emitArgs = Build-Arguments $emitCtor @{

        runtimeMetadataVersion = $emitOptionsType.GetField('CurrentMetadataVersion', 'Public,Static').GetValue($null)

        extensionEmitValues    = $extensionValues

        # MUST stay false. With concurrent emit the package outputter switches to its FilePackageWriter

        # and writes the emitted code to a FOLDER BESIDE THE PROJECT instead of into the package. The

        # package is then silently incomplete and the server rejects it with "Specified part does not

        # exist in the package" - naming neither the cause nor the stray files left in the source tree.

        concurrentEmit         = $false

    }

    $emitOptions = $emitCtor.Invoke($emitArgs)

    if (-not (Test-Path -LiteralPath $OutputFolder)) { New-Item -Path $OutputFolder -ItemType Directory -Force | Out-Null }

    if (-not $FileName) {

        $invalid = [System.IO.Path]::GetInvalidFileNameChars()

        $clean = {

            param($value)

            (($value.ToCharArray() | ForEach-Object { if ($invalid -contains $_) { '_' } else { $_ } }) -join '').Trim()

        }

        $suffix = if ($Kind -eq 'Runtime') { '.runtime.app' } else { '.app' }

        # BC's own naming. Naming it after the project folder produces 'app.app' for the usual

        # app/ + test/ layout, and two projects emitted into one folder overwrite each other.

        $FileName = "$(& $clean $manifestType.GetProperty('AppPublisher').GetValue($manifest))_" +

        "$(& $clean $manifestType.GetProperty('AppName').GetValue($manifest))_" +

        "$(& $clean "$($manifestType.GetProperty('AppVersion').GetValue($manifest))")$suffix"

    }

    $outputPath = Join-Path $OutputFolder $FileName

    $stream = [System.IO.File]::Create($outputPath)

    try {

        if ($Kind -eq 'Runtime') {

            $runtimeOptions = (Get-CompilerType 'RuntimeBuildOptions').GetConstructors()[0].Invoke(@([bool]$IncludeAlCode))

            $outputterArgs = @($manifest, $stream, $compilation, $emitOptions, $runtimeOptions, $projectFull)

            $outputterType = Get-CompilerType 'RuntimePackageModuleOutputter'

        }

        else {

            $outputterArgs = @($manifest, $stream, $compilation, $emitOptions, $projectFull)

            $outputterType = Get-CompilerType 'PackageModuleOutputter'

        }

        # Match on the argument count AND on the second parameter being a Stream: the runtime outputter

        # also has overloads taking a package writer and an original package, and picking one by

        # position alone would bind silently to the wrong shape.

        $outputterCtor = $outputterType.GetConstructors() | Where-Object {

            $_.GetParameters().Count -eq $outputterArgs.Count -and $_.GetParameters()[1].ParameterType.Name -eq 'Stream'

        } | Select-Object -First 1

        if (-not $outputterCtor) { throw "This BC version's $($outputterType.Name) has no constructor taking a stream with $($outputterArgs.Count) arguments." }

        $outputter = $outputterCtor.Invoke($outputterArgs)

        $emitResult = $compilationType.GetMethod('Emit').Invoke(

            $compilation, @($emitOptions, $outputter, [System.Threading.CancellationToken]::None))

        $emitErrors = @($emitResult.Diagnostics | Where-Object { "$($_.Severity)" -eq 'Error' })

        if (-not $emitResult.Success -or $emitErrors.Count -gt 0) {

            $detail = ($emitErrors | Select-Object -First 5 | ForEach-Object { "  $($_.Id): $($_.GetMessage())" }) -join [Environment]::NewLine

            throw "The compiler could not produce the package.$([Environment]::NewLine)$detail"

        }

        # The outputter finishes the package on Dispose. Disposing it BEFORE the stream is not

        # stylistic: the package is written during teardown, and a closed stream loses it.

        if ($outputter -is [System.IDisposable]) { $outputter.Dispose() }

    }

    finally {

        $stream.Dispose()

    }

    [PSCustomObject]@{

        Path    = $outputPath

        Bytes   = (Get-Item -LiteralPath $outputPath).Length

        Objects = $sourceCount

    }

}