Modules/businessdev.ALbuild.Containers/Public/Get-BcArtifact.ps1
|
function Get-BcArtifact { <# .SYNOPSIS Downloads and caches a Business Central artifact (application + platform packages). .DESCRIPTION Downloads the application package for the given artifact URL and, unless suppressed, the matching platform package (the same URL with the country replaced by 'platform'). Each package is a ZIP; it is extracted into the artifact cache and reused on subsequent calls. .PARAMETER ArtifactUrl The artifact URL (see Find-BcArtifactUrl). .PARAMETER CacheFolder Root cache folder. Defaults to the configured ArtifactCacheFolder. .PARAMETER IncludePlatform Also download/extract the platform package. Default: $true. .PARAMETER Force Re-download even if a cached copy exists. .PARAMETER KeepOnlyLatestBuild Keep only one artifact per Major.Minor.Build in this cache: before downloading, remove the cached versions that share the first three version parts and are older than this one. Intended for pipelines resolving with Select 'Latest', where every run picks a build the cache has never seen and the previous one is dead weight the moment it lands. Pruning happens BEFORE the download so the space is actually available for it - that is the failure this addresses, an agent whose drive filled up mid-extraction. Defaults to the KeepOnlyLatestArtifactBuild configuration setting, so an agent can opt in machine-wide with Set-ALbuildConfig. .EXAMPLE Get-BcArtifact -ArtifactUrl (Find-BcArtifactUrl -Country w1 -Select Latest) .EXAMPLE Get-BcArtifact -ArtifactUrl $url -KeepOnlyLatestBuild Downloads the artifact and leaves only it behind for its Major.Minor.Build. .OUTPUTS PSCustomObject with ApplicationPath, PlatformPath, Version, Country. #> [CmdletBinding()] [OutputType([PSCustomObject])] param( [Parameter(Mandatory, Position = 0, ValueFromPipeline)] [ValidateNotNullOrEmpty()] [string] $ArtifactUrl, [string] $CacheFolder, [bool] $IncludePlatform = $true, [switch] $Force, [switch] $KeepOnlyLatestBuild ) process { if (-not $CacheFolder) { $CacheFolder = Get-ALbuildConfig -Name 'ArtifactCacheFolder' } $info = Get-BcArtifactVersion -ArtifactUrl $ArtifactUrl $pruneSuperseded = [bool]$KeepOnlyLatestBuild if (-not $PSBoundParameters.ContainsKey('KeepOnlyLatestBuild')) { $pruneSuperseded = [bool](Get-ALbuildConfig -Name 'KeepOnlyLatestArtifactBuild') } if ($pruneSuperseded) { # Before the download: the point is to free the space this download needs. $null = Remove-BcSupersededArtifact -CacheFolder $CacheFolder -Type $info.Type -KeepVersion $info.Version.ToString() -Confirm:$false } $downloadAndExtract = { param([string] $url, [string] $targetFolder, [bool] $force, [string] $cacheRoot, [bool] $isPlatform) $completeMarker = Join-Path $targetFolder '.albuild-complete' # Fast path: a fully-extracted artifact (marker present) is reused as-is. if ((Test-Path -LiteralPath $completeMarker) -and -not $force) { return $targetFolder } # A tree somebody ELSE produced is adopted, not re-downloaded. The BC container image writes # its artifact straight into the mounted cache and leaves no marker of ours; judging that only # by our own marker meant fetching multiple GB again for a copy that was already on the disk - # and it is the reason the estate carried the same artifact twice, in two cache roots, for # 1.8 TB. Repair-BcArtifactCache has adopted such folders on the container path all along; # this is the same judgement on the host path. if (-not $force -and (Test-Path -LiteralPath $targetFolder)) { $existing = Test-BcArtifactFolderIntact -Path $targetFolder -IsPlatform:$isPlatform if ($existing.IsIntact) { try { $inventory = Get-BcArtifactInventory -Path $targetFolder Set-Content -LiteralPath $completeMarker -Encoding UTF8 -Value ( [PSCustomObject]@{ completedOn = (Get-Date -Format 'o'); files = $inventory.Files; bytes = $inventory.Bytes; adopted = $true } | ConvertTo-Json -Compress) Write-ALbuildLog -Level Verbose "Adopted the artifact already cached at '$targetFolder' ($($inventory.Files) file(s)) instead of downloading it again." } catch { Write-ALbuildLog -Level Verbose "Could not fingerprint '$targetFolder': $($_.Exception.Message)." } return $targetFolder } } # CONCURRENCY: self-hosted agents share this cache and parallel builds race a brand-new # artifact the first time it appears. The old code extracted IN PLACE into the shared final # folder (and deleted it when the marker was missing), so a parallel build could read a # half-extracted folder - missing apps -> "package ... could not be found" at compile. # Fix: (1) serialise same-host extraction of the SAME artifact with a global mutex (avoids # redundant multi-GB downloads), and (2) publish ATOMICALLY - extract to a private temp folder # on the same volume, then rename it into place. A partial extraction is never visible under # $targetFolder, and a complete folder is never deleted out from under a concurrent reader. # The lock name is computed by Get-ALbuildCacheLockName so that Clear-ALbuildCache locks the # SAME mutex before it removes this folder. Do not inline the hash here again - two copies # drift, and a prune that no longer matches deletes the folder out from under this reader. $mutex = New-Object System.Threading.Mutex($false, (Get-ALbuildCacheLockName -Path $targetFolder)) $held = $false try { try { $held = $mutex.WaitOne([TimeSpan]::FromMinutes(30)) } catch [System.Threading.AbandonedMutexException] { $held = $true } # prior holder crashed; we own it now # Re-check under the lock: another process may have completed the extraction while we waited. if ((Test-Path -LiteralPath $completeMarker) -and -not $force) { return $targetFolder } $tempZip = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.Guid]::NewGuid().ToString() + '.zip') # Stage under a SHORT directory at the cache ROOT - deliberately NOT a long # "<final>.staging-<guid>" suffix beside the DEEP final folder. BC artifacts nest paths # close to the 260-char Windows MAX_PATH limit (e.g. the platform's WebClient assets), so a # long staging suffix at the <type>\<version>\ level pushes the deepest files past MAX_PATH # on agents without long-path support and Expand-Archive fails with a misleading # "Cannot find path ... because it does not exist". A cache-root staging path is SHORTER # than the final path, so anything that fits the final folder also fits staging - while # staying on the SAME volume (rename is atomic). The id is only 8 hex so '\.staging\<id>' # (18 chars) is <= even the SHORTEST possible final suffix ('\onprem\1.0.0.0\w1'), keeping # the guarantee for every artifact; 32 bits is ample against the rare cross-host race the # mutex does not cover (same-host extraction is already serialised above). $stagingRoot = Join-Path $cacheRoot '.staging' New-Item -Path $stagingRoot -ItemType Directory -Force | Out-Null $staging = Join-Path $stagingRoot ([System.Guid]::NewGuid().ToString('N').Substring(0, 8)) try { Write-ALbuildLog "Downloading artifact $url ..." Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction SilentlyContinue $maxDownloadAttempts = 5 $attempt = 0 while ($true) { $attempt++ try { # Save-BcRemoteFile prefers curl (much faster than Invoke-WebRequest for the # large artifact ZIPs) and falls back to Invoke-WebRequest when curl is missing. Save-BcRemoteFile -Url $url -OutFile $tempZip # Validate the download is a COMPLETE zip. A proxy/firewall on a flaky agent can # truncate a large artifact so the transfer "finishes" but the file is short; # the tell is a missing central directory, which otherwise only surfaces later as # Expand-Archive "End of Central Directory record could not be found". Opening the # archive reads that directory (at the end of the file) and throws if it is # missing, so a truncated download is caught here and re-downloaded. $zipCheck = [System.IO.Compression.ZipFile]::OpenRead($tempZip) try { $null = $zipCheck.Entries.Count } finally { $zipCheck.Dispose() } break } catch { if ($attempt -ge $maxDownloadAttempts) { throw "Failed to download a complete artifact '$url' after $maxDownloadAttempts attempt(s): $($_.Exception.Message). If this persists, a proxy/firewall on the agent is likely truncating large downloads (whitelist *.azurefd.net / *.blob.core.windows.net)." } Write-ALbuildLog -Level Warning "Artifact download attempt $attempt/$maxDownloadAttempts failed ($($_.Exception.Message)); retrying in $(5 * $attempt)s..." if (Test-Path -LiteralPath $tempZip) { Remove-Item -LiteralPath $tempZip -Force -ErrorAction SilentlyContinue } Start-Sleep -Seconds (5 * $attempt) } } New-Item -Path $staging -ItemType Directory -Force | Out-Null try { Expand-Archive -LiteralPath $tempZip -DestinationPath $staging -Force } catch { # The usual causes on a freshly set-up agent: a path exceeding the 260-char Windows # MAX_PATH limit (BC artifacts nest deeply - enable long paths with # LongPathsEnabled=1 or use a shorter ArtifactCacheFolder), or the disk running out # of space mid-extraction. Surface that instead of the raw path error. throw "Failed to extract artifact into '$staging': $($_.Exception.Message) " + '(this is typically a Windows MAX_PATH path-length limit - enable long paths ' + "(LongPathsEnabled=1) or shorten the artifact cache folder - or low disk space on the cache drive)." } # Marker goes INTO the staging folder, so it and the content become visible together. # It records a FINGERPRINT (file count + total bytes), not just a timestamp: a # timestamp only proves the extraction finished once, while the fingerprint lets a # later run detect that content went missing afterwards - which is what silently # produced a container without Applications.<country>. $inventory = Get-BcArtifactInventory -Path $staging Set-Content -LiteralPath (Join-Path $staging '.albuild-complete') -Encoding UTF8 -Value ( [PSCustomObject]@{ completedOn = (Get-Date -Format 'o') files = $inventory.Files bytes = $inventory.Bytes } | ConvertTo-Json -Compress) # Publish atomically. If a complete copy already exists (race lost, and not -force), # keep it. Only a stale/partial final (no marker) is replaced - never a complete one, so # a concurrent reader is never left without files. if ((Test-Path -LiteralPath $completeMarker) -and -not $force) { return $targetFolder } $parent = Split-Path -Parent $targetFolder if ($parent -and -not (Test-Path -LiteralPath $parent)) { New-Item -Path $parent -ItemType Directory -Force | Out-Null } if (Test-Path -LiteralPath $targetFolder) { Remove-Item -LiteralPath $targetFolder -Recurse -Force -ErrorAction SilentlyContinue } try { Move-ALbuildDirectory -Path $staging -Destination $targetFolder } catch { # Lost a cross-host race: another agent published it. Use theirs if complete. if (Test-Path -LiteralPath $completeMarker) { return $targetFolder } throw } return $targetFolder } finally { if (Test-Path -LiteralPath $tempZip) { Remove-Item -LiteralPath $tempZip -Force -ErrorAction SilentlyContinue } if (Test-Path -LiteralPath $staging) { Remove-Item -LiteralPath $staging -Recurse -Force -ErrorAction SilentlyContinue } } } finally { if ($held) { $mutex.ReleaseMutex() } $mutex.Dispose() } } $versionFolder = Join-Path $CacheFolder ($info.Type) | Join-Path -ChildPath $info.Version.ToString() # The last argument says which SHAPE to expect: a country artifact and a platform artifact are # validated differently, and adopting one as the other would wave a broken folder through. $appFolder = & $downloadAndExtract $ArtifactUrl (Join-Path $versionFolder $info.Country) ([bool]$Force) $CacheFolder $false $platformFolder = $null if ($IncludePlatform) { $platformUrl = $ArtifactUrl -replace "/$([regex]::Escape($info.Country))$", '/platform' $platformFolder = & $downloadAndExtract $platformUrl (Join-Path $versionFolder 'platform') ([bool]$Force) $CacheFolder $true } # Mark this version as in use, so a concurrent job's prune leaves it alone while we read it. Set-BcArtifactLastUse -VersionFolder $versionFolder return [PSCustomObject]@{ ApplicationPath = $appFolder PlatformPath = $platformFolder Version = $info.Version Country = $info.Country Type = $info.Type } } } |