Includes/Download.provider.webscrapper.ps1
|
<#
Provider : webscrapper Dependencies : none (HTML parsed with regex, relative URLs resolved via [uri]) Downloads the page declared in download.url, extracts every http(s) link (<a> href attributes, absolute or relative resolved against the page), then selects the first link matching the patterns of the download.assets[] list (tested in order). Patterns are tested against the full URL (patterns like 'product=firefox-latest-ssl&os=win64' therefore work on query strings). With version: latest and a named group 'ver' in the matched pattern (e.g. 'soft-(?<ver>\d+\.\d+\.\d+)-x64\.zip'), every matching link is kept and the one carrying the highest captured version wins ; the captured value feeds {{ VERSION }} and {{ TAGNAME }}. Without a 'ver' group, the first matching link wins (page order). WARNING : this provider depends on the structure of the targeted page. Re-check it if the vendor changes its download page. #> <# .SYNOPSIS Extract all http(s) links from an HTML page. .DESCRIPTION Returns absolute URLs (relative hrefs resolved against the page URL). mailto:, javascript: and other schemes are dropped. #> function Get-LinksFromWebPage { [CmdletBinding()] [OutputType([string[]])] Param ( [Parameter(Mandatory = $true)][string]$Url ) Begin { Write-EnterFunction } Process { $html = (Invoke-WebRequest -Uri $Url -UseBasicParsing).Content $links = @() foreach ($m in [regex]::Matches($html, '<a\s[^>]*href\s*=\s*["'']([^"'']+)["'']')) { $raw = $m.Groups[1].Value if ($raw -match '^(javascript|mailto):') { continue } try { # absolute hrefs pass through, relative ones are resolved # against the page url. The base MUST be cast to [uri] : with # two plain strings, PowerShell binds the Uri(string, bool) # overload and silently drops the second argument. $abs = [uri]::new([uri]$Url, $raw).AbsoluteUri } catch { continue } if ($abs -notmatch '^https?://') { continue } $links += $abs } # de-duplicate, keep order return @($links | Select-Object -Unique) } End { Write-LeaveFunction } } <# .SYNOPSIS Select the link of a web page matching the ordered pattern list. .DESCRIPTION Returns @{ Name ; Url ; Spec } where Name is the leaf of the link (or a derived name) and Spec the matched asset spec (for signature-file lookup). #> function Get-AssetFromWebScrapper { [CmdletBinding()] [OutputType([hashtable])] Param ( # Resolved package definition [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][object]$pkgdef, # Template values (uppercase keys, mustache compatible) [Parameter(Mandatory = $false)][hashtable]$mustacheValues = @{} ) Begin { Write-EnterFunction } Process { $pageUrl = Render-MustacheField -value $pkgdef.download.url -mustacheValues $mustacheValues Write-Verbose "Scraping page : $pageUrl" $links = Get-LinksFromWebPage -Url $pageUrl if (-not $links.Count) { Write-Fatal "No http(s) link found on page : $pageUrl" } Write-Verbose "Found $($links.Count) links on page" # version latest + a (?<ver>) capture group in the pattern : the # highest captured version wins instead of the first page link $pickLatest = ($pkgdef.version -eq 'latest') $match = Select-GenericPackageAsset -Names $links -Assets $links -AssetSpecs @($pkgdef.download.assets) -mustacheValues $mustacheValues -PickHighestVersion:$pickLatest if (-not $match) { Write-Fatal "No link of page $pageUrl matches the declared patterns" } # a captured version becomes the effective VERSION / TAGNAME if ($match.CapturedVersion) { if ($mustacheValues) { $mustacheValues.VERSION = $match.CapturedVersion $mustacheValues.TAGNAME = $match.CapturedVersion } } $link = $links[$match.Index] $name = Get-AssetLocalName -Url $link -Codename $pkgdef.codename -Version ($mustacheValues.VERSION ?? $pkgdef.version) return @{ Name = $name Url = $link Spec = $match.Spec } } End { Write-LeaveFunction } } <# .SYNOPSIS Get asset information from a web scraper page without downloading. #> function Get-AssetInfoFromWebScrapper { [CmdletBinding()] [OutputType([array])] Param ( [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][object]$pkgdef, [Parameter(Mandatory = $false)][string]$CachePath, [Parameter(Mandatory = $false)][hashtable]$mustacheValues = @{} ) Begin { Write-EnterFunction $assets = @() } Process { $asset = Get-AssetFromWebScrapper -pkgdef $pkgdef -mustacheValues $mustacheValues if ($asset) { $assets += @{ name = $asset.Name download_url = $asset.Url local_path = "$CachePath/$($asset.Name)" } } } End { Write-LeaveFunction return $assets } } <# .SYNOPSIS Download an asset from a scraped web page. .DESCRIPTION Scrapes the page, selects the matching link, downloads the file (skipping when already cached) and verifies the signature-file when declared (the signature link is looked up on the same page). #> function Download-AssetFromWebScrapper { [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseApprovedVerbs", "", Justification="Download is a more intuitive verb for this function because Get- is too generic.")] [CmdletBinding()] [OutputType([String])] Param ( # Resolved package definition [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][object]$pkgdef, # Destination path (cache directory) [Parameter()][String]$DestinationPath, # Template values (uppercase keys, mustache compatible) [Parameter(Mandatory = $false)][hashtable]$mustacheValues = @{} ) Begin { Write-EnterFunction } Process { # Resolve DestinationPath: parameter > pkgdef.download.destination > temp path if ([string]::IsNullOrEmpty($DestinationPath)) { if ($pkgdef.download.destination) { $DestinationPath = $pkgdef.download.destination } else { $DestinationPath = [io.path]::GetTempPath() } } $asset = Get-AssetFromWebScrapper -pkgdef $pkgdef -mustacheValues $mustacheValues if (-not $asset) { return $null } $destFile = "$DestinationPath/$($asset.Name)" if (Test-FileExist $destFile) { Write-Verbose "Asset $($asset.Name) already exists in $DestinationPath, skipping download" return $destFile } Write-Verbose "Downloading $($asset.Name) from $($asset.Url) to $DestinationPath" $null = Download-File -Url $asset.Url -To $DestinationPath -Filename $asset.Name -PassThru if (-not (Test-FileExist $destFile)) { Write-Fatal "Error downloading asset $($asset.Name)" return $null } # Signature-file verification (companion link from the same page) if ($asset.Spec -and $asset.Spec.'signature-file') { $sigPattern = Render-MustacheField -value $asset.Spec.'signature-file'.pattern -mustacheValues $mustacheValues $pageUrl = Render-MustacheField -value $pkgdef.download.url -mustacheValues $mustacheValues $links = Get-LinksFromWebPage -Url $pageUrl $sigLink = @($links | Where-Object { $_ -match $sigPattern }) | Select-Object -First 1 if (-not $sigLink) { Write-Fatal "Signature link matching '$sigPattern' not found on page $pageUrl" return $null } $sigName = Get-AssetLocalName -Url $sigLink -Codename $pkgdef.codename -Version ($mustacheValues.VERSION ?? $pkgdef.version) $sigFile = "$DestinationPath/$sigName" if (-not (Test-FileExist $sigFile)) { $null = Download-File -Url $sigLink -To $DestinationPath -Filename $sigName -PassThru } if (-not (Test-GenericPackageSignature -AssetPath $destFile -SignaturePath $sigFile -Algorithm $asset.Spec.'signature-file'.algorithm)) { return $null } } return $destFile } End { Write-LeaveFunction } } |