Public/Save-NSPFortinetFirmware.ps1
|
function Save-NSPFortinetFirmware { <# .SYNOPSIS Downloads firmware files listed by Get-NSPFortinetFirmware into <Destination>\<Model>\<Version>, several at a time, checks each against Fortinet's published SHA-512, and skips files already there. .DESCRIPTION Each file is saved as <Destination>\<Model>\<Version>\<Name>, for example D:\Firmware\1001F\7.6.7\FGT_1001F-v7.6.7.M-build3704-FORTINET.out. A file with no model in its name goes under the product name instead (D:\Firmware\FortiGate\7.6.7\...). Up to -ThrottleLimit files download at once (default 4). Each download link is single-use and is requested only when a download slot is free. Downloads go to a .partial file and only get their real name once they match the checksum the portal publishes (SHA-512, or MD5 when that is all there is). A file the portal has no checksum for yet (a new patch answers 'N/A' for a while) is checked against the listed size. A file already there with the listed size is skipped without being read, so a library in OneDrive or SharePoint isn't pulled back down to the PC on every run. -Verify hashes those files too and downloads any that don't match again; -Force downloads everything again. One file failing is reported (Status Failed, with Error) and the rest carry on. Progress shows as a bar and as a line per file ([ 3/42] 60F 7.6.7 downloading: ..., and a line with the time, speed and checksum result as each one finishes), with a summary at the end. -InformationAction SilentlyContinue turns the lines off. Run Connect-NSPFortinetSupport first. Returns Model, Version, Name, Status (Downloaded, Skipped, Verified, Failed), Checksum (SHA512, MD5, or Size), Path and Error. .PARAMETER InputObject Files from Get-NSPFortinetFirmware. .PARAMETER Destination The library root that holds the model folders. .PARAMETER ThrottleLimit How many files to download at once. Default 4; 1 downloads one at a time. .PARAMETER Verify Hash files already there against the portal's checksum; download again on a mismatch. .PARAMETER Force Download files that are already there. .EXAMPLE Get-NSPFortinetFirmware -Version 7.6 -Model 1001F -Filter *.out | Save-NSPFortinetFirmware -Destination D:\Firmware .EXAMPLE Get-NSPFortinetFirmware -Version 7.4, 7.6 -Model 60F, 100F -Filter *.out | Save-NSPFortinetFirmware -Destination D:\Firmware -ThrottleLimit 6 #> [CmdletBinding(SupportsShouldProcess)] [OutputType([pscustomobject])] param( [Parameter(Mandatory, ValueFromPipeline)][object[]]$InputObject, [Parameter(Mandatory)][string]$Destination, [ValidateRange(1, 16)][int]$ThrottleLimit = 4, [switch]$Verify, [switch]$Force ) begin { # A line per file ([3/42] ...) unless the caller turned information output off. if (-not $PSBoundParameters.ContainsKey('InformationAction')) { $InformationPreference = 'Continue' } $queue = [Collections.Generic.List[object]]::new() # The portal lists sizes in KB; allow for 1000 or 1024 and rounding. (Repeated in $download.) $sizeMatches = { param([long]$Bytes, [long]$SizeKB) $SizeKB -lt 1 -or ($Bytes -ge $SizeKB * 990 -and $Bytes -le ($SizeKB + 1) * 1034) } # The checksum Fortinet publishes for a file name: Algorithm (SHA512, MD5) and Hash, or $null. # A freshly released patch answers 'N/A' until its checksums are published. $checksumOf = { param($File) if ($File.PSObject.Properties['HasChecksum'] -and -not $File.HasChecksum) { return $null } $published = Invoke-NSPFortinetSupportApi -Path 'ticket/api/tk/download/checksum' -Body @{ fileName = $File.Name } if ([string]$published.shA512Code -match '^[0-9a-fA-F]{128}$') { return [pscustomobject]@{ Algorithm = 'SHA512'; Hash = [string]$published.shA512Code } } if ([string]$published.md5Code -match '^[0-9a-fA-F]{32}$') { return [pscustomobject]@{ Algorithm = 'MD5'; Hash = [string]$published.md5Code } } $null } # One download, run in its own runspace: no module functions or session there, only what is # passed in. The link itself is the authorization, so no token is needed. $download = { param([string]$Url, [string]$UserAgent, [string]$Partial, [string]$Target, [long]$SizeKB, [string]$Algorithm, [string]$Hash) $timer = [Diagnostics.Stopwatch]::StartNew() try { $client = New-Object Net.WebClient $client.Headers['User-Agent'] = $UserAgent try { $client.DownloadFile($Url, $Partial) } finally { $client.Dispose() } $length = (Get-Item -LiteralPath $Partial).Length if (-not ($SizeKB -lt 1 -or ($length -ge $SizeKB * 990 -and $length -le ($SizeKB + 1) * 1034))) { throw "Downloaded $length bytes, but the portal lists $SizeKB KB." } if ($Algorithm -and (Get-FileHash -LiteralPath $Partial -Algorithm $Algorithm).Hash -ne $Hash) { throw "It does not match its published $Algorithm checksum; the download is corrupt and was discarded." } Move-Item -LiteralPath $Partial -Destination $Target -Force [pscustomobject]@{ Error = ''; Length = $length; Seconds = $timer.Elapsed.TotalSeconds } } catch { Remove-Item -LiteralPath $Partial -Force -ErrorAction SilentlyContinue $message = if ($_.Exception.InnerException) { $_.Exception.InnerException.Message } else { $_.Exception.Message } [pscustomobject]@{ Error = $message; Length = 0; Seconds = $timer.Elapsed.TotalSeconds } } } } process { # Collected first, so progress can say "3 of 42". foreach ($file in $InputObject) { if (-not ($file.Product -and $file.Version -and $file.Name -and $file.FilePath)) { throw 'Pipe files from Get-NSPFortinetFirmware (each needs Product, Version, Name and FilePath).' } $queue.Add($file) } } end { $total = $queue.Count $width = "$total".Length $started = Get-Date $stats = @{ Downloaded = 0; Failed = 0; Bytes = 0L; Done = 0 } $fail = { param($Item, [string]$Message) $stats.Failed++ $Item.Result.Status = 'Failed' $Item.Result.Checksum = '' $Item.Result.Error = $Message Write-Warning "$($Item.Prefix) FAILED: $($Item.File.Name): $Message" $Item.Result } # Pass 1, one at a time: what is already there, what -Verify passes, and what to download. $pending = [Collections.Generic.List[object]]::new() $index = 0 foreach ($file in $queue) { $index++ $modelFolder = if ($file.Model) { $file.Model } else { $file.Product } $folder = Join-Path (Join-Path $Destination $modelFolder) $file.Version $target = Join-Path $folder $file.Name $item = [pscustomobject]@{ File = $file Prefix = "[{0,$width}/$total] {1} {2}" -f $index, $modelFolder, $file.Version Folder = $folder Target = $target Checksum = $null Result = [pscustomobject]@{ PSTypeName = 'NSP.FortiGate.FortinetFirmwareDownload'; Model = $file.Model; Version = $file.Version; Name = $file.Name; Status = 'Skipped'; Checksum = 'Size'; Path = $target; Error = '' } } try { if (-not $Force -and (Test-Path -LiteralPath $target) -and (& $sizeMatches (Get-Item -LiteralPath $target).Length $file.SizeKB)) { if (-not $Verify) { Write-Information "$($item.Prefix) already there: $($file.Name)" $item.Result.Checksum = '' $item.Result continue } Write-Information "$($item.Prefix) verifying: $($file.Name)" $item.Checksum = & $checksumOf $file if (-not $item.Checksum -or (Get-FileHash -LiteralPath $target -Algorithm $item.Checksum.Algorithm).Hash -eq $item.Checksum.Hash) { if ($item.Checksum) { $item.Result.Status = 'Verified'; $item.Result.Checksum = $item.Checksum.Algorithm } $item.Result continue } Write-Warning "$target does not match its published $($item.Checksum.Algorithm); downloading it again." } if ($PSCmdlet.ShouldProcess($target, "Download $($file.Name)")) { $pending.Add($item) } } catch { & $fail $item $_.Exception.Message } } if (-not $pending.Count) { if ($total) { Write-Information ("{0} file(s): nothing to download, in {1:hh\:mm\:ss}." -f $total, ((Get-Date) - $started)) } return } # Pass 2: up to $ThrottleLimit downloads at once. This thread keeps the portal calls (token # renewal, checksums, single-use links); the runspaces only download, check and rename. # Windows PowerShell 5.1 (.NET Framework) allows 2 connections per host by default, and the # downloads and the portal calls share support.fortinet.com: two downloads would starve the # rest. Raise it for the process, and for the host's ServicePoint if one already exists. $connections = $ThrottleLimit + 4 if ([Net.ServicePointManager]::DefaultConnectionLimit -lt $connections) { [Net.ServicePointManager]::DefaultConnectionLimit = $connections } $servicePoint = [Net.ServicePointManager]::FindServicePoint([uri]'https://support.fortinet.com/') if ($servicePoint.ConnectionLimit -lt $connections) { $servicePoint.ConnectionLimit = $connections } $pool = [runspacefactory]::CreateRunspacePool(1, $ThrottleLimit) $pool.Open() $active = [Collections.Generic.List[object]]::new() $next = 0 try { while ($next -lt $pending.Count -or $active.Count) { while ($active.Count -lt $ThrottleLimit -and $next -lt $pending.Count) { $item = $pending[$next] $next++ try { if (-not $item.Checksum) { $item.Checksum = & $checksumOf $item.File } $url = (Invoke-NSPFortinetSupportApi -Path 'ticket/api/tk/download/firmwareDownload' -Body @{ productId = $item.File.Product; filePath = $item.File.FilePath }).url $null = New-Item -ItemType Directory -Force -Path $item.Folder $shell = [powershell]::Create() $shell.RunspacePool = $pool $algorithm = if ($item.Checksum) { $item.Checksum.Algorithm } else { '' } $hash = if ($item.Checksum) { $item.Checksum.Hash } else { '' } $null = $shell.AddScript($download).AddArgument($url).AddArgument($script:NSPFortinetSupportSession.UserAgent).AddArgument("$($item.Target).partial").AddArgument($item.Target).AddArgument([long]$item.File.SizeKB).AddArgument($algorithm).AddArgument($hash) Write-Information ("$($item.Prefix) downloading: $($item.File.Name) ({0:N1} MB)" -f ($item.File.SizeKB / 1024)) $active.Add([pscustomobject]@{ Item = $item; Shell = $shell; Handle = $shell.BeginInvoke() }) } catch { $stats.Done++ & $fail $item $_.Exception.Message } } Start-Sleep -Milliseconds 250 foreach ($job in @($active.ToArray() | Where-Object { $_.Handle.IsCompleted })) { $null = $active.Remove($job) $stats.Done++ $item = $job.Item try { $outcome = @($job.Shell.EndInvoke($job.Handle))[0] } catch { $outcome = [pscustomobject]@{ Error = $_.Exception.Message; Length = 0; Seconds = 0 } } finally { $job.Shell.Dispose() } if (-not $outcome) { $outcome = [pscustomobject]@{ Error = 'The download returned nothing.'; Length = 0; Seconds = 0 } } if ($outcome.Error) { & $fail $item $outcome.Error; continue } $seconds = [Math]::Max([double]$outcome.Seconds, 0.1) $check = if ($item.Checksum) { "$($item.Checksum.Algorithm) OK" } else { 'size OK (Fortinet has not published a checksum yet; -Verify later)' } Write-Information ("$($item.Prefix) done: $($item.File.Name) in {0:N0} s ({1:N1} MB/s), $check" -f $seconds, ($outcome.Length / 1MB / $seconds)) $stats.Downloaded++ $stats.Bytes += $outcome.Length $item.Result.Status = 'Downloaded' if ($item.Checksum) { $item.Result.Checksum = $item.Checksum.Algorithm } $item.Result } $percent = [int]($stats.Done * 100 / $pending.Count) Write-Progress -Activity 'Fortinet firmware' -Status "$($stats.Done) of $($pending.Count) downloads done, $($active.Count) in progress" -PercentComplete $percent } } finally { # Ctrl+C or an error: stop what is still running and leave no .partial files behind. foreach ($job in $active) { try { $job.Shell.Stop() } catch { Write-Verbose "Stopping a download: $($_.Exception.Message)" } $job.Shell.Dispose() Remove-Item -LiteralPath "$($job.Item.Target).partial" -Force -ErrorAction SilentlyContinue } $pool.Dispose() Write-Progress -Activity 'Fortinet firmware' -Completed } Write-Information ("{0} file(s): {1} downloaded ({2:N2} GB), {3} failed, {4} already there or not needed, in {5:hh\:mm\:ss}." -f $total, $stats.Downloaded, ($stats.Bytes / 1GB), $stats.Failed, ($total - $stats.Downloaded - $stats.Failed), ((Get-Date) - $started)) } } |