Public/Get-DatabricksDBFSFile.ps1
|
<#
.SYNOPSIS Get a listing of files and folders within DBFS .DESCRIPTION Get a listing of files and folders within DBFS .PARAMETER BearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER Region Azure Region - must match the URL of your Databricks workspace, example northeurope .PARAMETER DBFSFile The Databricks DBFS file to download .PARAMETER TargetFile Local file to download to .EXAMPLE PS C:\> Get-DatabricksDBFSFile -BearerToken $BearerToken -Region $Region -DBFSFile /test/config.txt -TargetFile ./output/config.txt .NOTES Author: Simon D'Morias / Data Thirst Ltd #> Function Get-DatabricksDBFSFile { param( [parameter(Mandatory = $false)][string]$BearerToken, [parameter(Mandatory = $false)][string]$Region, [parameter(Mandatory = $true)][string]$DBFSFile, [parameter(Mandatory = $true)][string]$TargetFile ) if ($PSVersionTable.PSVersion.Major -lt 6){ Throw "This command requires PowerShell 6 or higher" } [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = GetHeaders $PSBoundParameters $size = 1048576 $body = @{'path' = $DBFSFile} $chunkStart = 0 [byte[]]$finalFile = $null $chunkEnd = $chunkStart + $size $bytesRead = $size while ($bytesRead -eq $size){ $body['offset'] = $chunkStart $body['length'] = $size $BodyText = $Body | ConvertTo-Json -Depth 10 $chunk = Invoke-RestMethod -Uri "$global:DatabricksURI/api/2.0/dbfs/read" -Body $BodyText -Method 'GET' -Headers $Headers $finalFile += [Convert]::FromBase64String($chunk.data) $chunkStart = $chunkEnd + 1 $chunkEnd = $chunkStart + $size $bytesRead = $chunk.bytes_read } Set-Content -Path $TargetFile -Value $finalFile -AsByteStream } |