Private/Get-Notebooks.ps1
|
Function Get-Notebooks ($FolderContents, $OriginalPath, $LocalOutputPath, $Format = "SOURCE" ) { $threadJobs = @() $throttleLimit = GetCpuCount $scriptBlock = { param($DatabricksFile, $Format = "SOURCE", $Headers, $uri, $LocalExportPath, $tempLocalExportPath) Try { New-Item -Force -path $tempLocalExportPath -Type File | Out-Null Invoke-RestMethod -Method Get -Uri $uri -Headers $Headers -OutFile $tempLocalExportPath $Response = @() $Response = Get-Content $tempLocalExportPath -Encoding UTF8 $NewResponse = $Response -ne "# Databricks notebook source" Remove-Item $tempLocalExportPath if ($Format -eq "SOURCE") { $ResponseString = ($NewResponse.replace("[^`r]`n", "`n") -Join "`n") } else{ $ResponseString = $NewResponse } if ((Test-Path -PathType Leaf -Path $LocalExportPath) -eq $true) { Set-Content -path $LocalExportPath -value $ResponseString | Out-Null } else { New-Item -force -path $LocalExportPath -value $ResponseString -type file | Out-Null } } Catch { Write-Error $_.ErrorDetails.Message Throw } } if ($Format -eq "DBC") { Set-LocalNotebook $OriginalPath "dbc" $LocalOutputPath "DBC" return } $Headers = GetHeaders $null $FolderContent = $FolderContents.objects ForEach ($Object In $FolderContent) { if ($Object.object_type -eq "DIRECTORY") { $FolderName = ($Object.path).Replace($OriginalPath, "") Write-Verbose "Folder Name: $FolderName!" $SubfolderContents = Get-FolderContents $Object.path Get-Notebooks $SubfolderContents ($Object.path + "/") $LocalOutputPath $Format } elseif ($Object.object_type -eq "NOTEBOOK") { $Notebook = $Object.path $NotebookLanguage = $Object.language Write-Verbose "Calling Writing of $Notebook ($NotebookLanguage)" $DatabricksFileForUrl = Format-DataBricksFileName -DataBricksFile $Notebook $uri = "$global:DatabricksURI/api/2.0/workspace/export?path=" + $DatabricksFileForUrl + "&format=$Format&direct_download=true" switch ($Format) { "SOURCE" { $FileExtentions = @{"PYTHON" = ".py"; "SCALA" = ".scala"; "SQL" = ".sql"; "R" = ".r" } $FileExt = $FileExtentions[$NotebookLanguage] } "HTML" { $FileExt = ".html" } "JUPYTER" { $FileExt = ".ipynb" } "DBC" { $FileExt = ".dbc" } } $LocalExportPath = $Notebook.Replace($ExportPath + "/", "") + $FileExt $tempLocalExportPath = $Notebook.Replace($ExportPath + "/", "") + ".temp" + $FileExt $LocalExportPath = Join-Path $LocalOutputPath $LocalExportPath $tempLocalExportPath = Join-Path $LocalOutputPath $tempLocalExportPath $threadJobs += Start-ThreadJob -Name $Notebook -ScriptBlock $ScriptBlock -ThrottleLimit $throttleLimit -ArgumentList @($Notebook, $Format, $Headers, $uri, $LocalExportPath, $tempLocalExportPath) #Set-LocalNotebook $Notebook $NotebookLanguage $LocalOutputPath $Format } else { Write-Warning "Unknown Type $Object.object_type" } } if ($threadJobs -ne 0) { Wait-Job -Job $threadJobs | Out-Null $toThrow = $null foreach ($threadJob in $threadJobs) { $getState = Get-Job $threadJob.Name | Select-Object -Last 1 if ($getState.State -eq 'Failed') { $toThrow = 1 Write-Host ($threadJob.ChildJobs[0].JobStateInfo.Reason.Message) -ForegroundColor Red } else { Write-Verbose "$($getState.Name) has $($getState.State)" } } if ($null -ne $toThrow) { Write-Error "Oh dear one of the jobs has failed. Check the details of the jobs above." } } } |