modules/Downloader/RedashMultipart/RedashQueryApi.psm1
|
function Get-RedashContent(){ [CmdletBinding()] param( [string] $domain, [int] $queryId, [hashtable] $filterParams, [string] $fileExtension, [hashtable] $headers, [string] $outFile, [bool] $oldest, [bool] $descending, [int] $limitRows ) #region Internal functions function Download([string] $domain, [string] $queryId, [hashtable] $filterParams, [string] $fileExtension, [hashtable] $authHeaders, [int] $chunk, [string] $source){ $urlInit = '{0}/api/queries/{1}/results' $urlJob = '{0}/api/jobs/{1}' $urlResult = '{0}/api/query_results/{1}.{2}' if ($chunk -eq 1) { Write-Host ("Download {0} audit log..." -f $source) } # Initialize the download try { $urlInit = ($urlInit -f @($domain, $queryId)) $json = (Invoke-RestMethod -Method POST -Uri $urlInit -Headers $authHeaders -Body (ConvertTo-Json $filterParams)) } catch { throw ('Web request to ''{0}'' has failed! (Error: {1})' -f @($urlInit, $_)) } # if there is job executed behind on Redash if ($json.job){ $jobId = $json.job.id $urlJob = ($urlJob -f @($domain, $jobId)) Write-Host ('Waiting for redash {0}' -f $jobId) -NoNewline while ($json.job.status -lt 3) { Start-Sleep -Seconds 1 Write-Host '.' -NoNewline try { $json = Invoke-RestMethod -Method GET -Uri $urlJob -Headers $authHeaders } catch { Write-Host '' throw ('Problems during the redash request: {0}' -f $_) } } if ($json.job.status -gt 3){ Write-Host '' throw ('Redash request failed! {0}' -f $json.job.message) } $urlResult = ($urlResult -f @($domain, $json.job.query_result_id, $fileExtension)) } else { Write-Host ('Download from redash cache {0}' -f $json.query_result.id) -NoNewline $urlResult = ($urlResult -f @($domain, $json.query_result.id, $fileExtension)) } # Job is successfully executed (status: 3) $content = $null Write-Host ("[{0}]" -f @($chunk)) -NoNewline try { $content = (Invoke-WebRequest -Method GET -Uri $urlResult -Headers $authHeaders -TimeoutSec 300) } catch { Write-Host '' throw ('Error while downloading the logs, message: {0}' -f $_.Exception.Message) } Write-Host ' :: Downloaded!' return $content } function Merge-JsonContent([PSCustomObject] $downloaded, [PSCustomObject] $new, [bool] $oldest, [bool] $descending){ if ($descending) { $new.query_result.data.rows = ($new.query_result.data.rows | Sort-Object { [int]$_.id }) } if (-not $downloaded) { $downloaded = $new } else { if ($oldest) { $downloaded.query_result.data.rows += $new.query_result.data.rows } else { $downloaded.query_result.data.rows = $new.query_result.data.rows + $downloaded.query_result.data.rows } } return $downloaded } function Merge-CsvContent([string[]] $downloaded, [string[]] $new, [bool] $oldest, [bool] $descending){ # add the header line if (-not $downloaded) { $downloaded += $new[0] } # exclude the header for all others $new = $new[1..($new.Count-1)] if ($descending) { [array]::Reverse($new) } if ($oldest) { $downloaded += $new } elseif ($this.limitRange -eq $this.RANGE_NEWEST) { $downloaded = $new + $downloaded } return $downloaded } function NextDates([object] $content, [datetime] $from, [datetime] $to, [string] $fileExtension, [bool] $oldest){ $bottom = $null $top = $null switch ($fileExtension) { 'csv' { if (-not $content[1]) { return $from, $from } $top = [datetime]($content[1].Split(',')[4]) $last = $content.Count - 1 while (-not $content[$last]) { $last-- } $bottom = [datetime]($content[$last].Split(',')[4]) } 'json' { if ($content.query_result.data.rows.Count -eq 0) { return $from, $from } $bottom = $content.query_result.data.rows[0].created_at $last = $content.query_result.data.rows.Count - 1 while (-not $content.query_result.data.rows[$last].created_at) { $last-- } $top = $content.query_result.data.rows[$last].created_at } } if ($oldest){ if ($bottom -gt $top ){ $from = $bottom } else { $from = $top} } else { if ($bottom -gt $top ){ $to = $top } else { $to = $bottom } } return @($from, $to) } function NextMessageId([object] $content, [long] $messageId, [string] $fileExtension, [bool] $oldest){ Write-Debug ("fileExtension: {0}, oldest: {1}" -f @($fileExtension, $oldest)) $retValue = $messageId $bottom = $null; $top = $null switch ($fileExtension) { 'csv' { if (-not $content[1]) { return $messageId } $top = [long]($content[1].Split(',')[0]) $last = $content.Count - 1 while (-not $content[$last]) { $last-- } $bottom = [long]($content[$last].Split(',')[0]) } 'json' { if ($content.query_result.data.rows.Count -eq 0) { return $messageId } $bottom = $content.query_result.data.rows[0].id $last = $content.query_result.data.rows.Count - 1 while (-not $content.query_result.data.rows[$last].id) { $last-- } $top = $content.query_result.data.rows[$last].id } } Write-Debug ("Bottom: {0}, top: {1}" -f @($bottom, $top)) if ($oldest){ if ($bottom -gt $top ){ $retValue = $bottom } else { $retValue = $top} } else { if ($bottom -gt $top ){ $retValue = $top } else { $retValue = $bottom } } return $retValue } #endregion $temp = $outFile.split('-') $source = $temp[$temp.Count-1].Split('.')[0] $downloadContent = $null $messageId = 0 $count = 1; $shouldBreak = $false while (-not $shouldBreak) { Write-Debug "Filter message id: $(Convertto-Json $filterParams)" $content = (Download -domain $domain -queryId $queryId $filterParams $filterParams -fileExtension $fileExtension -authHeaders $headers ` -chunk $count -source $source) Write-Debug "Limit rows: $limitRows" switch ($fileExtension) { 'csv' { $content = ($content -split '\r?\n') Write-Debug ("Total rows (csv): {0}" -f ($content.Count-1)) if (($content.Count-1) -lt $limitRows){ $shouldBreak = $true; Write-Debug "Should stop (csv)" } $downloadContent = (Merge-CsvContent $downloadContent $content $oldest $descending) } 'json' { $content = ($content | ConvertFrom-Json) Write-Debug ("Total rows (json): {0}" -f ($content.query_result.data.rows.Count)) if (($content.query_result.data.rows.Count) -lt $limitRows) { $shouldBreak = $true; Write-Debug "Should stop (json)" } $downloadContent = (Merge-JsonContent $downloadContent $content $oldest $descending) } } $count++ $messageId = (NextMessageId $content $messageId $fileExtension $oldest) $filterParams.parameters.messageId = ([string]$messageId) #Write-Debug "Filter message id: $($filterParams.messageId)" } switch ($fileExtension) { 'csv' { $downloadContent | Out-File -Path $outFile } 'json' { $downloadContent.query_result.data.rows | ConvertTo-Json -Depth 100 | Out-File -Path $outFile } } } |