Public/Datascripts.ps1
|
$DataTypeConfig = @{ MatchPattern = '\.(json)|(csv)|(xlsx?)' BadExtensionErrorMessage = 'The file you attempted to upload has an invalid type. Valid types are JSON, CSV, and Excel files.' FileNotFoundError = 'The file you specified cannot be found.' } ## Fields that older WS responses returned but the /api/ endpoints may not. ## Injected as $null so downstream consumers (Extensions.ps1, on-disk JSON sidecars) don't ## see "property not found" failures. Confirm against a live 6.6+ server before pruning. $Script:TMDatascriptPlaceholderFields = @('useWithAssetActions', 'unlink', 'shared', 'createdBy') Function Get-TMDatascript { <# .SYNOPSIS Gets DataScripts from TransitionManager. .DESCRIPTION Retrieves DataScripts and optionally filters them by name or provider. The function can also save the script code locally while retrieving the DataScript metadata. .PARAMETER TMSession A TMSession object or session name to use for the request. Defaults to `'Default'`. .PARAMETER Name The DataScript name to retrieve. .PARAMETER ProviderName One or more Provider names used to filter the returned DataScripts. .PARAMETER ResetIDs Switch indicating that identifiers on the returned objects should be refreshed or normalized during retrieval. .PARAMETER SaveCodePath A local folder path where the DataScript code should be saved. .PARAMETER Passthru Switch indicating that the retrieved DataScript objects should be returned. .EXAMPLE Get-TMDatascript -Name 'Load CMDB Data' Retrieves the DataScript named `Load CMDB Data`. .EXAMPLE Get-TMDatascript -SaveCodePath '.\Datascripts' -Passthru Retrieves DataScripts, saves their code locally, and returns the objects. .NOTES Use `-SaveCodePath` when you want both metadata and local script files. #> [alias("Get-TMETLScript")] [CmdletBinding(DefaultParameterSetName = 'Default')] param( [Parameter(Mandatory = $false)] [PSObject]$TMSession = 'Default', [Parameter(Mandatory = $false)] [String]$Name, [Parameter(Mandatory = $false)] [string]$Version, [Parameter(Mandatory = $false)] [String[]]$ProviderName, [Parameter(Mandatory = $false)] [Switch]$ResetIDs, [Parameter(Mandatory = $false)] [Switch]$Full, [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')] [String]$SaveCodePath, [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')] [Switch]$Passthru ) $TMSession = Get-TMSession $TMSession ## List all DataScripts. Each list row lacks etlSourceCode (and a few other fields); ## per-id hydration below pulls the full record. $Result = @(Invoke-TMRestMethod -Api 'dataScript' -Method Get -TMSession $TMSession) if ($Result.Count -eq 0) { if ($PSBoundParameters.ContainsKey('Name')) { throw "Datascript '$Name' does not exist in project '$($TMSession.UserContext.Project.Name)'" } if ($PSBoundParameters.ContainsKey('ProviderName')) { throw "No Datascript(s) found with given ProviderName: '$ProviderName'" } return $null } ## Filter on list-row fields BEFORE the per-id hydration so we don't make extra round-trips. if ($PSBoundParameters.ContainsKey('Version')) { $Result = @($Result | Where-Object semVer -eq $Version) if (-not $Result.Count) { return $null } } if ($ProviderName) { $Result = @($Result | Where-Object { $_.provider.name -in $ProviderName }) } elseif ($Name) { $Result = @($Result | Where-Object { $_.name -eq $Name }) } if (-not $Result.Count) { return $null } ## Hydrate each remaining row with the full record (incl. etlSourceCode, formSpec, formModel) for ($i = 0; $i -lt $Result.Count; $i++) { if ($Full.IsPresent -or -not [String]::IsNullOrEmpty($SaveCodePath)) { try { $Hydrated = Invoke-TMRestMethod -Api "dataScript/$($Result[$i].id)" -Method Get -TMSession $TMSession } catch { return $_ } } else { $Hydrated = $Result[$i] } foreach ($field in $Script:TMDatascriptPlaceholderFields) { if ($Hydrated.PSObject.Properties.Name -notcontains $field) { Add-Member -InputObject $Hydrated -NotePropertyName $field -NotePropertyValue $null -Force } } $Result[$i] = $Hydrated } if ($ResetIDs) { for ($i = 0; $i -lt $Result.Count; $i++) { $Result[$i].id = $null $Result[$i].provider.id = $null } } ## Save the Code Files to a folder if ($SaveCodePath) { ## Save Each of the Script Source Data foreach ($Item in $Result) { ## Get a FileName safe version of the Provider Name $SafeProviderName = Get-FilenameSafeString -String $Item.provider.name $SafeScriptName = Get-FilenameSafeString -String $Item.name $Item.semVer = ($Item.semVer.trim().Length -gt 0) ? $Item.semVer.trim() : '1.0.0' $SafeScriptName += " - $($Item.semVer)" ## Create the Provider Action Folder path $ProviderPath = Join-Path $SaveCodePath $SafeProviderName Test-FolderPath -FolderPath $ProviderPath ## Create a File name for the Datascript $ProviderScriptConfig = Join-Path $ProviderPath ($SafeScriptName + '.json') $ProviderScriptPath = Join-Path $ProviderPath ($SafeScriptName + '.groovy') ## Copy the code from the Datascript $DatascriptCode = $Item.etlSourceCode ? $Item.etlSourceCode.toString() : "" ## Remove the ETL Code from the Item, so the JSON sidecar stays a config file $Item.PSObject.Properties.Remove('etlSourceCode') ## Null out server-specific fields so the .json sidecar is portable across projects/servers. ## Per TME-459: do NOT remove fields; only null the ones that are server-bound. $dateCreated = $Item.dateCreated $lastUpdated = $Item.lastUpdated $Item.dateCreated = $null $Item.lastUpdated = $null if ($Item.PSObject.Properties.Name -contains 'createdBy') { $Item.createdBy = $null } ## Start Writing the Content of the Script (Force to overwrite any existing files) Set-Content -Path $ProviderScriptConfig -Force -Value (ConvertTo-Json -InputObject $Item -Depth 5) Get-Item -Path $ProviderScriptConfig | ForEach-Object { $_.CreationTime = ($dateCreated ?? (Get-Date)) $_.LastWriteTime = ($lastUpdated ?? (Get-Date)) } Set-Content -Path $ProviderScriptPath -Force -Value $DatascriptCode Get-Item -Path $ProviderScriptPath | ForEach-Object { $_.CreationTime = ($dateCreated ?? (Get-Date)) $_.LastWriteTime = ($lastUpdated ?? (Get-Date)) } } } if ($Passthru -or !$SaveCodePath) { return $Result } } Function New-TMDatascript { <# .SYNOPSIS Creates or updates a DataScript in TransitionManager. .DESCRIPTION Uses the supplied DataScript definition object to create a new TM DataScript. When `-Update` is provided, an existing DataScript can be updated instead of only creating a new record. .PARAMETER TMSession A TMSession object or session name to use for the request. Defaults to `'Default'`. .PARAMETER Datascript The DataScript definition object to create or update. .PARAMETER Update Switch indicating that an existing DataScript should be updated. .EXAMPLE $datascript = Read-TMDatascriptFile -Path '.\Datascripts\LoadCmdb.ps1' New-TMDatascript -Datascript $datascript -Update Creates or updates a DataScript from a local script definition. #> [alias("New-TMETLScript")] param( [Parameter(Mandatory = $false)][PSObject]$TMSession = 'Default', [Alias("ETLScript")] [Parameter(Mandatory = $true)][PSObject]$Datascript, [Parameter(Mandatory = $false)][switch]$Update, [Parameter(Mandatory = $false)][switch]$PassThru ) $TMSession = Get-TMSession $TMSession $DatascriptName = $Datascript.name Add-Member -InputObject $DataScript -NotePropertyName semVer -NotePropertyValue ( [String]::IsNullOrWhiteSpace($DataScript.semVer) ? '1.0.0' : $DataScript.semVer) -Force ## Resolve provider id (auto-create the provider if missing). Provider commandlets are REST-based. if ([string]::IsNullOrWhiteSpace($Datascript.provider.name)) { Write-Error -Message "Provider name is blank for Datascript: $($Datascript.name)" return } $ProviderID = (Get-TMProvider -Name $Datascript.provider.name -TMSession $TMSession).id if (!$ProviderID) { $NowFormatted = (Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ' -AsUTC).ToString() $Provider = [PSCustomObject]@{ id = $null name = $Datascript.provider.name description = "" comment = "" dateCreated = $NowFormatted lastUpdated = $NowFormatted } $ProviderID = (New-TMProvider -Provider $Provider -PassThru -TMSession $TMSession).id } $ExistingMatches = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName -Version $Datascript.semVer -ErrorAction SilentlyContinue $ExistingDatascript = @($ExistingMatches | Where-Object { $_.provider.name -eq $Datascript.provider.name }) | Select-Object -First 1 if ($ExistingDatascript) { if (-not $Update.IsPresent) { Write-Host 'An existing Datascript with the same name, version and provider was found. This DataScript WAS NOT UPDATED' -ForegroundColor Yellow if ($PassThru) { return $ExistingDatascript } else { return } } else { $Datascript.semVer = Step-SemVer $Datascript.SemVer Write-Host ('An existing Datascript with the same name, version and provider was found. The version number is being incremented to {0}' -f $Datascript.semVer) -ForegroundColor Yellow } } ## Build the body. fromExtension is hardcoded to true to mark this as a REST-managed DataScript. $body = @{ name = $Datascript.name provider = $ProviderID description = $Datascript.description isAutoProcess = [bool]($Datascript.isAutoProcess ?? $false) etlSourceCode = $Datascript.etlSourceCode?.ToString() ?? '' semVer = $Datascript.semVer fromExtension = $true branchId = $Datascript.branchId ?? 0 status = $Datascript.status ?? 'Development' dataSourceType = $Datascript.dataSourceType ?? 'External' contextDomains = $Datascript.contextDomains ?? @() affectedDomains = $Datascript.affectedDomains ?? @() formSpec = $Datascript.formSpec formModel = $Datascript.formModel type = $Datascript.type ?? 'Transformation' mode = $Datascript.mode ?? 'IMPORT' shared = $Datascript.shared ?? 1 } try { Write-Verbose ('New-TMDatascript: POSTing upsert for [' + $Datascript.provider.name + '/' + $Datascript.name + ' v' + $Datascript.semVer + ']') $Response = Invoke-TMRestMethod -Api 'dataScript' -Method Post -BodyParams $body -TMSession $TMSession -ErrorAction SilentlyContinue } catch { if ($_ -like '*Cannot update or create DataScript because the name is not unique for this project, provider version and branch.') { Write-Host 'Datascript was not updated because the Name, Provider and Version Number are not unique.' -ForegroundColor Yellow return } else { Write-Host "Unable to create/update Datascript." return $_ } } $UpdatedDatascript = $Response foreach ($field in $Script:TMDatascriptPlaceholderFields) { if ($UpdatedDatascript.PSObject.Properties.Name -notcontains $field) { Add-Member -InputObject $UpdatedDatascript -NotePropertyName $field -NotePropertyValue $null -Force } } if ($PassThru) { return $UpdatedDatascript } } Function Invoke-TMDatascript { <# .SYNOPSIS Invokes a DataScript in TransitionManager. .DESCRIPTION Starts a DataScript by name and optionally supplies input data, file metadata, progress activity ids, batch queueing behavior, and notification options. .PARAMETER TMSession A TMSession object or session name to use for the request. Defaults to `'Default'`. .PARAMETER DatascriptName The name of the DataScript to invoke. .PARAMETER QueueBatches Boolean indicating whether generated import batches should be queued. .PARAMETER MonitorBatches Boolean indicating whether the function should monitor generated batches. .PARAMETER Data Optional in-memory data to submit to the DataScript run. .PARAMETER FilePath The source file path to associate with the DataScript run. .PARAMETER FileName The source file name to associate with the DataScript run. .PARAMETER ActivityId The root progress activity id to use for progress reporting. .PARAMETER ParentActivityId The parent progress activity id to use for nested progress reporting. .PARAMETER isAutoPost Switch indicating that the invocation should be treated as an auto-post operation. .PARAMETER SendNotification Switch indicating that a notification should be sent for the run. .EXAMPLE Invoke-TMDatascript -DatascriptName 'Load CMDB Data' -FilePath '.\input.csv' -QueueBatches $true Invokes the specified DataScript and queues any generated import batches. #> [alias("Invoke-TMETLScript", "Test-TMETLScript", "Test-TMDatascript")] param( [Parameter(Mandatory = $false)][PSObject]$TMSession = 'Default', [Alias("ETLScriptName")] [Parameter(Mandatory = $true)][String]$DatascriptName, [Parameter(Mandatory = $false)][bool]$QueueBatches = $false, [Parameter(Mandatory = $false)][bool]$MonitorBatches = $false, [Parameter(Mandatory = $false)]$Data, [Parameter(Mandatory = $false)][String]$FilePath, [Parameter(Mandatory = $false)][String]$FileName, [Parameter(Mandatory = $false)][Int16]$ActivityId, [Parameter(Mandatory = $false)][Int16]$ParentActivityId = -1, [Parameter(Mandatory = $false)][switch]$isAutoPost, [Switch]$SendNotification ) $TMSession = Get-TMSession $TMSession if ($FilePath) { if ( -not (Test-Path -Path $FilePath -PathType Leaf) ) { throw $DataTypeConfig.FileNotFoundError } if ( -not $Data ) { $Data = Get-Content $FilePath -Raw $FileName = (Get-Item $FilePath).Name } if ( -not $FileName ) { $FileName = (Get-Item $FilePath).Name } $DataType = (Get-Item $FilePath).Extension if ($DataType -and $DataType -notmatch $DataTypeConfig.MatchPattern) { throw $DataTypeConfig.BadExtensionErrorMessage } } ## -QueueBatches and -isAutoPost both map to processFile's isAutoPost flag. When set, the server ## auto-queues the resulting import batches; we no longer need to PATCH /importBatch/queue per batch. $EffectiveAutoPost = $isAutoPost.IsPresent -or $QueueBatches ## Setup Write-Progress activity tree if ($ActivityId) { $ProgressIndicators = @( @{ Id = $ActivityId; Activity = 'TransitionManager Data Import'; ParentId = $ParentActivityId } @{ Id = ($ActivityId + 1); Activity = 'Datascript Data Transformation'; ParentId = $ActivityId } @{ Id = ($ActivityId + 2); Activity = 'Import Batches'; ParentId = $ActivityId } @{ Id = ($ActivityId + 3); Activity = 'Monitor Batches'; ParentId = $ActivityId } ) $ProgressIndicators | ForEach-Object { Write-Progress @_ -CurrentOperation 'Queued' -PercentComplete 0 } } ## Validate the DataScript exists and grab its id if ($ActivityId) { Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Validating Datascript Script' -Status 'Confirming Datascript Script exists in TransitionManager' -PercentComplete 5 } Write-Host 'Validating Datascript Script: ' -NoNewline Write-Host $DatascriptName -ForegroundColor Yellow $Datascript = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName if (-not $Datascript) { Throw 'The Datascript [' + $DatascriptName + '] does not exist' } if ($Datascript.Count -gt 1) { if ($Datascript.Status -contains 'Released') { $Datascript = $Datascript | Where-Object { $_.Status -eq "Released" } } $Datascript = $Datascript | Sort-Object { ($_.semVer -replace '-.+$') -as [version] } -Descending } $Datascript = @($Datascript) | Select-Object -First 1 ## Kick off processFile (multipart) -- replaces the prior /fileSystem/uploadFileETLAssetImport ## + /assetImport/initiateTransformData pair with a single call. $ProcessFileUri = 'https://{0}/tdstm/api/dataScript/processFile' -f $TMSession.TMServer $Form = @{ dataScriptId = ([string]$Datascript.id) project = ([string]$TMSession.UserContext.Project.Id) isAutoPost = ($EffectiveAutoPost.ToString().ToLower()) sendNotification = ($SendNotification.IsPresent.ToString().ToLower()) } if ($FilePath) { $Form.file = Get-Item -Path $FilePath } if ($ActivityId) { Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Starting Datascript Transformation' -PercentComplete 5 } Write-Host 'TransitionManager Data Import: ' -NoNewline Write-Host 'Starting Datascript Transformation' -ForegroundColor Yellow ## Invoke-WebRequest -Form builds multipart and sets its own boundary. The session's pre-set ## Content-Type: application/json header is temporarily cleared so PowerShell's multipart header ## wins. (See Authentication.ps1 ApplyHeaderTokens.) $TransformationStartTime = (Get-Date).ToUniversalTime() $TMCertSettings = @{ SkipCertificateCheck = $TMSession.AllowInsecureSSL } $PreservedContentType = $TMSession.TMRestSession.Headers['Content-Type'] $TMSession.TMRestSession.Headers.Remove('Content-Type') | Out-Null try { $response = Invoke-WebRequest -Method Post -Uri $ProcessFileUri -WebSession $TMSession.TMRestSession -Form $Form @TMCertSettings -Verbose:$false } catch { throw $_ } finally { if ($PreservedContentType) { $TMSession.TMRestSession.Headers['Content-Type'] = $PreservedContentType } } if ($response.StatusCode -ne 200) { Throw "processFile failed with status $($response.StatusCode)" } $processFileResult = $response.Content | ConvertFrom-Json $DatascriptProgressKey = $processFileResult.progressKey $BatchGroupGuid = $processFileResult.groupGuid ## --- Phase 1: poll until transform finishes --- if ($ActivityId) { Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Running Datascript Transformation' -PercentComplete 0 } $PhaseActivityId = $ActivityId ? ($ActivityId + 1) : 0 $Completed = $false while (-not $Completed) { try { $JobProgress = Invoke-TMRestMethod -Api "job/$DatascriptProgressKey/status" -Method Get -TMSession $TMSession } catch { throw $_ } switch ($JobProgress.status.ToString().ToUpper()) { 'PENDING' { $CurrentOperation = 'Datascript Pending'; $Status = 'Pending'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'QUEUED' { $CurrentOperation = 'Datascript Queued'; $Status = 'Queued'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'IN PROGRESS' { $CurrentOperation = 'Datascript Running'; $Status = 'Transforming Data'; $PercentComplete = $JobProgress.percentComp -gt 99 ? 99 : $JobProgress.percentComp; $SleepSeconds = 2; break } 'RUNNING' { $CurrentOperation = 'Datascript Running'; $Status = 'Transforming Data'; $PercentComplete = $JobProgress.percentComp -gt 99 ? 99 : $JobProgress.percentComp; $SleepSeconds = 2; break } 'COMPLETED' { $CurrentOperation = 'Datascript Processing Complete'; $Status = 'Creating Import Batches'; $Completed = $true; $PercentComplete = 99; $SleepSeconds = 0; break } 'FAILED' { Write-Host "Datascript Processing Failed $($JobProgress.detail)"; Throw $JobProgress.detail } Default { $CurrentOperation = 'State Unknown'; $Status = "Unknown ($($JobProgress.status)). Sleeping to try again."; $PercentComplete = $JobProgress.percentComp ?? 1; $SleepSeconds = 2 } } if ($ActivityId) { Write-Progress -Id $PhaseActivityId -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation $CurrentOperation -Status $Status -PercentComplete ($PercentComplete ?? 0) } else { Write-Host ('Status - ' + $JobProgress.status + ': ' + $JobProgress.percentComp + '%') } if (-not $Completed) { Start-Sleep -Seconds $SleepSeconds } } ## Transition to Activity + 2 if ($ActivityId) { Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Transformation Complete' -PercentComplete 100 -Completed Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Importing Batches' -CurrentOperation 'Beginning Import' -PercentComplete 5 } $TransformationCompletedTime = $JobProgress.lastUpdated $TransformationDuration = $TransformationCompletedTime - $TransformationStartTime if ($TransformationDuration.TotalMinutes -ge 2) { Write-Host ('Datascript Transformation completed in {0} minutes, Loading to Batch Import...' -f $TransformationDuration.TotalMinutes) } else { Write-Host ('Datascript Transformation completed in {0} Seconds, Loading to Batch Import...' -f $TransformationDuration.TotalSeconds) } ## TODO: TM REST endpoint changes are necessary TM-26655 ## Presently, the Transformation progressKey is know, but the Import Progress Key is unknowable. ## Workaround to ensure enough time has passed allowing batches to be created Start-Sleep -Seconds ($TransformationDuration.TotalSeconds + 5) <# ## --- Phase 2: poll the same progressKey for the batch-creation phase --- ## Today's WS server cycles the same key through transform + batch loading. Whether the REST ## server does the same (or returns Completed immediately because groupGuid was handed back ## at processFile time) is pending live verification. This loop is safe in both cases. $PhaseActivityId = $ActivityId ? ($ActivityId + 2) : 0 $Completed = $false while (-not $Completed) { try { $JobProgress = Invoke-TMRestMethod -Api "job/$DatascriptProgressKey/status" -Method Get -TMSession $TMSession } catch { throw $_ } switch ($JobProgress.status.ToString().ToUpper()) { 'PENDING' { $CurrentOperation = 'Loading Batches Pending'; $Status = 'Pending Batch Loading'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'QUEUED' { $CurrentOperation = 'Loading Batches Queued'; $Status = 'Queued'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'IN PROGRESS' { $CurrentOperation = 'Loading Batches'; $Status = 'Loading Batches'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'RUNNING' { $CurrentOperation = 'Loading Batches'; $Status = 'Loading Batches'; $PercentComplete = $JobProgress.percentComp; $SleepSeconds = 2; break } 'COMPLETED' { ## job/status.data.groupGuid may be populated on the second pass; fall back to the ## groupGuid returned by processFile (which is what -Monitor/-Queue scoping uses). if ($JobProgress.data -and $JobProgress.data.groupGuid) { $BatchGroupGuid = $JobProgress.data.groupGuid } $CurrentOperation = 'Batch Loading Complete' $Status = 'Loaded Import Batches' $Completed = $true $PercentComplete = 99 $SleepSeconds = 0 break } 'FAILED' { Write-Host "Batch Loading Processing Failed $($JobProgress.detail)"; Throw $JobProgress.detail } Default { $CurrentOperation = 'State Unknown'; $Status = "Unknown ($($JobProgress.status)). Sleeping to try again."; $PercentComplete = $JobProgress.percentComp ?? 1; $SleepSeconds = 2 } } if ($ActivityId) { $ProgressOptions = @{ Id = $PhaseActivityId ParentId = $ActivityId Activity = 'Import Batch Loading' CurrentOperation = $CurrentOperation Status = $Status PercentComplete = $PercentComplete } if ($Completed) { $ProgressOptions.Completed = $true } Write-Progress @ProgressOptions } else { Write-Host ('Status - ' + $JobProgress.status + ': ' + $JobProgress.percentComp + '%') } if (-not $Completed) { Start-Sleep -Seconds $SleepSeconds } } #> ## List the batches scoped to this invocation. The groupGuid filter is undocumented but ## expected to be honored by GET /api/importBatch -- if the live server doesn't honor it, ## fall back to filtering client-side by dataScript.id + a dateCreated window. # $Batches = @(Invoke-TMRestMethod -Api 'importBatch' -Method Get -ApiParams "groupGuid=$BatchGroupGuid" -TMSession $TMSession) $Batches = @(Invoke-TMRestMethod -Api 'importBatch' -Method Get -TMSession $TMSession) ## Filter the Batches to the ones that should match based on ETL script, start time and invoker $Batches = $Batches | Where-Object { $_.createdBy -eq $TMSession.UserContext.Person.FullName -and $_.dateCreated -gt $TransformationCompletedTime.AddSeconds(-10) -and $_.dataScript.id -eq $Datascript.id } if ($Batches.Count -eq 0) { Write-Host 'No Batches have been created' return } if ($EffectiveAutoPost) { Write-Host ('{0} Batches have Queued' -f $Batches.Count) ## Transition to Activity + 3 (per-batch monitoring layer) if ($ActivityId) { Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Batches Imported' -CurrentOperation 'Importing Batches Complete' -PercentComplete 100 -Completed } } ## Allow the batches to be monitored in the UI if ($EffectiveAutoPost -and $MonitorBatches) { if ($ActivityId) { Write-Progress -Id ($ActivityId + 3) -ParentId $ActivityId -Activity 'Monitoring Batches' -CurrentOperation 'Monitoring Batches' -PercentComplete 5 } ## Sort: non-Dependency first, then Dependency (preserves WS-era ordering) $BatchesToProcess = [System.Collections.ArrayList]@() $Batches | Where-Object { $_.domainClassName -ne 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null } $Batches | Where-Object { $_.domainClassName -eq 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null } for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) { $Batch = $BatchesToProcess[$i] if ($Batch.recordsSummary.count -gt 0) { if ($ActivityId) { ## Activity (Root) + 3 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer Write-Progress -Id ($ActivityId + 3 + $i + 1) ` -ParentId ($ActivityId + 3) ` -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) ` -CurrentOperation 'Monitoring Progress' ` -Status 'Importing' ` -PercentComplete 1 } Write-Host 'Monitoring Batch Status for ' -NoNewline Write-Host $Batch.domainClassName -ForegroundColor Yellow $Completed = $false while (-not $Completed) { try { $BatchProgress = Invoke-TMRestMethod -Api "importBatch/$($Batch.id)" -Method Get -TMSession $TMSession } catch { throw $_ } switch ($BatchProgress.status.code) { "RUNNING" { $CurrentOperation = 'Import Running' $Status = 'Importing Batch Data' $PercentComplete = $BatchProgress.recordsSummary.count ? ($BatchProgress.recordsSummary.processed / $BatchProgress.recordsSummary.count) * 100 : 1 $SleepSeconds = 2 break } "QUEUED" { $CurrentOperation = 'Batch Queued'; $Status = 'Batch Queued'; $PercentComplete = 1; $SleepSeconds = 2; break } "PENDING" { if ($BatchProgress.recordsSummary.erred -gt 0) { Write-Host ("There was an issue posting records {0} in batch number {1}. Review the batch for more information" -f $Batch.recordsSummary.erred, $Batch.id) -ForegroundColor Yellow $Completed = $true $Status = "Batch has errors" $PercentComplete = [Math]::Floor((($BatchProgress.recordsSummary.count - $BatchProgress.recordsSummary.pending) / $Batch.recordsSummary.count) * 100) break } $CurrentOperation = 'Batch Pending'; $Status = 'Batch Pending'; $PercentComplete = 1; $SleepSeconds = 2; break } "COMPLETED" { $Completed = $true; $CurrentOperation = 'Complete'; $Status = 'Complete'; $PercentComplete = 100; $SleepSeconds = 0; break } Default { $CurrentOperation = 'Status Unknown. Retrying'; $Status = 'Retrying'; $PercentComplete = 1; $SleepSeconds = 2; break } } if ($ActivityId) { $ProgressOptions = @{ Id = ($ActivityId + 3 + $i + 1) ParentId = ($ActivityId + 3) Activity = ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) CurrentOperation = $CurrentOperation Status = $Status PercentComplete = ($PercentComplete ?? 0) } if ($Completed) { $ProgressOptions.Completed = $true } Write-Progress @ProgressOptions } else { Write-Host ($Batch.domainClassName + ' Status: ' + $BatchProgress.status.code + ' - ' + [int]($PercentComplete ?? 0) + '%') } if (-not $Completed) { Start-Sleep -Seconds $SleepSeconds } } } Write-Host 'Batch Import Complete for ' -NoNewline Write-Host $Batch.domainClassName -ForegroundColor Yellow } ## Mark the Manage Batches Activity Complete if ($ActivityId) { Write-Progress -Id ($ActivityId + 3) ` -ParentId $ActivityId ` -Activity 'Batches Posted' ` -CurrentOperation 'Complete' ` -Status 'All Batches Imported' ` -PercentComplete 100 ` -Completed } } ## Mark the TM Import Activity complete if ($ActivityId) { Write-Progress -Id $ActivityId ` -ParentId $ParentActivityId ` -Activity 'TransitionManager Data Import' ` -CurrentOperation 'Complete' ` -Status 'All Data Imported' ` -PercentComplete 100 ` -Complete } if ($MonitorBatches.IsPresent) { Write-Host 'All Batches have Posted successfully' } } Function Read-TMDatascriptFile { <# .SYNOPSIS Reads a local DataScript file into a DataScript definition. .DESCRIPTION Loads a DataScript file from disk and converts it into the object shape expected by DataScript creation and update commands. .PARAMETER Path The path to the DataScript file to read. .EXAMPLE Read-TMDatascriptFile -Path '.\Datascripts\LoadCmdb.ps1' Reads a local DataScript file and returns the parsed DataScript definition. .NOTES The returned object can be passed to `New-TMDatascript`. #> param( [Parameter(Mandatory = $true)]$Path ) ## Resolve a sidecar JSON config if one exists next to the .groovy file. $DatascriptConfigJsonPath = $Path -Replace '.groovy', '.json' $DatascriptConfigFile = Get-Item -Path $DatascriptConfigJsonPath -ErrorAction 'SilentlyContinue' if ($DatascriptConfigFile) { ## Split layout: JSON metadata in .json, source in .groovy $TMDatascript = Get-Content -Path $DatascriptConfigFile | ConvertFrom-Json Add-Member -InputObject $TMDatascript -NotePropertyName etlSourceCode -NotePropertyValue (Get-Content -Path $Path -Raw) return $TMDatascript } ## Combined "ReferenceDesign" layout: JSON header block embedded in the .groovy file. $Content = Get-Content -Path $Path -Raw if (-not $Content) { return } $ContentLines = Get-Content -Path $Path New-Variable astTokens -Force New-Variable astErr -Force $ast = [System.Management.Automation.Language.Parser]::ParseInput($Content, [ref]$astTokens, [ref]$astErr) $ConfigBlockStartLine = $astTokens | Where-Object { $_.Text -like '/*********TransitionManager-ETL-Script*********' } | Select-Object -First 1 | Select-Object -ExpandProperty Extent | Select-Object -ExpandProperty StartLineNumber if (-not $ConfigBlockStartLine) { ## No metadata header -- derive name/provider from file path conventions. ## Convention: <root>/<ProviderName>/<DatascriptName>.groovy $DatascriptConfig = [pscustomobject]@{ DatascriptName = (Get-Item -Path $Path).BaseName Description = '' ProviderName = (Get-Item -Path $Path).Directory.BaseName IsAutoProcess = $false Target = $null Mode = 'Import' } $ConfigBlockEndLine = -1 } else { ## Parse the embedded JSON header block. $ConfigBlockEndLine = $astTokens | Where-Object { $_.Text -like '*********TransitionManager-ETL-Script*********/' } | Select-Object -First 1 | Select-Object -ExpandProperty Extent | Select-Object -ExpandProperty StartLineNumber $JsonConfigBlockStartLine = $ConfigBlockStartLine + 1 $JsonConfigBlockEndLine = $ConfigBlockEndLine - 1 $DatascriptConfigJson = $JsonConfigBlockStartLine..$JsonConfigBlockEndLine | ForEach-Object { $ContentLines[$_ - 1] } | Out-String if ($DatascriptConfigJson) { $DatascriptConfig = $DatascriptConfigJson | ConvertFrom-Json } else { $DatascriptConfig = [pscustomobject]@{ DatascriptName = (Get-Item -Path $Path).BaseName Description = '' ProviderName = (Get-Item -Path $Path).Directory.BaseName IsAutoProcess = $false Target = $null Mode = 'Import' } } } ## Pull the code block from after the metadata header (or the whole file if no header). $StartCodeBlockLine = $ConfigBlockEndLine + 1 $EndCodeBlockLine = $ast[-1].Extent.EndLineNumber $DatascriptStringBuilder = New-Object System.Text.StringBuilder $StartCodeBlockLine..$EndCodeBlockLine | ForEach-Object { $DatascriptStringBuilder.AppendLine($ContentLines[$_]) | Out-Null } $DatascriptCode = $DatascriptStringBuilder.ToString() $TMDatascript = [pscustomobject]@{ id = $null name = $DatascriptConfig.DatascriptName description = $DatascriptConfig.Description etlSourceCode = $DatascriptCode provider = @{ id = $null name = $DatascriptConfig.ProviderName } dateCreated = Get-Date lastUpdated = Get-Date } return $TMDatascript } |