TransitionManager

6.6.0

Public/Datascripts.ps1

## TM Datascripts

$DataTypeConfig = @{
    MatchPattern             = '\.(json)|(csv)|(xlsx?)'
    BadExtensionErrorMessage = 'The file you attempted to upload has an invalid type. Valid types are JSON, CSV, and Excel files.'
    FileNotFoundError        = 'The file you specified cannot be found.'
}
Function Get-TMDatascript {
    [alias("Get-TMETLScript")]
    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param(
        [Parameter(Mandatory = $false)]
        [PSObject]$TMSession = 'Default',

        [Parameter(Mandatory = $false)]
        [String]$Name,

        [Parameter(Mandatory = $false)]
        [String[]]$ProviderName,

        [Parameter(Mandatory = $false)]
        [Switch]$ResetIDs,

        [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')]
        [String]$SaveCodePath,

        [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')]
        [Switch]$Passthru
    )

    ## Get Session Configuration
    $TMSession = Get-TMSession $TMSession

    #Honor SSL Settings
    $TMCertSettings = $TMSession.AllowInsecureSSL ? @{SkipCertificateCheck = $true } : @{SkipCertificateCheck = $false }

    # Format the uri
    $Instance = $TMSession.TMServer.Replace('/tdstm', '').Replace('https://', '').Replace('http://', '')
    $uri = "https://$instance/tdstm/ws/dataScript/list?status=ALL"

    try {
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
    } catch {
        return $_
    }

    if ($response.StatusCode -in @(200, 204)) {
        $Result = ($response.Content | ConvertFrom-Json).data
        if ($Result.Count -eq 0) { return $false }
    } else {
        return "Unable to collect Datascripts."
    }

    ## Get each Datascript's Source Code in the list
    for ($i = 0; $i -lt $Result.Count; $i++) {

        $uri = "https://"
        $uri += $instance
        $uri += '/tdstm/ws/dataScript/' + $Result[$i].id
        try {
            $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
        } catch {
            return $_
        }

        if ($response.StatusCode -eq 200) {
            $Result[$i] = ($response.Content | ConvertFrom-Json).data.datascript
        } else {
            return "Unable to collect Datascripts."
        }
    }

    if ($ResetIDs) {
        for ($i = 0; $i -lt $Result.Count; $i++) {
            $Result[$i].id = $null
            $Result[$i].provider.id = $null
        }
    }

    ## Return the details -- Filter based on passed parameters
    if ($ProviderName) {
        $Result = $Result | Where-Object { $_.provider.name -in $ProviderName }
    } elseif ($Name) {
        $Result = $Result | Where-Object { $_.name -eq $Name }
    }

    ## Save the Code Files to a folder
    if ($SaveCodePath) {

        ## Save Each of the Script Source Data
        foreach ($Item in $Result) {

            ## Get a FileName safe version of the Provider Name
            $SafeProviderName = Get-FilenameSafeString -String $Item.provider.name
            $SafeScriptName = Get-FilenameSafeString -String $Item.name

            $Item.semVer = ($Item.semVer.trim().Length -gt 0) ? $Item.semVer.trim() : '1.0.0'
            $SafeScriptName += " - $($Item.semVer)"

            ## Create the Provider Action Folder path
            $ProviderPath = Join-Path $SaveCodePath $SafeProviderName
            Test-FolderPath -FolderPath $ProviderPath

            ## Create a File ame for the Action
            $ProviderScriptConfig = Join-Path $ProviderPath ($SafeScriptName + '.json')
            $ProviderScriptPath = Join-Path $ProviderPath ($SafeScriptName + '.groovy')

            ## Copy the code from the Datascript
            $DatascriptCode = $Item.etlSourceCode ? $Item.etlSourceCode.toString() : ""

            ## Remove the ETL Code fro the Item, so it can be converted to Json
            $Item.PSObject.Properties.Remove('etlSourceCode')

            ## Start Writing the Content of the Script (Force to overwrite any existing files)
            Set-Content -Path $ProviderScriptConfig -Force -Value (ConvertTo-Json -InputObject $Item -Depth 5)
            Set-Content -Path $ProviderScriptPath -Force -Value $DatascriptCode
        }
    }

    if ($Passthru -or !$SaveCodePath) {
        return $Result
    }
}

Function New-TMDatascript {
    [alias("New-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][PSObject]$TMSession = 'Default',
        [Alias("ETLScript")]
        [Parameter(Mandatory = $true)][PSObject]$Datascript,
        [Parameter(Mandatory = $false)][switch]$Update

    )
    ## Get Session Configuration
    $TMSession = Get-TMSession $TMSession

    #Honor SSL Settings
    $TMCertSettings = @{SkipCertificateCheck = $TMSession.AllowInsecureSSL }

    ## Look for an existing Script
    $DatascriptId = 0
    $DatascriptName = $Datascript.name
    $ExistingDatascript = Get-TMDatascript -Name $DatascriptName -TMSession $TMSession

    ## If the Script should not be updated, return it
    if ($ExistingDatascript) {

        $DataScriptId = $ExistingDatascript.id

        ## End the function if there is not an update to be made
        if ($Update) {

            $PostBodyJSON = @{
                name                = $Datascript.name ?? $ExistingDatascript.name
                description         = $Datascript.description ?? $ExistingDatascript.description
                mode                = $Datascript.mode ?? $ExistingDatascript.mode ?? 'IMPORT'
                semVer              = $Datascript.semVer ?? $ExistingDatascript.semVer ?? ''
                branchId            = $Datascript.branchId ?? $ExistingDatascript.branchId ?? 0
                branchedFromId      = $Datascript.branchedFromId ?? $ExistingDatascript.branchedFromId ?? 0
                branchedFromVersion = $Datascript.branchedFromVersion ?? $ExistingDatascript.branchedFromVersion ?? ''
                sortVersion         = $Datascript.sortVersion ?? $ExistingDatascript.sortVersion ?? ''
                status              = $Datascript.status ?? $ExistingDatascript.status ?? 'Development'
                dataSourceType      = $Datascript.dataSourceType ?? $ExistingDatascript.dataSourceType ?? 'External'
                contextDomains      = $Datascript.contextDomains ?? $ExistingDatascript.contextDomains ?? @()
                affectedDomains     = $Datascript.affectedDomains ?? $ExistingDatascript.affectedDomains ?? @()
                formSpec            = $Datascript.formSpec ?? $ExistingDatascript.formSpec ?? @()
                formModel           = $Datascript.formModel ?? $ExistingDatascript.formModel ?? @()
                type                = $Datascript.type ?? $ExistingDatascript.type ?? "Transformation"
                provider            = $ProviderID
                fromExtension       = $true

            } | ConvertTo-Json -Compress -Depth 100

            $uri = "https://"
            $uri += $TMSession.TMServer
            $uri += '/tdstm/ws/dataScript'

            ## Post the Datascript Object to the server
            Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession
            $response = Invoke-WebRequest -Method 'POST' -Uri $uri -WebSession $TMSession.TMWebSession -Body $PostBodyJSON @TMCertSettings
            if ($response.StatusCode -eq 200) {
                $responseContent = $response.Content | ConvertFrom-Json
                if ($responseContent.status -eq "success") {

                    ## Now that the shell exists, get it's ID
                    $DatascriptID = $responseContent.data.datascript.id
                    $UpdatedDatascript = $responseContent.data.datascript
                }
            }


        } else {

            ## If $PassThru is present, return the Existing Datascript script.
            if ($PassThru) {
                return $Datascript
            } else {
                return
            }
        }
    } else {

        ## Lookup Provider ID
        if ([String]::IsNullOrWhiteSpace($Datascript.provider.name)) {
            Write-Error -Message "Provider name is blank for Datascript: $($Datascript.name)"
            return
        }
        $ProviderID = (Get-TMProvider -Name $Datascript.provider.name -TMSession $TMSession).id
        if (!$ProviderID) {
            # Create the provider if it doesn't exist
            $NowFormatted = (Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ' -AsUTC).ToString()
            $Provider = [PSCustomObject]@{
                id          = $null
                name        = $Datascript.provider.name
                description = ""
                comment     = ""
                dateCreated = $NowFormatted
                lastUpdated = $NowFormatted
            }
            $ProviderID = (New-TMProvider -Provider $Provider -PassThru -TMSession $TMSession).id
        }

        try {

            ## There is not an existing Datascript, Create the basic
            ## Create the shell Datascript object, which must be created first
            $PostBodyJSON = @{
                name                = $Datascript.name
                description         = $Datascript.description
                mode                = $Datascript.mode
                provider            = $ProviderID
                fromExtension       = $true
                semVer              = $Datascript.semVer ?? ''
                branchId            = $Datascript.branchId ?? 0
                branchedFromId      = $Datascript.branchedFromId ?? 0
                branchedFromVersion = $Datascript.branchedFromVersion ?? ''
                sortVersion         = $Datascript.sortVersion ?? ''
                status              = $Datascript.status ?? 'Development'
                dataSourceType      = $Datascript.dataSourceType ?? 'External'
                contextDomains      = $Datascript.contextDomains ?? @()
                affectedDomains     = $Datascript.affectedDomains ?? @()
                formSpec            = $Datascript.formSpec ?? @()
                formModel           = $Datascript.formModel ?? @()
                type                = $Datascript.type ?? "Transformation"

            } | ConvertTo-Json -Compress -Depth 100

            $uri = "https://"
            $uri += $TMSession.TMServer
            $uri += '/tdstm/ws/dataScript'

            ## Post the Datascript Object to the server
            Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession
            $response = Invoke-WebRequest -Method 'POST' -Uri $uri -WebSession $TMSession.TMWebSession -Body $PostBodyJSON @TMCertSettings
            if ($response.StatusCode -eq 200) {
                $responseContent = $response.Content | ConvertFrom-Json
                if ($responseContent.status -eq "success") {

                    ## Now that the shell exists, get it's ID
                    $DatascriptID = $responseContent.data.datascript.id
                    $UpdatedDatascript = $responseContent.data.datascript
                }
            }
        } catch {
            Write-Host "Unable to create Datascript."
            return $_
        }

    }

    ## Now that we have an ID for the Datascript, send the source code content
    $PostBodyJSON = @{
        id     = $DatascriptID
        script = $Datascript.etlSourceCode.toString()
    } | ConvertTo-Json -Depth 100

    try {

        ## Post the Source to be saved for this script.
        $uri = "https://"
        $uri += $TMSession.TMServer
        $uri += '/tdstm/ws/dataScript/saveScript'
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -Body $PostBodyJSON @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq 'success') {
                $DatascriptID = $responseContent.data.datascript.id
                $UpdatedDatascript = $responseContent.data.datascript
            }
        } elseif ($response.StatusCode -eq 204) {
            Write-Host "Updating the source code responded with a 204..." -ForegroundColor Yellow
        } else {
            throw "There was an issue saving the source code to the datascript"
        }
    } catch {
        Write-Host "Unable to save Datascript Source to server"
        return $_
    }

    ## Post updates to the Form Spec (version 6.5.0 and higher)
    if ($Datascript.formSpec -and ([System.version]$TMSession.TMVersion -ge [System.Version]"6.5.0")) {
        $PostBodyJSON = @{
            spec  = $Datascript.formSpec ?? $ExistingDatascript.formSpec ?? @()
            model = $Datascript.formModel ?? $ExistingDatascript.formModel ?? @()
        } | ConvertTo-Json -Depth 100

        try {
            ## Post the Source to be saved for this script.
            $uri = "https://"
            $uri += $TMSession.TMServer
            $uri += "/tdstm/ws/dataScript/$($DatascriptID)/form"
            $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -Body $PostBodyJSON @TMCertSettings
            if ($response.StatusCode -eq 200) {
                $responseContent = $response.Content | ConvertFrom-Json
                if ($responseContent.status -eq 'success') {
                    if ($PassThru) {
                        $UpdatedDatascript = $responseContent.data.dataScript
                    }
                }
            } elseif ($response.StatusCode -eq 204) {
                ##
            } else {
                throw "There was an issue saving the source code to the datascript"
            }
        } catch {
            Write-Host "Unable to create Datascript."
            return $_
        }
    }

    ## Return the object if required
    if ($Passthru) {
        return $UpdatedDatascript
    }
}

Function Invoke-TMDatascript {
    [alias("Invoke-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][PSObject]$TMSession = 'Default',
        [Alias("ETLScriptName")]
        [Parameter(Mandatory = $true)][String]$DatascriptName,
        [Parameter(Mandatory = $false)][bool]$QueueBatches = $false,
        [Parameter(Mandatory = $false)][bool]$MonitorBatches = $false,
        [Parameter(Mandatory = $false)]$Data,
        [Parameter(Mandatory = $false)][String]$FilePath,
        [Parameter(Mandatory = $false)][String]$FileName,
        [Parameter(Mandatory = $false)][Int16]$ActivityId,
        [Parameter(Mandatory = $false)][Int16]$ParentActivityId = -1,
        [Parameter(Mandatory = $false)][switch]$isAutoPost,
        [Switch]$SendNotification
    )

    ## Get Session Configuration
    $TMSession = Get-TMSession $TMSession

    if ($FilePath) {
        if ( -not (Test-Path -Path $FilePath -PathType Leaf) ) {
            throw $DataTypeConfig.FileNotFoundError
        }

        ## If a File Path was passed and there is no data, Read the file
        if ( -not $Data ) {
            $Data = Get-Content $FilePath -Raw
            $FileName = (Get-Item $FilePath).Name
        }

        ## If a File Path was passed and there is no File Name, get the file name
        if ( -not $FileName ) {
            $FileName = (Get-Item $FilePath).Name
        }
    }

    #Honor SSL Settings
    $TMCertSettings = @{SkipCertificateCheck = $TMSession.AllowInsecureSSL }

    if ($ActivityId) {
        ## Parent ID is only used on the root 'TransitionManager Data Import' Activity
        ## ParentID + 1 = Transform Data
        ## ParentID + 2 = Import Batches
        ## ParentID + 3 + n for each batch

        ## Add the Datascript Processing Progress Indicators
        $ProgressIndicators = @()
        $ProgressIndicators += @{ Id = $ActivityId; Activity = 'TransitionManager Data Import'; ParentId = $ParentActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 1); Activity = 'Datascript Data Transformation'; ParentId = $ActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 2); Activity = 'Import Batches'; ParentId = $ActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 3); Activity = 'Monitor Batches'; ParentId = $ActivityId }

        #Write Progress Indicators for each in the Array
        $ProgressIndicators | ForEach-Object {
            Write-Progress @_ -CurrentOperation 'Queued' -PercentComplete 0
        }
    }

    ## Fix the Server URL
    $Instance = $TMSession.TMServer.Replace('/tdstm', '')
    $instance = $instance.Replace('https://', '')
    $instance = $instance.Replace('http://', '')

    $Boundary = '----------------------------540299933173025267350719'

    ## First Test to see if the Datascript Script exists or not
    if ($ActivityId) {

        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Validating Datascript Script' -Status 'Confirming Datascript Script exists in TransitionManager' -PercentComplete 5
    }
    Write-Host 'Validating Datascript Script: ' -NoNewline
    Write-Host $DatascriptName -ForegroundColor Yellow

    $Datascript = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName
    if (-Not $Datascript) { Throw  'The Datascript [' + $DatascriptName + '] does not exist' }

    ## If there is a file to upload to storage, upload it
    if ($FileName) {

        ## Build the Data Post
        $uri = "https://"
        $uri += $instance
        $uri += "/tdstm/ws/fileSystem/uploadFileETLAssetImport"

        ## Determine the data type of the file to be uploaded
        if ($FilePath) {
            $DataType = (Get-Item $FilePath).Extension
            if ($DataType -and $DataType -notmatch $DataTypeConfig.MatchPattern) {
                throw $DataTypeConfig.BadExtensionErrorMessage
            }
        } else {
            $DataType = [string]::Empty
        }

        ## FileName
        if (-not $FileName) {
            $FileName = ('DataForETL_' + (Get-Date -Format FileDateTime ) + $DataType)
        }

        ## Construct the PostBody and Form Data for the request
        $CRLF = "`r`n";
        $PostBody = ("-----------------------------$boundary",
            'Content-Disposition: form-data; name="uploadType"',
            '',
            'assetImport',
            "-----------------------------$boundary",
            ''
        ) -join $CRLF

        $FileUpload = @{
            Form = @{
                uploadType = 'assetImport'
                file       = Get-Item -Path $FilePath
                fileName   = $FileName
            }
        }
        Write-Host "Uploading File: " -NoNewline
        Write-Host $FileName -ForegroundColor Yellow

        ## Upload the Data File
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Uploading Data' -Status ('Uploading ' + $FileName) -PercentComplete 5
        }

        try {
            ## If the file is being uploaded, append the fileupload to the command
            if ($FileName) {
                $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings @FileUpload

            } else {

                $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings
            }

        } catch {
            throw $_
        }
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptdataFileName = $responseContent.data.filename
            } else {
                Throw "Unable to upload data to TM Datascript pre-transform storage."
            }
        }
    }

    ## With the file uploaded, Initiate the Datascript script on the server
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/assetImport/initiateTransformData"
    $uri += "?dataScriptId=" + $Datascript.id

    ## Attach the file if there was one
    if ($Filename) {
        $uri += "&filename=" + $DatascriptdataFileName
        $uri += "&originalFilename=" + $FileName
    }

    ## Deal with Version specific items
    $uri += "&sendNotification=" + $SendNotification.IsPresent.toString().ToLower()
    if ($isAutoPost.IsPresent) {
        $uri += '&isAutoPost=true'
    }

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Starting Datascript Transformation' -PercentComplete 5
    }
    Write-Host 'TransitionManager Data Import: ' -NoNewline
    Write-Host 'Starting Datascript Transformation' -ForegroundColor Yellow

    $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptProgressKey = $responseContent.data.progressKey
        } else {
            throw $responseContent.errors
        }
    } else {
        Throw $response
    }


    ## Supply Progress from Datascript Transformation Progress
    if ($ActivityId) {
        Write-Progress -Id $($ActivityId + 1) -Activity 'Datascript Data Transformation' -CurrentOperation 'Running Datascript Transformation' -PercentComplete 0 -ParentId $ActivityId
    }

    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    ## $ReportedLogItems = [System.Collections.ArrayList]@()
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {

                        $CurrentOperation = 'Datascript Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break

                    }
                    "Pending" {
                        $CurrentOperation = 'Datascript Pending'
                        $Status = 'Pending'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Datascript Running'
                        $Status = 'Transforming Data'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp -gt 99 ? 99 : $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        # $DatascriptOutputKey = $DatascriptProgress.detail
                        $CurrentOperation = 'Datascript Processing Complete'
                        $Status = 'Creating Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Datascript Processing Complete, Creating Import Batches."
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Datascript Processing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    Write-Progress -Id ($ActivityId + 1) `
                        -ParentId $ActivityId `
                        -Activity 'Datascript Data Transformation' `
                        -CurrentOperation $CurrentOperation `
                        -Status $Status `
                        -PercentComplete ($PercentComplete ?? 0)
                } else {
                    Write-Host $ProgressString
                }
                Start-Sleep -Seconds $SleepSeconds
            }
        }
    }

    ## Update Progress and transition to Activity + 2
    Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Transformation Complete' -PercentComplete 100 -Completed
    Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Importing Batches' -CurrentOperation 'Beginning Import' -PercentComplete 5

    Write-Host "Datascript Transformation is complete, Loading to Batch Import"

    ## TODO, this is where a Job Log could be pulled. With the DatascriptOutputKey now provided, the API like this could be used:
    ## https://tmad60.transitionmanager.net/tdstm/ws/assetImport/viewData?filename=DatascriptOutputData_BSItLiP036AOsI0cr8e1g2WmipbdtN0c.json and
    ## https://tmad60.transitionmanager.net/tdstm/ws/assetImport/viewData?filename=DatascriptOutputData_BSItLiP036AOsI0cr8e1g2WmipbdtN0c_Application.json
    ## Check the Datascript Script Editor Web Dev tools to see more

    ## Ensure the Batches loaded and the job is finished
    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {
                        $CurrentOperation = 'Loading Batches Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "Pending" {
                        $CurrentOperation = 'Loading Batches Pending'
                        $Status = 'Pending Batch Loading'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Loading Batches'
                        $Status = 'Loading Batches'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        $BatchGroupGuid = $DatascriptProgress.data.groupGuid
                        $CurrentOperation = 'Batch Loading Complete'
                        $Status = 'Loaded Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Completed Loading Import Batches."
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Batch Loading Processing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    $ProgressOptions = @{
                        Id               = ($ActivityId + 2)
                        ParentId         = $ActivityId
                        Activity         = 'Import Batch Loading'
                        CurrentOperation = $CurrentOperation
                        Status           = $Status
                        PercentComplete  = $PercentComplete
                    }
                    if ($Completed) {
                        $ProgressOptions.Completed = $True
                    }
                    Write-Progress @ProgressOptions
                } else {
                    Write-Host $ProgressString
                }
                if (-Not $Completed) {
                    Start-Sleep -Seconds $SleepSeconds
                }
            }
        }
    }

    ## Assemble the batch list object. Dependencies are deliberately moved to the end
    $BatchesToProcess = [System.Collections.ArrayList]@()

    ## Get the Batches that were created during the import
    ## With the Datascript converted, Use it to create Import Batches
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/import/batches?groupGuid="
    $uri += $BatchGroupGuid

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the Transformed data filename to the Datascript engine to Import the Batches.
    $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $Batches = $responseContent.data
        } else {

        }
    } else {

    }

    if ($Batches.Count) {
        Write-Host $Batches.Length -ForegroundColor Yellow -NoNewline
        Write-Host ' Batches have been created'
    }

    ## Update Progress and transition to Activity + 3
    Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Batches Imported' -CurrentOperation 'Importing Batches Complete' -PercentComplete 100 -Completed
    Write-Progress -Id ($ActivityId + 3) -ParentId $ActivityId -Activity 'Monitoring Batches' -CurrentOperation 'Monitoring Batches' -PercentComplete 5

    ## OPTIONAL - If the switch was set to Queue the batches
    if ($QueueBatches) {

        ## Sort the batches to a desired order
        $Batches | Where-Object { $_.domainClass -ne 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null }
        $Batches | Where-Object { $_.domainClass -eq 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null }

        ## Write Progress to the Monitor Import Batches
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 3) `
                -ParentId $ActivityId `
                -Activity 'Manage Batches' `
                -CurrentOperation 'Queueing Batches' `
                -Status ('Queueing ' + [string]$Batches.Length + ' batches.') `
                -PercentComplete 5
        }
        Write-Host "Queueing Batches: " -NoNewline
        Write-Host $Batches.Length -ForegroundColor Yellow

        ## Add a Progress Activity for each of the Import Batches
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]

            ## Create Progress Results for the batches that were created
            if ($ActivityId) {

                ## Activity Explained: The $ActivityId is considered the (Root) + 2 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                    -ParentId ($ActivityId + 3) `
                    -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                    -CurrentOperation 'Queueing Batch' `
                    -Status ([String]$Batch.recordsSummary.count + ' ' + $Batch.domainClassName + ' records') `
                    -PercentComplete 10
            }
            Write-Host "Batch (id $($Batch.id)) Created with $([String]$Batch.recordsSummary.count) $($Batch.domainClassName) records"
        }

        ## Queue each Batch
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]
            if ($Batch.autoProcess -eq 0) {

                ## With the file uploaded, Initiate the Datascript script on the server
                $uri = "https://"
                $uri += $instance
                $uri += "/tdstm/ws/import/batches"

                Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

                $PostBody = @{
                    action = 'QUEUE'
                    ids    = $Batch.id
                } | ConvertTo-Json -Depth 100


                ## Post this batch to begin it's Queueing
                $response = Invoke-WebRequest -Method Patch -Uri $uri -WebSession $TMSession.TMWebSession -Body $PostBody @TMCertSettings
                if ($response.StatusCode -eq 200) {
                    $responseContent = $response.Content | ConvertFrom-Json
                    if ($responseContent.status -eq "success") {

                        ## Create Progress Results for the batches that were created
                        if ($ActivityId) {

                            ## Notify the Manage Batches Activity of a queued batch
                            Write-Progress -Id ($ActivityId + 3) `
                                -ParentId $ActivityId `
                                -Activity 'Monitor Batches' `
                                -CurrentOperation 'Queueing Batches' `
                                -Status ('Queued ' + $Batch.domainClass + ' Batch') `
                                -PercentComplete 5

                            ## Activity (Root) + 2 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                            Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                                -ParentId ($ActivityId + 3) `
                                -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                                -CurrentOperation 'Batch Queued'  `
                                -Status 'Queued' `
                                -PercentComplete 0
                        }
                        Write-Host "Batch Queued: " -NoNewline
                        Write-Host $Batch.domainClassName -ForegroundColor Yellow
                    } else {
                        Throw 'Failed to Queue Batch'
                    }
                } else {
                    Throw 'Failed to Queue Batch.'
                }
            }
        }

        ## If Monitoring isn't going to occur, status should be complete here
        if ($MonitorBatches -ne $true) {
            Write-Progress -Id ($ActivityId + 3) -ParentId $ActivityId -Activity 'Batches Queued' -CurrentOperation 'Batches have been queued' -PercentComplete 100 -Completed
        }
    }

    ## Allow the batch to be monitored and reported on in the UI
    if ($MonitorBatches) {
        $BatchStatus = @{
            CompletedBatches = 0
            TotalBatches     = $BatchesToProcess.Count
        }

        ## Monitor the batches to completion
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]
            if ($Batch.recordsSummary.count -gt 0) {
                ## Build the URL
                $uri = "https://"
                $uri += $instance
                $uri += "/tdstm/ws/import/batch/"
                $uri += $Batch.id
                $uri += "/progress"

                Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

                ## Poll for the status of the Import Batch
                if ($ActivityId) {

                    ## Activity (Root) + 3 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                    Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                        -ParentId ($ActivityId + 3) `
                        -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                        -CurrentOperation 'Monitoring Progress'  `
                        -Status 'Importing' `
                        -PercentComplete 1
                }
                Write-Host 'Monitoring Batch Status for ' -NoNewline
                Write-Host $Batch.domainClassName -ForegroundColor Yellow

                ## Start a loop to run while the batch is not complete.
                $Completed = $false
                while ($Completed -eq $false) {

                    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
                    $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
                    if ($response.StatusCode -eq 200) {
                        $responseContent = $response.Content | ConvertFrom-Json
                        if ($responseContent.status -eq "success") {

                            $BatchProgress = $responseContent.data

                            switch ($BatchProgress.status.code) {
                                "RUNNING" {
                                    $CurrentOperation = 'Import Running'
                                    $Status = 'Importing Batch Data'
                                    $ProgressString = $DomainClass + ' Status: Running - ' + $BatchProgress.progress + "%"
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break
                                }
                                "QUEUED" {
                                    $CurrentOperation = 'Batch Queued'
                                    $Status = 'Batch Queued'
                                    $ProgressString = $DomainClass + ' Status: Queued'
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break

                                }
                                "PENDING" {
                                    $CurrentOperation = 'Batch Pending'
                                    $Status = 'Batch Pending'
                                    $ProgressString = $DomainClass + ' Status: Pending'
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break

                                }
                                "COMPLETED" {
                                    $Completed = $true
                                    $CurrentOperation = 'Complete'
                                    $Status = 'Complete'
                                    $ProgressString = ($batch.domainClassName + " Status: Complete")
                                    $PercentComplete = 100
                                    $SleepSeconds = 0
                                    break

                                }
                                Default {
                                    $CurrentOperation = 'Status Unknown. Retrying'
                                    $Status = 'Retrying'
                                    $ProgressString = ($batch.domainClassName + "Status Unkown. Retrying")
                                    $PercentComplete = 1
                                    $SleepSeconds = 2
                                    break
                                }
                            }

                            ## Display the Status of this loop
                            if ($ActivityId) {

                                $ProgressOptions = @{
                                    Id               = ($ActivityId + 3 + $i + 1)
                                    ParentId         = ($ActivityId + 3)
                                    Activity         = ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count)
                                    CurrentOperation = $CurrentOperation
                                    Status           = $Status
                                    PercentComplete  = ($PercentComplete ?? 0)
                                }
                                ## Activity (Root) + 3 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                                if ($Completed) {
                                    $ProgressOptions.Completed = $true
                                }
                                Write-Progress @ProgressOptions

                            } else {
                                Write-Host $ProgressString
                            }

                            ## Sleep the expected duration
                            Start-Sleep -Seconds $SleepSeconds
                        }
                    }
                }
            }
            ## Now that this batch is done, increment the Completed Batches counter
            $BatchStatus.CompletedBatches++
            Write-Host 'Batch Import Complete for ' -NoNewline
            Write-Host $Batch.domainClassName -ForegroundColor Yellow
        }

        ## Mark the Manage Batches Activity Complete
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 3) `
                -ParentId $ActivityId `
                -Activity 'Batches Posted' `
                -CurrentOperation 'Complete'  `
                -Status 'All Batches Imported' `
                -PercentComplete 100 `
                -Completed
        }
    }


    ## Mark the TM Import Activity complete
    if ($ActivityId) {
        Write-Progress -Id $ActivityId `
            -ParentId $ParentActivityId `
            -Activity 'TransitionManager Data Import' `
            -CurrentOperation 'Complete'  `
            -Status 'All Data Imported' `
            -PercentComplete 100 `
            -Complete
    }

    if ($Batches.Count) {
        Write-Host 'All Batches have Imported successfully'
    }
}

Function Test-TMDatascript {
    [alias("Test-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][PSObject]$TMSession = 'Default',
        [Alias("ETLScriptName")]
        [Parameter(Mandatory = $true)][String]$DatascriptName,
        [Parameter(Mandatory = $false)][bool]$QueueBatches = $false,
        [Parameter(Mandatory = $false)][bool]$MonitorBatches = $false,
        [Parameter(Mandatory = $false)]$Data,
        [Parameter(Mandatory = $false)][String]$FilePath,
        [Parameter(Mandatory = $false)][String]$FileName,
        [Parameter(Mandatory = $false)][Int16]$ActivityId,
        [Parameter(Mandatory = $false)][Int16]$ParentActivityId = -1
    )

    ## Get Session Configuration
    $TMSession = Get-TMSession $TMSession

    if ($FilePath) {
        if ( -not (Test-Path -Path $FilePath -PathType Leaf) ) {
            throw $DataTypeConfig.FileNotFoundError
        }

        ## If a File Path was passed and there is no data, Read the file
        if ( -not $Data ) {
            $Data = Get-Content $FilePath -Raw
            $FileName = (Get-Item $FilePath).Name
        }

        ## If a File Path was passed and there is no File Name, get the file name
        if ( -not $FileName ) {
            $FileName = (Get-Item $FilePath).Name
        }
    }

    #Honor SSL Settings
    $TMCertSettings = @{SkipCertificateCheck = $TMSession.AllowInsecureSSL }

    if ($ActivityId) {
        ## Parent ID is only used on the root 'TransitionManager Data Import' Activity
        ## ParentID + 1 = Transform Data
        ## ParentID + 2 = Import Batches
        ## ParentID + 3 + n for each batch

        ## Add the Datascript Processing Progress Indicators
        $ProgressIndicators = @()
        $ProgressIndicators += @{ Id = $ActivityId; Activity = 'Test Datascript Data Transformation'; ParentId = $ParentActivityId }

    }

    ## Fix the Server URL
    $Instance = $TMSession.TMServer.Replace('/tdstm', '')
    $instance = $instance.Replace('https://', '')
    $instance = $instance.Replace('http://', '')

    $Boundary = '----------------------------540299933173025267350719'

    ## First Test to see if the Datascript exists or not
    if ($ActivityId) {

        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Validating Datascript' -Status 'Confirming Datascript exists in TransitionManager' -PercentComplete 5
    }
    $Datascript = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName
    if (-Not $Datascript) { Throw  'The Datascript [' + $DatascriptName + '] does not exist' }

    ## Determine the data type of the file to be uploaded
    if ($FilePath) {

        ## Build the Data Post
        $uri = "https://"
        $uri += $instance
        $uri += "/tdstm/ws/fileSystem/uploadFileETLAssetImport"

        $DataType = (Get-Item $FilePath).Extension
        if ($DataType -notmatch $DataTypeConfig.MatchPattern) {
            throw $DataTypeConfig.BadExtensionErrorMessage
        }


        ## FileName
        $FileName = ('DataForETL_' + (Get-Date -Format FileDateTime ) + $DataType)

        ## Construct the PostBody and Form Data for the request
        $CRLF = "`r`n";
        $PostBody = ("-----------------------------$boundary",
            'Content-Disposition: form-data; name="uploadType"',
            '',
            'assetImport',
            "-----------------------------$boundary",
            ''
        ) -join $CRLF
        $FileUpload = @{
            Form = @{
                uploadType = 'assetImport'
                file       = Get-Item -Path $FilePath
                fileName   = $FileName
            }
        }
        ## Upload the Data File
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Uploading Data' -Status ('Uploading ' + $FileName) -PercentComplete 5
        }

        try {
            ## If the file is being uploaded, append the fileupload to the command
            if ($FileUpload) {
                $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings @FileUpload
            } else {
                $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSession.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings
            }

        } catch {
            throw $_
        }

        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptdataFileName = $responseContent.data.filename
            } else {
                Throw "Unable to upload data to TM Datascript pre-transform storage."
            }
        }
    }

    ## With the file uploaded, Initiate the Datascript script on the server
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/dataScript/initiateTestScript"

    ## Create a post body
    if ($FilePath) {
        $PostBody = [PSCustomObject]@{
            dataScriptId = $Datascript.id
            filename     = $DatascriptdataFileName
            script       = ($Datascript.etlSourceCode -replace $CRLF, '\n')
        } | ConvertTo-Json -Compress
    } else {
        $PostBody = [PSCustomObject]@{
            dataScriptId = $Datascript.id
            script       = ($Datascript.etlSourceCode)
        } | ConvertTo-Json -Compress
    }

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Starting Datascript Testing' -PercentComplete 5
    } else {
        Write-Host 'TransitionManager Data Import: Starting Datascript Test'
    }

    $response = Invoke-WebRequest -Method Post -Uri $uri -Body $PostBody -WebSession $TMSession.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptProgressKey = $responseContent.data.progressKey
        } else {
            throw $responseContent.errors
        }
    } else {
        Throw $response
    }


    ## Supply Progress from Datascript Transformation Progress
    if ($ActivityId) {
        Write-Progress -Id $($ActivityId + 1) -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Running Datascript Transformation' -PercentComplete 0 -ParentId $ActivityId
    } else {
        Write-Host 'Testing Datascript Data Transformation: Starting Datascript Processing'
    }

    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSession.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {

                        $CurrentOperation = 'Datascript Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break

                    }
                    "Pending" {
                        $CurrentOperation = 'Datascript Pending'
                        $Status = 'Pending'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Datascript Running'
                        $Status = 'Transforming Data'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        $CurrentOperation = 'Datascript Testing Complete'
                        $Status = 'Creating Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Datascript Testing Complete"
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Datascript Testing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    Write-Progress -Id ($ActivityId + 1) `
                        -ParentId $ActivityId `
                        -Activity 'Testing Datascript Data Transformation' `
                        -CurrentOperation $CurrentOperation `
                        -Status $Status `
                        -PercentComplete $PercentComplete
                } else {
                    Write-Host $ProgressString
                }
                Start-Sleep -Seconds $SleepSeconds
            }
        }
    }

    ## Update Progress and transition to Activity + 2
    Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Tested Datascript Data Transformation' -CurrentOperation 'Testing Complete' -PercentComplete 100 -Completed

    ## Mark the TM Import Activity complete
    if ($ActivityId) {
        Write-Progress -Id $ActivityId `
            -ParentId $ParentActivityId `
            -Activity 'TransitionManager Data Import' `
            -CurrentOperation 'Complete'  `
            -Status 'All Data Imported' `
            -PercentComplete 100 `
            -Complete
    }
}

Function Read-TMDatascriptFile {
    param(
        [Parameter(Mandatory = $true)]$Path
    )

    ## First order of business is to determine if this Script file has a counterpart Json file
    $DatascriptConfigJsonPath = $Path -Replace '.groovy', '.json'
    $DatascriptConfigFile = Get-Item -Path $DatascriptConfigJsonPath -ErrorAction 'SilentlyContinue'

    ## Check if there is a config JSON file, if so, get the Action from the 2 files
    if ($DatascriptConfigFile) {

        ## Get the Recipe Object that doesn't have the source code
        $TMDatascript = Get-Content -Path $DatascriptConfigFile | ConvertFrom-Json
        Add-Member -InputObject $TMDatascript -NotePropertyName etlSourceCode -NotePropertyValue (Get-Content -Path $Path -Raw)
    }

    ## Read the ReferenceDesign (Combined) file
    else {

        ## Name the Input File
        $Content = Get-Content -Path $Path -Raw

        ## Ignore Empty Files
        if (-Not $Content) {
            return
        }

        $ContentLines = Get-Content -Path $Path

        ## Create Automation Token Variables Parse the Script File
        New-Variable astTokens -Force
        New-Variable astErr -Force
        $ast = [System.Management.Automation.Language.Parser]::ParseInput($Content, [ref]$astTokens, [ref]$astErr)

        ##
        ## Assess the Script Parts to get delineating line numbers
        ##

        ## Locate the Delimiting line
        $ConfigBlockStartLine = $astTokens | `
                Where-Object { $_.Text -like '/*********TransitionManager-ETL-Script*********' } |`
                Select-Object -First 1 | `
                Select-Object -ExpandProperty Extent | `
                Select-Object -ExpandProperty StartLineNumber

        ## Test if the file has been written with MetaData
        if (-Not $ConfigBlockStartLine) {

            ## This file does not have metadata. Create the minimum data for the constructor below
            $DatascriptConfig = @{
                DatascriptName = (Get-Item -Path $Path).BaseName
                Description    = ""
                ProviderName   = (Get-Item -Path $Path).Directory.BaseName
                IsAutoProcess  = $false
                Target         = $null
                Mode           = 'Import'
            }

            ## Add Variables that will be used in the construction
            $DatascriptCode = $ContentLines
            $ConfigBlockEndLine = -1
        } else {

            ## This File has metadata - Work to parse it

            $ConfigBlockEndLine = $astTokens | `
                    Where-Object { $_.Text -like '*********TransitionManager-ETL-Script*********/' } |`
                    Select-Object -First 1 | `
                    Select-Object -ExpandProperty Extent | `
                    Select-Object -ExpandProperty StartLineNumber

            ## Adjust the Line Numbers to capture just the JSON
            $JsonConfigBlockStartLine = $ConfigBlockStartLine + 1
            $JsonConfigBlockEndLine = $ConfigBlockEndLine - 1


            ##
            ## Read the Script Header to gather the configurations
            ##

            ## Get all of the lines in the header comment
            $DatascriptConfigJson = $JsonConfigBlockStartLine..$JSONConfigBlockEndLine | ForEach-Object {

                ## Return the line for collection
                $ContentLines[$_ - 1]

            } | Out-String

            ## Convert the JSON string to an Object
            if ($DatascriptConfigJson) {
                $DatascriptConfig = $DatascriptConfigJson | ConvertFrom-Json
            } else {
                $DatascriptConfig = @{
                    DatascriptName = (Get-Item -Path $Path).BaseName
                    Description    = ""
                    ProviderName   = (Get-Item -Path $Path).Directory.Parent.Parent.BaseName
                    IsAutoProcess  = $false
                    Target         = $null
                    Mode           = 'Import'
                }
            }
        }

        ##
        ## Read the Script Block
        ##

        ## Note where the Configuration Code is located
        $StartCodeBlockLine = $ConfigBlockEndLine + 1
        $EndCodeBlockLine = $ast[-1].Extent.EndLineNumber

        ## Create a Text StrinBuilder to collect the Script into
        $DatascriptStringBuilder = New-Object System.Text.StringBuilder

        ## For each line in the Code Block, add it to the Datascript Script Code StringBuilder
        $StartCodeBlockLine..$EndCodeBlockLine | ForEach-Object {
            $DatascriptStringBuilder.AppendLine($ContentLines[$_]) | Out-Null
        }

        ## Convert the StringBuilder to a Multi-Line String
        $DatascriptCode = $DatascriptStringBuilder.ToString()

        ## Assemble the Action Object
        $TMDatascript = [pscustomobject]@{

            ## Primary Information
            id            = $null
            name          = $DatascriptConfig.DatascriptName
            description   = $DatascriptConfig.Description

            ## Source Code
            etlSourceCode = $DatascriptCode

            ## Provider
            provider      = @{
                id   = $null
                name = $DatascriptConfig.ProviderName
            }

            ## Other details for the Datascript Script
            dateCreated   = Get-Date
            lastUpdated   = Get-Date
        }
    }

    ## Return the Action Object
    return $TMDatascript
}