TransitionManager

6.2.0

lib/Datascripts.ps1

## TM Datascripts
Function Get-TMDatascript {
    [alias("Get-TMETLScript")]
    [CmdletBinding(DefaultParameterSetName = 'Default')]
    param(
        [Parameter(Mandatory = $false)]
        [String]$TMSession = "Default",

        [Parameter(Mandatory = $false)]
        [String]$Name,

        [Parameter(Mandatory = $false)]
        [String[]]$ProviderName,

        [Parameter(Mandatory = $false)]
        [String]$Server = $global:TMSessions[$TMSession].TMServer,

        [Parameter(Mandatory = $false)]
        [Bool]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,

        [Parameter(Mandatory = $false)]
        [Switch]$ResetIDs,

        [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')]
        [String]$SaveCodePath,

        [Parameter(Mandatory = $false, ParameterSetName = 'SaveCode')]
        [Switch]$Passthru
    )

    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    #Honor SSL Settings
    $TMCertSettings = $TMSessionConfig.AllowInsecureSSL ? @{SkipCertificateCheck = $true } : @{SkipCertificateCheck = $false }

    # Format the uri
    $instance = $Server.Replace('/tdstm', '').Replace('https://', '').Replace('http://', '')

    if ($TMSessionConfig.TMVersion -ge '6.1.0') {
        $uri = "https://$instance/tdstm/ws/dataScript/list"
    }
    else {
        $uri = "https://$instance/tdstm/ws/dataingestion/datascript/list"
    }


    try {
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
    }
    catch {
        return $_
    }

    if ($response.StatusCode -in @(200, 204)) {
        $Result = ($response.Content | ConvertFrom-Json).data
        if ($Result.Count -eq 0) { return $false }
    }
    else {
        return "Unable to collect Datascripts."
    }

    ## Get each Datascript's Source Code in the list
    for ($i = 0; $i -lt $Result.Count; $i++) {

        $uri = "https://"
        $uri += $instance
        if ($TMSessionConfig.TMVersion -ge '6.1.0') {
            $uri += '/tdstm/ws/dataScript/' + $Result[$i].id
        }
        else {
            $uri += '/tdstm/ws/dataingestion/datascript/' + $Result[$i].id
        }
        try {
            $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
        }
        catch {
            return $_
        }

        if ($response.StatusCode -eq 200) {
            $Result[$i] = ($response.Content | ConvertFrom-Json).data.datascript
        }
        else {
            return "Unable to collect Datascripts."
        }
    }

    if ($ResetIDs) {
        for ($i = 0; $i -lt $Result.Count; $i++) {
            $Result[$i].id = $null
            $Result[$i].provider.id = $null
        }
    }

    ## Return the details -- Filter based on passed parameters
    if ($ProviderName) {
        $Result = $Result | Where-Object { $_.provider.name -in $ProviderName }
    }
    elseif ($Name) {
        $Result = $Result | Where-Object { $_.name -eq $Name }
    }

    ## Save the Code Files to a folder
    if ($SaveCodePath) {

        ## Save Each of the Script Source Data
        foreach ($Item in $Result) {

            ## Get a FileName safe version of the Provider Name
            $SafeProviderName = $Item.provider.name -replace '\:', '-' -replace '\/', '-' -replace '\|', '-' -replace '\\', '-'
            $SafeScriptName = $Item.name -replace '\:', '-' -replace '\/', '-' -replace '\|', '-' -replace '\\', '-'

            ## Create the Provider Action Folder path
            $ProviderPath = Join-Path $SaveCodePath $SafeProviderName
            Test-FolderPath -FolderPath $ProviderPath

            ## Create a File ame for the Action
            $ProviderScriptConfig = Join-Path $ProviderPath ($SafeScriptName + '.json')
            $ProviderScriptPath = Join-Path $ProviderPath ($SafeScriptName + '.groovy')

            ## Copy the code from the Datascript
            $DatascriptCode = $Item.etlSourceCode ? $Item.etlSourceCode.toString() : ""

            ## Remove the ETL Code fro the Item, so it can be converted to Json
            $Item.PSObject.Properties.Remove('etlSourceCode')

            ## Start Writing the Content of the Script (Force to overwrite any existing files)
            Set-Content -Path $ProviderScriptConfig -Force -Value (ConvertTo-Json -InputObject $Item -Depth 5)
            Set-Content -Path $ProviderScriptPath -Force -Value $DatascriptCode
        }
    }

    if ($Passthru -or !$SaveCodePath) {
        return $Result
    }
}

Function New-TMDatascript {
    [alias("New-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScript")]
        [Parameter(Mandatory = $true)][PSObject]$Datascript,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,
        [Parameter(Mandatory = $false)][switch]$Update

    )
    if ($global:TMSessions[$TMSession].TMVersion -like '4.6.*') {
        New-TMDatascript46 @PSBoundParameters
    }
    elseif ($global:TMSessions[$TMSession].TMVersion -ge '4.7.*' -and $global:TMSessions[$TMSession].TMVersion -lt '6.1.0') {
        New-TMDatascript47 @PSBoundParameters
    }
    else {
        New-TMDatascript61 @PSBoundParameters
    }
}
Function New-TMDatascript46 {
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScript")]
        [Parameter(Mandatory = $true)][PSObject]$Datascript,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL
    )

    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    #Honor SSL Settings
    if ($TMSessionConfig.AllowInsecureSSL) {
        $TMCertSettings = @{SkipCertificateCheck = $true }
    }
    else {
        $TMCertSettings = @{SkipCertificateCheck = $false }
    }

    $DatascriptName = $Datascript.name
    $ExistingDatascript = Get-TMDatascript -Name $DatascriptName -TMSession $TMSession
    if ($ExistingDatascript) {
        # Write-Host "Datascript Exists: "$DatascriptName
        if ($PassThru) { return $ExistingDatascript } else { return }
    }

    ## Action 0 (Renumber) - Confirm the name is unique
    $uri = "https://"
    $uri += $Server
    $uri += '/tdstm/ws/dataingestion/datascript/validateUnique'

    $PostBodyJSON = @{
        name       = $DatascriptName
        providerId = (Get-TMProvider -Name $Datascript.provider.name -TMSession $TMSession).id
    } | ConvertTo-Json -Depth 100

    try {
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $isUnique = $responseContent.data.isUnique
                if ($isUnique -ne $true) {
                    $thisDatascript = Get-TMDatascript -Name $DatascriptName -TMSession $TMSession
                }
            }
        }
    }
    catch {
        Write-Host "Unable to determine if Datascript Name is unique."
        return $_
    }


    ## Action 2, Create the Datascript so the Script code can be saved into it.
    $uri = "https://"
    $uri += $Server
    $uri += '/tdstm/ws/dataingestion/datascript'

    ## If there is an existing Datascript, update it
    if ($thisDatascript) {
        $Datascript.id = $thisDatascript.id
        $uri += '/' + $Datascript.id
    }
    else {
        $Datascript.PSObject.properties.Remove('id')
    }

    ## Lookup Cross References
    $ProviderID = (Get-TMProvider -TMSession $TMSession | Where-Object { $_.name -eq $Datascript.provider.name }).id

    $PostBodyJSON = @{
        name        = $DatascriptName
        mode        = $Datascript.mode
        description = $Datascript.description
        providerId  = $ProviderID
        # etlSourceCode = $Datascript.etlSourceCode
    } | ConvertTo-Json -Depth 100

    try {
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {

                $NewDatascript = $responseContent.data.datascript
                $PostBodyJSON = @{
                    id     = $NewDatascript.id
                    script = $Datascript.etlSourceCode
                } | ConvertTo-Json -Depth 100

                $uri = "https://"
                $uri += $Server
                $uri += '/tdstm/ws/dataingestion/dataScript/saveScript'

                $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
                if ($response.StatusCode -eq 200) {
                    $responseContent = $response.Content | ConvertFrom-Json
                    if ($responseContent.status -eq 'success') {
                        if ($PassThru) { return $responseContent.data.dataScript } else { return }
                    }
                }
            }
        }
    }
    catch {
        Write-Host "Unable to create Datascript."
        return $_
    }

}
Function New-TMDatascript47 {
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScript")]
        [Parameter(Mandatory = $true)][PSObject]$Datascript,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,
        [Parameter(Mandatory = $false)][switch]$Update

    )
    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    #Honor SSL Settings
    if ($TMSessionConfig.AllowInsecureSSL) {
        $TMCertSettings = @{SkipCertificateCheck = $true }
    }
    else {
        $TMCertSettings = @{SkipCertificateCheck = $false }
    }

    ## Look for an existing Script
    $DatascriptName = $Datascript.name
    $ExistingDatascript = Get-TMDatascript -Name $DatascriptName -TMSession $TMSession

    ## If the Script should not be updated, return it
    if ($ExistingDatascript) {

        ## End the function if there is not an update to be made
        if ($Update) {

            ## Record the DatascriptId for the update
            $DatascriptID = $ExistingDatascript.id

        }
        else {

            ## If $PassThru is present, return the Existing Datascript script.
            if ($PassThru) {
                return $ExistingDatascript
            }
            else {
                return
            }

        }
    }
    else {

        ## Lookup Provider ID
        if ([String]::IsNullOrWhiteSpace($Datascript.provider.name)) {
            Write-Error -Message "Provider name is blank for Datascript: $($Datascript.name)"
            return
        }
        $ProviderID = (Get-TMProvider -Name $Datascript.provider.name -TMSession $TMSession).id
        if (!$ProviderID) {
            # Create the provider if it doesn't exist
            $NowFormatted = (Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ' -AsUTC).ToString()
            $Provider = [PSCustomObject]@{
                id          = $null
                name        = $Datascript.provider.name
                description = ""
                comment     = ""
                dateCreated = $NowFormatted
                lastUpdated = $NowFormatted
            }
            $ProviderID = (New-TMProvider -Provider $Provider -PassThru -TMSession $TMSession).id
        }

        try {

            ## There is not an existing Datascript, Create the basic
            ## Create the shell Datascript object, which must be created first
            $PostBodyJSON = @{
                description = $Datascript.description
                mode        = $Datascript.mode
                name        = $Datascript.name
                providerId  = $ProviderID
            } | ConvertTo-Json -Compress

            $uri = "https://"
            $uri += $Server
            $uri += '/tdstm/ws/dataingestion/datascript'

            ## Post the Datascript Object to the server
            Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession
            $response = Invoke-WebRequest -Method 'POST' -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
            if ($response.StatusCode -eq 200) {
                $responseContent = $response.Content | ConvertFrom-Json
                if ($responseContent.status -eq "success") {

                    ## Now that the shell exists, get it's ID
                    $DatascriptID = $responseContent.data.datascript.id
                }
            }
        }
        catch {
            Write-Host "Unable to create Datascript."
            return $_
        }

    }

    ## Now that we have an ID for the Datascript, send the source code content
    $PostBodyJSON = @{
        id     = $DatascriptID
        script = $Datascript.etlSourceCode.ToString()
    } | ConvertTo-Json -Depth 100

    try {

        ## Post the Source to be saved for this script.
        $uri = "https://"
        $uri += $Server
        $uri += '/tdstm/ws/dataScript/saveScript'
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq 'success') {
                if ($PassThru) {
                    return $responseContent.data.dataScript
                }
            }
        }
        elseif ($response.StatusCode -eq 204) {
            return
        }
    }
 catch {
        Write-Host "Unable to save Datascript Source to server"
        return $_
    }

}

Function New-TMDatascript61 {
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScript")]
        [Parameter(Mandatory = $true)][PSObject]$Datascript,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,
        [Parameter(Mandatory = $false)][switch]$Update

    )
    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    #Honor SSL Settings
    if ($TMSessionConfig.AllowInsecureSSL) {
        $TMCertSettings = @{SkipCertificateCheck = $true }
    }
    else {
        $TMCertSettings = @{SkipCertificateCheck = $false }
    }

    ## Look for an existing Script
    $DatascriptName = $Datascript.name
    $ExistingDatascript = Get-TMDatascript -Name $DatascriptName -TMSession $TMSession

    ## If the Script should not be updated, return it
    if ($ExistingDatascript) {

        ## End the function if there is not an update to be made
        if ($Update) {

            ## Record the DatascriptId for the update
            $DatascriptID = $ExistingDatascript.id

        }
        else {

            ## If $PassThru is present, return the Existing Datascript script.
            if ($PassThru) {
                return $ExistingDatascript
            }
            else {
                return
            }

        }
    }
    else {

        ## Lookup Provider ID
        if ([String]::IsNullOrWhiteSpace($Datascript.provider.name)) {
            Write-Error -Message "Provider name is blank for Datascript: $($Datascript.name)"
            return
        }
        $ProviderID = (Get-TMProvider -Name $Datascript.provider.name -TMSession $TMSession).id
        if (!$ProviderID) {
            # Create the provider if it doesn't exist
            $NowFormatted = (Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ' -AsUTC).ToString()
            $Provider = [PSCustomObject]@{
                id          = $null
                name        = $Datascript.provider.name
                description = ""
                comment     = ""
                dateCreated = $NowFormatted
                lastUpdated = $NowFormatted
            }
            $ProviderID = (New-TMProvider -Provider $Provider -PassThru -TMSession $TMSession).id
        }

        try {

            ## There is not an existing Datascript, Create the basic
            ## Create the shell Datascript object, which must be created first
            $PostBodyJSON = @{
                description = $Datascript.description
                mode        = $Datascript.mode
                name        = $Datascript.name
                provider    = $ProviderID
            } | ConvertTo-Json -Compress

            $uri = "https://"
            $uri += $Server
            $uri += '/tdstm/ws/dataScript'

            ## Post the Datascript Object to the server
            Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession
            $response = Invoke-WebRequest -Method 'POST' -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
            if ($response.StatusCode -eq 200) {
                $responseContent = $response.Content | ConvertFrom-Json
                if ($responseContent.status -eq "success") {

                    ## Now that the shell exists, get it's ID
                    $DatascriptID = $responseContent.data.datascript.id
                }
            }
        }
        catch {
            Write-Host "Unable to create Datascript."
            return $_
        }

    }

    ## Now that we have an ID for the Datascript, send the source code content
    $PostBodyJSON = @{
        id     = $DatascriptID
        script = $Datascript.etlSourceCode.toString()
    } | ConvertTo-Json -Depth 100

    try {

        ## Post the Source to be saved for this script.
        $uri = "https://"
        $uri += $Server
        $uri += '/tdstm/ws/dataScript/saveScript'
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBodyJSON @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq 'success') {
                if ($PassThru) {
                    return $responseContent.data.dataScript
                }
            }
        }
        elseif ($response.StatusCode -eq 204) {
            return
        }
    }
 catch {
        Write-Host "Unable to save Datascript Source to server"
        return $_
    }

}

Function Invoke-TMDatascript {
    [alias("Invoke-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScriptName")]
        [Parameter(Mandatory = $true)][String]$DatascriptName,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,
        [Parameter(Mandatory = $false)][bool]$QueueBatches = $false,
        [Parameter(Mandatory = $false)][bool]$MonitorBatches = $false,
        [Parameter(Mandatory = $true)][String]$DataType,
        [Parameter(Mandatory = $false)]$Data,
        [Parameter(Mandatory = $false)][String]$FilePath,
        [Parameter(Mandatory = $false)][String]$FileName,
        [Parameter(Mandatory = $false)][Int16]$ActivityId,
        [Parameter(Mandatory = $false)][Int16]$ParentActivityId = -1,
        [Parameter(Mandatory = $false)][switch]$isAutoPost,
        [Switch]$SendNotification
    )
    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    ## If a File Path was passed and there is no data, Read the file
    if ($FilePath -and -not $Data) {
        $Data = Get-Content $FilePath -Raw
        $FileName = (Get-Item $FilePath).Name
    }

    ## If a File Path was passed and there is no File Name, get the file name
    if ($FilePath -and -not $FileName) {
        $FileName = (Get-Item $FilePath).Name
    }

    #Honor SSL Settings
    if ($TMSessionConfig.AllowInsecureSSL) {
        $TMCertSettings = @{SkipCertificateCheck = $true }
    }
    else {
        $TMCertSettings = @{SkipCertificateCheck = $false }
    }

    if ($ActivityId) {
        ## Parent ID is only used on the root 'TransitionManager Data Import' Activity
        ## ParentID + 1 = Transform Data
        ## ParentID + 2 = Import Batches
        ## ParentID + 3 + n for each batch

        ## Add the Datascript Processing Progress Indicators
        $ProgressIndicators = @()
        $ProgressIndicators += @{ Id = $ActivityId; Activity = 'TransitionManager Data Import'; ParentId = $ParentActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 1); Activity = 'Datascript Data Transformation'; ParentId = $ActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 2); Activity = 'Import Batches'; ParentId = $ActivityId }
        $ProgressIndicators += @{ Id = ($ActivityId + 3); Activity = 'Monitor Batches'; ParentId = $ActivityId }

        #Write Progress Indicators for each in the Array
        $ProgressIndicators | ForEach-Object {
            Write-Progress @_ -CurrentOperation 'Queued' -PercentComplete 0
        }
    }

    ## Fix the Server URL
    $instance = $Server.Replace('/tdstm', '')
    $instance = $instance.Replace('https://', '')
    $instance = $instance.Replace('http://', '')

    $Boundary = '----------------------------540299933173025267350719'

    ## First Test to see if the Datascript Script exists or not
    if ($ActivityId) {

        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Validating Datascript Script' -Status 'Confirming Datascript Script exists in TransitionManager' -PercentComplete 5
    }
    Write-Host 'Validating Datascript Script: ' -NoNewline
    Write-Host $DatascriptName -ForegroundColor Yellow

    $Datascript = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName
    if (-Not $Datascript) { Throw  'The Datascript [' + $DatascriptName + '] does not exist' }

    ## Build the Data Post
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/fileSystem/uploadFileETLAssetImport"

    ## FileName
    if (-not $FileName) {
        $FileName = ('DataForETL_' + (Get-Date -Format FileDateTime ) + '.' + $DataType)
    }

    ## Construct the PostBody and Form Data for the request
    $CRLF = "`r`n";
    $PostBody = ("-----------------------------$boundary",
        'Content-Disposition: form-data; name="uploadType"',
        '',
        'assetImport',
        "-----------------------------$boundary",
        ''
    ) -join $CRLF
    $FileUpload = @{
        Form = @{
            uploadType = 'assetImport'
            file       = Get-Item -Path $FilePath
            fileName   = $FileName
        }
    }

    ## Upload the Data File
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Uploading Data' -Status ('Uploading ' + $FileName) -PercentComplete 5
    }
    Write-Host "Uploading File: " -NoNewline
    Write-Host $FileName -ForegroundColor Yellow

    try {
        ## If the file is being uploaded, append the fileupload to the command
        if ($FileUpload) {
            $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings @FileUpload

        }
        else {

            $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings
        }

    }
    catch {
        throw $_
    }
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptdataFileName = $responseContent.data.filename
        }
        else {
            Throw "Unable to upload data to TM Datascript pre-transform storage."
        }
    }

    ## With the file uploaded, Initiate the Datascript script on the server
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/assetImport/initiateTransformData"
    $uri += "?dataScriptId=" + $Datascript.id
    $uri += "&filename=" + $DatascriptdataFileName

    ## Deal with Version specific items
    if ($TMSessionConfig.TMVersion -ge '6.0.2.1') {
        $uri += "&originalFilename=" + $FileName
        $uri += "&sendNotification=" + $SendNotification.IsPresent.toString().ToLower()
    }
    if ($TMSessionConfig.TMVersion -ge '6.1.0' -and $isAutoPost.IsPresent) {
        $uri += '&isAutoPost=true'
    }

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Starting Datascript Transformation' -PercentComplete 5
    }
    Write-Host 'TransitionManager Data Import: ' -NoNewline
    Write-Host 'Starting Datascript Transformation' -ForegroundColor Yellow

    $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptProgressKey = $responseContent.data.progressKey
        }
        else {
            throw $responseContent.errors
        }
    }
    else {
        Throw $response
    }


    ## Supply Progress from Datascript Transformation Progress
    if ($ActivityId) {
        Write-Progress -Id $($ActivityId + 1) -Activity 'Datascript Data Transformation' -CurrentOperation 'Running Datascript Transformation' -PercentComplete 0 -ParentId $ActivityId
    }

    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    ## $ReportedLogItems = [System.Collections.ArrayList]@()
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {

                        $CurrentOperation = 'Datascript Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break

                    }
                    "Pending" {
                        $CurrentOperation = 'Datascript Pending'
                        $Status = 'Pending'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Datascript Running'
                        $Status = 'Transforming Data'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp -gt 99 ? 99 : $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        $DatascriptOutputKey = $DatascriptProgress.detail
                        $CurrentOperation = 'Datascript Processing Complete'
                        $Status = 'Creating Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Datascript Processing Complete, Creating Import Batches."
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Datascript Processing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    Write-Progress -Id ($ActivityId + 1) `
                        -ParentId $ActivityId `
                        -Activity 'Datascript Data Transformation' `
                        -CurrentOperation $CurrentOperation `
                        -Status $Status `
                        -PercentComplete ($PercentComplete ?? 0)
                }
                else {
                    Write-Host $ProgressString
                }
                Start-Sleep -Seconds $SleepSeconds
            }
        }
    }

    ## Update Progress and transition to Activity + 2
    Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Transformation Complete' -PercentComplete 100 -Completed
    Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Importing Batches' -CurrentOperation 'Beginning Import' -PercentComplete 5

    Write-Host "Datascript Transformation is complete, Loading to Batch Import"

    ## TODO, this is where a Job Log could be pulled. With the DatascriptOutputKey now provided, the API like this could be used:
    ## https://tmad60.transitionmanager.net/tdstm/ws/assetImport/viewData?filename=DatascriptOutputData_BSItLiP036AOsI0cr8e1g2WmipbdtN0c.json and
    ## https://tmad60.transitionmanager.net/tdstm/ws/assetImport/viewData?filename=DatascriptOutputData_BSItLiP036AOsI0cr8e1g2WmipbdtN0c_Application.json
    ## Check the Datascript Script Editor Web Dev tools to see more

    # TM version 6.0 and lower seperate transforming and posting the Datascript. Version 6.1 does not. If 6.1 runs both the transform and the post steps we will get each import batch in TM twice.
    if ($TMSessionConfig.TMVersion -lt '6.1.0') {
        ## With the Datascript converted, Use it to create Import Batches
        $uri = "https://"
        $uri += $instance
        $uri += "/tdstm/ws/assetImport/loadData?filename="
        $uri += $DatascriptOutputKey
        $uri += "&dataScriptId="
        $uri += $Datascript.id

        Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

        ## Post the Transformed data filename to the Datascript engine to Import the Batches.
        $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $ImportJob = $responseContent.data
                $DatascriptProgressKey = $ImportJob.progressKey
            }
            else {
                throw 'Failed to Load Import Batch'
            }
        }
        else {
            Throw 'Failed to Load Import Batch.'
        }
    }

    ## Ensure the Batches loaded and the job is finished
    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {
                        $CurrentOperation = 'Loading Batches Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "Pending" {
                        $CurrentOperation = 'Loading Batches Pending'
                        $Status = 'Pending Batch Loading'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Loading Batches'
                        $Status = 'Loading Batches'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        $BatchGroupGuid = $DatascriptProgress.data.groupGuid
                        $CurrentOperation = 'Batch Loading Complete'
                        $Status = 'Loaded Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Completed Loading Import Batches."
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Batch Loading Processing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    $ProgressOptions = @{
                        Id               = ($ActivityId + 2)
                        ParentId         = $ActivityId
                        Activity         = 'Import Batch Loading'
                        CurrentOperation = $CurrentOperation
                        Status           = $Status
                        PercentComplete  = $PercentComplete
                    }
                    if ($Completed) {
                        $ProgressOptions.Completed = $True
                    }
                    Write-Progress @ProgressOptions
                }
                else {
                    Write-Host $ProgressString
                }
                if (-Not $Completed) {
                    Start-Sleep -Seconds $SleepSeconds
                }
            }
        }
    }

    ## Assemble the batch list object. Dependencies are deliberately moved to the end
    $BatchesToProcess = [System.Collections.ArrayList]@()

    ## Get the Batches that were created during the import
    ## With the Datascript converted, Use it to create Import Batches
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/import/batches?groupGuid="
    $uri += $BatchGroupGuid

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the Transformed data filename to the Datascript engine to Import the Batches.
    $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $Batches = $responseContent.data
        }
        else {

        }
    }
    else {

    }

    Write-Host $Batches.Length -ForegroundColor Yellow -NoNewline
    Write-Host ' Batches have been created'

    ## Update Progress and transition to Activity + 3
    Write-Progress -Id ($ActivityId + 2) -ParentId $ActivityId -Activity 'Batches Imported' -CurrentOperation 'Importing Batches Complete' -PercentComplete 100 -Completed
    Write-Progress -Id ($ActivityId + 3) -ParentId $ActivityId -Activity 'Monitoring Batches' -CurrentOperation 'Monitoring Batches' -PercentComplete 5

    ## OPTIONAL - If the switch was set to Queue the batches
    if ($QueueBatches) {

        ## Sort the batches to a desired order
        $Batches | Where-Object { $_.domainClass -ne 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null }
        $Batches | Where-Object { $_.domainClass -eq 'Dependency' } | ForEach-Object { $BatchesToProcess.Add($_) | Out-Null }

        ## Write Progress to the Monitor Import Batches
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 3) `
                -ParentId $ActivityId `
                -Activity 'Manage Batches' `
                -CurrentOperation 'Queueing Batches' `
                -Status ('Queueing ' + [string]$Batches.Length + ' batches.') `
                -PercentComplete 5
        }
        Write-Host "Queueing Batches:" -NoNewline
        Write-Host $Batches.Length -ForegroundColor Yellow

        ## Add a Progress Activity for each of the Import Batches
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]

            ## Create Progress Results for the batches that were created
            if ($ActivityId) {

                ## Activity Explained: The $ActivityId is considered the (Root) + 2 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                    -ParentId ($ActivityId + 3) `
                    -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                    -CurrentOperation 'Queueing Batch' `
                    -Status ([String]$Batch.recordsSummary.count + ' ' + $Batch.domainClassName + ' records') `
                    -PercentComplete 10
            }
            Write-Host "Batch (id $($Batch.id)) Created with $([String]$Batch.recordsSummary.count) $($Batch.domainClassName) records"
        }

        ## Queue each Batch
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]
            if ($Batch.autoProcess -eq 0) {

                ## With the file uploaded, Initiate the Datascript script on the server
                $uri = "https://"
                $uri += $instance
                $uri += "/tdstm/ws/import/batches"

                Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

                $PostBody = @{
                    action = 'QUEUE'
                    ids    = $Batch.id
                } | ConvertTo-Json -Depth 100


                ## Post this batch to begin it's Queueing
                $response = Invoke-WebRequest -Method Patch -Uri $uri -WebSession $TMSessionConfig.TMWebSession -Body $PostBody @TMCertSettings
                if ($response.StatusCode -eq 200) {
                    $responseContent = $response.Content | ConvertFrom-Json
                    if ($responseContent.status -eq "success") {

                        ## Create Progress Results for the batches that were created
                        if ($ActivityId) {

                            ## Notify the Manage Batches Activity of a queued batch
                            Write-Progress -Id ($ActivityId + 3) `
                                -ParentId $ActivityId `
                                -Activity 'Monitor Batches' `
                                -CurrentOperation 'Queueing Batches' `
                                -Status ('Queued ' + $Batch.domainClass + ' Batch') `
                                -PercentComplete 5

                            ## Activity (Root) + 2 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                            Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                                -ParentId ($ActivityId + 3) `
                                -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                                -CurrentOperation 'Batch Queued'  `
                                -Status 'Queued' `
                                -PercentComplete 0
                        }
                        Write-Host "Batch Queued: " -NoNewline
                        Write-Host $Batch.domainClassName -ForegroundColor Yellow
                    }
                    else {
                        Throw 'Failed to Queue Batch'
                    }
                }
                else {
                    Throw 'Failed to Queue Batch.'
                }
            }
        }

        ## If Monitoring isn't going to occur, status should be complete here
        if ($MonitorBatches -ne $true) {
            Write-Progress -Id ($ActivityId + 3) -ParentId $ActivityId -Activity 'Batches Queued' -CurrentOperation 'Batches have been queued' -PercentComplete 100 -Completed
        }
    }

    ## Allow the batch to be monitored and reported on in the UI
    if ($MonitorBatches) {
        $BatchStatus = @{
            CompletedBatches = 0
            TotalBatches     = $BatchesToProcess.Count
        }

        ## Monitor the batches to completion
        for ($i = 0; $i -lt $BatchesToProcess.Count; $i++) {

            ## Set Variable for Batch
            $Batch = $BatchesToProcess[$i]
            if ($Batch.recordsSummary.count -gt 0) {
                ## Build the URL
                $uri = "https://"
                $uri += $instance
                $uri += "/tdstm/ws/import/batch/"
                $uri += $Batch.id
                $uri += "/progress"

                Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

                ## Poll for the status of the Import Batch
                if ($ActivityId) {

                    ## Activity (Root) + 3 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                    Write-Progress -Id ($ActivityId + 3 + $i + 1) `
                        -ParentId ($ActivityId + 3) `
                        -Activity ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count) `
                        -CurrentOperation 'Monitoring Progress'  `
                        -Status 'Importing' `
                        -PercentComplete 1
                }
                Write-Host 'Monitoring Batch Status for ' -NoNewline
                Write-Host $Batch.domainClassName -ForegroundColor Yellow

                ## Start a loop to run while the batch is not complete.
                $Completed = $false
                while ($Completed -eq $false) {

                    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
                    $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
                    if ($response.StatusCode -eq 200) {
                        $responseContent = $response.Content | ConvertFrom-Json
                        if ($responseContent.status -eq "success") {

                            $BatchProgress = $responseContent.data

                            switch ($BatchProgress.status.code) {
                                "RUNNING" {
                                    $CurrentOperation = 'Import Running'
                                    $Status = 'Importing Batch Data'
                                    $ProgressString = $DomainClass + ' Status: Running - ' + $BatchProgress.progress + "%"
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break
                                }
                                "QUEUED" {
                                    $CurrentOperation = 'Batch Queued'
                                    $Status = 'Batch Queued'
                                    $ProgressString = $DomainClass + ' Status: Queued'
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break

                                }
                                "PENDING" {
                                    $CurrentOperation = 'Batch Pending'
                                    $Status = 'Batch Pending'
                                    $ProgressString = $DomainClass + ' Status: Pending'
                                    $PercentComplete = $BatchProgress.progress
                                    $SleepSeconds = 2
                                    break

                                }
                                "COMPLETED" {
                                    $Completed = $true
                                    $CurrentOperation = 'Complete'
                                    $Status = 'Complete'
                                    $ProgressString = ($batch.domainClassName + " Status: Complete")
                                    $PercentComplete = 100
                                    $SleepSeconds = 0
                                    break

                                }
                                Default {
                                    $CurrentOperation = 'Status Unknown. Retrying'
                                    $Status = 'Retrying'
                                    $ProgressString = ($batch.domainClassName + "Status Unkown. Retrying")
                                    $PercentComplete = 1
                                    $SleepSeconds = 2
                                    break
                                }
                            }

                            ## Display the Status of this loop
                            if ($ActivityId) {

                                $ProgressOptions = @{
                                    Id               = ($ActivityId + 3 + $i + 1)
                                    ParentId         = ($ActivityId + 3)
                                    Activity         = ($Batch.domainClassName + ' batch: ' + $Batch.id + ' | Total: ' + $Batch.recordsSummary.count)
                                    CurrentOperation = $CurrentOperation
                                    Status           = $Status
                                    PercentComplete  = ($PercentComplete ?? 0)
                                }
                                ## Activity (Root) + 3 (to move to Monitoring Batches) + I for the looping + 1 to add a new layer
                                if ($Completed) {
                                    $ProgressOptions.Completed = $true
                                }
                                Write-Progress @ProgressOptions

                            }
                            else {
                                Write-Host $ProgressString
                            }

                            ## Sleep the expected duration
                            Start-Sleep -Seconds $SleepSeconds
                        }
                    }
                }
            }
            ## Now that this batch is done, increment the Completed Batches counter
            $BatchStatus.CompletedBatches++
            Write-Host 'Batch Import Complete for ' -NoNewline
            Write-Host $Batch.domainClassName -ForegroundColor Yellow
        }

        ## Mark the Manage Batches Activity Complete
        if ($ActivityId) {
            Write-Progress -Id ($ActivityId + 3) `
                -ParentId $ActivityId `
                -Activity 'Batches Posted' `
                -CurrentOperation 'Complete'  `
                -Status 'All Batches Imported' `
                -PercentComplete 100 `
                -Completed
        }
    }


    ## Mark the TM Import Activity complete
    if ($ActivityId) {
        Write-Progress -Id $ActivityId `
            -ParentId $ParentActivityId `
            -Activity 'TransitionManager Data Import' `
            -CurrentOperation 'Complete'  `
            -Status 'All Data Imported' `
            -PercentComplete 100 `
            -Complete
    }
    Write-Host 'All Batches have Imported successfully'
}

Function Test-TMDatascript {
    [alias("Test-TMETLScript")]
    param(
        [Parameter(Mandatory = $false)][String]$TMSession = "Default",
        [Alias("ETLScriptName")]
        [Parameter(Mandatory = $true)][String]$DatascriptName,
        [Parameter(Mandatory = $false)][String]$Server = $global:TMSessions[$TMSession].TMServer,
        [Parameter(Mandatory = $false)]$AllowInsecureSSL = $global:TMSessions[$TMSession].AllowInsecureSSL,
        [Parameter(Mandatory = $false)][bool]$QueueBatches = $false,
        [Parameter(Mandatory = $false)][bool]$MonitorBatches = $false,
        [Parameter(Mandatory = $true)][String]$DataType,
        [Parameter(Mandatory = $false)]$Data,
        [Parameter(Mandatory = $false)][String]$FilePath,
        [Parameter(Mandatory = $false)][String]$FileName,
        [Parameter(Mandatory = $false)][Int16]$ActivityId,
        [Parameter(Mandatory = $false)][Int16]$ParentActivityId = -1
    )

    ## Get Session Configuration
    $TMSessionConfig = $global:TMSessions[$TMSession]
    if (-not $TMSessionConfig) {
        Write-Host 'TMSession: [' -NoNewline
        Write-Host $TMSession -ForegroundColor Cyan
        Write-Host '] was not Found. Please use the New-TMSession command.'
        Throw "TM Session Not Found. Use New-TMSession command before using features."
    }

    ## If a File Path was passed and there is no data, Read the file
    if ($FilePath -and -not $Data) {
        $Data = Get-Content $FilePath -Raw
        $FileName = (Get-Item $FilePath).Name
    }

    ## If a File Path was passed and there is no File Name, get the file name
    if ($FilePath -and -not $FileName) {
        $FileName = (Get-Item $FilePath).Name
    }

    #Honor SSL Settings
    if ($TMSessionConfig.AllowInsecureSSL) {
        $TMCertSettings = @{SkipCertificateCheck = $true }
    }
    else {
        $TMCertSettings = @{SkipCertificateCheck = $false }
    }

    if ($ActivityId) {
        ## Parent ID is only used on the root 'TransitionManager Data Import' Activity
        ## ParentID + 1 = Transform Data
        ## ParentID + 2 = Import Batches
        ## ParentID + 3 + n for each batch

        ## Add the Datascript Processing Progress Indicators
        $ProgressIndicators = @()
        $ProgressIndicators += @{ Id = $ActivityId; Activity = 'Test Datascript Data Transformation'; ParentId = $ParentActivityId }

    }

    ## Fix the Server URL
    $instance = $Server.Replace('/tdstm', '')
    $instance = $instance.Replace('https://', '')
    $instance = $instance.Replace('http://', '')

    $Boundary = '----------------------------540299933173025267350719'

    ## First Test to see if the Datascript exists or not
    if ($ActivityId) {

        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Datascript Data Transformation' -CurrentOperation 'Validating Datascript' -Status 'Confirming Datascript exists in TransitionManager' -PercentComplete 5
    }
    $Datascript = Get-TMDatascript -TMSession $TMSession -Name $DatascriptName
    if (-Not $Datascript) { Throw  'The Datascript [' + $DatascriptName + '] does not exist' }

    ## Build the Data Post
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/fileSystem/uploadFileETLAssetImport"

    ## FileName
    if (-not $FileName) {
        $FileName = ('DataForETL_' + (Get-Date -Format FileDateTime ) + '.' + $DataType)
    }

    ## Construct the PostBody and Form Data for the request
    $CRLF = "`r`n";
    $PostBody = ("-----------------------------$boundary",
        'Content-Disposition: form-data; name="uploadType"',
        '',
        'assetImport',
        "-----------------------------$boundary",
        ''
    ) -join $CRLF
    $FileUpload = @{
        Form = @{
            uploadType = 'assetImport'
            file       = Get-Item -Path $FilePath
            fileName   = $FileName
        }
    }
    ## Upload the Data File
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Uploading Data' -Status ('Uploading ' + $FileName) -PercentComplete 5
    }

    try {
        ## If the file is being uploaded, append the fileupload to the command
        if ($FileUpload) {
            $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings @FileUpload
        }
        else {
            $response = Invoke-WebRequest -Method Post -Uri $uri -WebSession $TMSessionConfig.TMWebSession -ContentType ('multipart/form-data; boundary=---------------------------' + $boundary) @TMCertSettings
        }

    }
    catch {
        throw $_
    }
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptdataFileName = $responseContent.data.filename
        }
        else {
            Throw "Unable to upload data to TM Datascript pre-transform storage."
        }
    }

    ## With the file uploaded, Initiate the Datascript script on the server
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/dataScript/initiateTestScript"

    ## Create a post body
    $PostBody = [PSCustomObject]@{
        dataScriptId = $Datascript.id
        filename     = $DatascriptdataFileName
        script       = ($Datascript.etlSourceCode -replace $CRLF, '\n')
    } | ConvertTo-Json -Compress

    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript
    if ($ActivityId) {
        Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Starting Datascript Testing' -PercentComplete 5
    }
    else {
        Write-Host 'TransitionManager Data Import: Starting Datascript Test'
    }

    $response = Invoke-WebRequest -Method Post -Uri $uri -Body $PostBody -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
    if ($response.StatusCode -eq 200) {
        $responseContent = $response.Content | ConvertFrom-Json
        if ($responseContent.status -eq "success") {
            $DatascriptProgressKey = $responseContent.data.progressKey
        }
        else {
            throw $responseContent.errors
        }
    }
    else {
        Throw $response
    }


    ## Supply Progress from Datascript Transformation Progress
    if ($ActivityId) {
        Write-Progress -Id $($ActivityId + 1) -Activity 'Testing Datascript Data Transformation' -CurrentOperation 'Running Datascript Transformation' -PercentComplete 0 -ParentId $ActivityId
    }
    else {
        Write-Host 'Testing Datascript Data Transformation: Starting Datascript Processing'
    }

    ## The Datascript is underway, Setup a progress URL
    $uri = "https://"
    $uri += $instance
    $uri += "/tdstm/ws/progress/" + $DatascriptProgressKey
    Set-TMHeaderContentType -ContentType JSON -TMSession $TMSession

    ## Poll for the status of the Datascript engine
    ## TODO: This should be converted to a function for polling the job engine. It's nearly dupilcated now in the Import Batch watching.
    $Completed = $false
    while ($Completed -eq $false) {

        ## Post the data starting the Datascript process. A progress key is provided. This progress key is what can be used to poll for the status of the Datascript script
        $response = Invoke-WebRequest -Method Get -Uri $uri -WebSession $TMSessionConfig.TMWebSession @TMCertSettings
        if ($response.StatusCode -eq 200) {
            $responseContent = $response.Content | ConvertFrom-Json
            if ($responseContent.status -eq "success") {
                $DatascriptProgress = $responseContent.data

                switch ($DatascriptProgress.status) {
                    "Queued" {

                        $CurrentOperation = 'Datascript Queued'
                        $Status = 'Queued'
                        $ProgressString = 'Status - Queued: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break

                    }
                    "Pending" {
                        $CurrentOperation = 'Datascript Pending'
                        $Status = 'Pending'
                        $ProgressString = 'Status - Pending: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "RUNNING" {
                        $CurrentOperation = 'Datascript Running'
                        $Status = 'Transforming Data'
                        $ProgressString = 'Status - Running: ' + $DatascriptProgress.percentComp + '%'
                        $PercentComplete = $DatascriptProgress.percentComp
                        $SleepSeconds = 2
                        Break
                    }
                    "COMPLETED" {
                        $CurrentOperation = 'Datascript Testing Complete'
                        $Status = 'Creating Import Batches'
                        $Completed = $true
                        $SleepSeconds = 0
                        $PercentComplete = 99
                        $ProgressString = "Status - Datascript Testing Complete"
                        Break
                    }
                    "Failed" {
                        $CurrentOperation = 'Failed'
                        $Status = $DatascriptProgress.status
                        Write-Host "Datascript Testing Failed "$DatascriptProgress.detail
                        Throw $DatascriptProgress.detail
                    }
                    Default {
                        $CurrentOperation = 'State Unknown'
                        $Status = 'Unknown. Sleeping to try again.'
                        $ProgressString = "Unknown Status: " + $DatascriptProgress.status
                        $PercentComplete = 99
                        $SleepSeconds = 2
                        Break
                    }
                }

                ## Notify the user of the Datascript Progress
                if ($ActivityId) {
                    Write-Progress -Id ($ActivityId + 1) `
                        -ParentId $ActivityId `
                        -Activity 'Testing Datascript Data Transformation' `
                        -CurrentOperation $CurrentOperation `
                        -Status $Status `
                        -PercentComplete $PercentComplete
                }
                else {
                    Write-Host $ProgressString
                }
                Start-Sleep -Seconds $SleepSeconds
            }
        }
    }

    ## Update Progress and transition to Activity + 2
    Write-Progress -Id ($ActivityId + 1) -ParentId $ActivityId -Activity 'Tested Datascript Data Transformation' -CurrentOperation 'Testing Complete' -PercentComplete 100 -Completed

    ## Mark the TM Import Activity complete
    if ($ActivityId) {
        Write-Progress -Id $ActivityId `
            -ParentId $ParentActivityId `
            -Activity 'TransitionManager Data Import' `
            -CurrentOperation 'Complete'  `
            -Status 'All Data Imported' `
            -PercentComplete 100 `
            -Complete
    }
}

Function Read-TMDatascriptFile {
    param(
        [Parameter(Mandatory = $true)]$Path
    )

    ## First order of business is to determine if this Script file has a counterpart Json file
    $DatascriptConfigJsonPath = $Path -Replace '.groovy', '.json'
    $DatascriptConfigFile = Get-Item -Path $DatascriptConfigJsonPath -ErrorAction 'SilentlyContinue'

    ## Check if there is a config JSON file, if so, get the Action from the 2 files
    if ($DatascriptConfigFile) {

        ## Get the Recipe Object that doesn't have the source code
        $TMDatascript = Get-Content -Path $DatascriptConfigFile | ConvertFrom-Json
        Add-Member -InputObject $TMDatascript -NotePropertyName etlSourceCode -NotePropertyValue (Get-Content -Path $Path -Raw)
    }

    ## Read the ReferenceDesign (Combined) file
    else {

        ## Name the Input File
        $Content = Get-Content -Path $Path -Raw

        ## Ignore Empty Files
        if (-Not $Content) {
            return
        }

        $ContentLines = Get-Content -Path $Path

        ## Create Automation Token Variables Parse the Script File
        New-Variable astTokens -Force
        New-Variable astErr -Force
        $ast = [System.Management.Automation.Language.Parser]::ParseInput($Content, [ref]$astTokens, [ref]$astErr)

        ##
        ## Assess the Script Parts to get delineating line numbers
        ##

        ## Locate the Delimiting line
        $ConfigBlockStartLine = $astTokens | `
            Where-Object { $_.Text -like '/*********TransitionManager-ETL-Script*********' } |`
            Select-Object -First 1 | `
            Select-Object -ExpandProperty Extent | `
            Select-Object -ExpandProperty StartLineNumber

        ## Test if the file has been written with MetaData
        if (-Not $ConfigBlockStartLine) {

            ## This file does not have metadata. Create the minimum data for the constructor below
            $DatascriptConfig = @{
                DatascriptName = (Get-Item -Path $Path).BaseName
                Description    = ""
                ProviderName   = (Get-Item -Path $Path).Directory.BaseName
                IsAutoProcess  = $false
                Target         = $null
                Mode           = 'Import'
            }

            ## Add Variables that will be used in the construction
            $DatascriptCode = $ContentLines
            $ConfigBlockEndLine = -1
        }
        else {

            ## This File has metadata - Work to parse it

            $ConfigBlockEndLine = $astTokens | `
                Where-Object { $_.Text -like '*********TransitionManager-ETL-Script*********/' } |`
                Select-Object -First 1 | `
                Select-Object -ExpandProperty Extent | `
                Select-Object -ExpandProperty StartLineNumber

            ## Adjust the Line Numbers to capture just the JSON
            $JsonConfigBlockStartLine = $ConfigBlockStartLine + 1
            $JsonConfigBlockEndLine = $ConfigBlockEndLine - 1


            ##
            ## Read the Script Header to gather the configurations
            ##

            ## Get all of the lines in the header comment
            $DatascriptConfigJson = $JsonConfigBlockStartLine..$JSONConfigBlockEndLine | ForEach-Object {

                ## Return the line for collection
                $ContentLines[$_ - 1]

            } | Out-String

            ## Convert the JSON string to an Object
            if ($DatascriptConfigJson) {
                $DatascriptConfig = $DatascriptConfigJson | ConvertFrom-Json
            }
            else {
                $DatascriptConfig = @{
                    DatascriptName = (Get-Item -Path $Path).BaseName
                    Description    = ""
                    ProviderName   = (Get-Item -Path $Path).Directory.Parent.Parent.BaseName
                    IsAutoProcess  = $false
                    Target         = $null
                    Mode           = 'Import'
                }
            }
        }

        ##
        ## Read the Script Block
        ##

        ## Note where the Configuration Code is located
        $StartCodeBlockLine = $ConfigBlockEndLine + 1
        $EndCodeBlockLine = $ast[-1].Extent.EndLineNumber

        ## Create a Text StrinBuilder to collect the Script into
        $DatascriptStringBuilder = New-Object System.Text.StringBuilder

        ## For each line in the Code Block, add it to the Datascript Script Code StringBuilder
        $StartCodeBlockLine..$EndCodeBlockLine | ForEach-Object {
            $DatascriptStringBuilder.AppendLine($ContentLines[$_]) | Out-Null
        }

        ## Convert the StringBuilder to a Multi-Line String
        $DatascriptCode = $DatascriptStringBuilder.ToString()

        ## Assemble the Action Object
        $TMDatascript = [pscustomobject]@{

            ## Primary Information
            id            = $null
            name          = $DatascriptConfig.DatascriptName
            description   = $DatascriptConfig.Description

            ## Source Code
            etlSourceCode = $DatascriptCode

            ## Provider
            provider      = @{
                id   = $null
                name = $DatascriptConfig.ProviderName
            }

            ## Other details for the Datascript Script
            dateCreated   = Get-Date
            lastUpdated   = Get-Date
        }
    }

    ## Return the Action Object
    return $TMDatascript
}