WttStudioUtils.psm1
|
Import-Module $PSScriptRoot\Logger.psm1 Import-Module $PSScriptRoot\PsHelper.psm1 Import-Module $PSScriptRoot\Utils.psm1 $script:WorkFlowCommandLine = Join-Path $env:WTTSTDIOATLAS "WorkFlowCommandLine.exe" $script:WTTCL = Join-Path $env:WTTSTDIOATLAS "WTTCL.exe" $script:WTTDimUpdate = Join-Path $env:WTTSTDIOATLAS "WTTDimUpdate.exe" function Test-WttInstalled() { if (-not (Test-Path $script:WorkFlowCommandLine)) { throw "Wtt studio install not found, cannot continue" } } $Global:DataStoreConnections = @{} $Global:IdentityServer = "ATLASIdentity.redmond.corp.microsoft.com" $Global:IdentityDb = "WTTIdentity" $Script:WttInitialized = $false function Initialize-Wtt { Test-WttInstalled if ($Script:WttInitialized) { return } Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMAsset.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMBase.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMAssetConfig.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMIdentity.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMJobs.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMParameter.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMResource.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMSQLProvider.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMDimension.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS WTTOMStage.dll) -Scope Global Import-Module (Join-Path $env:WTTSTDIOATLAS ConditionalWorkflowParameterStageTemplate.dll) -Scope Global $Script:WttInitialized = $true } function Get-WttConnection { param( [Parameter(Mandatory = $true)][string] $DataStore ) Initialize-Wtt $connection = $Global:DataStoreConnections[$DataStore] if ($connection -ne $null) { return $connection } $connInfo = New-Object -TypeName "Microsoft.DistributedAutomation.SqlIdentityConnectInfo" ` -ArgumentList @($Global:IdentityServer , $Global:IdentityDb) $serviceName = [Microsoft.DistributedAutomation.Jobs.JobsRuntimeDataStore]::ServiceName $dsConnection = [Microsoft.DistributedAutomation.Enterprise]::Connect($DataStore, $serviceName, $connInfo) $Global:DataStoreConnections[$DataStore] = $dsConnection return $dsConnection } function Get-ActiveWttWorkFlowInstance { param( [Parameter(Mandatory = $true)][string]$DataStore, [Parameter(Mandatory = $false)][string]$InstanceId, [Parameter(Mandatory = $false)][string]$Name ) if ([String]::IsNullOrEmpty($InstanceId) -and [String]::IsNullOrEmpty($Name)) { throw "Get-ActiveWttWorkFlowInstance: Either InstanceId or Name must be specified" } Write-TraceLog "Get-ActiveWttWorkFlowInstance: DataStore: $DataStore, Name: $Name, InstanceId: $InstanceId" $query = New-Object Microsoft.DistributedAutomation.Query ([Microsoft.DistributedAutomation.Workflow.WorkflowInstance]) if (-not [String]::IsNullOrEmpty($InstanceId)) { $query.AddExpression('Id', 'Equals', $InstanceId) } if (-not [String]::IsNullOrEmpty($Name)) { $query.AddExpression('Name', 'Equals', $Name) } $query.AddConjunction('And') $query.AddExpression('Status', 'Equals', 'Active') $dsConnection = Get-WttConnection -DataStore $DataStore $instance = $dsConnection.Query($query) | select -First 1 if ($instance -eq $null) { Write-TraceLog "Get-ActiveWttWorkFlowInstance: No active workflow instance found" return $null } Write-TraceLog "Get-ActiveWttWorkFlowInstance: Active workflow instance found" return $instance } function Get-WttResultCollectionsByName { param( [Parameter(Mandatory = $true)][string]$DataStore, [Parameter(Mandatory = $true)][string]$Name ) $dsConnection = Get-WttConnection -DataStore $DataStore $query = New-Object -TypeName Microsoft.DistributedAutomation.Query -ArgumentList @([Microsoft.DistributedAutomation.Jobs.ResultSummary]) $query.AddExpression("Name", [Microsoft.DistributedAutomation.QueryOperator]::Equals, $Name) $resColl = $dsConnection.Query($query) if ($resColl.Count -eq 0) { return $null } $retval = @() foreach ($r in $resColl) { $retval += $r } return $retval } function Get-WttMachine() { param( [Parameter(Mandatory = $true)][string]$DataStore, [Parameter(Mandatory = $true)][string]$Name ) $connection = Get-WttConnection -DataStore $DataStore $query = New-Object -TypeName Microsoft.DistributedAutomation.Query -ArgumentList @([Microsoft.DistributedAutomation.Asset.Resource]) $e = [Microsoft.DistributedAutomation.QueryOperator]::Equals $query.AddExpression("Name", $e, $Name) $resColl = $connection.Query($query) return $resColl[0] } function Get-WttPoolForMachine() { param( [string]$DataStore, [string]$Name ) $machine = Get-WttMachine -DataStore $DataStore -Name $Name if ($machine -eq $null) { Write-Error "No machine '$machineName' found" } $query = New-Object -TypeName Microsoft.DistributedAutomation.Query -ArgumentList @([Microsoft.DistributedAutomation.Asset.ResourcePool]) $e = [Enum]::Parse([Microsoft.DistributedAutomation.QueryOperator], "Equals") $query.AddExpression("Id", $e, $machine.ResourcePoolId) $connection = Get-WttConnection -DataStore $DataStore $pools = $connection.Query($query); if ($pools -eq $null -or $pools.Count -eq 0) { return $null } return $pools[0].FullPath } function Test-IsRetriableError { param( [string]$Result, [switch]$RetryOnNull ) if ($Result -eq $null -or [String]::IsNullOrEmpty($Result)) { return $RetryOnNull.IsPresent } if ($Result.Contains("A connection to the data store")) { # ERROR Message : A connection to the data store 'WTTIDENTITY' could not be established. return $true } return $false } function Invoke-WttWorkflow() { param( [int][parameter(Mandatory = $true)]$ID, [string][parameter(Mandatory = $true)]$MachinePool, [string[]][parameter(Mandatory = $true)]$Machines, [HashTable][parameter(Mandatory = $false)] $Params, # @ {"Param1" = "Value1"; "Param2" = "Value2"} [string][parameter(Mandatory = $true)]$MachineDataStore, [string][parameter(Mandatory = $true)]$JobDataStore, [bool][parameter(Mandatory = $false)]$WaitForExit, [string][parameter(Mandatory = $true)]$ResultCollection ) $CommandArgs = " /IdentityServer:$Global:IdentityServer /IdentityDatabase:WTTIDENTITY /ID:$ID /DataStore:$JobDataStore /ResourceDataStore:$MachineDataStore /MachinePool:'$MachinePool' " if ($Params -ne $nul) { foreach ($param in $Params.Keys) { $paramVal = $Params[$param]; $CommandArgs += " /CommonParam:$param='$paramVal'" } } if ($WaitForExit) { $CommandArgs += " /runandwait" } else { $CommandArgs += " /run" } $CommandArgs += " /MailTo:" + (whoami).Split("\")[1] + "@microsoft.com" if ($Machines.Count -gt 1) { $MachineList = $Machines -join "," } else { $MachineList = $Machines[0] } $workFlowInstanceIds = @() foreach ($node in $Machines) { $rCol = $ResultCollection + "-" + $node $machineCmd = $CommandArgs + " /ResultCollection:$rCol" $machineCmd = $machineCmd + " /Machine:$node" $cmd = '&' + "'" + $WorkFlowCommandLine + "'" + $machineCmd Write-TraceLog "Invoke-WttWorkflow: Command: $cmd" $bytes = [Text.Encoding]::Unicode.GetBytes($cmd) $encodedCommand = [Convert]::ToBase64String($bytes) $retryCount = 3 $retriableError = $false while ($retryCount -gt 0) { if ($retriableError) { Start-Sleep 60 } $result = powershell.exe -noprofile -encodedCommand $encodedCommand Write-TraceLog "Invoke-WttWorkflow: Result: $result" $retriableError = Test-IsRetriableError -Result $result -RetryOnNull if ($retriableError) { Write-TraceLog "Invoke-WttWorkflow: Retriable error detected, will retry" } else { break } $retryCount-- } $line = $result | Select-String "Created Workflow Instance ID:" if ($line -eq $null) { throw "Invoke-WttWorkflow: Failed to create workflow instance. Result: $result" } $id = $line.ToString().Split(":")[1].Trim() $workFlowInstanceIds += $id } return $workFlowInstanceIds } function Invoke-WttJob() { param( [string][parameter(Mandatory = $true)]$MachinePool, [string[]][parameter(Mandatory = $true)]$Machines, [string][parameter(Mandatory = $true)]$ID, [string][parameter(Mandatory = $true)]$MachineDataStore , [string][parameter(Mandatory = $false)]$JobDataStore = $MachineDataStore, [HashTable][parameter(Mandatory = $false)] $Params, # @ {"Param1" = "Value1"; "Param2" = "Value2"} [string][parameter(Mandatory = $true)]$ResultCollection, [switch][parameter(Mandatory = $false)]$Wait, [switch][parameter(Mandatory = $false)]$ThrowOnFailure ) $CommandArgs = " schedulejob /IdentityServer:$Global:IdentityServer /IdentityDatabase:WTTIDENTITY /SourceDataStore:$JobDataStore /DestinationDataStore:$MachineDataStore /MachinePool:$MachinePool " $CommandArgs += " /ResultCollection:$ResultCollection" $CommandArgs += " /RunAll:True" if ($Wait.IsPresent -or $ThrowOnFailure.IsPresent) { $CommandArgs += " /Wait" } if ($Params -ne $nul) { $tmpParamsAll = ""; foreach ($param in $Params.Keys) { $tmpParamsAll += " /parameter " + '`{' + " /name:$param /value:$($Params[$param]) " + '`} ' } $CommandArgs += " /Job " + '`{' + " /Id:$ID $tmpParamsAll " + '`} ' } else { $CommandArgs += " /JobId:$ID" } if ($Machines.Count -gt 1) { $MachineList = $Machines -join "," } else { $MachineList = $Machines[0] } $CommandArgs += " /MachineList:$MachineList" $cmd = '&' + "'" + $WTTCL + "'" + $CommandArgs Write-TraceLog "Invoke-WttJob: Command: $cmd" $bytes = [Text.Encoding]::Unicode.GetBytes($cmd) $encodedCommand = [Convert]::ToBase64String($bytes) $retryCount = 3 $retriableError = $false while ($retryCount -gt 0) { if ($retriableError) { Start-Sleep 60 } $result = powershell.exe -noprofile -encodedCommand $encodedCommand Write-TraceLog "Invoke-WttJob: Result: $result" $retriableError = Test-IsRetriableError -Result $result if ($retriableError) { Write-TraceLog "Invoke-WttJob: Retriable error detected, will retry" } else { break } $retryCount-- } [array]$jobIdResults = $result | Select-String "Schedule Created with Id" if ($jobIdResults -eq $null -or $jobIdResults.Count -eq 0) { throw "Invoke-WttJob: No results found" } [array]$scheduleIds = $jobIdResults | ForEach-Object { $_.ToString().Split(" ") | Select -Last 1 } if ($scheduleIds -eq $null -or $scheduleIds.Count -eq 0) { throw "Invoke-WttJob: No scheduleIds found" } if ($ThrowOnFailure) { $scheduleIds | ForEach-Object { $success = Test-WttJobSuccess -ScheduleID $_ -MachineDataStore $MachineDataStore if (-not $success) { throw "Invoke-WttJob: Job $ID failed" } } } } function Test-WttJobSuccess { param( [string][parameter(Mandatory = $true)]$ScheduleID, [string][parameter(Mandatory = $true)]$MachineDataStore ) $params = @( "schedulestatus", "/IdentityServer:ATLASIDENTITY", "/IdentityDatabase:WTTIDENTITY", "/ScheduleID:$ScheduleID", "/DataStore:$MachineDataStore" ) $params = " " + ($params -Join " ") $cmd = '&' + "'" + $WTTCL + "'" + $params Write-TraceLog "Test-WttJobSuccess: Command: $cmd" $retryCount = 3 $resultIdFound = $false while ($retryCount -gt 0) { if ($retriableError) { Start-Sleep 60 } $result = Invoke-PowershellCommand -cmd $cmd Write-TraceLog "Test-WttJobSuccess: Schedule $ScheduleID Results: $result" $line = 0; foreach ($r in $result) { if ($r.StartsWith("ResultID")) { $resultIdFound = $true break } elseif ((Test-IsRetriableError -Result $r)) { Write-TraceLog "Test-WttJobSuccess: Retrying $ScheduleID" $retriableError = $true break } $line++ } if (-not $retriableError) { break } $retryCount-- } if ($resultIdFound -eq $false) { throw "Test-WttJobSuccess: Failed to find ResultID in the output. Result: $result" } $resultLineNumber = $line + 2 if ($result.Count -lt $resultLineNumber) { throw "Test-WttJobSuccess: Failed to find result in the output. Result: $result" } $resultLine = $result[$line + 2] $resultLineEntires = $resultLine.split((@(" ")), [StringSplitOptions]::RemoveEmptyEntries) if ($resultLineEntires.Count -lt 3) { throw "Test-WttJobSuccess: Failed to find result status in the output. ResultLine: $resultLine" } return ($resultLineEntires[2] -eq "Completed") } function Get-WttJobStatus { param( [string]$DataStore, [string]$ScheduleID ) $CommandArgs = " schedulestatus /DataStore:$DataStore /ScheduleID:$ScheduleID" $cmd = '&' + "'" + $WTTCL + "'" + $CommandArgs Write-TraceLog "Get-WttJobStatus: Command: $cmd" $bytes = [Text.Encoding]::Unicode.GetBytes($cmd) $encodedCommand = [Convert]::ToBase64String($bytes) $result = powershell.exe -noprofile -encodedCommand $encodedCommand $status = $result | Select-String "Schedule status :" $status.ToString().Split(":")[1].Trim() } function Wait-ForWttJob { param( [string]$DataStore, [string]$ScheduleID ) $status = Get-WttJobStatus -DataStore $DataStore -ScheduleID $ScheduleID while ($true) { if ($status -eq "InProgress") { Write-TraceLog "Wait-ForWttJob: ScheduleID $ScheduleID" Start-Sleep -Seconds 30 $status = Get-WttJobStatus -DataStore $DataStore -ScheduleID $ScheduleID } else { Write-TraceLog "Wait-ForWttJob: ScheduleID $ScheduleID completed with $status" break } } } function Wait-ForWttWorkflow { param( [string]$DataStore, [string]$ResultCollectionName, [int] $TotalExpectedJobs ) Write-TraceLog "Wait-ForWttWorkflow: ResultCollectionName $ResultCollectionName" while ($true) { $resultCol = Get-WttResultCollectionsByName -DataStore $DataStore -Name $ResultCollectionName if ($resultCol -eq $null) { Write-TraceLog "Wait-ForWttWorkflow: ResultCollection $ResultCollectionName not found, will wait" Start-Sleep -Seconds 60 continue } $notComplete = $false foreach ($r in $resultCol) { Write-TraceLog "Wait-ForWttWorkflow: ResultCollection $($r.Name) Status $($r.ResultSummaryStatusId) Completed Jobs $($r.CompletedResults)" $o = ($r | Select-Object -Property InProgressResults, InvestigateResults, CancelledResults, CompletedResults, TotalResults, RuntimeLeft) Write-TraceLog "Wait-ForWttWorkflow: $o" if ($r.InvestigateResults -ne 0) { throw "Wait-ForWttWorkflow: $ResultCollectionName failed" } if ($TotalExpectedJobs -ne 0) { $percent = [Math]::Round(($r.CompletedResults / $TotalExpectedJobs) * 100, 0) Write-TraceLog "Wait-ForWttWorkflow: $percent% completed" } if ($r.ResultSummaryStatusId -ne "Completed") { $notComplete = $true } else { $instance = Get-ActiveWttWorkFlowInstance -DataStore $DataStore -Name $ResultCollectionName if ($instance -ne $null) { Write-TraceLog "Wait-ForWttWorkflow: Active instance found, will wait for $ResultCollectionName" $notComplete = $true } else { Write-TraceLog "Wait-ForWttWorkflow: $ResultCollectionName completed" } } } if ($notComplete) { Write-TraceLog "Wait-ForWttWorkflow: Sleeping for 2 minutes" Start-Sleep -Seconds 120 } else { break } } } |