JobQueuing.psm1
|
Function Get-Method { <# .DESCRIPTION This function provides details around the JQ methods available for jobs or queues. #> Param ( [Parameter(Mandatory=$false)] [ValidateSet('Queue','Job',ignorecase=$true)] [string]$Explain ) begin { if(!$Explain){$Explain = 'Queue'} } process { switch($Explain){ Queue { $out = @' This document explains the JQQueue methods. =========================================== JQQueue objects can be created using the New-JQQueue function. Methods are extrememly valuable in using the JQQueue. They extends the functionality and allow for an extensable queue, rather than a one-time function. =========================================== .Add The Add method can be used to append new jobs to the job queue. The jobs must be created using the New-JQJob function for this to function properly. The method will accept both an array of JQJobs, or a single JQJob. Usage: $newJob = New-JQJob -ScriptBlock {Start-Sleep 1}; $Queue.Add($newJob); .SetMode The SetMode method is used to change the current operating mode of the queue. If the queue is created in Background mode, it can be changed to progress mode with this method. Usage: $Queue.SetMode('Progress'); .Start The Start method is used to run the jobs within the queue. Using the Start method will immediately kick off as many jobs are allowed (max) by the queue. When the queue is started it will show the progress if the mode is set to progress. Note: The Start method can be called whenever the queue is not in a running state. If the queue finishes and more jobs are added the start method may be called. Usage: $Queue.Start(); .Play The Play method is used internally and should not be called by users. Usage: NA .ShowProgress The show progress method is used to display a progress bar for the queue. This will only work is the queue mode is set to Progress. If it is set to background then ShowProgress will not work properly. The queue must be running to show progress. Usage: $Queue.ShowProgress(); .Stop The stop method sends a signal to the queue to stop running jobs. All currently running jobs will complete, but no additional jobs will run when the queue is cleared. This does not alter the health state of the queue, and the start method may be called again at any time. Note: The stop method is not immediate. It will take some time to complete all existing jobs and exit. Currently there is no force stop metho supported to stop running jobs gracefully within the queue. Usage: $Queue.Stop(); .Clean The clean method is used to clear queue jobs and events from session. It is strongly recommended that either the Destroy or Clear methods are used after the queue is complete and no longer being used in a script. The user can deside whether to clean everything, or just events. Usage: $Queue.Clean(); #This will clear everything $Queue.Clean($true); #This will only clear events and not jobs .Destroy The destroy method cleans up all events and sessions, and also completely clears out the queue. Once a queue has been destroyed it cannot be used. Usage: $Queue.Destroy(); '@ } Job { $out = @' This document explains the JQJob methods. =========================================== JQJob objects can be created using the New-JQJob function. Methods are extrememly valuable in using JQJobs. They allow for quick and simple execution of common tasks. Aside from the usability, JQJobs also provide compatability with JQQueues. =========================================== .ChangeRunOnComplete The ChangeRunOnComplete method grants the ability to alter the RunOnComplete value See the help document for JQJobs for more details on this paramater. `Get-Help New-JQJob -Paramater RunOnComplete` Usage: $Job.ChangeRunOnComplete.({Start notepad.exe}); .Reset This method blanks out all properties of the job, setting them back to their defaults. This is intended to be used by internal processes. It is not recommended that this method be used. Usage: $Job.Reset(); .Start This method starts the job. Any RunOnComplete action specified will also be bound at this point. Usage: $Job.Start(); .Stop This method stops the job and removes registered events. Usage: $Job.Stop(); .Receive Recieves all data from powershell job. The data is stored in the Result propery of the object. Usage: $Job.Receive(); .Remove Removes the job. This is equivillent to using the Remove-JQJob function and specifying the name of the job. Usage: $Job.Remove(); '@ } } Write-Host $out; } } Function Get-Job { <# .DESCRIPTION This function returns JQJobs based on their names. #> Param ( [Parameter(Mandatory=$false)] [string]$Name ) $ErrorActionPreference = "SilentlyContinue"; try { if(!$global:_JQJobs){ return; } else { if($Name){ $global:_JQJobs | ? {$_.Name -eq $Name} } else { $global:_JQJobs; } } } catch {return;} } Function New-Job { <# .SYNOPSIS Creates a job-like object without actually starting a job. Essentially pre-stages a job. .DESCRIPTION Pre-stages a Powershell job. This creates a PowerShell extended job object, but does not start it immediately by default. Note: This job object has the capabilities of a basic powershell job. Future releases may support more advanced properties. The object extends the managability of the job and can be passed to a queue. .PARAMETER ArgumentList Specifies the arguments (parameter values) for the script that is specified by the FilePath parameter. Because all of the values that follow the ArgumentList parameter name are interpreted as being values of ArgumentList, the ArgumentList parameter should be the last parameter in the command. .PARAMETER InitializationScript Specifies commands that run before the job starts. Enclose the commands in braces ( { } ) to create a script block. Use this parameter to prepare the session in which the job runs. For example, you can use it to add functions, snap-ins, and modules to the session. .PARAMETER JobName Specifies a friendly name for the new job. You can use the name to identify the job to other job cmdlets, such as Stop-Job. The default friendly name is Job#, where "#" is an ordinal number that is incremented for each job. .PARAMETER ScriptBlock Specifies the commands to run in the background job. Enclose the commands in braces ( { } ) to create a script block. This parameter is required. .PARAMETER RunOnComplete Specified scriptblock is run whenever the job execution completes. This is independant of results, there is currently no built in mechanism to pass job results to the RunOnComplete script. .EXAMPLE Create a job object named "testJob" and assign it to the $job variable. The job does not run. $job = New-JQJob -Name "testJob" -ScriptBlock { Start-Sleep 5; Write-Output "I waited 5 seconds"; } Now that we have a test job we can use the .Start() method to start it. $job.Start(); If you need to stop the job before it completes you can use the .Stop() method. $job.Stop(); Once the job is complete you can receive the job. $job.Receive(); To remove the job completely use the .Remove() method $job.Remove(); .LINK PowerShell jobs - https://blogs.technet.microsoft.com/heyscriptingguy/2012/12/31/using-windows-powershell-jobs/ New-Job Get-Job Receive-Job Stop-Job #> param ( [Parameter(Mandatory=$true,ValueFromPipeline=$true)] [scriptblock]$ScriptBlock, [Parameter(Mandatory=$false)] [string]$JobName, [Parameter(Mandatory=$false)] [Object[]]$ArgumentList, [Parameter(Mandatory=$false)] [scriptblock]$InitializationScript, [Parameter(Mandatory=$false)] [scriptblock]$RunOnComplete ) begin { $dontrun = $false; Function Get-RandomJobName { Return "Job-" + (Get-Random -Minimum -500 -Maximum 2500000); } Function Get-TestedRandomJobName { $JobName = Get-RandomJobName; if((Get-Job -Name $JobName).Count -gt 0){ Get-TestedRandomJobName; } else { $JobName; } } if(!$global:_JQJobs){ $global:_JQJobs = [System.Collections.ArrayList]@(); } if(!$JobName){ $JobName = Get-TestedRandomJobName; } else { if((Get-JQJob -Name $JobName).Count -gt 0){ $dontrun = $true; Write-Error "Unable to create job. Job named $JobName already exists, please use a different name or leave name blank for a randomly generated name."; return; } } } process { if($dontrun){return;} $OutputOBJ = New-Object -TypeName psobject -Property @{ Name = $JobName; Script = $ScriptBlock; CompleteScript = $RunOnComplete; CompleteSourceID = $null; ArgumentList = $ArgumentList; InitializationScript = $InitializationScript; Status = 'Ready'; PSJob = [scriptblock]{}; Result = $null; JobType = 'JQJob' Errors = [System.Collections.ArrayList]@(); } Add-Member -InputObject $OutputOBJ ScriptMethod Start { if($this.Status -eq 'Removed'){ Write-Error "Unable to start job. It has been removed. Please create a new job."; $this.Error.Add("Unable to start job. It has been removed. Please create a new job."); return; } $context = $this; $Error.Clear(); $RemoveTaskNames = ($context.Name + "_Postjob_Action_"+((Get-Job).Count+22)); $context.CompleteSourceID = $RemoveTaskNames; $RemoveTasks = [string]{ $newState = ($Event.SourceArgs.JobStateInfo[-1]).State; $global:test += $Event; Unregister-Event -SourceIdentifier $RemoveTaskNames; Remove-Job -Name $RemoveTaskNames; $tJob = Get-JQJob $Event.SourceArgs.Name $tJob.Status = $tJob.psjob.state; } $RemoveTasks = $RemoveTasks.Replace('$RemoveTaskNames',$RemoveTaskNames); if($context.CompleteScript){ $JobCompleteAction = [scriptblock]::Create($RemoveTasks + $context.CompleteScript.ToString()); } try { $NJ = Start-Job -Name ($context.Name) -ScriptBlock ($context.Script) -ArgumentList ($context.ArgumentList) -InitializationScript ($context.InitializationScript); if($context.CompleteScript){ $rJ = Register-ObjectEvent -InputObject $NJ -SourceIdentifier $RemoveTaskNames -EventName StateChanged -Action $JobCompleteAction; } $context.PSJob = $NJ; $context.Status = $NJ.State; return $context.PSJob; } catch { Write-Error "Errors occured during job execution. Please see errors property for more details."; $context.Error.Add($Error); return; } } Add-Member -InputObject $OutputOBJ ScriptMethod ChangeRunOnComplete { Param ( [scriptblock]$CompleteScript ) $this.CompleteScript = $CompleteScript; } Add-Member -InputObject $OutputOBJ ScriptMethod Stop { $Error.Clear(); $context = $this; if($context.CompleteSourceID){ $RemoveTaskNames = $context.CompleteSourceID; Unregister-Event -SourceIdentifier $RemoveTaskNames; Get-Job -Name $RemoveTaskNames | Stop-Job | Remove-Job | Out-Null; } try { Stop-Job -Name $context.Name | Out-Null; $context.Status = $context.PSJob.State; return $context.PSJob; } catch { Write-Error "Errors while trying to halt the job. Please see errors property for more details."; $context.Error.Add($Error); } } Add-Member -InputObject $OutputOBJ ScriptMethod Reset { Param([Parameter(Mandatory=$false)][switch]$ForceClear) if($this.Status -ne 'Running'){ Remove-Job -Name $this.Name; $this.Status = 'Ready'; $this.Result = $null; $this.Errors = [System.Collections.ArrayList]@(); $this.PSJob = [scriptblock]{}; } else { if($ForceClear.IsPresent -eq $false){ Write-Error 'Job is currently running and data cannot yet be cleared. To clear the job while it is still running (this will stop the job) use the force override. Example: $job.Clear($true)'; $this.Error.Add('Job is currently running and data cannot yet be cleared. To clear the job while it is still running (this will stop the job) use the force override. Example: $job.Clear($true)'); } else { $this.Stop() | Out-Null; Remove-Job -Name ($this.Name) -Force; $this.Status = 'Ready'; $this.Result = $null; $this.Errors = [System.Collections.ArrayList]@(); $this.PSJob = [scriptblock]{}; } } } Add-Member -InputObject $OutputOBJ ScriptMethod Remove { $context = $this; $context.Reset($true); $context.Status = 'Removed'; $global:_JQJobs.Remove((Get-JQJob -Name ($context.Name))); ($context.PSOBject.Properties | ? {$_.MemberType -eq 'NoteProperty'}).Name | % {$context.PSOBject.Properties.Remove($_)} if(($global:_JQJobs).Count -eq 0){ Remove-Variable _JQJobs; } } Add-Member -InputObject $OutputOBJ ScriptMethod Receive { Param ( [switch]$force ) $Error.Clear(); if(($this.PSJob.State -ne 'Running') -or ($force.IsPresent)){ try { $Error.Clear(); $OJ = Receive-Job -Name $this.JobName; $this.Status = 'Data recieved'; $this.Result = $OJ; return $OJ; } catch { Write-Error "Errors retrieving data. Please see errors property for more details."; $this.Error.Add($Error); } } else { Write-Error 'Job is currently running and data cannot yet be retrieved. To receive data while job is still running use the force override. Example: $job.Receive($true)'; $this.Error.Add('Job is currently running and data cannot yet be retrieved. To receive data while job is still running use the force override. Example: $job.Receive($true)'); } } $global:_JQJobs.Add($OutputOBJ) | Out-Null; return $OutputOBJ; } } Function Remove-Job { <# .DESCRIPTION This function removes JQJobs based on their name, or based on pipeline input from the Get-JQJob function. #> param ( [Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] [string]$Name ) begin { } process { $ErrorActionPreference = "SilentlyContinue"; if(!$global:_JQJobs){ return $true; } if(!$Name){ Remove-Variable _JQJobs -Scope Global; return $true; } else { try { $remMe = ($global:_JQJobs | Where {$_.Name -eq $Name}); $global:_JQJobs.Remove($remMe) | Out-Null; return $true; } catch { return; } if($global:_JQJobs.Count -eq 0){Remove-Variable _JQJobs -Scope Global; return $true;} } } } Function Get-Queue { <# .DESCRIPTION This function retrieves existing queues based on their name. #> Param ( [Parameter(Mandatory=$false)] [string]$Name ) $ErrorActionPreference = "SilentlyContinue"; try { if(!$global:_JQQueues){ return; } else { if($Name){ $global:_JQQueues | ? {$_.Name -eq $Name} } elseif ($global:_JQQueues) { $global:_JQQueues; } else {return;} } } catch {return; } } Function New-Queue { <# .SYNOPSIS Creates an object which can be used to orginize and manage jobs. .DESCRIPTION Build a powershell object which can then be manipulated and used to optimize jobs. All jobs run in the background, but the queue object will track their status and can be configured to run actions on job completion, or retrieve results on completion. .PARAMETER JQJobs Optional paramater which accepts and array of jobs created using the New-JQJob function. Example below will create a new jobqueue with two jobs pre-adde Ex: New-JQQueue -JQJobs ((New-JQJob -ScriptBlock {Start-Sleep 10} ),(New-JQJob -ScriptBlock {Start-Sleep 10} )) .PARAMETER QueueName Optional paramater which, when specifies, builds the queue using a given name, rather than a default name. Useful when a script doesn't pass the job queue object into new contexts. .PARAMETER ActiveLimit Optional paramater which determines how many jobs can be running at any given time. If this option is not specified then the job limit will be determined based off of a few factors. If fewer than 8 jobs are included in the JQJobs param, then the limit will be set to 8 by default. If 8 or more jobs are present, then the script will determine an appropriate active limit based on the current system resource usage. .PARAMETER Mode Optional paramater which is "Background" by default. This determines how the jobqueue itself functions. If set to its default of background, then the jobqueue will be non-intrusive. If the mode is set to "Progress" then the queue progress bar will be displayed. [WARNING: This will block further script execution until all jobs are complete.] [NOTE: Sending ctrl+c to the shell will stop the progress monitor and current script execution, but will *NOT* stop the queue. This is by design.] .PARAMETER Start If the -Start switch is present, then the job queue will immediately execute all jobs which were added to the queue using the -JQJobs param. If no jobs are found, the queue will simply enter a ready state and await further input. .PARAMETER ResultAction Optional paramater which is set to "None" by default. Possible options: None, Store, Active This determines what will be done upon completing each job in the queue. If this option is set to None, then no additional action is done upon completion of jobs in the queue (other than continuing on to the next job). If the option is set to Store, then each job will be received upon completion, and stored within the queue's "JobResults" property [NOTE: Please test this functionality fully before relying on it. There are many considerations to keep in mind.] Lastly, if the option is set to Active, then the ActiveResScript param will be required. The scriptblock specified with the ActiveResScript param will be run each time a job is completed. [NOTE: Results of the job will be sent to the ActiveResScript scriptblock as $args] .PARAMETER ActiveResScript Required paramater IF ResultAction is set to Active. This param determines what script will be run each time a job completes. Job results will be recieved and sent to the script, which can make sure of the results using the $args variable. The jobs are recieved with the -Keep switch. This means that all results will still be available within the job after the ActiveResScript executes. .PARAMETER WhatIf Switch which, if present, prevents Queue object from being created. Rather, it will output the steps used to create the object. [WARNING: This is only very loosely implemented. NEVER rely on -WhatIf in production] .EXAMPLE $ja = @(); 1..3 | % { $jb = New-JQJob -ScriptBlock { $randTime = Get-Random -Minimum 5 -Maximum 30; Start-Sleep $randTime; Write-Output "test"; }; $ja += $jb; } $Queue = New-Queue -ActiveLimit 1 -Mode Progress -JQJobs $ja -ResultAction Active -ActiveResScript {$args | Out-File "C:\Tmp\jobstest.txt" -Append} -Start; .LINK PowerShell jobs - https://blogs.technet.microsoft.com/heyscriptingguy/2012/12/31/using-windows-powershell-jobs/ New-Job Get-Job Receive-Job Stop-Job .OUTPUTS This function outputs an object with usable methods and properties. It also adds the object to a global variable which can be accessed in any scope within the current session. It is not recommended to use the global variable directly. Instead, use the Get-JQQueue function. .NOTES It is important to understand that this function, while highly effective by itself, relies on the use of methods after it has been created. The methods may be listed using the `Get-JQMethod Queue` function. #> param ( [Parameter(Mandatory=$false,ValueFromPipeline=$true)] [psobject[]]$JQJobs, [Parameter(Mandatory=$false)] [string]$QueueName, [Parameter(Mandatory=$false)] [int]$ActiveLimit, [Parameter(Mandatory=$false)] [ValidateSet('Progress','Background',ignorecase=$true)] [string]$Mode, [Parameter(Mandatory=$false)] [switch]$Start, [Parameter(Mandatory=$false)] [ValidateSet('None','Store','Active',ignorecase=$true)] [string]$ResultAction, [Parameter(Mandatory=$false)] [scriptblock]$ActiveResScript, #[switch]$useSQLL, [Parameter(Mandatory=$false)] [switch]$WhatIf ) begin { #region PrivateFunctions function Install-PSSQLite { $WebClient = New-Object System.Net.WebClient; $WebClient.DownloadFile("https://github.com/RamblingCookieMonster/PSSQLite/archive/master.zip","$env:USERPROFILE\downloads\PSSQLite-Master.zip"); $WebClient.Dispose(); $File = "$env:USERPROFILE\downloads\PSSQLite-Master.zip"; $shell = new-object -com shell.application; $zip = $shell.NameSpace($File); New-Item -ItemType Directory -Path "$env:USERPROFILE\Downloads\Master" -ErrorAction SilentlyContinue; foreach($item in $zip.items()){ $shell.Namespace("$env:USERPROFILE\Downloads\Master").copyhere($item); #$shell.Namespace($destination).copyhere($item); } Copy-Item -Path "$env:USERPROFILE\Downloads\Master\PSSQLite-master\PSSQLite" -Recurse -Destination "$env:SystemRoot\System32\WindowsPowerShell\v1.0\Modules" -Confirm:$false -ErrorAction SilentlyContinue; Unblock-File "$env:SystemRoot\System32\WindowsPowerShell\v1.0\Modules\PSSQLite\*" -Confirm:$false; } #endregion if(!$QueueName){ $QueueName = "DefaultContext"; } if(!$global:_JQQueues){ [System.Collections.ArrayList]$global:_JQQueues = @(); } if(!$JQJobs){ $JQJobs = @(); } if(!$ResultAction){ $ResultAction = "None"; } if($ResultAction -eq 'Active'){ if(!$ActiveResScript){Write-Error "Missing paramater 'ActiveResScript.' This paramater is required in order to activley utilize job results. (Results will be passed as an argument to the scriptblock)"; return;} } if(!$ActiveLimit){ # If there are less than or equal to 8 jobs in the queue just set the active limit to 8, most systems should be able to handle that (so long as the scriptblock isn't insane) if($JQJobs.Count -lt 8) { if($WhatIf.IsPresent){ Write-Output ""; } $ActiveLimit = 8; } #If there are more than 8 jobs, then calculate how many should run at a time. Check the memory and CPU usage to determine whether to raise or lower the limit. #This is only done once, so hopeully users don't go and add too much load after the fact. else { $CPULoad = [int](((Get-Counter '\Processor(_Total)\% Processor Time' | select readings) -split ":")[1] -replace("}","") -replace(" ","")); $RamAvail = [int](((Get-Counter '\Memory\Available MBytes' | select readings) -split ":")[1] -replace("}","") -replace(" ","")); $RamTotal = (Get-WmiObject -Class Win32_ComputerSystem).TotalPhysicalMemory/1mb; $RamInUse = ($RamTotal - $RamAvail) / $RamTotal; $ActiveLimit = [math]::Floor((($JQJobs.Count/6) * ((1 - ($CPULoad*0.1)) + (1 - $RamInUse )))); } } if(!$Mode){ $Mode = "Background"; } try { $SQLLiteAvail = $true; if ((Get-Module -ListAvailable PSSQLite|Measure).Count -lt 1) { try { Install-PSSQLite } catch { $SQLLiteAvail = $false; } } if ($SQLLiteAvail) { $EXPolicy = Get-ExecutionPolicy; Set-ExecutionPolicy Bypass; Import-Module PSSQLite -ErrorAction Stop; Set-ExecutionPolicy $EXPolicy; } } catch { $SQLLiteAvail = $false; } #if($useSQLL -and !$SQLLiteAvail){ throw "SQL Lite not available. Failed to create queue."; } } process { $QueueName = "$QueueName"+(([int[]]($_JQQueues.Name -replace("[aA-zZ]",'')) | measure -Maximum).Maximum+1); $JQ = New-Object -TypeName psobject -Property @{ Name = "JQ_$QueueName"; Jobs = $JQJobs.Count; JobsCompleted = 0; ActiveLimit = $ActiveLimit; StartedBy = ([Environment]::UserDomainName)+"\"+([Environment]::UserName); Status = "Building"; Created = (Get-Date); RunningJobs = [System.Collections.ArrayList]@(); CompleteJobs = [System.Collections.ArrayList]@(); FailedJobs = [System.Collections.ArrayList]@(); PendingJobs = [System.Collections.ArrayList]$JQJobs; JobResults = [System.Collections.ArrayList]@(); Mode = $Mode; index = $global:_JQQueues.count; NextJob = 0; Errors = [System.Collections.ArrayList]@(); ResultAction = $ResultAction; ActiveResScript = $ActiveResScript; SQLitePath = "$PSScriptRoot\db"; UseSQLL = $useSQLL.IsPresent; SQLJobsAdded=0; } Add-Member -InputObject $JQ ScriptMethod Add { Param( [Parameter(Mandatory=$false,ValueFromPipeline=$false)] [psobject[]]$Addition ) $Context = $this; foreach($add in $Addition){ if(!$add.PSObject.Properties['Name']){ Write-Error "Failed to add job to queue, invalid job format. Please use New-Job to create the jobs you pass to this method."; return;} if(!$add.PSObject.Properties['Script']){ Write-Error "Failed to add job to queue, invalid job format. Please use New-Job to create the jobs you pass to this method."; return;} if(!$add.PSObject.Properties['ArgumentList']){ Write-Error "Failed to add job to queue, invalid job format. Please use New-Job to create the jobs you pass to this method."; return;} if(!$add.PSObject.Properties['InitializationScript']){ Write-Error "Failed to add job to queue, invalid job format. Please use New-Job to create the jobs you pass to this method."; return;} $Context.PendingJobs.Add($add) | Out-Null; $Context.Jobs++; } Write-Output ([String]$Addition.Count+" jobs added."); } Add-Member -InputObject $JQ ScriptMethod SetMode { Param( [ValidateSet('Progress','Background',ignorecase=$true)] $mode ); $this.Mode = "$mode"; } Add-Member -InputObject $JQ ScriptMethod Start { $this.Status = 'Running'; $this.Play(); if($this.Mode -eq 'Progress'){ $this.ShowProgress(); } } Add-Member -InputObject $JQ ScriptMethod Play { if($this.Status -in @('destroyed')){ $this.Errors.Add(@{Message="Unable to play queue, as it has been destroyed. To stop a queue without making it unusable, use the 'Stop' method."; Source="Queue Player"}); return; } $JobQueue = $this; $ThisQueue = $JobQueue.Name; $JobCompleteAction = { $sourceID = $Event.SourceIdentifier; $PrevState = ($Event.SourceArgs.PreviousJobStateInfo[-1]).State; $newState = ($Event.SourceArgs.JobStateInfo[-1]).State; $aThisQueue = ($Event.SourceIdentifier -split ("_"))[1]; $aJobQueue = ($global:_JQQueues | ? {$_.Name -eq "JQ_$aThisQueue"} ); $CompletedJob = ($aJobQueue.RunningJobs | ? {$_.sourceID -eq $sourceID}); $CompletedJobindex = $aJobQueue.RunningJobs.IndexOf($CompletedJob); $Job = Get-Job -Name $CompletedJob.idName; if($Job.State -ne $newState){ $aJobQueue.Errors.Add(@{Message="Job status doesn't match status returned by event";Job=$CompletedJob.idName;}) } $CompletedJob.end = Get-Date; # Remove event listener Get-EventSubscriber -SourceIdentifier $sourceID | Unregister-Event -Force -ErrorAction SilentlyContinue | Out-Null; if($aJobQueue.ResultAction -eq 'Store'){ $thisRes = New-Object -TypeName psobject -Property @{ Job = $CompletedJob.idName; JobCmd = $CompletedJob.Job.Command; Result = $CompletedJob.Job | Receive-Job; } $aJobQueue.JobResults.Add($thisRes); } elseif($aJobQueue.ResultAction -eq 'Active'){ $Error.Clear(); try { $recJ = $CompletedJob.Job | Receive-Job -Keep; Start-Job -ScriptBlock $aJobQueue.ActiveResScript -ArgumentList $recJ -Name ($CompletedJob.idName+"_PostAction") -ErrorAction Stop; } catch { $aJobQueue.Errors.Add(@{Message=$Error;Job=$CompletedJob.idName;}); } } # Add to the list of completed jobs and pull out of the running jobs list. if($newState -eq 'Failed'){ $aJobQueue.FailedJobs.Add($CompletedJob)|Out-Null; } else { $aJobQueue.CompleteJobs.Add($CompletedJob)|Out-Null; } $aJobQueue.JobsCompleted++; $aJobQueue.RunningJobs.RemoveAt($CompletedJobindex)|Out-Null; $aJobQueue.Play(); } while(($JobQueue.RunningJobs.Count -lt $JobQueue.ActiveLimit) -and ($JobQueue.Status -in @('ready','running','resuming')) -and $JobQueue.PendingJobs.Count -gt 0){ $runningJob = 0; $rjIndex = $JobQueue.NextJob; $JobName = ($ThisQueue+"_Child$rjIndex"); #$nJ = Start-Job -Name $JobName -ScriptBlock ($JobQueue.PendingJobs[$runningJob].Script) -ArgumentList ($JobQueue.PendingJobs[$runningJob].ArgumentList) -InitializationScript ($JobQueue.PendingJobs[$runningJob].InitializationScript); $nJ = $JobQueue.PendingJobs[$runningJob].Start(); $rJ = Register-ObjectEvent -InputObject $nJ -EventName StateChanged -Action $JobCompleteAction -SourceIdentifier ($ThisQueue+"_Monitor$rjIndex") -ErrorVariable $global:evtErr; $runObj = New-Object -TypeName psobject -Property @{ id=$rjIndex; job=$nJ; queue=$JobQueue.Name; idName = $JobName; sourceID = ($ThisQueue+"_Monitor$rjIndex"); index=$JobQueue.RunningJobs.Count; start = Get-Date; end=$null; } $JobQueue.PendingJobs.RemoveAt($runningJob) | Out-Null; $JobQueue.RunningJobs.Add($runObj) | Out-Null; $JobQueue.NextJob = $JobQueue.NextJob+1; } if($JobQueue.PendingJobs.Count -le 0){ $JobQueue.Status = 'finishing'; } if($JobQueue.Status -in @('finishing') -and $JobQueue.RunningJobs.Count -eq 0){ $JobQueue.Status = 'Complete'; $JobQueue.Clean($true); } if($JobQueue.Status -in @('stopping') -and $JobQueue.RunningJobs.Count -eq 0){ $JobQueue.Status = 'Stopped'; $JobQueue.Clean($true); } if($JobQueue.Status -in @('Complete','Stopped') -and $JobQueue.RunningJobs.Count -eq 0){ $JobQueue.Clean($true); } } Add-Member -InputObject $JQ ScriptMethod Clean { Param([Parameter(Mandatory=$false)][switch]$EventsOnly) $actions = "events have been cleaned"; Get-EventSubscriber | ? {$_.SourceIdentifier -match ($this.Name+"_Monitor")} | Unregister-Event -Force; Get-Job | ? { $_.Name -match ($this.Name+"_Monitor") }| Remove-Job; if(!$EventsOnly.IsPresent){ Get-Job | ? { $_.Name -match ($this.Name) }| Remove-Job; $actions = "events and jobs have been cleaned."; } Write-Output "Queue cleaned, $actions"; } Add-Member -InputObject $JQ ScriptMethod ShowProgress { while(($this.Mode -eq "Progress") -and ($this.Status -in @('Running','finishing','stopping'))){ if($this.Status -in @('finishing','stopping')){ $prog = 100; } else { $prog = ($this.JobsCompleted/$this.jobs)*100+1; if($prog -gt 100){ $prog = 100; }if($prog -lt 1){ $prog = 1; } } $statusMsg = ("Status:" + $this.Status + " | " + $this.PendingJobs.Count + " of " + $this.jobs +" remaining" + "| " + $this.RunningJobs.Count + " Jobs running"); if($this.Errors.count -gt 0){ $statusMsg += ("| Errors:" + $this.Errors.Count); } Write-Progress -Activity "Running jobs in queue" -Status $statusMsg -PercentComplete $prog; } } Add-Member -InputObject $JQ ScriptMethod Stop { $this.Status = 'Stopping'; } Add-Member -InputObject $JQ ScriptMethod Destroy { $removeObj = ($global:_JQQueues | Where {$_.Name -eq $this.Name}); $remove = $global:_JQQueues.IndexOf($removeObj); $global:_JQQueues.RemoveAt($remove); #$this.Clean(); $this.Status = 'Destroyed'; $this = $null; } if($SQLLiteAvail){ Add-Member -InputObject $JQ ScriptMethod SLiteCreate { $SQLLiteDB = ($this.SQLitePath + "\pending.SQLite"); $DataSource = $SQLLiteDB; $Query = "CREATE TABLE jobs (jobID INTEGER PRIMARY KEY, JobName TEXT UNIQUE, ScriptBlock NVARCHAR(MAX), ArgumentList NVARCHAR(MAX), CompleteScript NVARCHAR(MAX), InitializationScript NVARCHAR(MAX), Status TEXT, Added_Date DATETIME, Completed_Date DATETIME)"; New-Item -ItemType Directory -Path $this.SQLitePath -Force; Invoke-SqliteQuery -Query $Query -DataSource $DataSource; } Add-Member -InputObject $JQ ScriptMethod SLitePush { $context = $this; $SQLLiteDB = ($this.SQLitePath + "\pending.SQLite"); $DataSource = $SQLLiteDB; $insertQ = "INSERT OR IGNORE INTO NAMES (JobName, ScriptBlock, ArgumentList, CompleteScript, InitializationScript, Status) VALUES (@JobName, @ScriptBlock, @ArgumentList, @CompleteScript, @InitializationScript, @Status, @Added_Date, @Completed_Date)"; foreach($pending in $context.PendingJobs){ Invoke-SqliteQuery -DataSource $DataSource -Query $insertQ -SqlParameters @{ jobID = $context.SQLJobsAdded; JobName = $pending.Name; ScriptBlock = $pending.Script; ArgumentList = $pending.ArgumentList; CompleteScript = $pending.CompleteScript; InitializationScript = $pending.InitializationScript; Status = $pending.Status; Added_Date = (Get-Date); Completed_Date = $null; } $context.SQLJobsAdded++; } } Add-Member -InputObject $JQ ScriptMethod SLitePull { $context = $this; $SQLLiteDB = ($this.SQLitePath + "\pending.SQLite"); $DataSource = $SQLLiteDB; } } $JQ.Status = "Ready"; $global:_JQQueues.Add($JQ) | Out-Null; if($Start){ $JQ.Start(); } return $JQ; } } Function Remove-Queue { <# .DESCRIPTION This function removes existing queues based on name, or from the pipeline input object from Get-JQQueue. #> param ( [Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] [string]$Name ) begin { } process { $ErrorActionPreference = "SilentlyContinue"; if(!$global:_JQQueues){ return $true; } if(!$Name){ Remove-Variable _JQQueues -Scope Global; return $true; } else { try { $remMe = ($global:_JQQueues | Where {$_.Name -eq $Name}); $remMe.Destroy(); $global:_JQQueues.Remove($remMe) | Out-Null; return $true; if($global:_JQQueues.Count -eq 0){Remove-Variable _JQQueues -Scope Global; return $true;} } catch { return; } } } } |