Assets/JobRunner.ps1
|
<# .SYNOPSIS Master Wrapper for executing Ops jobs. .DESCRIPTION Runs PowerShell or Node.js scripts, handles logging, error catching, and alerting. .PARAMETER ScriptPath Full path to the script to execute. .PARAMETER JobName Name of the job for logging and alerting. .PARAMETER LogLevel Logging level (DEBUG, INFO, WARNING, ERROR). .PARAMETER ScriptArguments Arguments to pass to the script. A hashtable is splatted as named parameters, an array is passed positionally. .PARAMETER EmailRecipients List of email addresses for failure alerts. .PARAMETER SmtpServer SMTP server used for failure alerts. Defaults to $env:OPS_SMTP_SERVER. .PARAMETER SmtpFrom Sender address for failure alerts. Defaults to $env:OPS_SMTP_FROM. .PARAMETER SmtpPort SMTP port. Defaults to $env:OPS_SMTP_PORT, then to the Send-MailMessage default (25). .PARAMETER SmtpUseSsl Use SSL/TLS for the SMTP connection. Defaults to $env:OPS_SMTP_USESSL. .PARAMETER AlertWebhookUrl Webhook URL for failure alerts (e.g., Slack, Teams). .PARAMETER RequiredSecrets List of secret names to inject as environment variables. .PARAMETER PayloadBase64 Base64-encoded UTF-8 JSON holding the parameters above. Used by New-OpsJob.ps1 so that arguments survive the scheduled-task command line intact. #> param ( [string]$ScriptPath, [string]$JobName, [ValidateSet("DEBUG", "INFO", "WARNING", "ERROR")] [string]$LogLevel = "INFO", [object]$ScriptArguments = @(), [string[]]$EmailRecipients = @(), [string]$SmtpServer, [string]$SmtpFrom, [string]$SmtpPort, [switch]$SmtpUseSsl, [string]$AlertWebhookUrl, [string[]]$RequiredSecrets = @(), [string]$PayloadBase64 ) $ErrorActionPreference = "Stop" # --- Payload decoding ------------------------------------------------------- if (-not [string]::IsNullOrWhiteSpace($PayloadBase64)) { try { $Json = [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($PayloadBase64)) $Payload = $Json | ConvertFrom-Json } catch { throw "Could not decode -PayloadBase64: $($_.Exception.Message)" } if ($Payload.JobName) { $JobName = $Payload.JobName } if ($Payload.ScriptPath) { $ScriptPath = $Payload.ScriptPath } if ($Payload.AlertWebhookUrl) { $AlertWebhookUrl = $Payload.AlertWebhookUrl } if ($Payload.EmailRecipients) { $EmailRecipients = @($Payload.EmailRecipients) } if ($Payload.RequiredSecrets) { $RequiredSecrets = @($Payload.RequiredSecrets) } if ($null -ne $Payload.ScriptArguments) { # ConvertFrom-Json turns a JSON object into a PSCustomObject; splatting needs # a hashtable, so convert it back. if ($Payload.ScriptArguments -is [System.Management.Automation.PSCustomObject]) { $ArgTable = @{} foreach ($Property in $Payload.ScriptArguments.PSObject.Properties) { $ArgTable[$Property.Name] = $Property.Value } $ScriptArguments = $ArgTable } else { $ScriptArguments = @($Payload.ScriptArguments) } } } if ([string]::IsNullOrWhiteSpace($JobName)) { throw "JobName is required (pass -JobName or -PayloadBase64)." } if ([string]::IsNullOrWhiteSpace($ScriptPath)) { throw "ScriptPath is required (pass -ScriptPath or -PayloadBase64)." } $env:OPS_LOG_LEVEL = $LogLevel if ([string]::IsNullOrWhiteSpace($SmtpServer)) { $SmtpServer = $env:OPS_SMTP_SERVER } if ([string]::IsNullOrWhiteSpace($SmtpFrom)) { $SmtpFrom = $env:OPS_SMTP_FROM } if ([string]::IsNullOrWhiteSpace($SmtpPort)) { $SmtpPort = $env:OPS_SMTP_PORT } if (-not $SmtpUseSsl -and $env:OPS_SMTP_USESSL -in @("1", "true", "True", "yes")) { $SmtpUseSsl = $true } # --- Event log -------------------------------------------------------------- $EventSource = "WinBatchOrchestrator" $CanWriteEventLog = $false try { # SourceExists throws on access-denied as well as returning $false, and the old # "Application" fallback was itself an unregistered source that always failed. $CanWriteEventLog = [System.Diagnostics.EventLog]::SourceExists($EventSource) } catch { $CanWriteEventLog = $false } function Write-OpsEvent { param( [string]$Message, [ValidateSet("Information", "Warning", "Error")][string]$EntryType = "Information", [int]$EventId = 100 ) if (-not $CanWriteEventLog) { return } try { Write-EventLog -LogName Application -Source $EventSource -EntryType $EntryType -EventId $EventId -Message $Message -ErrorAction Stop } catch { Write-Warning "Could not write to the Application event log: $($_.Exception.Message)" } } # --- Utils & logging -------------------------------------------------------- Import-Module (Join-Path $PSScriptRoot "OpsUtils.psm1") -Force # The Ops root defaults to C:\Ops. OPS_ROOT overrides it for development and # testing, matching the dashboard server's existing convention. $OpsRoot = if ([string]::IsNullOrWhiteSpace($env:OPS_ROOT)) { "C:\Ops" } else { $env:OPS_ROOT } $LogDir = Join-Path $OpsRoot "Logs" if (-not (Test-Path $LogDir)) { New-Item -ItemType Directory -Path $LogDir -Force | Out-Null } # The job name reaches the filesystem here; strip anything that would escape the # log directory or produce an unopenable file. $SafeJobName = ($JobName -replace '[\\/:*?"<>|]', '_') $Timestamp = Get-Date -Format "yyyyMMdd-HHmmss" $LogFile = Join-Path $LogDir "$SafeJobName-$Timestamp.log" $JsonLogFile = Join-Path $LogDir "$SafeJobName.history.json" # Appending log for history $env:OPS_LOG_FILE = $JsonLogFile Start-Transcript -Path $LogFile -Append | Out-Null # --- Concurrency locking ---------------------------------------------------- # Mutex names are namespaced by '\', so a job name containing one would target # the wrong (or an invalid) kernel object. $MutexName = "Global\OpsJob-$SafeJobName" $Mutex = $null $HaveMutex = $false try { $Mutex = New-Object System.Threading.Mutex($false, $MutexName) } catch { Write-Warning "Could not create Mutex. Proceeding without concurrency check. $($_.Exception.Message)" } if ($null -ne $Mutex) { try { $HaveMutex = $Mutex.WaitOne(0, $false) } catch [System.Threading.AbandonedMutexException] { # A previous run was killed while holding the lock; we now own it. Write-Warning "Previous run of '$JobName' terminated without releasing its lock." $HaveMutex = $true } if (-not $HaveMutex) { Write-OpsLog -Message "Job $JobName is already running. Skipping execution." -Level "WARNING" $Mutex.Dispose() Stop-Transcript | Out-Null exit 0 } } Write-OpsLog -Message "Starting Job: $JobName" -Level "INFO" Write-OpsLog -Message "Script: $ScriptPath" -Level "INFO" Write-OpsEvent -Message "Starting OpsJob: $JobName" -EntryType Information -EventId 100 $ExitCode = 0 $ErrorMessage = "" $InjectedSecretVars = @() try { # --- Secret injection ----------------------------------------------------- foreach ($SecretName in $RequiredSecrets) { $SecretPath = Join-Path (Join-Path $OpsRoot "Secrets") "$SecretName.xml" if (Test-Path $SecretPath) { try { $Cred = Import-Clixml -Path $SecretPath if ($Cred -isnot [System.Management.Automation.PSCredential]) { throw "'$SecretPath' does not contain a PSCredential." } $EnvVarName = "SECRET_$($SecretName.ToUpperInvariant())" Set-Item -Path "env:$EnvVarName" -Value $Cred.GetNetworkCredential().Password $InjectedSecretVars += $EnvVarName # Never log the value, only that injection happened. Write-OpsLog -Message "Injected secret: $SecretName as `$env:$EnvVarName" -Level "DEBUG" } catch { Write-OpsLog -Message "Failed to load secret '$SecretName': $($_.Exception.Message)" -Level "WARNING" } } else { Write-OpsLog -Message "Secret not found: $SecretName" -Level "WARNING" } } if (-not (Test-Path $ScriptPath)) { throw "Script not found: $ScriptPath" } $Extension = [System.IO.Path]::GetExtension($ScriptPath).ToLowerInvariant() if ($Extension -eq ".ps1") { Write-OpsLog -Message "Executing PowerShell script..." -Level "INFO" $global:LASTEXITCODE = 0 # Splatting a hashtable binds named parameters; splatting an array binds # positionally. Both are what the caller asked for. & $ScriptPath @ScriptArguments # A script that ends with `exit <n>` sets $LASTEXITCODE without throwing; # without this check the job would be reported as a success. if ($LASTEXITCODE -ne 0) { throw "PowerShell script exited with code $LASTEXITCODE" } } elseif ($Extension -eq ".js") { Write-OpsLog -Message "Executing Node.js script..." -Level "INFO" $TargetDir = Split-Path -Parent $ScriptPath # Node.js Dependency Check if (Test-Path (Join-Path $TargetDir "package.json")) { if (-not (Test-Path (Join-Path $TargetDir "node_modules"))) { Write-OpsLog -Message "Installing Node.js dependencies..." -Level "INFO" # npm on Windows is npm.cmd; Start-Process does not apply PATHEXT, so # -FilePath "npm" fails with "The system cannot find the file specified". # Calling it inline also keeps its output inside the transcript. Push-Location $TargetDir try { $global:LASTEXITCODE = 0 & cmd.exe /c "npm install --production" 2>&1 | ForEach-Object { Write-Host $_ } if ($LASTEXITCODE -ne 0) { Write-OpsLog -Message "npm install failed with code $LASTEXITCODE" -Level "WARNING" } } finally { Pop-Location } } } $FinalArguments = @() if ($ScriptArguments -is [System.Collections.IDictionary]) { foreach ($key in $ScriptArguments.Keys) { $val = $ScriptArguments[$key] if ($val -is [switch] -or $val -is [bool]) { if ($val) { $FinalArguments += "--$key" } } else { $FinalArguments += "--$key" $FinalArguments += [string]$val } } } else { $FinalArguments = @($ScriptArguments) } # Invoked inline rather than via Start-Process: Start-Process -NoNewWindow # writes to the console directly, so node's output never reached the # transcript, and its -ArgumentList string does not quote arguments # containing spaces. $global:LASTEXITCODE = 0 & node $ScriptPath @FinalArguments 2>&1 | ForEach-Object { Write-Host $_ } if ($LASTEXITCODE -ne 0) { throw "Node.js script exited with code $LASTEXITCODE" } } else { throw "Unsupported script extension: $Extension" } Write-OpsLog -Message "Job completed successfully." -Level "INFO" Write-OpsEvent -Message "OpsJob Success: $JobName" -EntryType Information -EventId 101 } catch { $ExitCode = 1 $ErrorMessage = $_.Exception.Message Write-OpsLog -Message "Job Failed: $ErrorMessage" -Level "ERROR" Write-OpsEvent -Message "OpsJob Failed: $JobName`nError: $ErrorMessage" -EntryType Error -EventId 102 # Alerts are sent from the finally block, after Stop-Transcript. Sending them # here attaches a log file that Start-Transcript still holds open, which fails # with "the process cannot access the file because it is being used by # another process". } finally { # Do not leave decrypted secrets in the environment of anything that inherits # from this process later in the session. foreach ($EnvVarName in $InjectedSecretVars) { Remove-Item -Path "env:$EnvVarName" -ErrorAction SilentlyContinue } if ($null -ne $Mutex) { # ReleaseMutex throws if this thread never acquired it. if ($HaveMutex) { try { $Mutex.ReleaseMutex() } catch { Write-Warning "Could not release mutex: $($_.Exception.Message)" } } $Mutex.Dispose() } Stop-Transcript | Out-Null # --- Alerting --------------------------------------------------------------- # Runs after the transcript is closed so $LogFile is complete and unlocked. if ($ExitCode -ne 0) { if ($EmailRecipients.Count -gt 0) { if ([string]::IsNullOrWhiteSpace($SmtpServer) -or [string]::IsNullOrWhiteSpace($SmtpFrom)) { # Previously hardcoded to smtp.example.com, so every alert silently failed. Write-OpsLog -Message "Email recipients configured but no SMTP server/sender set. Set OPS_SMTP_SERVER and OPS_SMTP_FROM (or pass -SmtpServer/-SmtpFrom) to enable email alerts." -Level "WARNING" } else { Write-OpsLog -Message "Sending alert email..." -Level "INFO" $Subject = "FAILURE: Job $JobName" $Body = "Job $JobName failed.`n`nError: $ErrorMessage`n`nLog: $LogFile" $MailParams = @{ To = $EmailRecipients From = $SmtpFrom Subject = $Subject Body = $Body SmtpServer = $SmtpServer Encoding = [System.Text.Encoding]::UTF8 } if (-not [string]::IsNullOrWhiteSpace($SmtpPort)) { $MailParams["Port"] = [int]$SmtpPort } if ($SmtpUseSsl) { $MailParams["UseSsl"] = $true } if (Test-Path -Path $LogFile) { $MailParams["Attachments"] = $LogFile } try { # Send-MailMessage is obsolete but is the only in-box option on PS 5.1. Send-MailMessage @MailParams -ErrorAction Stop } catch { Write-OpsLog -Message "Failed to send email: $($_.Exception.Message)" -Level "ERROR" # A locked or oversized attachment should not cost us the alert itself. if ($MailParams.ContainsKey("Attachments")) { $MailParams.Remove("Attachments") try { Send-MailMessage @MailParams -ErrorAction Stop Write-OpsLog -Message "Sent alert email without the log attachment." -Level "WARNING" } catch { Write-OpsLog -Message "Failed to send email without attachment: $($_.Exception.Message)" -Level "ERROR" } } } } } if (-not [string]::IsNullOrWhiteSpace($AlertWebhookUrl)) { Write-OpsLog -Message "Sending webhook alert..." -Level "INFO" $WebhookPayload = @{ text = "FAILURE: Job $JobName" attachments = @(@{ color = "danger" title = "Job Failed: $JobName" text = "Error: $ErrorMessage" fields = @( @{ title = "Script"; value = $ScriptPath; short = $false } @{ title = "Log"; value = $LogFile; short = $false } ) }) } | ConvertTo-Json -Depth 5 try { Invoke-RestMethod -Uri $AlertWebhookUrl -Method Post -Body $WebhookPayload -ContentType "application/json" -ErrorAction Stop | Out-Null } catch { Write-OpsLog -Message "Failed to send webhook: $($_.Exception.Message)" -Level "ERROR" } } } exit $ExitCode } |