zvmRemoteScripts_utils.ps1
|
$ZAPPLIANCE_USER = "zadmin" #TODO Make global if possible, used as private variable elsewhere function Invoke-ZVMLScriptWithTimeout { <# .SYNOPSIS Executes the shell script on the ZVML VM with a timeout asynchronously Used to execute long-running Python scripts .OUTPUTS [VMScriptResultImpl] Result .ExitCode contains script success/failure Result .ScriptOutput must be used with StartsWith() or Contains() because output ends with extra \n newline character Result .TrimmedOutput contains clean script output #> param ( [ValidateNotNullOrEmpty()] [string]$ScriptText, [ValidateNotNullOrEmpty()] [string]$ActionName, [ValidateRange(1, 60)] [int]$TimeoutMinutes = 30 ) Write-Host "Starting $($MyInvocation.MyCommand)..." $ZVM = Get-VM -Name $ZVM_VM_NAME if ($null -eq $ZVM) { throw "$ZVM_VM_NAME VM does not exist." } Write-Host "Invoking '$ActionName' with $TimeoutMinutes minutes timeout." # Start the script asynchronously $task = Invoke-VMScript -VM $ZVM -ScriptText $ScriptText -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -RunAsync # Calculate the timeout time $timeoutTime = (Get-Date).AddMinutes($TimeoutMinutes) while ((Get-Date) -lt $timeoutTime) { # Check the task state periodically switch ($task.State) { 'Success' { # The 'Success' state indicates that the remote script was executed, but does not reflect the script success or failure. Write-Host "$ActionName execution done." # task.Result is VMScriptResultImpl type, and we add a new dynamic property to it. $task.Result | Add-Member -MemberType NoteProperty -Name "TrimmedOutput" -Value $task.Result?.ScriptOutput?.TrimEnd("`n") return $task.Result } 'Error' { throw "$ActionName execution error: $($task.TerminatingError.Message)" # In case of wrong VM credentials, the error message would be "Failed to authenticate with the guest operating system using the supplied credentials." } default { # If the task is 'Running' or in any other state, wait briefly before rechecking. Start-Sleep -Seconds 10 } } } # If the loop exits, it means the timeout was reached # Note that the task is not aborted, so the script could eventually succeed #TODO Maybe ask user to restart the ZVML, to avoid leaving the script running in unclear state throw "Timeout, '$ActionName' did not complete within the allotted time of $TimeoutMinutes minutes." } function Invoke-ZVMLScript { <# .SYNOPSIS Executes the shell script on the ZVML VM synchronously .OUTPUTS [VMScriptResultImpl] Result .ExitCode contains script success/failure Result .ScriptOutput contains the clean script output #> param ( [ValidateNotNullOrEmpty()] [string]$ScriptText ) Write-Host "Starting $($MyInvocation.MyCommand)..." $ZVM = Get-VM -Name $ZVM_VM_NAME if ($null -eq $ZVM) { throw "$ZVM_VM_NAME VM does not exist." } $res = Invoke-VMScript -VM $ZVM -ScriptText $ScriptText -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction Stop if ($null -eq $res) { throw "Failed to invoke VM script." } return [PSCustomObject]@{ ExitCode = $res.ExitCode ScriptOutput = $res.ScriptOutput.TrimEnd("`n") } } function Assert-ZertoInitialized { Write-Host "Starting $($MyInvocation.MyCommand)..." $startTime = Get-Date $action = { $ZVM = Get-VM -Name $ZVM_VM_NAME if ($null -eq $ZVM) { throw "$ZVM_VM_NAME VM does not exist." } $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER)) { throw "ZVMA not initialized, authentication failed." } #TODO: This single check is enough to determine if ZVM is initialized, split between null, when authentication fails and when ZVM is not initialized $zvmInitStatusFile = "/opt/zerto/zvr/initialization-files/zvm_initialized" $res = Invoke-VMScript -VM $ZVM -ScriptText "[ -e $zvmInitStatusFile ] && echo true || echo false" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne 'true')) { throw "ZVMA not initialized, initialization file not found." } } Invoke-Retry -Action $action -ActionName "TestZvmaInitialized" -RetryIntervalSeconds 120 Write-Host "Zerto initialization took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds." } function Set-ZertoVmPassword { <# .SYNOPSIS Sets the ZVM VM console password and updates the PersistentSecrets #> param( [ValidateNotNullOrEmpty()] [SecureString]$NewPassword ) Write-Host "Starting $($MyInvocation.MyCommand)..." $newPasswordText = ConvertFrom-SecureString -SecureString $NewPassword -AsPlainText $action = { $ZVM = Get-VM -Name $ZVM_VM_NAME if ($null -eq $ZVM) { throw "$ZVM_VM_NAME VM does not exist." } # Clear the password history and change password $cmdClearHistoryChangePassword = "sudo truncate -s 0 /etc/security/opasswd; echo '$($ZAPPLIANCE_USER):$newPasswordText' | sudo chpasswd" # We need to write result to variable to avoid module logging issues # We need SilentlyContinue because when an in-guest script changes its own account password, the authenticated session may no longer be valid by the time the cmdlet attempts to finalize and return a result. # This causes a "Failed to authenticate..." error, even though the password was successfully changed. # For the same reason, ScriptOutput will also be unavailable. $empty = Invoke-VMScript -VM $ZVM -ScriptText $cmdClearHistoryChangePassword -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $newPasswordText -ErrorAction SilentlyContinue if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER)) { throw "Failed to change ZVML VM password." } $PersistentSecrets.ZappliancePassword = $newPasswordText } Invoke-Retry -Action $action -ActionName "ChangeZvmlVmPassword" -RetryIntervalSeconds 20 -RetryCount 3 } function Set-ZertoConfiguration ([string]$DNS, [bool]$IsVaio) { Write-Host "Starting $($MyInvocation.MyCommand)..." # This was commented for 10u7+ because static DNS should now be set correctly by appliance from ovf, fixed in ZER-150945. # If no issues are found in production, this code can be removed in the next release. # Set-DnsConfiguration -DNS $DNS # Stop-ZVM # Start-ZVM Write-Host "Configuring Zerto, this might take a while..." $startTime = Get-Date $scriptLocation = "/opt/zerto/zlinux/avs/configure_zerto.py" $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --avsClientSecret '$($PersistentSecrets.AvsClientSecret)'$($IsVaio ? ' --isVaio' : '')" $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Configure ZVM" Write-Host "Zerto configuration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds." if ($result.ScriptOutput.Contains("Success")) { Write-Host "Zerto configured successfully." } elseif ($result.ScriptOutput.Contains("Warning:")) { $message = $result.ScriptOutput Write-Host $message Write-Warning $message } elseif ($result.ScriptOutput.Contains("Error:")) { $cleanErrMsg = $result.ScriptOutput -replace "Error: ", "" throw $cleanErrMsg } else { throw "An unexpected error occurred while configuring Zerto. Please reinstall Zerto." } } function Update-ZertoConfiguration { param( [ValidateNotNullOrEmpty()][string] $AzureTenantId, [ValidateNotNullOrEmpty()][string] $AzureClientID, [ValidateNotNullOrEmpty()][string] $AvsSubscriptionId, [ValidateNotNullOrEmpty()][string] $AvsResourceGroup, [ValidateNotNullOrEmpty()][string] $AvsCloudName ) Write-Host "Starting $($MyInvocation.MyCommand)..." Write-Host "Reconfiguring Zerto, this might take a while..." $scriptLocation = "/opt/zerto/zlinux/avs/reconfigure_zvm.py" $ZertoUserWithDomain = "$ZERTO_USER_NAME@$DOMAIN" $commandToExecute = "sudo python3 $scriptLocation " + "--avsClientSecret '$($PersistentSecrets.AvsClientSecret)' " + "--azureTenantId '$AzureTenantId' " + "--azureClientID '$AzureClientID' " + "--avsSubscriptionId '$AvsSubscriptionId' " + "--avsResourceGroup '$AvsResourceGroup' " + "--avsCloudName '$AvsCloudName' " + "--vcIp '$VC_ADDRESS' " + "--vcUsername '$ZertoUserWithDomain' " + "--vcPassword '$($PersistentSecrets.ZertoPassword)' " + "--zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)'" $startTime = Get-Date $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Reconfigure ZVM" Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds." if ($result.ScriptOutput.Contains("Success")) { Write-Host "Zerto reconfigured successfully." } elseif ($result.ScriptOutput.Contains("Warning:")) { $message = $result.ScriptOutput Write-Host $message Write-Warning $message } elseif ($result.ScriptOutput.Contains("Error:")) { $cleanErrMsg = $result.ScriptOutput -replace "Error: ", "" throw $cleanErrMsg } else { throw "An unexpected error occurred while reconfiguring Zerto. Please reinstall Zerto." } } function Test-ZertoPassword { <# .SYNOPSIS Checks validity of 'admin' and 'zadmin' passwords stored in PersistentSecrets .DESCRIPTION The Zerto 'admin' password is checked explicitly – try_zerto_login.py will return "Success" if the password is valid. The Console 'zadmin' password is checked implicitly – Invoke-ZVMLScriptWithTimeout will fail with authentication error if the password is invalid. #> Write-Host "Starting $($MyInvocation.MyCommand)..." $scriptLocation = "/opt/zerto/zlinux/avs/try_zerto_login.py" $commandToExecute = "sudo python3 $scriptLocation --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)'" try { $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Validate Zerto password" -TimeoutMinutes 5 if ($result.ScriptOutput.Contains("Success")) { Write-Host "Zerto password is valid." } else { throw "Provided Zerto password is not valid." } } catch { throw "Zerto password validation failed. Problem: $_" } } enum PasswordsValidationResult { PasswordsValid = 0; ZertoPasswordInvalid = 1; ConsolePasswordInvalidOrExpired = 2; } function Test-ZertoPasswordResult { <# .SYNOPSIS Checks validity of 'admin' and 'zadmin' passwords stored in PersistentSecrets and returns a result code. .OUTPUTS [PasswordsValidationResult] PasswordsValid - Both passwords are valid ZertoPasswordInvalid - The 'admin' password is not valid ConsolePasswordInvalidOrExpired - The 'zadmin' password is not valid Throws an error if any other issue occurs during validation. #> Write-Host "Starting $($MyInvocation.MyCommand)..." try { Test-ZertoPassword return [PasswordsValidationResult]::PasswordsValid } catch { if ($_ -match "Provided Zerto password is not valid") { Write-Host 'The Zerto "admin" password is not valid.' return [PasswordsValidationResult]::ZertoPasswordInvalid } if ($_ -match "Failed to authenticate with the guest operating system using the supplied credentials") { Write-Host 'The Console "zadmin" password is not valid.' return [PasswordsValidationResult]::ConsolePasswordInvalidOrExpired } throw $_ } } function Update-VcPasswordInZvm { param ( [ValidateNotNullOrEmpty()][string] $NewVcPassword, [ValidateNotNullOrEmpty()][string] $ZertoAdminPassword, [ValidateNotNullOrEmpty()][string] $ClientSecret ) process { Write-Host "Starting $($MyInvocation.MyCommand)..." $scriptLocation = "/opt/zerto/zlinux/avs/change_vc_password.py" $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$NewVcPassword' --zertoAdminPassword '$ZertoAdminPassword' --avsClientSecret '$ClientSecret'" $startTime = Get-Date $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Change VC password in ZVM" -TimeoutMinutes 20 Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds." if ($result.ScriptOutput.Contains("Success")) { Write-Host "The new VC password in ZVM set successfully." } else { if ($result.ScriptOutput.Contains("Error:")) { $cleanErrMsg = $result.ScriptOutput -replace "Error: ", "" throw $cleanErrMsg #TODO: Standardize error messages for Error and Unknown cases, here and elsewhere, review unit tests } throw "Unexpected error occurred while updating VC password in ZVM." } } } function Update-ClientCredentialsInZvm { param ( [ValidateNotNullOrEmpty()][string] $NewClientId, [ValidateNotNullOrEmpty()][string] $NewClientSecret ) process { Write-Host "Starting $($MyInvocation.MyCommand)..." $scriptLocation = "/opt/zerto/zlinux/avs/change_azure_client_credentials.py" # change_azure_client_credentials is available starting ZVM 10u5p2 $scriptExists = Test-FileExistsInZVM -FileLocation $scriptLocation #TODO: Consider extracting check to caller method, method should not return bool value if ($scriptExists -eq $false) { return $false # ZVMA version does not support updating Client Credentials } $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)' --azureClientId '$NewClientId' --avsClientSecret '$NewClientSecret'" $startTime = Get-Date $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Change Azure client credentials in ZVM" -TimeoutMinutes 20 Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds." if ($result.ScriptOutput.Contains("Success")) { Write-Host "New Azure client credentials set successfully in ZVM." } else { if ($result.ScriptOutput.Contains("Error:")) { $cleanErrMsg = $result.ScriptOutput -replace "Error: ", "" throw $cleanErrMsg } throw "Unexpected error occurred while changing Azure client credentials in ZVM." } return $true } } <# function Set-DnsConfiguration($DNS) { #TODO: Once the u7 is published, this method can be removed, because static DNS should be set correctly by appliance from ovf, fixed in ZER-150945 Write-Host "Starting $($MyInvocation.MyCommand)..." try { $action = { $ZVM = Get-VM -Name $ZVM_VM_NAME if ($null -eq $ZVM) { throw "$ZVM_VM_NAME VM does not exist." } $setDnsCommand = "grep -qxF 'nameserver $DNS' /etc/resolv.conf || echo 'nameserver $DNS' | sudo tee -a /etc/resolv.conf" $res = Invoke-VMScript -VM $ZVM -ScriptText $setDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue $checkDnsCommand = "grep -qF 'nameserver $DNS' /etc/resolv.conf && echo 'true' || echo 'false'" $res = Invoke-VMScript -VM $ZVM -ScriptText $checkDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue if ($null -eq $res -or $res.ScriptOutput.Trim() -ne "true") { throw "Failed to force set DNS" } $lockFileCommand = 'sudo chattr +i /etc/resolv.conf' $res = Invoke-VMScript -VM $ZVM -ScriptText $lockFileCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword Write-Host "DNS successfully set" } Invoke-Retry -Action $action -ActionName "SetDNS" -RetryCount 4 -RetryIntervalSeconds 30 } catch { $message = "Failed to set DNS. Configuration may fail. Problem: $_" Write-Host $message Write-Warning $message } } #> function Get-Nameservers { Write-Host "Starting $($MyInvocation.MyCommand)..." try { # Unlock resolv.conf file which could have been locked during the installation of previous versions, no need to lock again, fixed in ZER-150945 $commandUnlockFile = 'sudo chattr -i /etc/resolv.conf' $res = Invoke-ZVMLScript -ScriptText $commandUnlockFile if ($res.ExitCode -ne 0) { throw "Failed to unlock resolv.conf. $($res.ScriptOutput)" # ScriptOutput should contain bash error } # awk searches for lines starting with 'nameserver' and prints the second field (the IP address) $commandGetDnsEntries = "awk '/^[[:space:]]*nameserver[[:space:]]/ {print `$2}' /etc/resolv.conf || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandGetDnsEntries if ($res.ExitCode -ne 0) { throw "Failed to read resolv.conf, $($res.ScriptOutput)" # ScriptOutput should contain bash error } $output = $res.ScriptOutput $ips = $output -split "`n" return $ips | ForEach-Object { [System.Net.IPAddress]$_ } } catch { throw "Failed to get ZVML DNS nameserver entries. Problem: $_" } } function Add-Nameserver ([System.Net.IPAddress]$DnsIp) { Write-Host "Starting $($MyInvocation.MyCommand)..." try { # tee adds the line to the end of file $commandAddDnsEntry = "echo 'nameserver $DnsIp # AVS $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')' | sudo tee -a /etc/resolv.conf || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandAddDnsEntry if ($res.ExitCode -ne 0) { throw "Failed to edit resolv.conf, $($res.ScriptOutput)" # ScriptOutput should contain bash error } } catch { throw "Failed to add ZVML DNS nameserver entry. Problem: $_" } } function Remove-Nameserver ([System.Net.IPAddress]$DnsIp) { Write-Host "Starting $($MyInvocation.MyCommand)..." try { # sed /{pattern}/{command} with /d removes the matched line starting with 'nameserver' $commandRemoveDnsEntry = "sudo sed -i '/^[[:space:]]*nameserver[[:space:]]\+$DnsIp/d' /etc/resolv.conf || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandRemoveDnsEntry if ($res.ExitCode -ne 0) { throw "Failed to edit resolv.conf, $($res.ScriptOutput)" # ScriptOutput should contain bash error } } catch { throw "Failed to remove ZVML DNS nameserver entry. Problem: $_" } } function Check-Nameserver ([System.Net.IPAddress]$DnsIp) { Write-Host "Starting $($MyInvocation.MyCommand)..." try { $destinationHost = "management.azure.com" # dig queries DNS using the specified DNS server for the specified host $commandCheckDnsEntry = "dig @$DnsIp $destinationHost A +short +time=10 +retry=3 || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandCheckDnsEntry if ($res.ExitCode -ne 0) { throw "Failed to query DNS, $($res.ScriptOutput)" # ScriptOutput should contain bash error } } catch { throw "Failed to check ZVML DNS nameserver entry. Problem: $_" } } function Test-FileExistsInZVM ($FileLocation) { Write-Host "Starting $($MyInvocation.MyCommand)..." try { # test -f can only return 0 if file exists or 1 $commandFileExists = "test -f $FileLocation || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandFileExists if ($res.ExitCode -eq 0) { return $true } else { return $false } } catch { throw "Failed to check file in ZVM. Problem: $_" } } function Test-FeatureFlagEnabled { param( [ValidateNotNullOrEmpty()] [string]$Flag ) Write-Host "Starting $($MyInvocation.MyCommand)..." # -i: Makes the search case-insensitive. # -x: Matches only entire lines (not partial matches). # -F: Interprets the pattern as a fixed string, not a regular expression. # -q: Runs silently, suppressing output and returning only the exit code. $commandCheckFlag = "test -f '/opt/zerto/zlinux/avs/feature-flags.conf' && grep -ixFq '$Flag' '/opt/zerto/zlinux/avs/feature-flags.conf' || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandCheckFlag if ($res.ExitCode -ne 0) { throw "Your ZVMA version does not support '$Flag' feature in AVS. Please use a version which supports '$Flag' feature." } Write-Host "ZVMA version supports '$Flag' feature." } function Set-ZertoVmPasswordExpiration { <# .SYNOPSIS Resets the ZVM VM console password expiration #> Write-Host "Starting $($MyInvocation.MyCommand)..." try { # grep -Pq, -P to use Perl-compatible regular expressions, -q to be quiet without outputting the matching lines # returns 0 exit code if 'maxdays -1' is set and then 'Maximum -1' pattern is found $chageCommand = "sudo chage --maxdays -1 $ZAPPLIANCE_USER && chage -l $ZAPPLIANCE_USER | grep -Pq '^Maximum number of days.*-1$' || exit 1" $res = Invoke-ZVMLScript -ScriptText $chageCommand if ($res.ExitCode -ne 0) { throw "Failed to chage, $($res.ScriptOutput)" # ScriptOutput should contain bash error } } catch { throw "Failed to set password expiry. Problem: $_" } } function Set-AzureResourceGroup { <# .SYNOPSIS Sets the Azure Resource Group name in the ZVMA #> param ( [string]$resourceGroupName ) Write-Host "Starting $($MyInvocation.MyCommand)..." $scriptLocation = "/opt/zerto/zlinux/avs/change_resource_group.py" $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --avsClientSecret '$($PersistentSecrets.AvsClientSecret)' --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)' --resourceGroupName '$resourceGroupName'" $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName "Change resource group" if ($result.ScriptOutput.Contains("Success")) { Write-Host "Resource group set successfully." } elseif ($result.ScriptOutput.Contains("Error:")) { $cleanErrMsg = $result.ScriptOutput -replace "Error: ", "" throw $cleanErrMsg } else { throw "An unexpected error occurred while changing the resource group." } } function Enable-VAIOConfiguration { <# .SYNOPSIS Enable Zerto protection using VAIO (vSphere API for IO filtering) #> Write-Host "Starting $($MyInvocation.MyCommand)..." $scriptLocation = '/opt/zerto/zlinux/avs/enable_vaio.py' $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --avsClientSecret '$($PersistentSecrets.AvsClientSecret)' --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)'" $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Enable VAIO' if ($result.ScriptOutput.Contains('Success')) { Write-Host 'VAIO enabled successfully.' } elseif ($result.ScriptOutput.Contains('Error:')) { $cleanErrMsg = $result.ScriptOutput -replace 'Error: ', '' throw $cleanErrMsg } else { throw 'An unexpected error occurred while enabling VAIO.' } } function New-ZertoLogs { param ( [Parameter(Mandatory = $false)] [bool]$WithPrometheus ) Write-Host "Starting $($MyInvocation.MyCommand)..." try { $scriptParams = $WithPrometheus ? ' --with-prometheus' : '' $commandToExecute = "sudo python3 /opt/zerto/zlinux/avs/collect_logs.py$scriptParams" $res = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Create ZVML logs archive' -TimeoutMinutes 30 $LOG_FILE = 'LogPath' $scriptOutput = $res.TrimmedOutput Write-Host "ZVML logs collection script output:`n$scriptOutput" if (-not ($scriptOutput -match "(?m)^Success\.[^']+'(?<$LOG_FILE>[^']+?)'\s+created\.$")) { throw 'No log file created.' } $logFile = $matches[$LOG_FILE] return $logFile } catch { throw "Failed to create ZVML log archive. Problem: $_" } } function Cleanup-LogBundles { Write-Host "Starting $($MyInvocation.MyCommand)..." function Test-LogBundlesExist { # find all files in both directories, print0 to handle spaces in filenames, xargs -0 to handle null-terminated input, -r to avoid running stat if no files found $commandGetLogBundles = "find /var/log/zerto/zvr/bundles /var/log/zerto/zvr/temp-bundles -type f -print0 | xargs -0 -r stat -c '%n | Size %s bytes' || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandGetLogBundles if ($res.ExitCode -ne 0) { throw "Failed to get log bundles files, $($res.ScriptOutput)" # ScriptOutput should contain bash error } $foundFilesList = $res.ScriptOutput $filesExist = (-not [string]::IsNullOrWhiteSpace($foundFilesList)) return @($filesExist, $foundFilesList) } try { $filesExist, $foundFilesList = Test-LogBundlesExist if (-not $filesExist) { Write-Host "No log bundle files found. Nothing to cleanup." return } Write-Host "Found log bundle files:" Write-Host $foundFilesList $totalBytes = 0 $lines = $foundFilesList -split "`n" foreach ($line in $lines) { if ($line -match "Size (\d+) bytes") { $sizeBytes = [long]$matches[1] $totalBytes += $sizeBytes } } $totalMB = [math]::Round($totalBytes / 1MB, 2) Write-Host "Total bundles size: $totalMB MB" Write-Host "Deleting log bundles..." # find all files in both directories, -mindepth 1 to avoid deleting the directories themselves, -delete to remove the files $commandDeleteLogBundles = "find /var/log/zerto/zvr/bundles /var/log/zerto/zvr/temp-bundles -mindepth 1 -delete || exit 1" $res = Invoke-ZVMLScript -ScriptText $commandDeleteLogBundles if ($res.ExitCode -ne 0) { throw "Failed to delete log bundles files, $($res.ScriptOutput)" } $filesExist, $foundFilesList = Test-LogBundlesExist if ($filesExist) { throw "Failed to delete some log bundles files: `n$foundFilesList" } Write-Host "Log bundles cleanup complete. Freed $totalMB MB of disk space." } catch { throw "Failed to cleanup log bundles. Problem: $_" } } function Invoke-ZvmaNetDiagnostics { <# .SYNOPSIS Performs network connectivity diagnostics for ZVMA #> param( [ValidateNotNullOrEmpty()] [string]$TargetUri ) Write-Host "Starting $($MyInvocation.MyCommand)..." Write-Host "### Executing connectivity diagnostics for '$TargetUri'" try { $uri = [System.Uri]$TargetUri $hostname = $uri.Host Write-Host "## DNS lookup result:`n$(Invoke-ZvmaNsLookup -TargetHost $hostname)" Write-Host "## DNS dig lookup result:`n$(Invoke-ZvmaDigLookup -TargetHost $hostname)" Write-Host "## Network traceroute result:`n$(Invoke-ZvmaTraceroute -TargetHost $hostname)" Write-Host "## TCP netcat connectivity test result:`n$(Invoke-ZvmaNetcat -TargetHost $hostname)" Write-Host "## TLS connectivity test result:`n$(Invoke-ZvmaOpenSslCheck -TargetHost $hostname)" Write-Host "## HTTP connectivity test result:`n$(Invoke-ZvmaCurl -TargetHost $TargetUri)" } catch { throw "Failed to run connectivity diagnostics. Problem: $_" } } function Invoke-ZvmaNsLookup { <# .SYNOPSIS Executes nslookup command on the ZVMA to test DNS resolution #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost ) Write-Host "Starting $($MyInvocation.MyCommand)..." return Invoke-ZvmaNetDiagnosticCommand -Command "nslookup" -TargetHost $TargetHost -CommandDescription "DNS lookup" } function Invoke-ZvmaDigLookup { <# .SYNOPSIS Executes dig command on the ZVMA for DNS resolution and detailed diagnostics #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost ) Write-Host "Starting $($MyInvocation.MyCommand)..." return Invoke-ZvmaNetDiagnosticCommand -Command "dig" -TargetHost $TargetHost -CommandDescription "DNS dig lookup" } function Invoke-ZvmaTraceroute { <# .SYNOPSIS Executes traceroute command on the ZVMA to trace network path #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost, [bool]$UseTcp = $true, [int]$Port = 443 ) Write-Host "Starting $($MyInvocation.MyCommand)..." $commandArgs = "-p $Port" if ($UseTcp) { $commandArgs += " -T " } return Invoke-ZvmaNetDiagnosticCommand -Command "sudo traceroute" -TargetHost $TargetHost -CommandArgs $commandArgs -CommandDescription "Network traceroute" } function Invoke-ZvmaNetcat { <# .SYNOPSIS Executes netcat command on the ZVMA to test TCP port connectivity .PARAMETER ZeroIo Enables netcat's zero-I/O mode (-z). When $true (default), netcat checks if a port is open without sending data. Set to $false to make a full connection and allow data transfer. #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost, [int]$Port = 443, [bool]$IsVerbose = $true, [bool]$ZeroIo = $true ) Write-Host "Starting $($MyInvocation.MyCommand)..." $commandArgs = "$Port" if ($ZeroIo) { $commandArgs += " -z" } if ($IsVerbose) { $commandArgs += " -v" } return Invoke-ZvmaNetDiagnosticCommand -Command "nc" -TargetHost $TargetHost -CommandArgs $commandArgs -CommandDescription "TCP netcat connectivity test" } function Invoke-ZvmaOpenSslCheck { <# .SYNOPSIS Executes openssl s_client command on the ZVMA to test TLS connectivity #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost, [int]$Port = 443, [int]$TimeoutSeconds = 10, [ValidateSet("tls1", "tls1_1", "tls1_2", "tls1_3")] [string]$TlsVersion = "tls1_3" ) Write-Host "Starting $($MyInvocation.MyCommand)..." return Invoke-ZvmaNetDiagnosticCommand -Command "timeout $TimeoutSeconds openssl s_client -connect" -TargetHost "$($TargetHost):$Port" -CommandArgs "-$TlsVersion" -CommandDescription "TLS connectivity test" } function Invoke-ZvmaCurl { <# .SYNOPSIS Executes curl command on the ZVMA to test HTTP connectivity and verbose output #> param ( [ValidateNotNullOrEmpty()] [string]$TargetHost, [bool]$IsVerbose = $true ) Write-Host "Starting $($MyInvocation.MyCommand)..." $commandText = "curl" if ($IsVerbose) { $commandText += " -v" } return Invoke-ZvmaNetDiagnosticCommand -Command $commandText -TargetHost "$TargetHost" -CommandDescription "HTTP connectivity test" } function Invoke-ZvmaNetDiagnosticCommand { <# .SYNOPSIS Executes a network diagnostics command on the ZVMA #> param ( [ValidateNotNullOrEmpty()] [string]$Command, [ValidateNotNullOrEmpty()] [string]$TargetHost, [string]$CommandArgs = "", [ValidateNotNullOrEmpty()] [string]$CommandDescription ) Write-Host "Starting Invoke-ZvmaNetDiagnosticCommand for $CommandDescription..." try { $fullCommand = "$Command $TargetHost $CommandArgs" Write-Host "Executing command: $fullCommand" $res = Invoke-ZVMLScript -ScriptText $fullCommand Write-Host "$CommandDescription for $TargetHost completed." return $res.ScriptOutput } catch { $errorMessage = "Failed to perform $CommandDescription on ZVM. Problem: $_" Write-Error $errorMessage return "Error: $errorMessage" } } |