zvm_scripts_utils.ps1

$ZAPPLIANCE_USER = "zadmin" #TODO Make global if possible, used as private variable elsewhere
$NOT_ALLOWED_CHARACTERS_IN_SHELL_PARAMS = [regex]"['\x00-\x1F\x7F]" # The single-quote, null-byte, or control characters like \n \r \t are not allowed

function Build-SafeShellPyCommand {
    <#
        .SYNOPSIS
        Builds a safe shell command string with proper parameter quoting for executing a Python script in ZVML
 
        .DESCRIPTION
        In POSIX-compliant shells (e.g., bash), single quotes prevent all shell interpretation, including variable expansion and command substitution.
        Characters inside single-quoted strings are treated literally.
        When a parameter is correctly enclosed in single quotes, it is not subject to shell injection by character content alone. This is a key security property.
        A single quote (') cannot appear inside a single-quoted string and cannot be escaped.
        As a result, the only character that can break single-quote safety is another single quote - so we do not allow it in parameters.
        Other control characters (like newlines) are also disallowed to prevent unexpected behavior in the shell.
    #>

    param(
        [ValidateNotNullOrEmpty()]
        [string]$PyScriptPath,

        [ValidateNotNull()]
        [System.Collections.IDictionary]$PyScriptParams
    )

    function Validate-BashSingleQuote([string]$ParamName, [string]$ParamValue) {
        if ($ParamValue -match $NOT_ALLOWED_CHARACTERS_IN_SHELL_PARAMS) {
            throw "Script parameter '$ParamName' contains not allowed characters."
        }
        return $ParamValue
    }

    $commandParts = @("sudo python3 $PyScriptPath")

    foreach ($param in $PyScriptParams.GetEnumerator()) {
        if ($param.Value -is [bool]) {
            if ($param.Value -eq $true) {
                $commandParts += "--$($param.Key)"
            }
        }
        else {
            $safeValue = Validate-BashSingleQuote -ParamName $param.Key -ParamValue $param.Value
            $commandParts += "--$($param.Key) '$safeValue'"
        }
    }

    $command = $commandParts -join ' '

    return $command
}

function Invoke-ZVMLScriptWithTimeout {
    <#
        .SYNOPSIS
        Executes the shell script on the ZVML VM with a timeout asynchronously
        Used to execute long-running Python scripts
 
        .OUTPUTS
        [VMScriptResultImpl]
        Result .ExitCode contains script success/failure
        Result .ScriptOutput must be used with StartsWith() or Contains() because output ends with extra \n newline character
        Result .TrimmedOutput contains clean script output
    #>

    param (
        [ValidateNotNullOrEmpty()]
        [string]$ScriptText,

        [ValidateNotNullOrEmpty()]
        [string]$ActionName,

        [ValidateRange(1, 60)]
        [int]$TimeoutMinutes = 30
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $ZVM = Get-VM -Name $ZVM_VM_NAME
    if ($null -eq $ZVM) {
        throw "$ZVM_VM_NAME VM does not exist."
    }

    Write-Host "Executing '$ActionName' with $TimeoutMinutes minutes timeout."
    $startTime = Get-Date
    # Run the script asynchronously
    $task = Invoke-VMScript -VM $ZVM -ScriptText $ScriptText -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -RunAsync

    # Calculate the timeout time
    $timeoutTime = (Get-Date).AddMinutes($TimeoutMinutes)

    while ((Get-Date) -lt $timeoutTime) {
        # Check the task state periodically
        switch ($task.State) {
            'Success' {
                # The 'Success' state indicates that the remote script was executed, but does not reflect the script success or failure.
                Write-Host "Executing '$ActionName' took $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."

                # task.Result is VMScriptResultImpl type, and we add a new dynamic property to it.
                $task.Result | Add-Member -MemberType NoteProperty -Name "TrimmedOutput" -Value $task.Result?.ScriptOutput?.TrimEnd("`n")

                return $task.Result
            }
            'Error' {
                throw "$ActionName execution error: $($task.TerminatingError.Message)"
                # In case of wrong VM credentials, the error message would be "Failed to authenticate with the guest operating system using the supplied credentials."
            }
            default {
                # If the task is 'Running' or in any other state, wait briefly before rechecking.
                Start-Sleep -Seconds 10
            }
        }
    }

    # If the loop exits, it means the timeout was reached
    # Note that the task is not aborted, so the script could eventually succeed
    #TODO Maybe ask user to restart the ZVML, to avoid leaving the script running in unclear state
    throw "Timeout, '$ActionName' did not complete within the allotted time of $TimeoutMinutes minutes."
}

function Invoke-ZVMLScript {
    <#
        .SYNOPSIS
        Executes the shell script on the ZVML VM synchronously
 
        .OUTPUTS
        [VMScriptResultImpl]
        Result .ExitCode contains script success/failure
        Result .ScriptOutput contains the clean script output
    #>

    param (
        [ValidateNotNullOrEmpty()]
        [string]$ScriptText
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $ZVM = Get-VM -Name $ZVM_VM_NAME
    if ($null -eq $ZVM) {
        throw "$ZVM_VM_NAME VM does not exist."
    }

    $res = Invoke-VMScript -VM $ZVM -ScriptText $ScriptText -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction Stop
    if ($null -eq $res) {
        throw "Failed to invoke VM script."
    }

    return [PSCustomObject]@{
        ExitCode     = $res.ExitCode
        ScriptOutput = $res.ScriptOutput.TrimEnd("`n")
    }
}

function Assert-ZertoInitialized {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $startTime = Get-Date
    $action = {
        $ZVM = Get-VM -Name $ZVM_VM_NAME
        if ($null -eq $ZVM) {
            throw "$ZVM_VM_NAME VM does not exist."
        }

        $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
        if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER)) {
            throw "ZVMA not initialized, authentication failed."
        }

        #TODO: This single check is enough to determine if ZVM is initialized, split between null, when authentication fails and when ZVM is not initialized
        $zvmInitStatusFile = "/opt/zerto/zvr/initialization-files/zvm_initialized"
        $res = Invoke-VMScript -VM $ZVM -ScriptText "[ -e $zvmInitStatusFile ] && echo true || echo false" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
        if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne 'true')) {
            throw "ZVMA not initialized, initialization file not found."
        }
    }
    Invoke-Retry -Action $action -ActionName 'Test ZVMA initialized' -RetryIntervalSeconds 120
    Write-Host "Zerto initialization completed, duration: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."
}

function Set-ZertoVmPassword {
    <#
    .SYNOPSIS
        Sets the ZVM VM console password and updates the PersistentSecrets
    #>

    param(
        [ValidateNotNullOrEmpty()]
        [SecureString]$NewPassword
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."
    $newPasswordText = ConvertFrom-SecureString -SecureString $NewPassword -AsPlainText
    $action = {
        $ZVM = Get-VM -Name $ZVM_VM_NAME
        if ($null -eq $ZVM) {
            throw "$ZVM_VM_NAME VM does not exist."
        }

        # Clear the password history and change password
        $cmdClearHistoryChangePassword = "sudo truncate -s 0 /etc/security/opasswd; echo '$($ZAPPLIANCE_USER):$newPasswordText' | sudo chpasswd"
        # We need to write result to variable to avoid module logging issues
        # We need SilentlyContinue because when an in-guest script changes its own account password, the authenticated session may no longer be valid by the time the cmdlet attempts to finalize and return a result.
        # This causes a "Failed to authenticate..." error, even though the password was successfully changed.
        # For the same reason, ScriptOutput will also be unavailable.
        $empty = Invoke-VMScript -VM $ZVM -ScriptText $cmdClearHistoryChangePassword -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue

        $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $newPasswordText -ErrorAction SilentlyContinue
        if (($null -eq $res) -or ($res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER)) {
            throw "Failed to change ZVML VM password."
        }
        $PersistentSecrets.ZappliancePassword = $newPasswordText
    }
    Invoke-Retry -Action $action -ActionName 'Change ZVMA VM password' -RetryIntervalSeconds 20 -RetryCount 3
}

function Set-ZertoConfiguration {
    param(
        [bool]$IsVaio = $true
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $startTime = Get-Date

    # This was commented for 10u7+ because static DNS should now be set correctly by appliance from ovf, fixed in ZER-150945.
    # If no issues are found in production, this code can be removed in the next release.
    # Also removed [string]$DNS param
    #
    # Set-DnsConfiguration -DNS $DNS
    # Stop-ZVM
    # Start-ZVM

    Write-Host "Configuring Zerto, this might take a while..."
    if ($IsVaio) {
        Write-Host "Enabling VAIO Support."
    }

    $scriptPath = '/opt/zerto/zlinux/avs/configure_zerto.py'
    $scriptParameters = @{

        'vcPassword'      = $($PersistentSecrets.ZertoPassword)
        'avsClientSecret' = $($PersistentSecrets.AvsClientSecret)
        'isVaio'          = $IsVaio #TODO NOW test that bool is actually passed correctly both for true and false
    }
    $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

    $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Configure ZVM'

    if ($result.ScriptOutput.Contains("Success")) {
        Write-Host "Zerto configured successfully."
    }
    elseif ($result.ScriptOutput.Contains("Warning:")) {
        $message = $result.ScriptOutput
        Write-Host $message
        Write-Warning $message
    }
    elseif ($result.ScriptOutput.Contains("Error:")) {
        $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
        throw $cleanErrMsg
    }
    else {
        throw "An unexpected error occurred while configuring Zerto. Please reinstall Zerto."
    }
    Write-Host "Zerto configuration completed, duration: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."
}

function Update-ZertoConfiguration {
    param(
        [ValidateNotNullOrEmpty()][string]
        $AzureTenantId,

        [ValidateNotNullOrEmpty()][string]
        $AzureClientID,

        [ValidateNotNullOrEmpty()][string]
        $AvsSubscriptionId,

        [ValidateNotNullOrEmpty()][string]
        $AvsResourceGroup,

        [ValidateNotNullOrEmpty()][string]
        $AvsCloudName
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    Write-Host "Reconfiguring Zerto, this might take a while..."
    $ZertoUserWithDomain = "$ZERTO_USER_NAME@$DOMAIN"

    $scriptPath = '/opt/zerto/zlinux/avs/reconfigure_zvm.py'
    $scriptParameters = @{
        'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
        'avsClientSecret'    = $PersistentSecrets.AvsClientSecret
        'azureTenantId'      = $AzureTenantId
        'azureClientID'      = $AzureClientID
        'avsSubscriptionId'  = $AvsSubscriptionId
        'avsResourceGroup'   = $AvsResourceGroup
        'avsCloudName'       = $AvsCloudName
        'vcIp'               = $VC_ADDRESS
        'vcUsername'         = $ZertoUserWithDomain
        'vcPassword'         = $PersistentSecrets.ZertoPassword
    }
    $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

    $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Reconfigure ZVM'

    if ($result.ScriptOutput.Contains("Success")) {
        Write-Host "Zerto reconfigured successfully."
    }
    elseif ($result.ScriptOutput.Contains("Warning:")) {
        $message = $result.ScriptOutput
        Write-Host $message
        Write-Warning $message
    }
    elseif ($result.ScriptOutput.Contains("Error:")) {
        $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
        throw $cleanErrMsg
    }
    else {
        throw "An unexpected error occurred while reconfiguring Zerto. Please reinstall Zerto."
    }
}


function Test-ZertoPassword {
    <#
    .SYNOPSIS
        Checks validity of 'admin' and 'zadmin' passwords stored in PersistentSecrets
 
    .DESCRIPTION
        The Zerto 'admin' password is checked explicitly – try_zerto_login.py will return "Success" if the password is valid.
        The Console 'zadmin' password is checked implicitly – Invoke-ZVMLScriptWithTimeout will fail with authentication error if the password is invalid.
    #>

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $scriptPath = '/opt/zerto/zlinux/avs/try_zerto_login.py'
    $scriptParameters = @{
        'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
    }
    $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

    try {
        $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Validate Zerto password' -TimeoutMinutes 5
        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "Zerto password is valid."
        }
        else {
            throw "Provided Zerto password is not valid."
        }
    }
    catch {
        throw "Zerto password validation failed. Problem: $_"
    }
}

enum PasswordsValidationResult {
    PasswordsValid = 0;
    ZertoPasswordInvalid = 1;
    ConsolePasswordInvalidOrExpired = 2;
}

function Test-ZertoPasswordResult {
    <#
    .SYNOPSIS
        Checks validity of 'admin' and 'zadmin' passwords stored in PersistentSecrets and returns a result code.
 
    .OUTPUTS
        [PasswordsValidationResult]
        PasswordsValid - Both passwords are valid
        ZertoPasswordInvalid - The 'admin' password is not valid
        ConsolePasswordInvalidOrExpired - The 'zadmin' password is not valid
        Throws an error if any other issue occurs during validation.
    #>

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        Test-ZertoPassword
        return [PasswordsValidationResult]::PasswordsValid
    }
    catch {
        if ($_ -match "Provided Zerto password is not valid") {
            Write-Host 'The Zerto "admin" password is not valid.'
            return [PasswordsValidationResult]::ZertoPasswordInvalid
        }
        if ($_ -match "Failed to authenticate with the guest operating system using the supplied credentials") {
            Write-Host 'The Console "zadmin" password is not valid.'
            return [PasswordsValidationResult]::ConsolePasswordInvalidOrExpired
        }
        throw $_
    }

}

function Update-VcPasswordInZvm {
    param (
        [ValidateNotNullOrEmpty()][string] $NewVcPassword,
        [ValidateNotNullOrEmpty()][string] $ZertoAdminPassword,
        [ValidateNotNullOrEmpty()][string] $ClientSecret
    )
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $scriptPath = '/opt/zerto/zlinux/avs/change_vc_password.py'
        $scriptParameters = @{
            'zertoAdminPassword' = $ZertoAdminPassword
            'vcPassword'         = $NewVcPassword
            'avsClientSecret'    = $ClientSecret
        }
        $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

        $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Change VC password in ZVM' -TimeoutMinutes 20

        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "The new VC password in ZVM set successfully."
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg #TODO: Standardize error messages for Error and Unknown cases, here and elsewhere, review unit tests
            }
            throw "Unexpected error occurred while updating VC password in ZVM."
        }
    }
}

function Update-ClientCredentialsInZvm {
    param (
        [ValidateNotNullOrEmpty()][string] $NewClientId,
        [ValidateNotNullOrEmpty()][string] $NewClientSecret
    )
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $scriptPath = '/opt/zerto/zlinux/avs/change_azure_client_credentials.py' # change_azure_client_credentials is available starting ZVM 10u5p2

        $scriptExists = Test-FileExistsInZVM -FileLocation $scriptPath #TODO: Consider extracting check to caller method, method should not return bool value
        if ($scriptExists -eq $false) {
            return $false # ZVMA version does not support updating Client Credentials
        }

        $scriptParameters = @{
            'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
            'vcPassword'         = $PersistentSecrets.ZertoPassword
            'azureClientId'      = $NewClientId
            'avsClientSecret'    = $NewClientSecret
        }
        $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

        $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Change Azure client credentials in ZVM' -TimeoutMinutes 20

        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "New Azure client credentials set successfully in ZVM."
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg
            }
            throw "Unexpected error occurred while changing Azure client credentials in ZVM."
        }

        return $true
    }
}

<#
function Set-DnsConfiguration($DNS) {
    #TODO: Once the u7 is published, this method can be removed, because static DNS should be set correctly by appliance from ovf, fixed in ZER-150945
    Write-Host "Starting $($MyInvocation.MyCommand)..."
 
    try {
        $action = {
            $ZVM = Get-VM -Name $ZVM_VM_NAME
            if ($null -eq $ZVM) {
                throw "$ZVM_VM_NAME VM does not exist."
            }
            $setDnsCommand = "grep -qxF 'nameserver $DNS' /etc/resolv.conf || echo 'nameserver $DNS' | sudo tee -a /etc/resolv.conf"
            $res = Invoke-VMScript -VM $ZVM -ScriptText $setDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
 
            $checkDnsCommand = "grep -qF 'nameserver $DNS' /etc/resolv.conf && echo 'true' || echo 'false'"
            $res = Invoke-VMScript -VM $ZVM -ScriptText $checkDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
            if ($null -eq $res -or $res.ScriptOutput.Trim() -ne "true") {
                throw "Failed to force set DNS"
            }
 
            $lockFileCommand = 'sudo chattr +i /etc/resolv.conf'
            $res = Invoke-VMScript -VM $ZVM -ScriptText $lockFileCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword
            Write-Host "DNS successfully set"
        }
        Invoke-Retry -Action $action -ActionName "SetDNS" -RetryCount 4 -RetryIntervalSeconds 30
    }
    catch {
        $message = "Failed to set DNS. Configuration may fail. Problem: $_"
        Write-Host $message
        Write-Warning $message
    }
}
#>


function Test-FileExistsInZVM ($FileLocation) {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        # test -f can only return 0 if file exists or 1
        $commandFileExists = "test -f $FileLocation || exit 1"

        $res = Invoke-ZVMLScript -ScriptText $commandFileExists
        if ($res.ExitCode -eq 0) {
            return $true
        }
        else {
            return $false
        }
    }
    catch {
        throw "Failed to check file in ZVM. Problem: $_"
    }
}

function Test-FeatureFlagEnabled {
    param(
        [ValidateNotNullOrEmpty()]
        [string]$Flag
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    # -i: Makes the search case-insensitive.
    # -x: Matches only entire lines (not partial matches).
    # -F: Interprets the pattern as a fixed string, not a regular expression.
    # -q: Runs silently, suppressing output and returning only the exit code.
    $commandCheckFlag = "test -f '/opt/zerto/zlinux/avs/feature-flags.conf' && grep -ixFq '$Flag' '/opt/zerto/zlinux/avs/feature-flags.conf' || exit 1"

    $res = Invoke-ZVMLScript -ScriptText $commandCheckFlag
    if ($res.ExitCode -ne 0) {
        throw "Your ZVMA version does not support '$Flag' feature in AVS. Please use a version which supports '$Flag' feature."
    }

    Write-Host "ZVMA version supports '$Flag' feature."
}

function Set-ZertoVmPasswordExpiration {
    <#
    .SYNOPSIS
        Resets the ZVM VM console password expiration to never expire
    #>

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        # grep -Pq, -P to use Perl-compatible regular expressions, -q to be quiet without outputting the matching lines
        # returns 0 exit code if 'maxdays -1' is set and then 'Maximum -1' pattern is found
        $chageCommand = "sudo chage --maxdays -1 $ZAPPLIANCE_USER && chage -l $ZAPPLIANCE_USER | grep -Pq '^Maximum number of days.*-1$' || exit 1"

        $res = Invoke-ZVMLScript -ScriptText $chageCommand
        if ($res.ExitCode -ne 0) {
            throw "Failed to chage, $($res.ScriptOutput)" # ScriptOutput should contain bash error
        }
    }
    catch {
        throw "Failed to set password expiry. Problem: $_"
    }
}

function Set-AzureResourceGroup {
    <#
    .SYNOPSIS
        Sets the Azure Resource Group name in the ZVMA
    #>

    param (
        [string]$ResourceGroupName
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $scriptPath = '/opt/zerto/zlinux/avs/change_resource_group.py'
    $scriptParameters = @{
        'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
        'vcPassword'         = $PersistentSecrets.ZertoPassword
        'avsClientSecret'    = $PersistentSecrets.AvsClientSecret
        'resourceGroupName'  = $ResourceGroupName
    }
    $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

    $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Change resource group'

    if ($result.ScriptOutput.Contains("Success")) {
        Write-Host "Resource group set successfully."
    }
    elseif ($result.ScriptOutput.Contains("Error:")) {
        $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
        throw $cleanErrMsg
    }
    else {
        throw "An unexpected error occurred while changing the resource group."
    }
}

function Enable-VAIOConfiguration {
    <#
    .SYNOPSIS
        Enable Zerto protection using VAIO (vSphere API for IO filtering)
    #>

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $scriptPath = '/opt/zerto/zlinux/avs/enable_vaio.py'
    $scriptParameters = @{
        'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
        'vcPassword'         = $PersistentSecrets.ZertoPassword
        'avsClientSecret'    = $PersistentSecrets.AvsClientSecret
    }
    $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

    $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Enable VAIO'

    if ($result.ScriptOutput.Contains('Success')) {
        Write-Host 'VAIO enabled successfully.'
    }
    elseif ($result.ScriptOutput.Contains('Error:')) {
        $cleanErrMsg = $result.ScriptOutput -replace 'Error: ', ''
        throw $cleanErrMsg
    }
    else {
        throw 'An unexpected error occurred while enabling VAIO.'
    }
}

function New-ZertoLogs {
    param (
        [bool]$WithPrometheus = $false
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        $scriptPath = '/opt/zerto/zlinux/avs/collect_logs.py'
        $scriptParameters = @{
            'with-prometheus' = $WithPrometheus #TODO NOW test that bool is actually passed correctly both for true and false
        }
        $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

        $res = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Create ZVML logs archive' -TimeoutMinutes 30

        $LOG_FILE = 'LogPath'

        $scriptOutput = $res.TrimmedOutput

        Write-Host "ZVML logs collection script output:`n$scriptOutput"

        if (-not ($scriptOutput -match "(?m)^Success\.[^']+'(?<$LOG_FILE>[^']+?)'\s+created\.$")) {
            throw 'No log file created.'
        }

        $logFile = $matches[$LOG_FILE]

        return $logFile
    }
    catch {
        throw "Failed to create ZVML log archive. Problem: $_"
    }
}

function Cleanup-LogBundles {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    function Test-LogBundlesExist {
        # find all files in both directories, print0 to handle spaces in filenames, xargs -0 to handle null-terminated input, -r to avoid running stat if no files found
        $commandGetLogBundles = "find /var/log/zerto/zvr/bundles /var/log/zerto/zvr/temp-bundles -type f -print0 | xargs -0 -r stat -c '%n | Size %s bytes' || exit 1"

        $res = Invoke-ZVMLScript -ScriptText $commandGetLogBundles
        if ($res.ExitCode -ne 0) {
            throw "Failed to get log bundles files, $($res.ScriptOutput)" # ScriptOutput should contain bash error
        }
        $foundFilesList = $res.ScriptOutput

        $filesExist = (-not [string]::IsNullOrWhiteSpace($foundFilesList))

        return @($filesExist, $foundFilesList)
    }

    try {
        $filesExist, $foundFilesList = Test-LogBundlesExist

        if (-not $filesExist) {
            Write-Host "No log bundle files found. Nothing to cleanup."
            return
        }

        Write-Host "Found log bundle files:"
        Write-Host $foundFilesList

        $totalBytes = 0
        $lines = $foundFilesList -split "`n"
        foreach ($line in $lines) {
            if ($line -match "Size (\d+) bytes") {
                $sizeBytes = [long]$matches[1]
                $totalBytes += $sizeBytes
            }
        }

        $totalMB = [math]::Round($totalBytes / 1MB, 2)
        Write-Host "Total bundles size: $totalMB MB"

        Write-Host "Deleting log bundles..."

        # find all files in both directories, -mindepth 1 to avoid deleting the directories themselves, -delete to remove the files
        $commandDeleteLogBundles = "find /var/log/zerto/zvr/bundles /var/log/zerto/zvr/temp-bundles -mindepth 1 -delete || exit 1"

        $res = Invoke-ZVMLScript -ScriptText $commandDeleteLogBundles
        if ($res.ExitCode -ne 0) {
            throw "Failed to delete log bundles files, $($res.ScriptOutput)"
        }

        $filesExist, $foundFilesList = Test-LogBundlesExist
        if ($filesExist) {
            throw "Failed to delete some log bundles files: `n$foundFilesList"
        }

        Write-Host "Log bundles cleanup complete. Freed $totalMB MB of disk space."
    }
    catch {
        throw "Failed to cleanup log bundles. Problem: $_"
    }
}

function Set-ZVMTweak {
    <#
    .SYNOPSIS
        Sets ZVM tweak
    #>

    param(
        [ValidateNotNullOrEmpty()]
        [string]$TweakName,

        [ValidateNotNull()]
        [string]$TweakValue,

        [string]$TweakComment = 'AVS tweak'
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."


    try {
        $scriptPath = "/opt/zerto/zlinux/avs/set_tweak.py"
        $scriptExists = Test-FileExistsInZVM -FileLocation $scriptPath
        if ($scriptExists -eq $false) {
            throw "ZVMA version does not support automated tweaks configuration."
        }

        $scriptParameters = @{
            'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
            'tweakName'          = $TweakName
            'tweakValue'         = $TweakValue
            'tweakComment'       = $TweakComment
        }
        $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

        $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Set ZVM tweak' -TimeoutMinutes 3

        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "Tweak '$TweakName' set successfully."
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg
            }
            throw "Unexpected error occurred while setting tweak '$TweakName'."
        }
    }
    catch {
        throw "Failed to set tweak '$TweakName'. Problem: $_"
    }
}

function Get-ZVMTweak {
    <#
    .SYNOPSIS
        Gets ZVM tweak
    #>

    param(
        [ValidateNotNullOrEmpty()]
        [string]$TweakName
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        $scriptPath = "/opt/zerto/zlinux/avs/get_tweak.py"
        $scriptExists = Test-FileExistsInZVM -FileLocation $scriptPath
        if ($scriptExists -eq $false) {
            throw "ZVMA version does not support automated tweaks configuration."
        }

        $scriptParameters = @{
            'zertoAdminPassword' = $PersistentSecrets.ZertoAdminPassword
            'tweakName'          = $TweakName
        }
        $commandToExecute = Build-SafeShellPyCommand -PyScriptPath $scriptPath -PyScriptParams $scriptParameters

        $result = Invoke-ZVMLScriptWithTimeout -ScriptText $commandToExecute -ActionName 'Get ZVM tweak' -TimeoutMinutes 3

        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "Tweak '$TweakName' got successfully."
            return $result.ScriptOutput
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg
            }
            throw "Unexpected error occurred while getting tweak '$TweakName'."
        }
    }
    catch {
        throw "Failed to get tweak '$TweakName'. Problem: $_"
    }
}

function Invoke-CmdletWithPyScript {
    param (
        [string]$ScriptName,
        [scriptblock]$Action
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        #Сopies ScriptName script from PSEngine to ZVML
        $ZVML_AVS_SCRIPTS_PATH = '/opt/zerto/zlinux/avs'
        $PS_ENGINE_SCRIPTS_PATH = 'scripts'

        $SCRIPT_PATH = "$PS_ENGINE_SCRIPTS_PATH/$ScriptName"

        $psScriptFullPath = Join-Path -Path $PSScriptRoot $SCRIPT_PATH
        $ZVMLScriptPath = "$ZVML_AVS_SCRIPTS_PATH/$ScriptName"

        Copy-ItemFromPSEngineToZVM -Path $psScriptFullPath -Destination $ZVMLScriptPath

        #Executes scriptblock after script file was mounted to ZVML
        & $Action
    }
    catch {
        Write-Error "Error during script execution. Problem: $_"
        throw
    }
    finally {
        #TBH if script file must be removed afterwards
    }
}