zvmRemoteScripts_utils.ps1

$ZAPPLIANCE_USER = "zadmin"

function Invoke-ZVMScriptWithTimeout {
    param (
        [Parameter(Mandatory = $true)]
        [string]$ScriptText,

        [Parameter(Mandatory = $true)]
        [string]$ActionName,

        [int]$TimeoutMinutes = 30
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $ZVM = Get-VM -Name $ZVM_VM_NAME

    Write-Host "Invoking $ActionName with $TimeoutMinutes min timeout."
    # Start the script asynchronously
    $task = Invoke-VMScript -VM $ZVM -ScriptText $ScriptText -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -RunAsync

    # Calculate the timeout time
    $timeoutTime = (Get-Date).AddMinutes($TimeoutMinutes)

    while ((Get-Date) -lt $timeoutTime) {
        # Check the task status
        if ($task.State -eq 'Success') {
            # The 'Success' state indicates that the remote script was executed, but does not reflect the script's success or failure.
            Write-Output "$ActionName execution done."
            return $task.Result
        }
        elseif ($task.State -eq 'Error') {
            throw "$ActionName execution failed with error: $($task.ErrorMessage)"
        }

        # Wait for a short period before checking again
        Start-Sleep -Seconds 30
    }

    # If the loop exits, it means the timeout was reached
    throw "$ActionName did not complete within the allotted time of $TimeoutMinutes minutes."
}

function Assert-ZertoInitialized {
    Write-Host "Starting $($MyInvocation.MyCommand)..."
    $action = {
        $ZVM = Get-VM -Name $ZVM_VM_NAME
        if ($null -eq $ZVM) {
            Write-Error "$ZVM_VM_NAME doesn't exists" -ErrorAction Stop
        }
        $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
        if ($null -eq $res -or $res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER) {
            throw "ZVM failed to initialize"
        }

        $zvmInitStatusFile = "/opt/zerto/zvr/initialization-files/zvm_initialized"
        $res = Invoke-VMScript -VM $ZVM -ScriptText "[ -e $zvmInitStatusFile ] && echo true || echo false" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
        if ($null -eq $res -or $res.ScriptOutput.Trim() -ne 'true') {
            throw "ZVM failed to initialize"
        }
    }
    Invoke-Retry -Action $action -ActionName "TestZertoInitialized" -RetryCount 15 -RetryIntervalSeconds 60
}

function Set-ZertoVmPassword {
    param(
        [SecureString]$NewPassword
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."
    $passwordText = ConvertFrom-SecureString -SecureString $NewPassword -AsPlainText
    $action = {
        $ZVM = Get-VM -Name $ZVM_VM_NAME
        if ($null -eq $ZVM) {
            Write-Error "$ZVM_VM_NAME doesn't exists" -ErrorAction Stop
        }
        #We need to write result to variable to avoid module logging issues
        $passChange = Invoke-VMScript -VM $ZVM -ScriptText "echo '$($ZAPPLIANCE_USER):$passwordText' | sudo chpasswd" -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
        $res = Invoke-VMScript -VM $ZVM -ScriptText "whoami" -GuestUser $ZAPPLIANCE_USER -GuestPassword $passwordText -ErrorAction SilentlyContinue
        if ($null -eq $res -or $res.ScriptOutput.Trim() -ne $ZAPPLIANCE_USER) {
            throw "Failed to change ZVM VM password"
        }
        $PersistentSecrets.ZappliancePassword = $passwordText
    }
    Invoke-Retry -Action $action -ActionName "ChangeZvmVmPassword" -RetryCount 10 -RetryIntervalSeconds 60
}

function Set-ZertoConfiguration ($DNS) {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $startTime = Get-Date
    Write-Host "Waiting for Zerto to start, this might take a while..."
    Assert-ZertoInitialized
    Write-Host "Zerto initialization took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."
    Set-DnsConfiguration -DNS $DNS

    Stop-ZVM
    Start-ZVM

    Write-Host "Configuring Zerto, this might take a while..."
    $startTime = Get-Date
    #TODO: we need to rename key that holds VC user password. It sound confusing
    $scriptLocation = "/opt/zerto/zlinux/avs/configure_zerto.py"
    $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --avsClientSecret '$($PersistentSecrets.AvsClientSecret)'"
    $result = Invoke-ZVMScriptWithTimeout -ScriptText $commandToExecute -ActionName "Configure ZVM"
    Write-Host "Zerto configuration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."

    if ($result.ScriptOutput.Contains("Success")) {
        Write-Host "Zerto configured successfully."
    }
    elseif ($result.ScriptOutput.Contains("Warning:")) {
        $message = $result.ScriptOutput
        Write-Host $message
        Write-Warning $message
    }
    elseif ($result.ScriptOutput.Contains("Error:")) {
        $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
        throw $cleanErrMsg
    }
    else {
        throw "An unexpected error occurred while configuring Zerto. Please reinstall Zerto."
    }
}

function Update-ZertoConfiguration {
    param(
        [ValidateNotNullOrEmpty()][string]
        $AzureTenantId,

        [ValidateNotNullOrEmpty()][string]
        $AzureClientID,

        [ValidateNotNullOrEmpty()][string]
        $AvsSubscriptionId,

        [ValidateNotNullOrEmpty()][string]
        $AvsResourceGroup,

        [ValidateNotNullOrEmpty()][string]
        $AvsCloudName
    )
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    Write-Host "Validating Zerto VM password"
    Assert-ZertoInitialized
    Test-ZertoPassword

    Write-Host "Reconfiguring Zerto, this might take a while..."
    $startTime = Get-Date

    $scriptLocation = "/opt/zerto/zlinux/avs/reconfigure_zvm.py"
    $ZertoUserWithDomain = "$ZERTO_USER_NAME@$DOMAIN"

    $commandToExecute = "sudo python3 $scriptLocation " +
    "--avsClientSecret '$($PersistentSecrets.AvsClientSecret)' " +
    "--azureTenantId '$AzureTenantId' " +
    "--azureClientID '$AzureClientID' " +
    "--avsSubscriptionId '$AvsSubscriptionId' " +
    "--avsResourceGroup '$AvsResourceGroup' " +
    "--avsCloudName '$AvsCloudName' " +
    "--vcIp '$VC_ADDRESS' " +
    "--vcUsername '$ZertoUserWithDomain' " +
    "--vcPassword '$($PersistentSecrets.ZertoPassword)' " +
    "--zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)'"

    $result = Invoke-ZVMScriptWithTimeout -ScriptText $commandToExecute -ActionName "Reconfigure ZVM"
    Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."

    if ($result.ScriptOutput.Contains("Success")) {
        Write-Host "Zerto reconfigured successfully."
    }
    elseif ($result.ScriptOutput.Contains("Warning:")) {
        $message = $result.ScriptOutput
        Write-Host $message
        Write-Warning $message
    }
    elseif ($result.ScriptOutput.Contains("Error:")) {
        $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
        throw $cleanErrMsg
    }
    else {
        throw "An unexpected error occurred while reconfiguring Zerto. Please reinstall Zerto."
    }
}

function Test-ZertoPassword { #TODO: Pass password as parameter
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."
        $scriptLocation = "/opt/zerto/zlinux/avs/try_zerto_login.py"
        $commandToExecute = "sudo python3 $scriptLocation --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)'"
        try {
            $result = Invoke-ZVMScriptWithTimeout -ScriptText $commandToExecute -ActionName "Validate Zerto password" -TimeoutMinutes 5
            if ($result.ScriptOutput.Contains("Success")) {
                Write-Host "Zerto password is valid"
            }
            else {
                throw "Please provide valid Zerto password."
            }
        }
        catch {
            $errorMessage = "An error happened during Zerto password validataion. Problem: $_"
            Write-Host $errorMessage
            Write-Error $errorMessage -ErrorAction Stop
        }
    }
}

function Update-VcPasswordInZvm {
    param (
        [ValidateNotNullOrEmpty()][string] $NewVcPassword,
        [ValidateNotNullOrEmpty()][string] $ZertoAdminPassword,
        [ValidateNotNullOrEmpty()][string] $ClientSecret
    )
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $scriptLocation = "/opt/zerto/zlinux/avs/change_vc_password.py"
        $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$NewVcPassword' --zertoAdminPassword '$ZertoAdminPassword' --avsClientSecret '$ClientSecret'"

        $startTime = Get-Date
        $result = Invoke-ZVMScriptWithTimeout -ScriptText $commandToExecute -ActionName "Change VC password in ZVM" -TimeoutMinutes 20
        Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."

        if ($result.ScriptOutput.Contains("Success")) {
                Write-Host "New VC password set successfully in ZVM."
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg
            }
            throw "Unexpected error occurred while updating VC password in ZVM."
        }
    }
}

function Update-ClientCredentialsInZvm {
    param (
        [ValidateNotNullOrEmpty()][string] $NewClientId,
        [ValidateNotNullOrEmpty()][string] $NewClientSecret
    )
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $scriptLocation = "/opt/zerto/zlinux/avs/change_azure_client_credentials.py" # change_azure_client_credentials is available starting ZVM 10u5p2

        $scriptExists = Test-FileExistsInZVM -FileLocation $scriptLocation #TODO: Consider extracting check to caller method, method should not return bool value
        if ($scriptExists -eq $false) {
            return $false # ZVMA version does not support updating Client Credentials
        }

        $commandToExecute = "sudo python3 $scriptLocation --vcPassword '$($PersistentSecrets.ZertoPassword)' --zertoAdminPassword '$($PersistentSecrets.ZertoAdminPassword)' --azureClientId '$NewClientId' --avsClientSecret '$NewClientSecret'"

        $startTime = Get-Date
        $result = Invoke-ZVMScriptWithTimeout -ScriptText $commandToExecute -ActionName "Change Azure client credentials in ZVM" -TimeoutMinutes 20
        Write-Host "Zerto reconfiguration took: $((Get-Date).Subtract($startTime).TotalSeconds.ToString("F0")) seconds."

        if ($result.ScriptOutput.Contains("Success")) {
            Write-Host "New Azure client credentials set successfully in ZVM."
        }
        else {
            if ($result.ScriptOutput.Contains("Error:")) {
                $cleanErrMsg = $result.ScriptOutput -replace "Error: ", ""
                throw $cleanErrMsg
            }
            throw "Unexpected error occurred while updating Azure client credentials in ZVM."
        }

        return $true
    }
}

function Set-DnsConfiguration($DNS) {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {
        $action = {
            $ZVM = Get-VM -Name $ZVM_VM_NAME
            if ($null -eq $ZVM) {
                Write-Error "$ZVM_VM_NAME doesn't exists" -ErrorAction Stop
            }
            $setDnsCommand = "grep -qxF 'nameserver $DNS' /etc/resolv.conf || echo 'nameserver $DNS' | sudo tee -a /etc/resolv.conf"
            $res = Invoke-VMScript -VM $ZVM -ScriptText $setDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue

            $checkDnsCommand = "grep -qF 'nameserver $DNS' /etc/resolv.conf && echo 'true' || echo 'false'"
            $res = Invoke-VMScript -VM $ZVM -ScriptText $checkDnsCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction SilentlyContinue
            if ($null -eq $res -or $res.ScriptOutput.Trim() -ne "true") {
                throw "Failed to force set DNS"
            }

            $lockFileCommand = 'sudo chattr +i /etc/resolv.conf'
            $res = Invoke-VMScript -VM $ZVM -ScriptText $lockFileCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword
            Write-Host "DNS successfully set"
        }
        Invoke-Retry -Action $action -ActionName "SetDNS" -RetryCount 4 -RetryIntervalSeconds 30
    }
    catch {
        Write-Warning "Failed to set DNS. Problem: $_. Configuration may fail."
    }
}

function Test-FileExistsInZVM($FileLocation) {
    Write-Host "Starting $($MyInvocation.MyCommand)..."

    try {

        $ZVM = Get-VM -Name $ZVM_VM_NAME
        if ($null -eq $ZVM) {
            throw "$ZVM_VM_NAME VM does not exist."
        }

        $existsFileCommand = "test -f $FileLocation && echo 'true' || echo 'false'"

        $res = Invoke-VMScript -VM $ZVM -ScriptText $existsFileCommand -GuestUser $ZAPPLIANCE_USER -GuestPassword $PersistentSecrets.ZappliancePassword -ErrorAction Stop

        switch (${res}?.ScriptOutput?.Trim()) {
            { $_ -eq $null } { throw "Unknown error." }
            { $_ -eq "true" } { return $true }
            { $_ -eq "false" } { return $false }
            default { throw "Unexpected output '$_'." }
        }

    }
    catch {
        throw "Failed to check file in ZVM. Problem: $_"
    }
}