Start-RobustCloudCommand.ps1
|
################################################################################# # # The sample scripts are not supported under any Microsoft standard support # program or service. The sample scripts are provided AS IS without warranty # of any kind. Microsoft further disclaims all implied warranties including, without # limitation, any implied warranties of merchantability or of fitness for a particular # purpose. The entire risk arising out of the use or performance of the sample scripts # and documentation remains with you. In no event shall Microsoft, its authors, or # anyone else involved in the creation, production, or delivery of the scripts be liable # for any damages whatsoever (including, without limitation, damages for loss of business # profits, business interruption, loss of business information, or other pecuniary loss) # arising out of the use of or inability to use the sample scripts or documentation, # even if Microsoft has been advised of the possibility of such damages. # ################################################################################# # # Created by Matbyrd@microsoft.com # ################################################################################# Param( [switch]$Agree, [Parameter(Mandatory=$true)] [string]$LogFile, [Parameter(Mandatory=$true)] $Recipients, [Parameter(Mandatory=$true)] [ScriptBlock]$ScriptBlock, $Credential, [int]$ManualThrottle=0, [double]$ActiveThrottle=.25, [int]$ResetSeconds=870, [string]$IdentifyingProperty, [Switch]$NonInteractive ) # Writes output to a log file with a time date stamp Function Write-Log { Param ([string]$string) # Get the current date [string]$date = Get-Date -Format G # Write everything to our log file ( "[" + $date + "] - " + $string) | Out-File -FilePath $LogFile -Append # If NonInteractive true then suppress host output if (!($NonInteractive)){ ( "[" + $date + "] - " + $string) | Write-Host } } # Sleeps X seconds and displays a progress bar Function Start-SleepWithProgress { Param([int]$sleeptime) # Loop Number of seconds you want to sleep For ($i=0;$i -le $sleeptime;$i++){ $timeleft = ($sleeptime - $i); # Progress bar showing progress of the sleep Write-Progress -Activity "Sleeping" -CurrentOperation "$Timeleft More Seconds" -PercentComplete (($i/$sleeptime)*100) -Status " " # Sleep 1 second start-sleep 1 } Write-Progress -Completed -Activity "Sleeping" -Status " " } # Setup a new O365 Powershell Session Function New-CleanO365Session { # If we don't have a credential then prompt for it $i = 0 while (($Credential -eq $Null) -and ($i -lt 5)){ $script:Credential = Get-Credential -Message "Please provide your Exchange Online Credentials" $i++ } # If we still don't have a credentail object then abort if ($Credential -eq $null){ Write-log "[Error] - Failed to get credentials" Write-Error -Message "Failed to get credentials" -ErrorAction Stop } Write-Log "Removing all PS Sessions" # Destroy any outstanding PS Session Get-PSSession | Remove-PSSession -Confirm:$false # Force Garbage collection just to try and keep things more agressively cleaned up due to some issue with large memory footprints [System.GC]::Collect() # Sleep 15s to allow the sessions to tear down fully Write-Log ("Sleeping 15 seconds for Session Tear Down") Start-SleepWithProgress -SleepTime 15 # Clear out all errors $Error.Clear() # Create the session Write-Log "Creating new PS Session" $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri "https://outlook.office365.com/powershell-liveid/" -Credential $Credential -Authentication Basic -AllowRedirection # Check for an error while creating the session if ($Error.Count -gt 0){ Write-Log "[ERROR] - Error while setting up session" Write-log $Error # Increment our error count so we abort after so many attempts to set up the session $ErrorCount++ # if we have failed to setup the session > 3 times then we need to abort because we are in a failure state if ($ErrorCount -gt 3){ Write-log "[ERROR] - Failed to setup session after multiple tries" Write-log "[ERROR] - Aborting Script" exit } # If we are not aborting then sleep 60s in the hope that the issue is transient Write-Log "Sleeping 60s so that issue can potentially be resolved" Start-SleepWithProgress -sleeptime 60 # Attempt to set up the sesion again New-CleanO365Session } # If the session setup worked then we need to set $errorcount to 0 else { $ErrorCount = 0 } # Import the PS session $null = Import-PSSession $session -AllowClobber # Set the Start time for the current session Set-Variable -Scope script -Name SessionStartTime -Value (Get-Date) } # Verifies that the connection is healthy # Goes ahead and resets it every $ResetSeconds number of seconds either way Function Test-O365Session { # Get the time that we are working on this object to use later in testing $ObjectTime = Get-Date # Reset and regather our session information $SessionInfo = $null $SessionInfo = Get-PSSession # Make sure we found a session if ($SessionInfo -eq $null) { Write-Log "[ERROR] - No Session Found" Write-log "Recreating Session" New-CleanO365Session } # Make sure it is in an opened state if not log and recreate elseif ($SessionInfo.State -ne "Opened"){ Write-Log "[ERROR] - Session not in Open State" Write-log ($SessionInfo | fl | Out-String ) Write-log "Recreating Session" New-CleanO365Session } # If we have looped thru objects for an amount of time gt our reset seconds then tear the session down and recreate it elseif (($ObjectTime - $SessionStartTime).totalseconds -gt $ResetSeconds){ Write-Log ("Session Has been active for greater than " + $ResetSeconds + " seconds" ) Write-Log "Rebuilding Connection" # Estimate the throttle delay needed since the last session rebuild # Amount of time the session was allowed to run * our activethrottle value # Divide by 2 to account for network time, script delays, and a fudge factor # Subtract 15s from the results for the amount of time that we spend setting up the session anyway [int]$DelayinSeconds = ((($ResetSeconds * $ActiveThrottle) / 2) - 15) # If the delay is >15s then sleep that amount for throttle to recover if ($DelayinSeconds -gt 0){ Write-Log ("Sleeping " + $DelayinSeconds + " addtional seconds to allow throttle recovery") Start-SleepWithProgress -SleepTime $DelayinSeconds } # If the delay is <15s then the sleep already built into New-CleanO365Session should take care of it else { Write-Log ("Active Delay calculated to be " + ($DelayinSeconds + 15) + " seconds no addtional delay needed") } # new O365 session and reset our object processed count New-CleanO365Session } else { # If session is active and it hasn't been open too long then do nothing and keep going } # If we have a manual throttle value then sleep for that many milliseconds if ($ManualThrottle -gt 0){ Write-log ("Sleeping " + $ManualThrottle + " milliseconds") Start-Sleep -Milliseconds $ManualThrottle } } # If the $identifyingProperty has not been set then we attempt to locate a value for tracking modified objects Function Get-ObjectIdentificationProperty { Param($object) Write-Log "Trying to identify a property for displaying per object progress" # Common properties to check [array]$PropertiesToCheck = "DisplayName","Name","Identity","PrimarySMTPAddress","Alias","GUID" # Set our counter to 0 $i = 0 # While we haven't found an ID property continue checking while ([string]::IsNullOrEmpty($IdentifyingProperty)) { # If we have gone thru the list then we need to throw an error because we don't have Identity information # Set the string to bogus just to ensure we will exit the while loop if ($i -gt ($PropertiesToCheck.length -1)) { Write-Log "[ERROR] - Unable to find a common identity parameter in the input object" # Create an error message that has all of the valid property names that we are looking for $PropertiesToCheck | foreach { [string]$PropertiesString = $PropertiesString + "`"" + $_ + "`", " } $PropertiesString = $PropertiesString.TrimEnd(", ") [string]$errorstring = "Objects does not contain a common identity parameter " + $PropertiesString + " please use -IdentifyingProperty to set the identity value" # Throw error Write-Error -Message $errorstring -ErrorAction Stop } # Get the property we are testing out of our array [string]$Property = $PropertiesToCheck[$i] # Check the properties of the object to see if we have one that matches a well known name # If we have found one set the value to that property if ($object.$Property -ne $null) { Write-log ("Found " + $Property + " to use for displaying per object progress") Set-Variable -Scope script -Name IdentifyingProperty -Value $Property } # Increment our position counter $i++ } } # Gather and print out information about how fast the script is running Function Get-EstimatedTimeToCompletion { param([int]$ProcessedCount) # Increment our count of how many objects we have processed $ProcessedCount++ # Every 100 we need to estimate our completion time and write that out if (($ProcessedCount % 100) -eq 0){ # Get the current date $CurrentDate = Get-Date # Average time per object in seconds $AveragePerObject = (((($CurrentDate) - $ScriptStartTime).totalseconds) / $ProcessedCount) # Write out session stats and estimated time to completion Write-Log ("[STATS] - Total Number of Objects: " + $ObjectCount) Write-Log ("[STATS] - Number of Objects processed: " + $ProcessedCount) Write-Log ("[STATS] - Average seconds per object: " + $AveragePerObject) Write-Log ("[STATS] - Estimated completion time: " + $CurrentDate.addseconds((($ObjectCount - $ProcessedCount) * $AveragePerObject))) } # Return number of objects processed so that the variable in incremented return $ProcessedCount } #################### # Main Script #################### # Force use of at least version 3 of powershell https://technet.microsoft.com/en-us/library/hh847765.aspx #Requires -version 3 # Turns on strict mode https://technet.microsoft.com/library/03373bbe-2236-42c3-bf17-301632e0c428(v=wps.630).aspx Set-StrictMode -Version 2 # Write creation date of script for version information Write-Log "Created 05/10/2016" # Statement to ensure that you have looked at the disclaimer or that you have removed this line so you don't have too if ($Agree -ne $true){ Write-Error "Please run the script with -Agree to indicate that you have read and agreed to the sample script disclaimer at the top of the script file" -ErrorAction Stop } else { Write-log "Agreed to Disclaimer" } # Log the script block for debugging purposes Write-log $ScriptBlock # Setup our first session to O365 $ErrorCount = 0 New-CleanO365Session # Get when we started the script for estimating time to completion $ScriptStartTime = Get-Date # Get the object count and write it out to be used in esitmated time to completion + logging [int]$ObjectCount = $Recipients.count [int]$ObjectsProcessed = 0 # If we don't have an identifying property then try to find one if ([string]::IsNullOrEmpty($IdentifyingProperty)) { # Call our function for finding an identifying property and pass in the first recipient object Get-ObjectIdentificationProperty -object $Recipients[0] } # Go thru each recipient object and execute the script block foreach ($object in $Recipients) { # Set our initial while statement values $TryCommand = $true $errorcount = 0 # Try the command 3 times and exit out if we can't get it to work # Record the error and restart the session each time it errors out while ($TryCommand) { Write-log ("Running scriptblock for " + ($object.$IdentifyingProperty).tostring()) # Clear all errors $Error.Clear() # Test our connection and rebuild if needed Test-O365Session # Invoke the script block Invoke-Command -InputObject $object -ScriptBlock $ScriptBlock # Test for errors if ($Error.Count -gt 0) { # Write that we failed Write-Log ("[ERROR] - Failed on object " + ($object.$IdentifyingProperty).tostring()) write-log $Error # Increment error count $errorcount++ # Handle if we keep failing on the object if ($errorcount -ge 3) { Write-Log ("[ERROR] - Oject has failed three times " + ($object.$IdentifyingProperty).tostring()) Write-Log ("[ERROR] - Skipping Object") # Increment the object processed count / Estimate time to completion $ObjectsProcessed = Get-EstimatedTimeToCompletion -ProcessedCount $ObjectsProcessed # Set trycommand to false so we abort the while loop $TryCommand = $false } # Otherwise try the command again else { Write-Log ("Rebuilding session and trying again") # Create a new session in case the error was due to a session issue New-CleanO365Session } } else { # Since we didn't get an error don't run again $TryCommand = $false # Increment the object processed count / Estimate time to completion $ObjectsProcessed = Get-EstimatedTimeToCompletion -ProcessedCount $ObjectsProcessed } } } Write-Log "Script Complete Destroying PS Sessions" # Destroy any outstanding PS Session Get-PSSession | Remove-PSSession -Confirm:$false <# .SYNOPSIS Generic wrapper script that tries to ensure that a script block successfully finishes execution in O365 against a large object count. Works well with intense operations that may cause throttling .DESCRIPTION Wrapper script that tries to ensure that a script block successfully finishes execution in O365 against a large object count. It accomplishs this by doing the following: * Monitors the health of the Remote powershell session and restarts it as needed. * Restarts the session every X number seconds to ensure a valid connection. * Attempts to work past session related errors and will skip objects that it can't process. * Attempts to calculate throttle exhaustion and sleep a sufficient time to allow throttle recovery .PARAMETER Agree Verifies that you have read and agree to the disclaimer at the top of the script file. .PARAMETER AutomaticThrottle Calculated value based on your tenants powershell recharge rate. You tenant recharge rate can be calculated using a Micro Delay Warning message. Look for the following line in your Micro Delay Warning Message Balance: -1608289/2160000/-3000000 The middle value is the recharge rate. Divide this value by the number of milliseconds in an hour (3600000) And subtract the result from 1 to get your AutomaticThrottle value 1 - (2160000 / 3600000) = 0.4 Default Value is .25 .PARAMETER Credential Credential object for logging into Exchange Online Shell. Prompts if there is non provided. .PARAMETER IdentifyingProperty What property of the objects we are processing that will be used to identify them in the log file and host If the value is not set by the user the script will attempt to determine if one of the following properties is present "DisplayName","Name","Identity","PrimarySMTPAddress","Alias","GUID" If the value is not set and we are not able to match a well known property the script will generate an error and terminate. .PARAMETER LogFile Location and file name for the log file. .PARAMETER ManualThrottle Manual delay of X number of milliseconds to sleep between each cmdlets call. Should only be used if the AutomaticThrottle isn't working to introduce sufficent delay to prevent Micro Delays .PARAMETER NonInteractive Suppresses output to the screen. All output will still be in the log file. .PARAMETER Recipients Array of objects to operate on. This can be mailboxes or any other set of objects. Input must be an array! Anything comming in from the array can be accessed in the script block using $input.property .PARAMETER ResetSeconds How many seconds to run the script block before we rebuild the session with O365. .PARAMETER ScriptBlock The script that you want to robustly execute against the array of objects. The Recipient objects will be provided to the cmdlets in the script block and can be accessed with $input as if you were pipelining the object. .LINK http://EHLO.Link .OUTPUTS Creates the log file specified in -logfile. Logfile contains a record of all actions taken by the script. .EXAMPLE invoke-command -scriptblock {Get-mailbox -resultsize unlimited | select-object -property Displayname,PrimarySMTPAddress,Identity} -session (get-pssession) | export-csv c:\temp\mbx.csv $mbx = import-csv c:\temp\mbx.csv $cred = get-Credential .\Start-RobustCloudCommand.ps1 -Agree -Credential $cred -recipients $mbx -logfile C:\temp\out.log -ScriptBlock {Set-Clutter -identity $input.PrimarySMTPAddress.tostring() -enable:$false} Gets all mailboxes from the service returning only Displayname,Identity, and PrimarySMTPAddress. Exports the results to a CSV Imports the CSV into a variable Gets your O365 Credential Executes the script setting clutter to off .EXAMPLE invoke-command -scriptblock {Get-mailbox -resultsize unlimited | select-object -property Displayname,PrimarySMTPAddress,Identity} -session (get-pssession) | export-csv c:\temp\recipients.csv $recipients = import-csv c:\temp\recipients.csv $cred = Get-Credential .\Start-RobustCloudCommand.ps1 -Agree -Credential $cred -recipients $recipients -logfile C:\temp\out.log -ScriptBlock {Get-MobileDeviceStatistics -mailbox $input.PrimarySMTPAddress.tostring() | Select-Object -Property @{Name = "PrimarySMTPAddress";Expression={$input.PrimarySMTPAddress.tostring()}},DeviceType,LastSuccessSync,FirstSyncTime | Export-Csv c:\temp\stats.csv -Append } Gets All Recipients and exports them to a CSV (for restartability) Imports the CSV into a variable Gets your O365 Credentials Executs the script to gather EAS Device statistics and output them to a csv file #> |