Public/Submit-LMDataModelConcurrent.ps1

<#
.SYNOPSIS
Submits a data model for ingest by PushMetrics, can also be configured to submit models in parallel.

.DESCRIPTION
Uses models generated by Build-LMDataModel to submit to PushMetrics for ingestion.

.PARAMETER ModelObject
Existing model already converted from JSON or directly from output of Build-LMDataModel.

.PARAMETER DatasourceSuffix
The suffix appended to all created PushMetrics DSes, defaults to _PMv1 if not set.

.PARAMETER ForceGraphProvisioning
Will force and attempt to provision datasource graphs regarless if they already exist or not

.PARAMETER ConcurrencyLimit
Number of models to process in parallel, defaults to 1. Running too many concurrently can result in 429 errors.

.PARAMETER BearerToken
Logic Monitor bearer token for connecting to the targeted portal

.PARAMETER AccountName
LogicMontior Portal to submit models to

.EXAMPLE
Submit-LMDataModelConcurrent -ModelObject $Model -DatasourceSuffix "_PMv1" -ForceGraphProvisioning -BearerToken XXXXXXX -AccountName portal_name

.INPUTS
None. You cannot pipe objects to this command.

.LINK
Module repo: https://github.com/stevevillardi/Logic.Monitor.SE

.LINK
PSGallery: https://www.powershellgallery.com/packages/Logic.Monitor.SE
#>

Function Submit-LMDataModelConcurrent{
    [CmdletBinding()]
    Param(
        [Parameter(ValueFromPipeline,Mandatory)]
        [ValidateScript({ 
            If(Test-Json $_ -ErrorAction SilentlyContinue){$TestObject = $_ | ConvertFrom-Json -Depth 10}
            Else{ $TestObject = $_}

            $RequiredProperties= @("Datasources","Properties","DisplayName","HostName","SimulationType")
            $Members= Get-Member -InputObject $TestObject -MemberType NoteProperty
            If($Members){
                $MissingProperties= Compare-Object -ReferenceObject $Members.Name -DifferenceObject $RequiredProperties -PassThru | Where-Object {$_.SideIndicator -eq "=>"}
            }
            #Missing expected schema properties, dont continue
            If (!$MissingProperties){$True}
            Else{Throw [System.Management.Automation.ValidationMetadataException] "Missing schema properties: $($missingProperties -Join ",")"}
        })]
        $ModelObject,

        [String]$DatasourceSuffix = "_PMv1",

        [Switch]$ForceGraphProvisioning,

        [Int]$ConcurrencyLimit = 1,

        [Parameter(Mandatory)]
        [String]$BearerToken,

        [Parameter(Mandatory)]
        [String]$AccountName
    )
    Begin{}
    Process{
        #Loop through models and submit for ingest
        $ModelCount = ($ModelObject.Datasources | Measure-Object).Count

        Write-Host "=========================================================================" -ForegroundColor White
        Write-Host "| BEGIN MULTI-THREAD($ConcurrencyLimit) PROCESSING ($($ModelObject.DisplayName)) |" -ForegroundColor White
        Write-Host "=========================================================================" -ForegroundColor White
        Write-Host "Model contains $ModelCount datasource(s) for ingest, beinging processing."
        $ModelObject.Datasources | ForEach-Object -Parallel {
            Function New-LMSimulatedDataValue {
                [CmdletBinding()]
                Param(
                    $Datapoint,
                    $SimulationType,
                    $SeedValue
                )
            
                #Generate unique seed
                $HostSeedValue = [Math]::Abs($SeedValue.GetHashCode())
                #Set Defaults
                $TotalMin = 1440 #24 hours
                $Interval = 10 #Assumes running on 10 minute intervals
            
                [Int]$TimeSliceMin = Get-Date -Format %m
                [Int]$TimeSliceHour = Get-Date -Format %H
            
                $Value = Switch($SimulationType){
                    "replicaiton" {
                        #TODO
                    }
                    "8to5" {
                        #TODO
                    }
                    default {
                        Switch($Datapoint.MetricType){
                            "Rate" {
                                $MinValue = 0
                                $MaxValue = 125000
            
                                $Fuzz = Get-SecureRandom -Minimum -10 -Maximum 10
                                $TimeSlicePercent = ($TimeSliceHour / 24 + $TimeSliceMin / (60 * 24))
            
                                If($TimeSlicePercent -le .50){
                                    [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $TimeSlicePercent)) + $Fuzz)
                                }
                                Else{
                                    [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $(1 - $TimeSlicePercent))) + $Fuzz)
                                }
                            }
                            "Percentage" {
                                $MinValue = 0
                                $MaxValue = 100
            
                                $Fuzz = Get-SecureRandom -Minimum -10 -Maximum 10
                                $TimeSlicePercent = ($TimeSliceHour / 24 + $TimeSliceMin / (60 * 24))
            
                                If($TimeSlicePercent -le .50){
                                    $ValuePercent = [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $TimeSlicePercent)) + $Fuzz)
                                }
                                Else{
                                    $ValuePercent = [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $(1 - $TimeSlicePercent))) + $Fuzz)
                                }
                                If($ValuePercent -gt 100){$ValuePercent = 100}
            
                                $ValuePercent
                            }
                            "IO-Latency" {
                                $MinValue = 0
                                $MaxValue = 125000
            
                                $Fuzz = Get-SecureRandom -Minimum -10 -Maximum 10
                                $TimeSlicePercent = ($TimeSliceHour / 24 + $TimeSliceMin / (60 * 24))
            
                                If($TimeSlicePercent -le .50){
                                    [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $TimeSlicePercent)) + $Fuzz)
                                }
                                Else{
                                   [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $(1 - $TimeSlicePercent))) + $Fuzz)
                                }
                            }
                            "SpaceUsage" {
                                $GrowthFactor = Get-SecureRandom -Minimum 1.0 -Maximum 1.25
                                If(!$Datapoint.MinValue){$MinValue = 0}Else{$MinValue = $Datapoint.MinValue}
                                If(!$Datapoint.MaxValue){$MaxValue = 3221225472}Else{$MinValue = $Datapoint.MaxValue}
            
                                $TimeSlicePercent = ($TimeSliceHour / 24 + $TimeSliceMin / (60 * 24))
            
                                [Math]::Abs([Math]::Floor(($(Get-Random -Minimum $MinValue -Maximum $MaxValue -SetSeed $HostSeedValue) * $TimeSlicePercent)) * $GrowthFactor)
                            }
                            "Status" {
                                If($Datapoint.MinValue -and $Datapoint.MaxValue){
                                    Get-SecureRandom -Minimum $Datapoint.MinValue -Maximum $Datapoint.MaxValue
                                }
                                Else{
                                    Get-SecureRandom -Minimum 0 -Maximum 5
                                }
                            }
                            Default {
                                Get-SecureRandom -Minimum 0 -Maximum 1000
                            }
                        }
                    }
                }
                Return $Value
            }
            
            Function Generate-LMData {
                [CmdletBinding()]
                Param(
                    $Datapoint,
                    $Instance,
                    $SimulationType,
                    $SeedValue
                )
                #If we have instance data from our model, use that instead
                If($Instance.Data){
                    $FilteredData = $Instance.Data | Where-Object {$_."$($Datapoint.Name)" -ne "No Data"}
                    If($FilteredData){
            
                        $TotalDPs = ($FilteredData | Measure-Object).Count - 1
                        $Variance = 5 #Introduce some variation into slected index so we dont have as many duplicate polls when the sample size is smaller than 100
                        [Int]$TimeSlice = Get-Date -Format %Hmm 
                        $TimePercentage = $TimeSlice/2359
                        $IndexValue = [Math]::Floor($(Get-Random -Minimum $([decimal]($TotalDPs * $TimePercentage) - $Variance) -Maximum $([decimal]($TotalDPs * $TimePercentage) + $Variance)))
                        If($IndexValue -ge $TotalDPs){$IndexValue = -1} #If we go out of index, set to last item
                        If($IndexValue -lt 0){$IndexValue = -0} #If we go our of index set to first item
                        $Value = $FilteredData[$IndexValue]."$($Datapoint.Name)"
            
                        Write-Debug "Generated value of ($Value) for datapoint ($($Instance.Name)-$($Datapoint.Name)) using data provided with the model."
                    }
                    Else{
                        $Value = New-LMSimulatedDataValue -Datapoint $Datapoint -SimulationType $SimulationType -SeedValue $SeedValue
                        Write-Debug "No instance data found for datapoint ($($Instance.Name)-$($Datapoint.Name)) using generated value of $($Datapoint.MetricType):($Value) as fallback."
                    }
            
                }
                Else{
                    $Value = New-LMSimulatedDataValue -Datapoint $Datapoint -SimulationType $SimulationType -SeedValue $SeedValue
                    Write-Debug "Generated value of ($Value) for datapoint ($($Instance.Name)-$($Datapoint.Name)) using metric type ($($Datapoint.MetricType)) and model simulation type ($SimulationType)."
                }
                Return $Value
            }

            #Manually load modules since running as a job they are not automatically loaded
            Import-Module Microsoft.PowerShell.SecretStore -ErrorAction SilentlyContinue
            Import-Module Microsoft.PowerShell.SecretManagement -ErrorAction SilentlyContinue
            Import-Module Logic.Monitor -ErrorAction SilentlyContinue
            Import-Module Logic.Monitor.SE -ErrorAction SilentlyContinue

            #Dev module import for testing, not needed for production
            Import-Module /Users/steven.villardi/Documents/GitHub/Logic.Monitor/Dev.Logic.Monitor.psd1 -Force -ErrorAction SilentlyContinue
            Import-Module /Users/steven.villardi/Documents/GitHub/Logic.Monitor.SE/Dev.Logic.Monitor.SE.psd1 -Force -ErrorAction SilentlyContinue

            #Connect to portal
            Connect-LMAccount -BearerToken $using:BearerToken -AccountName $using:AccountName -SkipVersionCheck -DisableConsoleLogging -SkipCredValidation

            $StatusMessage = $null

            #Check if we are logged in and have valid api creds
            If ($(Get-LMAccountStatus).Type -ne "Bearer") {
                Write-Error "Push Metrics API only supports Bearer Token auth, please re-connect using a valid bearer token."
                return
            }

            $InstCount = ($_.Instances | Measure-Object).Count
            $DpCount = ($_.Datapoints | Measure-Object).Count
            $GCount = ($_.Graphs | Measure-Object).Count
            $OGCount = ($_.OverviewGraphs | Measure-Object).Count
            $StatusMessage += "`n" + "Model loaded for datasource $($_.Defenition.Name) using device $($Using:ModelObject.DisplayName) and simulation type $($Using:ModelObject.SimulationType)."
            $StatusMessage += "`n" +  "Model contains $InstCount instance(s), each with $DpCount datapoint(s) and $($GCount + $OGCount) graph definition(s)."

            #Loop through instances and generate instance and dp objects
            $InstanceArray = [System.Collections.Generic.List[object]]::New()
            Foreach($Instance in $_.Instances){
                Write-Debug "Processing datapoints for instance $($Instance.Name)."
                $Datapoints = [System.Collections.Generic.List[object]]::New()
                Foreach($Datapoint in $_.Datapoints){
                    $Value = Generate-LMData -Datapoint $Datapoint -Instance $Instance -SimulationType $Using:ModelObject.SimulationType -SeedValue $Using:ModelObject.HostName
                    $Datapoints.Add([PSCustomObject]@{
                        Name = $Datapoint.Name
                        Description = $Datapoint.Description
                        Value = $Value
                    })
                }
                $DatapointsArray = New-LMPushMetricDataPoint -Datapoints $Datapoints
                If($Instance.Properties){$Instance.Properties.PSObject.Properties | ForEach-Object -begin {$InstancePropertyHash=@{}} -process {$InstancePropertyHash."$($_.Name)" = $_.Value}}
                $InstanceArray.Add($(New-LMPushMetricInstance -Datapoints $DatapointsArray -InstanceName $Instance.Name -InstanceDisplayName $Instance.DisplayName -InstanceDescription $Instance.Description -InstanceProperties $InstancePropertyHash))
            }

            #Submit PushMetric to portal
            $DeviceHostName = $Using:ModelObject.HostName
            $DeviceDisplayName = $Using:ModelObject.DisplayName
            $DatasourceGroup =  $_.DatasourceGroupName
            $DatasourceDisplayName = $_.Defenition.displayName
            $DatasourceName = $_.Defenition.Name.Replace("-","") + $Using:DatasourceSuffix
            $ResourceIds = @{"system.hostname"=$DeviceHostName;"system.displayname"=$DeviceDisplayName}

            $StatusMessage += "`n" + "Submitting PushMetric to ingest."
            If($Using:ModelObject.Properties){$Using:ModelObject.Properties.PSObject.Properties | ForEach-Object -begin {$DevicePropertyHash=@{}} -process {$DevicePropertyHash."$($_.Name)" = $_.Value}}
            $Result = Send-LMPushMetric -Instances $InstanceArray -DatasourceGroup $DatasourceGroup -DatasourceDisplayName $DatasourceDisplayName -DatasourceName $DatasourceName -ResourceIds $ResourceIds -ResourceProperties $DevicePropertyHash -NewResourceHostName $DeviceHostName

            $StatusMessage += "`n" + "PushMetric submitted with status: $($Result.message) @($($Result.timestamp))"
            #Apply graph definitions if they do not exist yet
            Write-Debug "Checking if datasource $DatasourceName has been created yet"
            $PMDatasource = Get-LMDatasource -Name $DatasourceName
            If($PMDatasource.Id){
                Write-Debug "$DatasourceName found, checking if graph defentions have been created yet."
                $PMGraphs = Get-LMDatasourceGraph -DataSourceId $PMDatasource.Id
                $PMOverviewGraphs = Get-LMDatasourceOverviewGraph -DataSourceId $PMDatasource.Id

                If($PMDatasource.collectInterval -eq 60){
                    Write-Debug "$DatasourceName found with default collection interval, setting to 600 seconds."
                    Set-LMDatasource -Id $PMDatasource.Id -PollingIntervalInSeconds 600 | Out-Null
                }

                If((!$PMGraphs -or $Using:ForceGraphProvisioning) -and $_.Graphs){
                    Write-Debug "No instance graphs found or force creation specified, importing graph definitions from model."
                    Foreach($Graph in $_.Graphs){
                        Write-Debug "Importing instance graph $($Graph.Name)."
                        #Update datapointIDs in each graph so they match the new push module
                        Foreach($Datapoint in $Graph.datapoints){
                            $DPName = $Datapoint.Name
                            $DPIndex = $PMDatasource.datapoints.name.IndexOf($DPName)
                            $DPId = $PMDatasource.datapoints[$DPIndex].id

                            $Index = $Graph.datapoints.name.IndexOf($DPName)
                            If($Index -eq -1){
                                $Graph.datapoints[$Index].dataPointId = $null
                                $Graph.datapoints[$Index].dataSourceDataPointId = $null
                            }
                            Else{
                                $Graph.datapoints[$Index].dataPointId = $DPId
                                $Graph.datapoints[$Index].dataSourceDataPointId = $DPId
                            }
                        }
                        New-LMDatasourceGraph -RawObject $Graph -DatasourceId $PMDatasource.Id | Out-Null
                    }
                }
                Else{
                    Write-Debug "Existing instance graphs found or none included with selected model, skipping importing instance graph definitions."
                }
                If((!$PMOverviewGraphs -or $Using:ForceGraphProvisioning) -and $_.OverviewGraphs){
                    Write-Debug "No overview graphs found or force creation specified, importing graph definitions from model."
                    Foreach($OverviewGraph in $_.OverviewGraphs){
                        Write-Debug "Importing overview graph $($OverviewGraph.Name)."
                        #Update datapointIDs in each graph so they match the new push module
                        Foreach($Datapoint in $OverviewGraph.datapoints){
                            $DPName = $Datapoint.dataPointName
                            $DPIndex = $PMDatasource.datapoints.name.IndexOf($DPName)
                            $DPId = $PMDatasource.datapoints[$DPIndex].id

                            $Index = $OverviewGraph.datapoints.dataPointName.IndexOf($DPName)
                            If($Index -eq -1){
                                $OverviewGraph.datapoints[$Index].dataPointId = $null
                                $OverviewGraph.datapoints[$Index].dataSourceDataPointId = $null
                            }
                            Else{
                                $OverviewGraph.datapoints[$Index].dataPointId = $DPId
                                $OverviewGraph.datapoints[$Index].dataSourceDataPointId = $DPId
                            }
                        }
                        New-LMDatasourceOverviewGraph -RawObject $OverviewGraph -DatasourceId $PMDatasource.Id | Out-Null
                    }
                }
                Else{
                    Write-Debug "Existing overview graphs found or none included with selected model, skipping importing overview graph definitions."
                }
            }
            Else{
                Write-Debug "$DatasourceName not found, will recheck on next submission."
            }
            Write-Host $StatusMessage
        } -ThrottleLimit $ConcurrencyLimit
    }
    End{
        Write-Host "=========================================================================" -ForegroundColor White
        Write-Host "| END PROCESSING ($($ModelObject.DisplayName)) |" -ForegroundColor White
        Write-Host "=========================================================================" -ForegroundColor White
    }
}