Public/Build-LMDataModel.ps1
|
#Requires -Version 7 Function Build-LMDataModel { [CmdletBinding(DefaultParameterSetName="ModelDatasources")] [OutputType([System.Collections.Generic.List[object]])] Param( [Parameter(Mandatory,ParameterSetName="ModelDatasources")] [System.Collections.Generic.List[object]]$DatasourceNames, [Switch]$ReplicateModuleGraphs, [Switch]$IncludeAlertThresholds, [Parameter(ParameterSetName = 'GenerateInstances')] [Parameter(Mandatory,ParameterSetName="ModelDatasources")] [Switch]$GenerateInstances, [Parameter(ParameterSetName = 'GenerateInstances')] [Parameter(Mandatory,ParameterSetName="ModelDatasources")] [Int]$InstanceCount = 1, [ValidateSet("8to5","random","replication","replay_model")] [String]$SimulationType="random", [Parameter(Mandatory)] [String]$DeviceHostName, [String]$DeviceDisplayName = $DeviceHostName, [Parameter(Mandatory,ParameterSetName = 'ModelDevice')] [String]$ModelDeviceHostName, [Parameter(ParameterSetName = 'ModelDevice')] [Switch]$IncludeModelDeviceProperties, [Parameter(ParameterSetName = 'ModelDevice')] [String]$ModelDeviceInstanceCount = 24, [Parameter(ParameterSetName = 'ModelDevice')] [Switch]$IncludeModelDeviceData ) #If we have a model deivce make sure we can grab it from the connected portal fist If($ModelDeviceHostName){ $DatasourceNames = [System.Collections.Generic.List[object]]::New() $ModelDevice = Get-LMDevice -Name $ModelDeviceHostName | Select-Object -First 1 If($ModelDevice){ Write-Debug "Located model device, using device ($ModelDeviceHostName) as model candidate.(ID: $($ModelDevice.id))" $ModelDeviceDatasources = Get-LMDeviceDatasourceList -id $ModelDevice.Id | Where-Object {$_.instanceNumber -gt 0} $DatasourceNames.AddRange($ModelDeviceDatasources.datasourceName) } Else{ Write-Error "Unable to locate provided model device ($ModelDeviceHostName), ensure hostname is correct and try again." Return } } #If set to collect model data override simulation type to replay model If($IncludeModelDeviceData){$SimulationType = "replay_model"} $DataSourceModels = [System.Collections.Generic.List[object]]::New() #Loop through each provided DSName to generate models Foreach($DatasourceName in $DatasourceNames){ #Grab specified source DS for DatasourceName so we can model it for PushMetrics Write-Debug "Using provided datasource name ($DatasourceName) as model candidate." $DatasourceInfo = Get-LMDatasource -Name $DatasourceName $Datasource = $DatasourceInfo | Select-Object description,tags,name,displayName,collectInterval,hasMultiInstances,dataPoints #Do we have a valid DS to model If($Datasource){ #Extract what we need from the source datasource to use as basis for modeling $DatasourceDefenition = $Datasource | Select-Object description,tags,name,displayName,collectInterval,hasMultiInstances $Datapoints = $Datasource.datapoints $DatasourceGroupName = $Datasource.group #Estimate datapoint type for model generation $DatapointDefenition = [System.Collections.Generic.List[object]]::New() Foreach ($Datapoint in $Datapoints){ $MetricType = Resolve-MetricType -Datapoint $Datapoint.name If($Datapoint.postProcessorMethod -eq "expression"){ $Type = "complex" $ComplexDatapointFormula = $Datapoint.postProcessorParam } Else{ $Type = "standard" $ComplexDatapointFormula = "N/A" } Write-Debug "Detected $Type datapoint $($Datapoint.name) matching metric type: $MetricType" $DatapointDefenition.Add([PSCustomObject]@{ Name = $Datapoint.name MaxValue = $Datapoint.maxValue MinValue = $Datapoint.minValue Description = $Datapoint.description Type = $Type ComplexDatapointFormula = $ComplexDatapointFormula AlertForNoData = If($IncludeAlertThresholds){[Boolean]!$Datapoint.alertForNoData} AlertExpr = If($IncludeAlertThresholds){$Datapoint.alertExpr} AlertSubject = If($IncludeAlertThresholds){$Datapoint.alertSubject} AlertBody = If($IncludeAlertThresholds){$Datapoint.alertBody} MetricType = $MetricType }) } #Get Graph settings we will need to model our PushMetric DS after If($ReplicateModuleGraphs){ Write-Debug "Exporting existing datasource overview graph defenition(s) for model export." $DatasourceGraphModel = Get-LMDatasourceGraph -DatasourceName $Datasource.Name | Select-Object -ExcludeProperty id Write-Debug "Exporting existing datasource instance graph defenition(s) for model export." $DatasourceOverviewGraphModel = Get-LMDatasourceOverviewGraph -DatasourceName $Datasource.Name | Select-Object -ExcludeProperty id } #Generate Instances if not specified $Instances = [System.Collections.Generic.List[object]]::New() If($GenerateInstances -and [Boolean]$Datasource.hasMultiInstances){ Write-Debug "Generating $InstanceCount instance(s) for data model export." $Instances.AddRange($(Build-Instance -InstanceCount $InstanceCount -Datasource $DatasourceDefenition)) } ElseIf($GenerateInstances){ Write-Debug "Datasource $($Datasource.name) not enabled for multiple instances, default instance creation to single instance for data model export." $Instances.Add($(Build-Instance -SingleInstance -Datasource $DatasourceDefenition)) } ElseIf($ModelDeviceHostName -and $ModelDeviceDatasources){ $ModelInstances = Get-LMDeviceDatasourceInstance -DatasourceId $DatasourceInfo.Id -DeviceId $ModelDevice.Id Write-Debug "Datasource $($Datasource.name) is being model after device $($ModelDevice.DisplayName) , adding up to $ModelDeviceInstanceCount instances from pool of $(($ModelInstances | Measure-Object).Count) existing instance(s) to data model export." If($($ModelInstances | Measure-Object).Count -gt 1){ $Instances.AddRange($(Build-Instance -ModelInstance -InstanceList $($ModelInstances | Sort-Object -Property Id | Select-Object -First $ModelDeviceInstanceCount) -Datasource $DatasourceDefenition -IncludeModelData:$IncludeModelDeviceData -ModelDeviceHostName $ModelDeviceHostName)) } Else{ $Instances.Add($(Build-Instance -ModelInstance -InstanceList $ModelInstances -Datasource $DatasourceDefenition -IncludeModelData:$IncludeModelDeviceData -ModelDeviceHostName $ModelDeviceHostName)) } } #Combine our model info together for export $DataSourceModel = [PSCustomObject]@{ DatasourceName = $DatasourceName DatasourceGroupName = $DatasourceGroupName Instances = $Instances Defenition = $DatasourceDefenition Datapoints = $DatapointDefenition OverviewGraphs = $DatasourceOverviewGraphModel Graphs = $DatasourceGraphModel } } Write-Debug "Exporting data model ($DatasourceName)." $DataSourceModels.Add($DataSourceModel) } $PropertiesHash = @{} If($IncludeModelDeviceProperties){ $Properties = $($ModelDevice.autoProperties + $ModelDevice.customProperties) | Where-Object {$_.name -notmatch ".pass|.community|.key|.cert|.secret|.user|system.|.id"} Foreach ($Prop in $Properties){ $Prop.name = "autodiscovery." + $Prop.name.replace("auto.","") $PropertiesHash.Add($Prop.name,$Prop.value) } } $DeviceModel = [PSCustomObject]@{ HostName = $DeviceHostName DisplayName = $DeviceDisplayName Properties = $PropertiesHash SimulationType = $SimulationType Datasources = $DataSourceModels } Return $DeviceModel } Function Resolve-MetricType { Param( [Parameter(Mandatory)] [Object]$Datapoint ) Switch -Regex ($Datapoint){ '[Pp]kts|[Bb]ps|[Mm]bps|[Rr]ate|[Tt]ime|[Dd]uration' {Return "Rate"} '[Pp]ercent|[Uu]til' {Return "Percentage"} '[Ww]rite|[Rr]ead|[Ii]ops' {Return "IO-Latency"} '[Ff]ree|[Uu]sed|[Rr]eserved|[Aa]vailable' {Return "SpaceUsage"} '[Ss]tatus|[Ss]tate|[Ee]nabled' {Return "Status"} default {Return "Count"} } } Function Build-Instance { Param( [Switch]$SingleInstance, [Switch]$ModelInstance, [System.Collections.Generic.List[object]]$InstanceList, [Int]$InstanceCount, [Boolean]$IncludeModelData, [String]$ModelDeviceHostName, $Datasource ) $Instances = [System.Collections.Generic.List[object]]::New() #If single instance just return datasource name as instance If($SingleInstance){ Write-Debug "Single instance datasource detected, skipping instance generation usings datasource name as instance" $Instances.Add([PSCustomObject]@{ Name = ($Datasource.name -replace '[#\\/();=]', '_') DisplayName = ($Datasource.displayName -replace '[#\\/;=]', '_') Description = "" Properties = @{} Type = "SingleInstance" Data = @() }) Return $Instances } If($ModelInstance){ #Add generated instance to instance list Write-Debug "Instances found for export: ($(($InstanceList[0..9].Name -Join ","))...)" Foreach($Instance in $InstanceList){ $Data = [System.Collections.Generic.List[object]]::New() If($IncludeModelData){ Write-Debug "Extracting last 24 hours of model device data for instance ($($Instance.name))" $InstanceData = Get-LMDeviceData -DeviceName $ModelDeviceHostName -DatasourceName $Datasource.name -InstanceName $Instance.name -StartDate (Get-Date).AddHours(-24) -EndDate (Get-Date) If($InstanceData){ $DataCount = ($InstanceData | Measure-Object).Count Write-Debug "Added $DataCount time series metrics to data model for ($($Instance.name))" $Data.AddRange($InstanceData) } Else{ Write-Debug "No recent instance data found for ($($Instance.name))" } } If($IncludeModelDeviceProperties){ $PropertiesHash = @{} $Properties = $($Instance.customProperties + $Instance.autoProperties) | Where-Object {$_.name -notmatch ".pass|.community|.key|.cert|.secret|.user|system.|.id"} Foreach ($Prop in $Properties){ $Prop.name = "autodiscovery." + $Prop.name.replace("auto.","") $PropertiesHash.Add($Prop.name,$Prop.value) } } $Instances.Add([PSCustomObject]@{ Name = (($Instance.name -replace '[#\\/;=]', '_') -replace '[\[\]{}]','') DisplayName = If($Instance.displayName){(($Instance.displayName -replace '[#\\;=]', '_') -replace '[\[\]{}]','')}Else{(($Datasource.displayName -replace '[#\\;=]', '_'))} Description = $Instance.description Properties = $PropertiesHash Type = "DeviceModeled" Data = $Data }) } Return $Instances } Else{ If($Datasource.tags){ $Type = Switch -Regex ($Datasource.tags){ 'inode|disk|drive' {"Disk"} 'filesystem|file' {"FileSystem"} 'core|cpu|processor' {"CPU"} 'vlan' {"VLAN"} 'chassis|array|cluster|node' {"Node"} 'pdu|ups|battery' {"Battery"} 'psu|power' {"PowerSupply"} 'temperature|voltage' {"Sensor"} 'interface|port' {"Interface"} 'ec2|compute engine|vm' {"Server"} default {"Instance"} } } Else{ $Type = "Unknown" } Write-Debug "Datasource instance type detected as $Type, using type for instance generation" #For multiple instances loop through and generate insance types For (($i = 1); $i -le $InstanceCount; $i++){ #Generate instance name based on tags and dp names $Name = $Type + '{0:d2}' -f $i $DisplayName = $Type + '{0:d2}' -f $i $Description = "" #Add generated instance to instance list $Instances.Add([PSCustomObject]@{ Name = ($Name -replace '[#\\;=]', '_') DisplayName = ($DisplayName -replace '[#\\;=]', '_') Description = $Description Properties = @{} Type = $Type Data = @() }) } } Return $Instances } |