src/CheckCreateUpdate-TableDcr-Structure.ps1
|
Function CheckCreateUpdate-TableDcr-Structure { <# .SYNOPSIS Create or Update Azure Data Collection Rule (DCR) used for log ingestion to Azure LogAnalytics using Log Ingestion API (combined) .DESCRIPTION Combined function which will combine 3 functions in one call: Get-AzLogAnalyticsTableAzDataCollectionRuleStatus CreateUpdate-AzLogAnalyticsCustomLogTableDcr CreateUpdate-AzDataCollectionRuleLogIngestCustomLog .PARAMETER Data Data object .PARAMETER Tablename Specifies the table name in LogAnalytics .PARAMETER SchemaSourceObject This is the schema in hash table format coming from the source object .PARAMETER AzLogWorkspaceResourceId This is the Loganaytics Resource Id .PARAMETER DceName This is name of the Data Collection Endpoint to use for the upload Function will automatically look check in a global variable ($global:AzDceDetails) - or do a query using Azure Resource Graph to find DCE with name Goal is to find the log ingestion Uri on the DCE Variable $global:AzDceDetails can be build before calling this cmdlet using this syntax $global:AzDceDetails = Get-AzDceListAll -AzAppId $LogIngestAppId -AzAppSecret $LogIngestAppSecret -TenantId $TenantId -Verbose:$Verbose -Verbose:$Verbose .PARAMETER DcrName This is name of the Data Collection Rule to use for the upload Function will automatically look check in a global variable ($global:AzDcrDetails) - or do a query using Azure Resource Graph to find DCR with name Goal is to find the DCR immunetable id on the DCR Variable $global:AzDcrDetails can be build before calling this cmdlet using this syntax $global:AzDcrDetails = Get-AzDcrListAll -AzAppId $LogIngestAppId -AzAppSecret $LogIngestAppSecret -TenantId $TenantId -Verbose:$Verbose -Verbose:$Verbose .PARAMETER TableName This is tablename of the LogAnalytics table (and is also used in the DCR naming) .PARAMETER AzDcrSetLogIngestApiAppPermissionsDcrLevel Choose TRUE if you want to set Monitoring Publishing Contributor permissions on DCR level Choose FALSE if you would like to use inherited permissions from the resource group level (recommended) .PARAMETER LogIngestServicePricipleObjectId This is the object id of the Azure App service-principal NOTE: Not the object id of the Azure app, but Object Id of the service principal (!) .PARAMETER AzLogDcrTableCreateFromReferenceMachine Array with list of computers, where schema management can be done .PARAMETER AzLogDcrTableCreateFromAnyMachine True means schema changes can be made from any computer - FALSE means it can only happen from reference machine(s) .PARAMETER AzAppId This is the Azure app id .PARAMETER AzAppSecret This is the secret of the Azure app .PARAMETER TenantId This is the Azure AD tenant id .INPUTS None. You cannot pipe objects .OUTPUTS Output of REST PUT command. Should be 200 for success .EXAMPLE #------------------------------------------------------------------------------------------- # Variables #------------------------------------------------------------------------------------------- $TableName = 'InvClientComputerOSInfoTest4V2' # must not contain _CL $DcrName = "dcr-" + $AzDcrPrefixClient + "-" + $TableName + "_CL" $TenantId = "xxxxx" $LogIngestAppId = "xxxxx" $LogIngestAppSecret = "xxxxx" $DceName = "dce-log-platform-management-client-demo1-p" $LogAnalyticsWorkspaceResourceId = "/subscriptions/xxxxxx/resourceGroups/rg-logworkspaces/providers/Microsoft.OperationalInsights/workspaces/log-platform-management-client-demo1-p" $AzDcrPrefixClient = "clt1" $AzDcrSetLogIngestApiAppPermissionsDcrLevel = $false $AzDcrLogIngestServicePrincipalObjectId = "xxxxxx" $AzLogDcrTableCreateFromReferenceMachine = @() $AzLogDcrTableCreateFromAnyMachine = $true # building global variable with all DCEs, which can be viewed by Log Ingestion app $global:AzDceDetails = Get-AzDceListAll -AzAppId $LogIngestAppId -AzAppSecret $LogIngestAppSecret -TenantId $TenantId -Verbose:$Verbose # building global variable with all DCRs, which can be viewed by Log Ingestion app $global:AzDcrDetails = Get-AzDcrListAll -AzAppId $LogIngestAppId -AzAppSecret $LogIngestAppSecret -TenantId $TenantId -Verbose:$Verbose #------------------------------------------------------------------------------------------- # Collecting data (in) #------------------------------------------------------------------------------------------- Write-Output "" Write-Output "Collecting OS information" $DataVariable = Get-CimInstance -ClassName Win32_OperatingSystem #------------------------------------------------------------------------------------------- # Preparing data structure #------------------------------------------------------------------------------------------- # convert CIM array to PSCustomObject and remove CIM class information $DataVariable = Convert-CimArrayToObjectFixStructure -data $DataVariable # add CollectionTime to existing array $DataVariable = Add-CollectionTimeToAllEntriesInArray -Data $DataVariable # add Computer & UserLoggedOn info to existing array $DataVariable = Add-ColumnDataToAllEntriesInArray -Data $DataVariable -Column1Name Computer -Column1Data $Env:ComputerName -Column2Name UserLoggedOn -Column2Data $UserLoggedOn # Validating/fixing schema data structure of source data $DataVariable = ValidateFix-AzLogAnalyticsTableSchemaColumnNames -Data $DataVariable # Aligning data structure with schema (requirement for DCR) $DataVariable = Build-DataArrayToAlignWithSchema -Data $DataVariable #------------------------------------------------------------------------------------------- # Create/Update Schema for LogAnalytics Table & Data Collection Rule schema #------------------------------------------------------------------------------------------- CheckCreateUpdate-TableDcr-Structure -AzLogWorkspaceResourceId $LogAnalyticsWorkspaceResourceId ` -AzAppId $LogIngestAppId -AzAppSecret $LogIngestAppSecret -TenantId $TenantId ` -DceName $DceName -DcrName $DcrName -TableName $TableName -Data $DataVariable ` -LogIngestServicePricipleObjectId $AzDcrLogIngestServicePrincipalObjectId ` -AzDcrSetLogIngestApiAppPermissionsDcrLevel $AzDcrSetLogIngestApiAppPermissionsDcrLevel ` -AzLogDcrTableCreateFromAnyMachine $AzLogDcrTableCreateFromAnyMachine ` -AzLogDcrTableCreateFromReferenceMachine $AzLogDcrTableCreateFromReferenceMachine #------------------------------------------------------------------------------------------- # Output #------------------------------------------------------------------------------------------- Collecting OS information VERBOSE: Checking LogAnalytics table and Data Collection Rule configuration .... Please Wait ! VERBOSE: POST with -1-byte payload VERBOSE: received 1468-byte response of content type application/json; charset=utf-8 VERBOSE: GET with 0-byte payload VERBOSE: LogAnalytics table wasn't found ! VERBOSE: DCR was not found [ dcr-clt1-InvClientComputerOSInfoTest4V2_CL ] VERBOSE: POST with -1-byte payload VERBOSE: received 1468-byte response of content type application/json; charset=utf-8 VERBOSE: VERBOSE: Trying to update existing LogAnalytics table schema for table [ InvClientComputerOSInfoTest4V2_CL ] in VERBOSE: /subscriptions/fce4f282-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-logworkspaces/providers/Microsoft.OperationalInsights/works paces/log-platform-management-client-demo1-p VERBOSE: PATCH with -1-byte payload VERBOSE: PUT with -1-byte payload VERBOSE: received 7764-byte response of content type application/json; charset=utf-8 VERBOSE: VERBOSE: LogAnalytics Table doesn't exist or problems detected .... creating table [ InvClientComputerOSInfoTest4V2_CL ] in VERBOSE: /subscriptions/fce4f282-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-logworkspaces/providers/Microsoft.OperationalInsights/works paces/log-platform-management-client-demo1-p VERBOSE: PUT with -1-byte payload VERBOSE: received 7764-byte response of content type application/json; charset=utf-8 StatusCode : 200 StatusDescription : OK Content : {"properties":{"totalRetentionInDays":30,"archiveRetentionInDays":0,"plan":"Analytics","retentionInDaysAsDefault":tru e,"totalRetentionInDaysAsDefault":true,"schema":{"tableSubType":"DataCollectionRule... RawContent : HTTP/1.1 200 OK Pragma: no-cache Request-Context: appId=cid-v1:c7ec48f5-2684-46e8-accb-45e7dbec242b X-Content-Type-Options: nosniff api-supported-versions: 2015-03-20, 2015-11-01-preview, 2017-01-... Forms : {} Headers : {[Pragma, no-cache], [Request-Context, appId=cid-v1:c7ec48f5-2684-46e8-accb-45e7dbec242b], [X-Content-Type-Options, n osniff], [api-supported-versions, 2015-03-20, 2015-11-01-preview, 2017-01-01-preview, 2017-03-03-preview, 2017-03-15- preview, 2017-04-26-preview, 2020-03-01-preview, 2020-08-01, 2020-10-01, 2021-03-01-privatepreview, 2021-07-01-privat epreview, 2021-12-01-preview, 2022-09-01-privatepreview, 2022-10-01]...} Images : {} InputFields : {} Links : {} ParsedHtml : mshtml.HTMLDocumentClass RawContentLength : 7764 VERBOSE: POST with -1-byte payload VERBOSE: received 1468-byte response of content type application/json; charset=utf-8 VERBOSE: POST with -1-byte payload VERBOSE: received 1342-byte response of content type application/json; charset=utf-8 VERBOSE: Found required DCE info using Azure Resource Graph VERBOSE: VERBOSE: GET with 0-byte payload VERBOSE: received 898-byte response of content type application/json; charset=utf-8 VERBOSE: Found required LogAnalytics info VERBOSE: VERBOSE: GET with 0-byte payload VERBOSE: received 291-byte response of content type application/json; charset=utf-8 VERBOSE: VERBOSE: Creating/updating DCR [ dcr-clt1-InvClientComputerOSInfoTest4V2_CL ] with limited payload VERBOSE: /subscriptions/fce4f282-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-dcr-log-platform-management-client-demo1-p/providers/micros oft.insights/dataCollectionRules/dcr-clt1-InvClientComputerOSInfoTest4V2_CL VERBOSE: PUT with -1-byte payload VERBOSE: received 2094-byte response of content type application/json; charset=utf-8 StatusCode : 200 StatusDescription : OK Content : {"properties":{"immutableId":"dcr-3433400ee8ca4570b606a9a21f2eea79","dataCollectionEndpointId":"/subscriptions/fce4f2 82-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-dce-log-platform-management-client... RawContent : HTTP/1.1 200 OK Pragma: no-cache Vary: Accept-Encoding x-ms-ratelimit-remaining-subscription-resource-requests: 149 Request-Context: appId=cid-v1:2bbfbac8-e1b0-44af-b9c6-3a40669d37e3 x-ms-correla... Forms : {} Headers : {[Pragma, no-cache], [Vary, Accept-Encoding], [x-ms-ratelimit-remaining-subscription-resource-requests, 149], [Reques t-Context, appId=cid-v1:2bbfbac8-e1b0-44af-b9c6-3a40669d37e3]...} Images : {} InputFields : {} Links : {} ParsedHtml : mshtml.HTMLDocumentClass RawContentLength : 2094 VERBOSE: VERBOSE: Updating DCR [ dcr-clt1-InvClientComputerOSInfoTest4V2_CL ] with full schema VERBOSE: /subscriptions/fce4f282-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-dcr-log-platform-management-client-demo1-p/providers/micros oft.insights/dataCollectionRules/dcr-clt1-InvClientComputerOSInfoTest4V2_CL VERBOSE: PUT with -1-byte payload VERBOSE: received 4546-byte response of content type application/json; charset=utf-8 StatusCode : 200 StatusDescription : OK Content : {"properties":{"immutableId":"dcr-3433400ee8ca4570b606a9a21f2eea79","dataCollectionEndpointId":"/subscriptions/fce4f2 82-fcc6-43fb-94d8-bf1701b862c3/resourceGroups/rg-dce-log-platform-management-client... RawContent : HTTP/1.1 200 OK Pragma: no-cache Vary: Accept-Encoding x-ms-ratelimit-remaining-subscription-resource-requests: 148 Request-Context: appId=cid-v1:2bbfbac8-e1b0-44af-b9c6-3a40669d37e3 x-ms-correla... Forms : {} Headers : {[Pragma, no-cache], [Vary, Accept-Encoding], [x-ms-ratelimit-remaining-subscription-resource-requests, 148], [Reques t-Context, appId=cid-v1:2bbfbac8-e1b0-44af-b9c6-3a40669d37e3]...} Images : {} InputFields : {} Links : {} ParsedHtml : mshtml.HTMLDocumentClass RawContentLength : 4546 VERBOSE: VERBOSE: Waiting 10 sec to let Azure sync up so DCR rule can be retrieved from Azure Resource Graph VERBOSE: VERBOSE: Getting Data Collection Rules from Azure Resource Graph .... Please Wait ! VERBOSE: POST with -1-byte payload VERBOSE: received 1468-byte response of content type application/json; charset=utf-8 VERBOSE: POST with -1-byte payload VERBOSE: received 104224-byte response of content type application/json; charset=utf-8 #> [CmdletBinding()] param( [Parameter(mandatory)] [Array]$Data, [Parameter(mandatory)] [string]$AzLogWorkspaceResourceId, [Parameter(mandatory)] [string]$TableName, [Parameter(mandatory)] [string]$DcrName, [Parameter(mandatory)] [string]$DceName, [Parameter(mandatory)] [string]$LogIngestServicePricipleObjectId, [Parameter(mandatory)] [boolean]$AzDcrSetLogIngestApiAppPermissionsDcrLevel, [Parameter(mandatory)] [boolean]$AzLogDcrTableCreateFromAnyMachine, [Parameter(mandatory)] [AllowEmptyCollection()] [array]$AzLogDcrTableCreateFromReferenceMachine, [Parameter()] [string]$AzAppId, [Parameter()] [string]$AzAppSecret, [Parameter()] [string]$TenantId ) #------------------------------------------------------------------------------------------- # Create/Update Schema for LogAnalytics Table & Data Collection Rule schema #------------------------------------------------------------------------------------------- If ( ($AzAppId) -and ($AzAppSecret) ) { #----------------------------------------------------------------------------------------------- # Check if table and DCR exist - or schema must be updated due to source object schema changes #----------------------------------------------------------------------------------------------- # Get insight about the schema structure $Schema = Get-ObjectSchemaAsArray -Data $Data $StructureCheck = Get-AzLogAnalyticsTableAzDataCollectionRuleStatus -AzLogWorkspaceResourceId $AzLogWorkspaceResourceId -TableName $TableName -DcrName $DcrName -SchemaSourceObject $Schema ` -AzAppId $AzAppId -AzAppSecret $AzAppSecret -TenantId $TenantId -Verbose:$Verbose #----------------------------------------------------------------------------------------------- # Structure check = $true -> Create/update table & DCR with necessary schema #----------------------------------------------------------------------------------------------- If ($StructureCheck -eq $true) { If ( ( $env:COMPUTERNAME -in $AzLogDcrTableCreateFromReferenceMachine) -or ($AzLogDcrTableCreateFromAnyMachine -eq $true) ) # manage table creations { # build schema to be used for LogAnalytics Table $Schema = Get-ObjectSchemaAsHash -Data $Data -ReturnType Table -Verbose:$Verbose CreateUpdate-AzLogAnalyticsCustomLogTableDcr -AzLogWorkspaceResourceId $AzLogWorkspaceResourceId -SchemaSourceObject $Schema -TableName $TableName ` -AzAppId $AzAppId -AzAppSecret $AzAppSecret -TenantId $TenantId -Verbose:$Verbose # build schema to be used for DCR $Schema = Get-ObjectSchemaAsHash -Data $Data -ReturnType DCR CreateUpdate-AzDataCollectionRuleLogIngestCustomLog -AzLogWorkspaceResourceId $AzLogWorkspaceResourceId -SchemaSourceObject $Schema ` -DceName $DceName -DcrName $DcrName -TableName $TableName ` -LogIngestServicePricipleObjectId $LogIngestServicePricipleObjectId ` -AzDcrSetLogIngestApiAppPermissionsDcrLevel $AzDcrSetLogIngestApiAppPermissionsDcrLevel ` -AzAppId $AzAppId -AzAppSecret $AzAppSecret -TenantId $TenantId -Verbose:$Verbose } } } # create table/DCR } |