Framework/Core/STMapping/AzSKADOServiceMapping.ps1
|
using namespace System.Management.Automation Set-StrictMode -Version Latest class AzSKADOServiceMapping: CommandBase { [string] $OrgName [string] $ProjectName [string] $ProjectId [string] $BuildMappingsFilePath [string] $ReleaseMappingsFilePath [string] $RepositoryMappingsFilePath [string] $MappingType [string] $OutputFolderPath [string] $Auto = $false [switch] $UseCache #switch to read mapping details from cache [string] $StorageAccount; # Storage account name [string] $StorageRG;# Storage resource group name [string] $Container;# Storage Container to store ST mapping files [object] $StorageAccountCtx; # Power BI Report Storage settings to store ST mapping files [string] $ReportStorageAccount;# Storage account name for Dashboard [string] $ReportStorageRG;# Storage resource group name for Dashboard [string] $ReportContainer;#Storage Container to store ST mapping files use by Power Bi resports [object] $ReportStorageAccountCtx; [object] $Stopwatch;#Create a Stopwatch [string] $AzSKTempStatePath = [Constants]::AzSKTempFolderPath [ServiceMappingCacheHelper] $ServiceMappingCacheHelperObj; [int] $MappingExpirationLImit #Service id mapping expiration duration [bool] $resourceInCacheWithoutPipeline =$false #resource present in cache searched without pipeline reference $BuildSTDetails = @(); $ReleaseSTDetails =@(); $RepositorySTDetails =@(); $storageCachedData = @();#inmemory cached mapping data $lastDuration =0 #track previous resource scan duration $IncrementalScan = $false; AzSKADOServiceMapping([string] $organizationName, [string] $projectName, [string] $buildFileLocation, [string] $releaseFileLocation, [string] $repositoryFileLocation,[string] $mappingType,[string] $auto,[switch] $useCache, [switch] $IncrementalScan, [InvocationInfo] $invocationContext): Base($organizationName, $invocationContext) { $this.OrgName = $organizationName $this.ProjectName = $projectName $this.BuildMappingsFilePath = $buildFileLocation $this.ReleaseMappingsFilePath = $releaseFileLocation $this.RepositoryMappingsFilePath = $repositoryFileLocation $this.MappingType = $MappingType $this.Auto = $auto.ToLower(); $this.UseCache = $useCache $this.IncrementalScan = $IncrementalScan $this.StorageAccount = $env:StorageName; $this.StorageRG = $env:StorageRG; $this.Container = $env:Container; # Power BI Report Storage settings $this.ReportStorageAccount = $env:ReportStorageName; $this.ReportStorageRG = $env:ReportStorageRG; $this.ReportContainer = $env:ReportContainer; # Set Service id mapping expiration duration $this.MappingExpirationLimit = $env:MappingExpirationLimit; #get ServiceMapping cache helper instance $this.ServiceMappingCacheHelperObj = [ServiceMappingCacheHelper]::ServiceMappingCacheHelperInstance if (!$this.ServiceMappingCacheHelperObj) { $this.ServiceMappingCacheHelperObj = [ServiceMappingCacheHelper]::GetInstance($this.OrgName); } [ServiceMappingCacheHelper]::TelemetryLogging("scan started",$null); #get storage details if($this.Auto -eq 'true'){ if ($this.StorageRG -and $this.StorageAccount) { $keys = Get-AzStorageAccountKey -ResourceGroupName $this.StorageRG -Name $this.StorageAccount if ($null -eq $keys) { $this.PublishCustomMessage("Status: Storage account not found.", [MessageType]::Error); } else { #storage context to save ST files for ADO scanner $StorageContext = New-AzStorageContext -StorageAccountName $this.StorageAccount -StorageAccountKey $keys[0].Value -Protocol Https $this.StorageAccountCtx = $StorageContext.Context; } } if ($this.ReportStorageRG -and $this.ReportStorageAccount) { $keys = Get-AzStorageAccountKey -ResourceGroupName $this.ReportStorageRG -Name $this.ReportStorageAccount if ($null -eq $keys) { $this.PublishCustomMessage("Status: Storage account not found.", [MessageType]::Error); } else { #storage context to save ST files for Power Bi reports $ReportStorageContext = New-AzStorageContext -StorageAccountName $this.ReportStorageAccount -StorageAccountKey $keys[0].Value -Protocol Https $this.ReportStorageAccountCtx = $ReportStorageContext.Context; } } } } AzSKADOServiceMapping([string] $organizationName,[string] $projectName,[string] $mappingType,[InvocationInfo] $invocationContext): Base($organizationName, $invocationContext){ $this.OrgName = $organizationName $this.ProjectName = $projectName $this.MappingType = $mappingType $this.ServiceMappingCacheHelperObj = [ServiceMappingCacheHelper]::ServiceMappingCacheHelperInstance if (!$this.ServiceMappingCacheHelperObj) { $this.ServiceMappingCacheHelperObj = [ServiceMappingCacheHelper]::GetInstance($this.OrgName); } $projectURL = "https://dev.azure.com/{0}/_apis/projects/{1}?api-version=6.0" -f $this.OrgName, $this.ProjectName $response = [WebRequestHelper]::InvokeGetWebRequest($projectURL); $this.projectId = $response.id $this.IncrementalScan = $true } [MessageData[]] GetInactiveResourceDetails() { $this.storageCachedData = $this.ServiceMappingCacheHelperObj.GetWorkItemByHashAzureTable("All", "","","", $this.projectId) if([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile"){ $this.GetInactiveSecureFiles() } if([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup"){ $this.GetInactiveVariableGroups(); } [MessageData[]] $returnMsgs = @(); $returnMsgs += [MessageData]::new("Returning service mappings."); return $returnMsgs } hidden GetInactiveVariableGroups(){ #call the common ST mapping function with parameter false to ensure service trees are not fetched $this.FindSTForVGWithIncremental($false) } hidden GetInactiveSecureFiles(){ #call the common ST mapping function with parameter false to ensure service trees are not fetched $this.FindSTForSecureFileWithIncremental($false) } [MessageData[]] GetSTmapping() { $this.Stopwatch = [system.diagnostics.stopwatch]::StartNew() $this.Stopwatch.Start(); if(!$this.Auto -eq 'true') { if([string]::IsNullOrWhiteSpace($this.RepositoryMappingsFilePath) -or [string]::IsNullOrWhiteSpace($this.BuildMappingsFilePath) -or [string]::IsNullOrWhiteSpace($this.ReleaseMappingsFilePath)) { return "File Path not valid."; } if(![string]::IsNullOrWhiteSpace($this.BuildMappingsFilePath) -and ![string]::IsNullOrWhiteSpace($this.ReleaseMappingsFilePath)) { if(!(Test-Path $this.BuildMappingsFilePath) -or !(Test-Path $this.ReleaseMappingsFilePath)) { return "File Path not valid."; } } } $this.SaveScanDuration("Repository scan started", $false) $this.GetRepositoryMapping(); $this.SaveScanDuration("Repository scan ended",$true) #fetch all the cached mappings from cache and add to in-memory collection $this.storageCachedData = $this.ServiceMappingCacheHelperObj.GetWorkItemByHashAzureTable("All", "","","", $this.projectId) [ServiceMappingCacheHelper]::TelemetryLogging("GetSTmapping",$null); $this.GetBuildReleaseMapping(); if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "ServiceConnection") { $this.SaveScanDuration("Service Connections scan started", $false) $this.FetchSvcConnMapping(); $this.SaveScanDuration("Service Connections scan ended",$true) } if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "AgentPool") { $this.SaveScanDuration("Agent Pool scan started", $false) $this.FetchAgentPoolMapping(); $this.SaveScanDuration("Agent Pool scan ended",$true) } if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "Environment") { $this.SaveScanDuration("Environment scan started", $false) $this.FetchEnvironmentMapping(); $this.SaveScanDuration("Environment scan ended",$true) } if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup" -or $this.MappingType -eq "SecureFile") { $this.SaveScanDuration("VariableGroup/SecureFile scan started", $false) [ServiceMappingCacheHelper]::TelemetryLogging("GetSTmapping",$null); if($this.IncrementalScan){ if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup"){ $this.FindSTForVGWithIncremental($true); } if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile"){ $this.FindSTForSecureFileWithIncremental($true); } } else{ $this.FetchVarGrpSecureFileMapping(); } $this.SaveScanDuration("VariableGroup/SecureFile scan ended",$true) } if ([string]::IsNullOrWhiteSpace($this.MappingType) -or $this.MappingType -eq "All" -or $this.MappingType -eq "Feed") { $this.SaveScanDuration("Feed scan started", $false) $this.FetchFeedMapping(); $this.SaveScanDuration("Feed scan ended",$true) } [MessageData[]] $returnMsgs = @(); $returnMsgs += [MessageData]::new("Returning service mappings."); return $returnMsgs } hidden GetBuildReleaseMapping() { [ServiceMappingCacheHelper]::TelemetryLogging("GetBuildReleaseMapping",$null); $this.SaveScanDuration("Build's repo scan started", $false) if($this.Auto -eq 'true'){ $response = Get-AzStorageBlob -Blob 'BuildServiceMappingData.json' -Container $this.Container -Context $this.StorageAccountCtx $this.BuildSTDetails = $response.ICloudBlob.DownloadText() | ConvertFrom-Json } else { $this.BuildSTDetails = Get-content $this.BuildMappingsFilePath | ConvertFrom-Json } if ([Helpers]::CheckMember($this.BuildSTDetails, "data") -and ($this.BuildSTDetails.data | Measure-Object).Count -gt 0){ $this.BuildSTDetails.data = $this.BuildSTDetails.data | where-object {$_.ProjectName -eq $this.ProjectName} if (($this.BuildSTDetails.data | Measure-Object).Count -gt 0){ $this.ProjectId = $this.BuildSTDetails.data[0].projectId; } } if($this.UseCache) { if([Helpers]::CheckMember($this.storageCachedData[0],"ResourceID")){ $buildRepoList = $this.storageCachedData | Where-Object {($_.ResourceType -eq 'Repo') -and ($_.PipelineType -eq 'Build')} foreach($buildRepo in $buildRepoList) { $this.BuildSTDetails.data+=@([PSCustomObject] @{ buildDefinitionName = $buildRepo.PipelineName; buildDefinitionID = $buildRepo.PipelineID; serviceID = $buildRepo.ServiceTreeID; projectName = $this.ProjectName; projectID = $buildRepo.ProjectID; orgName = $buildRepo.OrgName } ) } } } else { # Get Build-Repo mappings try { $buildObjectListURL = ("https://dev.azure.com/{0}/{1}/_apis/build/definitions?queryOrder=lastModifiedDescending&api-version=6.0" +'&$top=10000') -f $($this.orgName), $this.projectName; $buildObjectList = $this.GetBuildReleaseObjects($buildObjectListURL,'Build'); $buildObjectList = $buildObjectList | Where-Object {$_.id -notin $this.BuildSTDetails.data.buildDefinitionID} $counter =0 foreach ($build in $buildObjectList) { try { $counter++ Write-Progress -Activity 'Build mappings...' -CurrentOperation $build.name -PercentComplete (($counter / $buildObjectList.count) * 100) $buildDefnObj = [WebRequestHelper]::InvokeGetWebRequest($build.url); $repositoryName = $buildDefnObj.repository.name; $repoSTData = $this.RepositorySTDetails.Data | Where-Object { ($_.repoName -eq $repositoryName)}; if($repoSTData -and $repoSTData.repoID -ne ""){ $this.BuildSTDetails.data+=@([PSCustomObject] @{ buildDefinitionName = $build.name; buildDefinitionID = $build.id; serviceID = $repoSTData.serviceID; projectName = $repoSTData.projectName; projectID = $repoSTData.projectID; orgName = $repoSTData.orgName } ) #Save repo mappings in azure table $this.AddMappinginfoInCache( $this.OrgName,$this.projectId,$build.id,$build.name, $repoSTData.serviceID,$build.createdDate,$repoSTData.repoID,$repositoryName,"Repo","Build",(Get-date).AddDays($this.MappingExpirationLimit)); } } catch{ } } [ServiceMappingCacheHelper]::TelemetryLogging("GetBuildReleaseMapping completed",$null); } catch { } } if($this.UseCache) { $this.ExportObjToJsonFile($this.BuildSTDetails, 'BuildSTData.json'); $this.ExportObjToJsonFileUploadToBlob($this.BuildSTDetails, 'BuildSTData.json'); } $this.SaveScanDuration("Build's repo scan ended", $true) $this.SaveScanDuration("Release's repo releases scan started", $false) if($this.Auto -eq 'true'){ $response = Get-AzStorageBlob -Blob 'ReleaseServiceMappingData.json' -Container $this.Container -Context $this.StorageAccountCtx $this.ReleaseSTDetails = $response.ICloudBlob.DownloadText() | ConvertFrom-Json } else { $this.ReleaseSTDetails = Get-content $this.ReleaseMappingsFilePath | ConvertFrom-Json } if ([Helpers]::CheckMember($this.ReleaseSTDetails, "data") -and ($this.ReleaseSTDetails.data | Measure-Object).Count -gt 0) { $this.ReleaseSTDetails.data = $this.ReleaseSTDetails.data | where-object {$_.ProjectName -eq $this.ProjectName} if (($this.ReleaseSTDetails.data | Measure-Object).Count -gt 0 -and [string]::IsNullOrWhiteSpace($this.ProjectId)) { $this.ProjectId = $this.ReleaseSTDetails.data[0].projectId } } [ServiceMappingCacheHelper]::TelemetryLogging("GetBuildReleaseMapping - release",$null); if($this.UseCache) { if([Helpers]::CheckMember($this.storageCachedData[0],"ResourceID")){ $releaseRepoList = $this.storageCachedData | Where-Object {($_.ResourceType -in ('Repo','ArtifactBuild')) -and ($_.PipelineType -eq 'Release')} foreach($releaseRepo in $releaseRepoList) { $this.ReleaseSTDetails.data+=@([PSCustomObject] @{ releaseDefinitionName = $releaseRepo.PipelineName; releaseDefinitionID = $releaseRepo.PipelineID; serviceID = $releaseRepo.ServiceTreeID; projectName = $this.ProjectName; projectID = $releaseRepo.ProjectID; orgName = $releaseRepo.OrgName } ) } } } else { # Get Release-Repo mappings try { $releaseObjectListURL = ("https://vsrm.dev.azure.com/{0}/{1}/_apis/release/definitions?api-version=6.0" ) -f $($this.orgName), $this.projectName; $releaseObjectList = $this.GetBuildReleaseObjects($ReleaseObjectListURL,'Release'); $releaseObjectList = $releaseObjectList | Where-Object {$_.id -notin $this.ReleaseSTDetails.data.releaseDefinitionID} $counter =0 foreach ($release in $releaseObjectList) { try { $counter++ Write-Progress -Activity 'Release mappings...' -CurrentOperation $release.name -PercentComplete (($counter / $releaseObjectList.count) * 100) $releaseDefnObj = [WebRequestHelper]::InvokeGetWebRequest($release.url); if($releaseDefnObj[0].artifacts) { $type = $releaseDefnObj[0].artifacts.type; switch ($type) { {($_ -eq "GitHubRelease") -or ($_ -eq "Git")}{ $repositoryName =$releaseDefnObj[0].artifacts.definitionReference.definition.name; $repoSTData = $this.RepositorySTDetails.Data | Where-Object { ($_.repoName -eq $repositoryName)}; if($repoSTData -and $repoSTData.repoID -ne ""){ $this.ReleaseSTDetails.data+=@([PSCustomObject] @{ releaseDefinitionName = $release.name; releaseDefinitionID = $release.id; serviceID = $repoSTData.serviceID; projectName = $repoSTData.projectName; projectID = $repoSTData.projectID; orgName = $repoSTData.orgName } ) #Save repo mappings in azure table $this.AddMappinginfoInCache( $this.OrgName,$this.projectId,$release.id,$release.name, $repoSTData.serviceID,$release.modifiedOn,$repoSTData.repoID,$repositoryName,"Repo","Release",(Get-date).AddDays($this.MappingExpirationLimit)); } } Build { $buildSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $releaseDefnObj[0].artifacts.definitionReference.definition.id) -and ($_.projectID -eq $releaseDefnObj[0].artifacts.definitionReference.project.id)}; If($buildSTData){ $this.ReleaseSTDetails.data+=@([PSCustomObject] @{ releaseDefinitionName = $release.name; releaseDefinitionID = $release.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) $this.AddMappinginfoInCache( $this.OrgName,$this.projectId,$release.id,$release.name, $buildSTData.serviceID,$release.modifiedOn,$buildSTData.buildDefinitionID,$buildSTData.buildDefinitionName,"ArtifactBuild","Release",(Get-date).AddDays($this.MappingExpirationLimit)); } } } } } catch{ } } } catch { } } if($this.UseCache) { $this.ExportObjToJsonFile($this.ReleaseSTDetails, 'ReleaseSTData.json'); $this.ExportObjToJsonFileUploadToBlob($this.ReleaseSTDetails, 'ReleaseSTData.json'); } $this.SaveScanDuration("Release's repo releases scan ended", $false) [ServiceMappingCacheHelper]::TelemetryLogging("GetBuildReleaseMapping - completed",$null); } hidden GetRepositoryMapping() { [ServiceMappingCacheHelper]::TelemetryLogging("GetRepositoryMapping - started",$null); if($this.Auto -eq 'true'){ $response = Get-AzStorageBlob -Blob 'RepoServiceMappingData.json' -Container $this.Container -Context $this.StorageAccountCtx $this.RepositorySTDetails = $response.ICloudBlob.DownloadText() | ConvertFrom-Json } else { $this.RepositorySTDetails = Get-content $this.RepositoryMappingsFilePath | ConvertFrom-Json } if ([Helpers]::CheckMember($this.RepositorySTDetails, "data") -and ($this.RepositorySTDetails.data | Measure-Object).Count -gt 0) { $this.RepositorySTDetails.data = $this.RepositorySTDetails.data | where-object {$_.ProjectName -eq $this.ProjectName} if (($this.RepositorySTDetails.data | Measure-Object).Count -gt 0) { $this.ProjectId = $this.RepositorySTDetails.data[0].projectId } } if($this.UseCache) { $this.ExportObjToJsonFile($this.RepositorySTDetails, 'RepositorySTData.json'); $this.ExportObjToJsonFileUploadToBlob($this.RepositorySTDetails, 'RepositorySTData.json'); } [ServiceMappingCacheHelper]::TelemetryLogging("GetRepositoryMapping - completed",$null); } hidden ExportObjToJsonFile($serviceMapping, $fileName) { $folderPath ="/" + $this.OrgName.ToLower() + "/" + $this.ProjectName.ToLower(); if($this.auto -eq "true"){ $this.OutputFolderPath = $this.AzSKTempStatePath + $folderPath; } else { $this.OutputFolderPath = [WriteFolderPath]::GetInstance().FolderPath + $folderPath; } If(!(test-path $this.OutputFolderPath)){ New-Item -ItemType Directory -Force -Path $this.OutputFolderPath } $serviceMapping | ConvertTo-Json -Depth 10 | Out-File (Join-Path $this.OutputFolderPath $fileName) -Encoding ASCII } hidden ExportObjToJsonFileUploadToBlob($serviceMapping, $fileName) { if($this.auto -eq "true"){ $fileName =$this.OrgName.ToLower() + "/" + $this.ProjectName.ToLower() + "/" + $fileName if ($null -ne $this.StorageAccountCtx){ Set-AzStorageBlobContent -Container $this.Container -File (Join-Path $this.AzSKTempStatePath $fileName) -Blob $fileName -Context $this.StorageAccountCtx -Force } if ($null -ne $this.ReportStorageAccountCtx){ Set-AzStorageBlobContent -Container $this.ReportContainer -File (Join-Path $this.AzSKTempStatePath $fileName) -Blob $fileName -Context $this.ReportStorageAccountCtx -Force } } } hidden [bool] FetchSvcConnMapping() { $svcConnSTMapping = @{ data = @(); }; try{ $serviceEndpointURL = ("https://dev.azure.com/{0}/{1}/_apis/serviceendpoint/endpoints?api-version=6.0-preview.4") -f $this.OrgName, $this.ProjectName; $serviceEndpointObj = [WebRequestHelper]::InvokeGetWebRequest($serviceEndpointURL) $Connections = $null if (([Helpers]::CheckMember($serviceEndpointObj, "count") -and $serviceEndpointObj[0].count -gt 0) -or (($serviceEndpointObj | Measure-Object).Count -gt 0 -and [Helpers]::CheckMember($serviceEndpointObj[0], "name"))) { $Connections = $serviceEndpointObj } $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of service connections for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total service connections to be mapped: $(($Connections | Measure-Object).Count)") $counter = 0 $apiURL = "https://{0}.visualstudio.com/_apis/Contribution/HierarchyQuery?api-version=5.0-preview.1" -f $this.OrgName $sourcePageUrl = "https://{0}.visualstudio.com/{1}/_settings/adminservices" -f $this.OrgName, $this.ProjectName; #generate access token with datastudio api audience $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) $Connections | ForEach-Object { $counter++ Write-Progress -Activity 'Service connection mappings...' -CurrentOperation $_.Name -PercentComplete (($counter / $Connections.count) * 100) $inputbody = "{'contributionIds':['ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider'],'dataProviderContext':{'properties':{'serviceEndpointId':'$($_.id)','projectId':'$($this.projectId)','sourcePage':{'url':'$($sourcePageUrl)','routeId':'ms.vss-admin-web.project-admin-hub-route','routeValues':{'project':'$($this.ProjectName)','adminPivot':'adminservices','controller':'ContributedPage','action':'Execute'}}}}}" | ConvertFrom-Json $responseObj = [WebRequestHelper]::InvokePostWebRequest($apiURL, $inputbody); try { if ([Helpers]::CheckMember($responseObj, "dataProviders") -and $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider") { #set true when STMapping not found in build & release STData files and need to recheck for azurerm type $unmappedSerConn = $true; $serviceConnEndPointDetail = $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider" if ($serviceConnEndPointDetail -and [Helpers]::CheckMember($serviceConnEndPointDetail, "serviceEndpointExecutionHistory") ) { $svcConnJobs = $serviceConnEndPointDetail.serviceEndpointExecutionHistory.data #Arranging in descending order of run time. $svcConnJobs = $svcConnJobs | Sort-Object startTime -Descending #Taking Unique runs $svcConnJobs = $svcConnJobs | Select-Object @{l = 'id'; e ={$_.definition.id}}, @{l = 'name'; e ={$_.definition.name}}, @{l = 'planType'; e ={$_.planType}} -Unique foreach ($job in $svcConnJobs) { if ($job.planType -eq "Build") { $buildSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $job.id) }; if($buildSTData){ $svcConnSTMapping.data += @([PSCustomObject] @{ serviceConnectionName = $_.Name; serviceConnectionID = $_.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) $unmappedSerConn = $false; break; } } elseif ($job.planType -eq "Release") { $releaseSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $job.id)}; if($releaseSTData){ $svcConnSTMapping.data += @([PSCustomObject] @{ serviceConnectionName = $_.Name; serviceConnectionID = $_.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) $unmappedSerConn = $false; break; } } } } if($serviceConnEndPointDetail -and $unmappedSerConn) { if ($serviceConnEndPointDetail.serviceEndpoint.type -eq "azurerm") { try { $responseObj = $this.GetServiceIdWithSubscrId($serviceConnEndPointDetail.serviceEndpoint.data.subscriptionId,$accessToken) if($responseObj) { $serviceId = $responseObj[2].Rows[0][4]; $svcConnSTMapping.data += @([PSCustomObject] @{ serviceConnectionName = $_.Name; serviceConnectionID = $_.id; serviceID = $serviceId; projectName = $_.serviceEndpointProjectReferences.projectReference.name; projectID = $_.serviceEndpointProjectReferences.projectReference.id; orgName = $this.OrgName } ) } } catch { } } } } } catch { #eat exception } } } catch { #eat exception } $this.PublishCustomMessage("Service mapping found: $(($svcConnSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($svcConnSTMapping.data, 'ServiceConnectionSTData.json'); $this.ExportObjToJsonFileUploadToBlob($svcConnSTMapping.data, 'ServiceConnectionSTData.json'); } return $true; } hidden [bool] FetchAgentPoolMapping() { $agentPoolSTMapping = @{ data = @(); }; try{ $agentPoolsDefnURL = ("https://{0}.visualstudio.com/{1}/_settings/agentqueues?__rt=fps&__ver=2") -f $this.OrgName, $this.ProjectName; $agentPoolsDefnsObj = [WebRequestHelper]::InvokeGetWebRequest($agentPoolsDefnURL); #generate access token with datastudio api audience $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) $taskAgentQueues = $null; if (([Helpers]::CheckMember($agentPoolsDefnsObj, "fps.dataProviders.data") ) -and (($agentPoolsDefnsObj.fps.dataProviders.data."ms.vss-build-web.agent-queues-data-provider") -and $agentPoolsDefnsObj.fps.dataProviders.data."ms.vss-build-web.agent-queues-data-provider".taskAgentQueues)) { $taskAgentQueues = $agentPoolsDefnsObj.fps.dataProviders.data."ms.vss-build-web.agent-queues-data-provider".taskAgentQueues | where-object{$_.pool.isLegacy -eq $false}; } $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of agent pool for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total agent pool to be mapped: $(($taskAgentQueues | Measure-Object).Count)") $counter = 0 $taskAgentQueues | ForEach-Object { $counter++ Write-Progress -Activity 'Agent pool mappings...' -CurrentOperation $_.Name -PercentComplete (($counter / $taskAgentQueues.count) * 100) $unmappedAgentPool = $true; $agtPoolId = $_.id $agtPoolName = $_.name $agentPoolsURL = "https://{0}.visualstudio.com/{1}/_settings/agentqueues?queueId={2}&__rt=fps&__ver=2" -f $this.orgName, $this.ProjectName, $agtPoolId $agentPool = [WebRequestHelper]::InvokeGetWebRequest($agentPoolsURL); if (([Helpers]::CheckMember($agentPool[0], "fps.dataProviders.data") ) -and ($agentPool[0].fps.dataProviders.data."ms.vss-build-web.agent-jobs-data-provider")) { $agentPoolJobs = $agentPool[0].fps.dataProviders.data."ms.vss-build-web.agent-jobs-data-provider".jobs | Where-Object { $_.scopeId -eq $this.ProjectId }; #Arranging in descending order of run time. $agentPoolJobs = $agentPoolJobs | Sort-Object queueTime -Descending #Taking unique runs $agentPoolJobs = $agentPoolJobs | Select-Object @{l = 'id'; e ={$_.definition.id}}, @{l = 'name'; e ={$_.definition.name}}, @{l = 'planType'; e ={$_.planType}} -Unique #If agent pool has been queued at least once foreach ($job in $agentPoolJobs){ if ($job.planType -eq "Build") { $buildSTData = $this.BuildSTDetails.data | Where-Object { ($_.buildDefinitionID -eq $job.id)}; if($buildSTData){ $agentPoolSTMapping.data += @([PSCustomObject] @{ agentPoolName = $_.Name; agentPoolID = $_.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) $unmappedAgentPool = $false; break; } } elseif ($job.planType -eq "Release") { $releaseSTData = $this.ReleaseSTDetails.data | Where-Object { ($_.releaseDefinitionID -eq $job.id)}; if($releaseSTData){ $agentPoolSTMapping.data += @([PSCustomObject] @{ agentPoolName = $_.Name; agentPoolID = $_.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) $unmappedAgentPool = $false; break; } } } } if($unmappedAgentPool) { $agentList = $agentPool[0].fps.dataProviders.data."ms.vss-build-web.agent-pool-data-provider".agents; $exit = $false $agentList | Where-Object {$exit -eq $false} | ForEach-Object { $agtName = $_.Name $responseObj = $this.GetAgentSubscrId($agtName) if($responseObj) { $logsRows = $responseObj.tables[0].rows; if($logsRows.count -gt 0){ $agentSubscriptionID = $logsRows[0][18]; try { $response = $this.GetServiceIdWithSubscrId($agentSubscriptionID,$accessToken) if($response){ $serviceId = $response[2].Rows[0][4]; $agentPoolSTMapping.data += @([PSCustomObject] @{ agentPoolName = $agtPoolName; agentPoolID = $agtPoolId; serviceID = $serviceId; projectName = $this.projectName; projectID = $this.projectId; orgName = $organizationName } ); $exit = $true } } catch { } } } } } } } catch { #eat exception } $this.PublishCustomMessage("Service mapping found: $(($agentPoolSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($agentPoolSTMapping.data, 'AgentPoolSTData.json'); $this.ExportObjToJsonFileUploadToBlob($agentPoolSTMapping.data, 'AgentPoolSTData.json'); } return $true; } hidden [bool] FetchVarGrpSecureFileMapping() { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - started",$null); $topNQueryString = '&$top=10000' [System.Collections.Generic.List[psobject]]$varGrps = @(); #This variable is used to store details returned from variable group file api(fetching all the variable group details in one call) [System.Collections.Generic.List[psobject]]$vgDetails = @(); [System.Collections.Generic.List[psobject]]$secureFiles = @(); #This variable is used to store details returned from secure file api(fetching all the secure file details in one call) [System.Collections.Generic.List[psobject]]$secureFileDetails = @(); #generate access token with datastudio api audience $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) $variableGroupSTMapping = @{ data = @(); }; $secureFileSTMapping = @{ data = @(); }; try { $releaseDefnURL = ("https://vsrm.dev.azure.com/{0}/{1}/_apis/release/definitions?api-version=6.0" +$topNQueryString) -f $($this.OrgName), $this.ProjectName; $releaseDefnsObj = [WebRequestHelper]::InvokeGetWebRequest($releaseDefnURL); if (([Helpers]::CheckMember($releaseDefnsObj, "count") -and $releaseDefnsObj[0].count -gt 0) -or (($releaseDefnsObj | Measure-Object).Count -gt 0 -and [Helpers]::CheckMember($releaseDefnsObj[0], "name"))) { $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of variable group/secure file using release for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total mappings to be evaluated: $(($releaseDefnsObj | Measure-Object).Count)") $counter = 0 if (($secureFileDetails | Measure-Object).count -eq 0) { $secureFilesURL = "https://dev.azure.com/{0}/{1}/_apis/distributedtask/securefiles?api-version=6.1-preview.1" -f $this.OrgName, $this.projectId; $secureFileDetails = [WebRequestHelper]::InvokeGetWebRequest($secureFilesURL); } if (($vgDetails | Measure-Object).count -eq 0) { $vgFilesURL = "https://dev.azure.com/{0}/{1}/_apis/distributedtask/variablegroups?api-version=6.0-preview.2" -f $this.OrgName, $this.projectId; $vgDetails = [WebRequestHelper]::InvokeGetWebRequest($vgFilesURL); } foreach ($relDef in $releaseDefnsObj) { $counter++ Write-Progress -Activity 'Variable group/secure file mappings via release...' -CurrentOperation $relDef.Name -PercentComplete (($counter / $releaseDefnsObj.count) * 100) try { $releaseObj = [WebRequestHelper]::InvokeGetWebRequest($relDef.url); #add var groups scoped at release scope. if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if((($releaseObj[0].variableGroups) | Measure-Object).Count -gt 0) { $varGrps.Add($releaseObj[0].variableGroups); } } #get var grps from each env of release pipeline foreach ($env in $releaseObj[0].environments) { if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if((($env.variableGroups) | Measure-Object).Count -gt 0) { $varGrps.Add($env.variableGroups); } } try { if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { $workflowtasks = @(); if([Helpers]::CheckMember($env, "deployPhases") ) { foreach ($deployPhase in $env.deployPhases) { if ([Helpers]::CheckMember($deployPhase,"workflowtasks")) { foreach ($workflowtask in $deployPhase.workflowtasks) { $workflowtasks += $workflowtask; } } } } foreach ($item in $workflowtasks) { if ([Helpers]::CheckMember($item, "inputs") -and [Helpers]::CheckMember($item.inputs, "secureFile")) { $secureFiles.Add($item.inputs.secureFile); } } } } catch { #eat exception } } if($this.UseCache) { # Find Service tree id for variable groups from cache $this.FindSTWithReleaseForVGSecFileCache($relDef, $varGrps,$secureFiles,$accessToken,$vgDetails,$secureFileDetails,$variableGroupSTMapping, $secureFileSTMapping) } else { $this.FindSTWithReleaseForVGSecFile($relDef, $varGrps,$secureFiles, $accessToken,$vgDetails, $secureFileDetails,$variableGroupSTMapping, $secureFileSTMapping) } } Catch{ $this.PublishCustomMessage($_.Exception.Message) } } $releaseDefnsObj = $null; } } catch{ #eat exception } try { $buildDefnURL = ("https://dev.azure.com/{0}/{1}/_apis/build/definitions?queryOrder=lastModifiedDescending&api-version=6.0" + $topNQueryString) -f $($this.OrgName), $this.ProjectName; $buildDefnsObj = [WebRequestHelper]::InvokeGetWebRequest($buildDefnURL) if (([Helpers]::CheckMember($buildDefnsObj, "count") -and $buildDefnsObj[0].count -gt 0) -or (($buildDefnsObj | Measure-Object).Count -gt 0 -and [Helpers]::CheckMember($buildDefnsObj[0], "name"))) { $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of variable group/secure file using build for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total mappings to be evaluated: $(($buildDefnsObj | Measure-Object).Count)") $counter = 0 foreach ($bldDef in $buildDefnsObj) { $counter++ Write-Progress -Activity 'Variable group/secure file mappings via build...' -CurrentOperation $bldDef.Name -PercentComplete (($counter / $buildDefnsObj.count) * 100) $buildObj = [WebRequestHelper]::InvokeGetWebRequest($bldDef.url.split('?')[0]); #getting secure files added in all the tasks. try { if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { $tasksSteps =@() if([Helpers]::CheckMember($buildObj, "process") -and [Helpers]::CheckMember($buildObj.process, "Phases") ) { foreach ($item in $buildObj.process.Phases) { if ([Helpers]::CheckMember($item, "steps")) { $tasksSteps += $item.steps; } } } foreach ($itemStep in $tasksSteps) { if ([Helpers]::CheckMember($itemStep, "inputs") -and [Helpers]::CheckMember($itemStep.inputs, "secureFile")) { $secureFiles += $itemStep.inputs.secureFile; } } } } catch { #eat exception } if($this.UseCache) { # Find Service tree id for variable groups from cache $this.FindSTWithBuildForVGSecFileCache($buildObj, $secureFiles, $accessToken,$vgDetails, $secureFileDetails, $variableGroupSTMapping, $secureFileSTMapping) } else { $this.FindSTWithBuildForVGSecFile($buildObj, $secureFiles, $accessToken,$vgDetails, $secureFileDetails, $variableGroupSTMapping, $secureFileSTMapping) } } $buildDefnsObj = $null; } } catch{ #eat exception } #Removing duplicate entries of the tuple (variableGroupId,serviceId) if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { $variableGroupSTMapping.data = $variableGroupSTMapping.data | Sort-Object -Unique variableGroupID,serviceID $this.PublishCustomMessage("Service mapping found: $(($variableGroupSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($variableGroupSTMapping.data, 'VariableGroupSTData.json'); $this.ExportObjToJsonFileUploadToBlob($variableGroupSTMapping.data, 'VariableGroupSTData.json'); } } #Removing duplicate entries of the tuple (securefile,serviceId) if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { $secureFileSTMapping.data = $secureFileSTMapping.data | Sort-Object -Unique secureFileID,serviceID $this.PublishCustomMessage("Service mapping found: $(($secureFileSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($secureFileSTMapping.data, 'SecureFileSTData.json'); $this.ExportObjToJsonFileUploadToBlob($secureFileSTMapping.data, 'SecureFileSTData.json'); } } return $true; } hidden [bool] FetchEnvironmentMapping() { $environmentSTMapping = @{ data = @(); }; try{ $environmentURL = 'https://dev.azure.com/{0}/{1}/_apis/distributedtask/environments?$top=10000&api-version=6.0-preview.1' -f $this.OrgName, $this.ProjectName; $environmentsObjList = @([WebRequestHelper]::InvokeGetWebRequest($environmentURL)); #generate access token with datastudio api audience $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) $unmappedEnv = $true; if ($environmentsObjList.count -gt 0 ) { $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of environments for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total environments to be mapped: $($environmentsObjList.count)") $counter = 0 $environmentsObjList | ForEach-Object{ $counter++ Write-Progress -Activity 'Environments mappings...' -CurrentOperation $_.Name -PercentComplete (($counter / $environmentsObjList.count) * 100) $apiURL = "https://dev.azure.com/{0}/{1}/_apis/distributedtask/environments/{2}/environmentdeploymentrecords?top=20&api-version=6.0-preview.1" -f $this.OrgName, $this.ProjectName, $_.id; $envDeploymenyRecords = @([WebRequestHelper]::InvokeGetWebRequest($apiURL)); if ($envDeploymenyRecords.Count -gt 0 -and [Helpers]::CheckMember($envDeploymenyRecords[0],"definition")) { $envDeploymenyRecords = $envDeploymenyRecords | Select-Object -First 10 foreach ($envJob in $envDeploymenyRecords){ if ([Helpers]::CheckMember($envJob, "planType") -and $envJob.planType -eq "Build") { $buildSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $envJob.definition.id) }; if($buildSTData){ $environmentSTMapping.data += @([PSCustomObject] @{ environmentName = $_.Name; environmentID = $_.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) $unmappedEnv =$false; break; } } elseif ([Helpers]::CheckMember($envJob, "planType") -and $envJob.planType -eq "Release") { $releaseSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $envJob.definition.id)}; if($releaseSTData){ $environmentSTMapping.data += @([PSCustomObject] @{ environmentName = $_.Name; environmentID = $_.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) $unmappedEnv =$false; break; } } } } if($unmappedEnv){ $envResourceApiURL = "https://dev.azure.com/{0}/{1}/_environments/{2}?view=resources&__rt=fps&__ver=2" -f $this.OrgName, $this.ProjectName, $_.id; $envResourceDetails = @([WebRequestHelper]::InvokeGetWebRequest($envResourceApiURL)); if ([Helpers]::CheckMember($envResourceDetails, "fps.dataProviders") -and $envResourceDetails.fps.dataProviders.data."ms.vss-environments-web.environment-resources-view-data-provider") { # Type 2 for VM $vmName = $envResourceDetails.fps.dataProviders.data."ms.vss-environments-web.environment-resources-view-data-provider".environment.resources | Where-Object type -eq 2 | Select-Object name; if($vmName){ $responseObj = $this.GetAgentSubscrId($vmName) if($responseObj) { $logsRows = $responseObj.tables[0].rows; if($logsRows.count -gt 0){ $agentSubscriptionID = $logsRows[0][18]; try { $response = $this.GetServiceIdWithSubscrId($agentSubscriptionID,$accessToken) if($response){ $serviceId = $response[2].Rows[0][4]; $environmentSTMapping.data += @([PSCustomObject] @{ environmentName = $_.Name; environmentID = $_.id; serviceID = $serviceId; projectName = $this.ProjectName; projectID = $this.ProjectId; orgName = $this.OrgName } ) $unmappedEnv = $false break; } } catch { } } } } if($unmappedEnv){ # Type 4 for AKS Cluster $clusterId = $envResourceDetails.fps.dataProviders.data."ms.vss-environments-web.environment-resources-view-data-provider".environment.resources | Where-Object type -eq 4 | Select-Object id; if($clusterId){ $clusterApiURL = "https://dev.azure.com/{0}/{1}/_environments/{2}/providers/kubernetes/{3}?__rt=fps&__ver=2" -f $this.OrgName, $this.ProjectName, $_.id, $clusterId; $clusterDetails = @([WebRequestHelper]::InvokeGetWebRequest($clusterApiURL)); if($clusterDetails -and [Helpers]::CheckMember($clusterDetails.fps.dataProviders.data,"ms.vss-environments-web.kubernetes-resource-data-provider")) { $subscripId = $clusterDetails.fps.dataProviders.data."ms.vss-environments-web.kubernetes-resource-data-provider".kubernetesEndpoint.data | Where-Object authorizationType -eq "AzureSubscription" | Select-Object azureSubscriptionId; if($subscripId){ $response = $this.GetServiceIdWithSubscrId($subscripId,$accessToken) if($response){ $serviceId = $response[2].Rows[0][4]; $environmentSTMapping.data += @([PSCustomObject] @{ environmentName = $_.Name; environmentID = $_.id; serviceID = $serviceId; projectName = $this.ProjectName; projectID = $this.ProjectId; orgName = $this.OrgName } ) break; } } } } } } } } } } catch { #eat exception } $this.PublishCustomMessage("Service mapping found: $(($environmentSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($environmentSTMapping.data, 'EnvironmentSTData.json'); $this.ExportObjToJsonFileUploadToBlob($environmentSTMapping.data, 'EnvironmentSTData.json'); } return $true; } hidden [bool] FetchFeedMapping() { $feedSTMapping = @{ data = @(); }; $feedDefnURL = 'https://feeds.dev.azure.com/{0}/{1}/_apis/packaging/feeds?api-version=6.0-preview.1' -f $this.OrgName, $this.ProjectName; $feedDefnsObj = @([WebRequestHelper]::InvokeGetWebRequest($feedDefnURL)); if ($feedDefnsObj.count -gt 0 ) { $this.PublishCustomMessage(([Constants]::DoubleDashLine)) $this.PublishCustomMessage("Generating service mappings of feeds for project [$($this.ProjectName)]...") $this.PublishCustomMessage("Total feeds to be mapped: $($feedDefnsObj.count)") $counter = 0 $feedDefnsObj | ForEach-Object { try{ $counter++ Write-Progress -Activity 'Feeds mappings...' -CurrentOperation $_.Name -PercentComplete (($counter / $feedDefnsObj.count) * 100) $feed = $_; #Get feed packages $packagesURL = $feed._links.packages.href; $feedPackages = @([WebRequestHelper]::InvokeGetWebRequest($packagesURL)); if ($feedPackages.count -gt 0) { $feedPackages = $feedPackages | Select-Object -First 10; foreach ($package in $feedPackages){ $provenanceURL = "https://feeds.dev.azure.com/{0}/{1}/_apis/packaging/Feeds/{2}/Packages/{3}/Versions/{4}/provenance?api-version=6.0-preview.1" -f $this.OrgName, $this.ProjectName, $feed.id, $package.id, $package.versions[0].id; $provenanceObj = @([WebRequestHelper]::InvokeGetWebRequest($provenanceURL)); if ($provenanceObj.Count -gt 0 -and [Helpers]::CheckMember($provenanceObj[0],"provenance.provenanceSource") -and [Helpers]::CheckMember($provenanceObj[0],"provenance.data")) { if ($provenanceObj[0].provenance.provenanceSource -eq "InternalBuild") { $definitionId = $provenanceObj[0].provenance.data."System.DefinitionId"; $buildSTData = $this.BuildSTDetails.Data | Where-Object { $_.buildDefinitionID -eq $definitionId }; if($buildSTData){ $feedSTMapping.data += @([PSCustomObject] @{ feedName = $feed.Name; feedID = $feed.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) break; } #if no details found in buildST file the try in repoST file if (!$buildSTData -and $this.RepositorySTDetails -and $this.RepositorySTDetails.count -gt 0) { $repoId = $provenanceObj[0].provenance.data."Build.Repository.Id"; $repoSTData = $this.RepositorySTDetails.Data | Where-Object { ($_.repoID -eq $repoId)}; if($repoSTData){ $feedSTMapping.data += @([PSCustomObject] @{ feedName = $feed.Name; feedID = $feed.id; serviceID = $repoSTData.serviceID; projectName = $repoSTData.projectName; projectID = $repoSTData.projectID; orgName = $repoSTData.orgName } ) break; } } } elseif ($provenanceObj[0].provenance.provenanceSource -eq "InternalRelease") { $definitionId = $provenanceObj[0].provenance.data."Release.DefinitionId"; $releaseSTData = $this.ReleaseSTDetails.Data | Where-Object { $_.releaseDefinitionID -eq $definitionId }; if($buildSTData){ $feedSTMapping.data += @([PSCustomObject] @{ feedName = $feed.Name; feedID = $feed.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) break; } } } } } } catch { #eat exception } } } $this.PublishCustomMessage("Service mapping found: $(($feedSTMapping.data | Measure-Object).Count)", [MessageType]::Info) if($this.UseCache) { $this.ExportObjToJsonFile($feedSTMapping.data, 'FeedSTData.json'); $this.ExportObjToJsonFileUploadToBlob($feedSTMapping.data, 'FeedSTData.json'); } return $true; } hidden [object] GetServiceIdWithSubscrId($subscriptionID,$accessToken) { $response = $null try { # call data studio to fetch azure subscription id and servce id mapping $apiURL = "https://genevareference.westcentralus.kusto.windows.net/v2/rest/query" $inputbody = '{"db": "Shared","csl": "DataStudio_ServiceTree_AzureSubscription_Snapshot | where SubscriptionId contains ''{0}''", "properties": {"Options": {"query_language": "csl","servertimeout": "00:04:00","queryconsistency": "strongconsistency","request_readonly": false,"request_readonly_hardline": false}}}' $inputbody = $inputbody.Replace("{0}", $subscriptionID) $header = @{ "Authorization" = "Bearer " + $accessToken } $response = [WebRequestHelper]::InvokeWebRequest([Microsoft.PowerShell.Commands.WebRequestMethod]::Post,$apiURL,$header,$inputbody,"application/json; charset=UTF-8"); } catch { } return $response } hidden [object] GetAgentSubscrId($agentName) { $response = $null try { #generate access token with datastudio api audience $accessToken = [ContextHelper]::GetLAWSAccessToken() # call data studio to fetch azure subscription id and servce id mapping $apiURL = "https://api.loganalytics.io/v1/workspaces/b32a5e40-0360-40db-a9d4-ec1083b90f0a/query?timespan=P7D" $inputbody = '{"query":"AzSK_ResourceInvInfo_CL| where Name_s =~ ''{0}''| where ResourceType == ''Microsoft.Compute/virtualMachines''","options":{"truncationMaxSize":67108864},"maxRows":30001,"workspaceFilters":{"regions":[]}}' $inputbody = $inputbody.Replace("{0}", $agentName) $header = @{ "Authorization" = "Bearer " + $accessToken } $response = [WebRequestHelper]::InvokeWebRequest([Microsoft.PowerShell.Commands.WebRequestMethod]::Post,$apiURL,$header,$inputbody,"application/json; charset=UTF-8"); } catch { } return $response } hidden [object] GetBuildReleaseObjects($resourceUrl,$resourceType) { $skipCount = 0 $applicableDefnsObj=@(); while (($resourceUrl)) { $skipCount = 10000; $responseAndUpdatedUri = [WebRequestHelper]::InvokeWebRequestForResourcesInBatch($resourceUrl, $resourceUrl, $skipCount,$resourceType); #API response with resources $resourceDefnsObj = @($responseAndUpdatedUri[0]); #updated URI: null when there is no continuation token $resourceDfnUrl = $responseAndUpdatedUri[1]; $applicableDefnsObj+=$resourceDefnsObj; if ( (($applicableDefnsObj | Measure-Object).Count -gt 0 -and [Helpers]::CheckMember($applicableDefnsObj[0], "name")) -or ([Helpers]::CheckMember($applicableDefnsObj, "count") -and $applicableDefnsObj[0].count -gt 0)) { $resourceUrl =$resourceDfnUrl; } else { break; } } Write-Progress -Activity "All $($resourceType)s fetched" -Status "Ready" -Completed $resourceDefnsObj = $null; Remove-Variable resourceDefnsObj; return $applicableDefnsObj; } #adding new mapping info hidden [void] AddMappinginfoInCache( [string] $orgName, [string] $projectID, [string] $pipelineID,[string] $pipelineName, [string] $serviceTreeID,[string] $pipelineLastModified,[string] $resourceID,[string] $resourceName,[string] $resourceType,[string] $pipelineType,$mappingExpiration) { if($this.IncrementalScan){ $hash = $this.ServiceMappingCacheHelperObj.GetHashedTag($this.projectId, "", "",$resourceID,$resourceType) } else{ $hash = $this.ServiceMappingCacheHelperObj.GetHashedTag($this.projectId, $pipelineID, $pipelineType,$resourceID,$resourceType) } $resourceInCache = $this.GetResourceDataFromCache($pipelineType,$pipelineID,$resourceType, $resourceID) if($resourceInCache) { $this.ServiceMappingCacheHelperObj.UpdateTableEntity($orgName,$projectID,$pipelineID,$pipelineName,$serviceTreeID,$pipelineLastModified, $resourceID, $resourceType, $resourceName, $pipelineType,$mappingExpiration, $this.IncrementalScan) #update mapping expiration date as per new scan $rowIndex = [array]::IndexOf($this.storageCachedData.RowKey,$hash) $this.storageCachedData[$rowIndex].MappingExpiration = $mappingExpiration } else { $this.ServiceMappingCacheHelperObj.InsertMappingInfoInTable($orgName,$projectID,$pipelineID,$pipelineName,$serviceTreeID,$pipelineLastModified,$resourceID,$resourceType,$resourceName,$pipelineType, $mappingExpiration, $this.IncrementalScan) #update in-memory cache with new record $this.storageCachedData+= @([PSCustomObject] @{"RowKey" =$hash; "OrgName" = $orgName; "ProjectID" = $projectID; "PipelineID" = $pipelineID;"PipelineName" = $pipelineName;"ServiceTreeID" = $serviceTreeID;"PipelineLastModified" = $pipelineLastModified;"ResourceID" = $resourceID;"ResourceType" = $resourceType;"ResourceName" = $resourceName;"PipelineType" = $pipelineType; "MappingExpiration" = $MappingExpiration}; ) } } #fetch resource mapping details from in-memory collection hidden [object] GetResourceDataFromCache($pipelineType,$pipelineID,$resourceType, $resourceID) { if(-not [Helpers]::CheckMember($this.storageCachedData[0],"ResourceID")){ return $null; } $resourceItem =@() if($this.IncrementalScan){ $hash = $this.ServiceMappingCacheHelperObj.GetHashedTag($this.projectId, "", "",$resourceID,$resourceType) } else{ $hash = $this.ServiceMappingCacheHelperObj.GetHashedTag($this.projectId, $pipelineID, $pipelineType,$resourceID,$resourceType) } $item = $this.storageCachedData | Where-Object -Property RowKey -eq $hash #Check resource id present in cache without mapped with pipeline id if((!$item) -and ($resourceType -notin ("Repo","ArtifactBuild"))){ $item = $this.storageCachedData | Where-Object {($_.ResourceID -eq $resourceID) -and ($_.ResourceType -eq $resourceType) -and ($_.ProjectID -eq $this.projectId) -and ($_.OrgName -eq $this.OrgName)} if($item) { $this.resourceInCacheWithoutPipeline = $true } } if($item){ return $item } return $resourceItem } # attribution of variable group/ secure file linked with build hidden [void] FindSTWithBuildForVGSecFile($buildObj, $secureFiles, $accessToken,$vgDetails, $secureFileDetails, $variableGroupSTMapping, $secureFileSTMapping) { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings for build",$null); #Variable to store current build STDATA $buildSTData = $null; if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if([Helpers]::CheckMember($buildObj[0],"variableGroups")) { $varGrps = @($buildObj[0].variableGroups) $apiURL = "https://{0}.visualstudio.com/_apis/Contribution/HierarchyQuery?api-version=5.0-preview.1" -f $this.OrgName $sourcePageUrl = "https://{0}.visualstudio.com/{1}/_settings/adminservices" -f $this.OrgName, $this.ProjectName; $varGrps | ForEach-Object{ $_ | ForEach-Object{ try { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings for variable group",$null); $buildSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $buildObj[0].id) -and ($_.projectName -eq $this.ProjectName) }; if($buildSTData) { $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $_.name; variableGroupID = $_.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) # add variable group mapping details in cache $this.AddMappinginfoInCache($buildSTData.orgName,$buildSTData.projectID,$buildObj.id,$buildObj.name, $buildSTData.serviceID,$buildObj.createdDate,$_.id,"VariableGroup","Build",(Get-date).AddDays($this.MappingExpirationLimit)); } else { if ($varGrps.Type -eq 'AzureKeyVault') { try { # get associated service connection id for variable group $servConnID = $varGrps[0].providerData.serviceEndpointId; # get azure subscription id from service connection $inputbody = "{'contributionIds':['ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider'],'dataProviderContext':{'properties':{'serviceEndpointId':'$($servConnID)','projectId':'$($this.projectId)','sourcePage':{'url':'$($sourcePageUrl)','routeId':'ms.vss-admin-web.project-admin-hub-route','routeValues':{'project':'$($this.ProjectName)','adminPivot':'adminservices','controller':'ContributedPage','action':'Execute'}}}}}" | ConvertFrom-Json $responseObj = [WebRequestHelper]::InvokePostWebRequest($apiURL, $inputbody); if ([Helpers]::CheckMember($responseObj, "dataProviders") -and $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider") { $serviceConnEndPointDetail = $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider" if ($serviceConnEndPointDetail.serviceEndpoint.type -eq "azurerm") { try { $responseObj = $this.GetServiceIdWithSubscrId($serviceConnEndPointDetail.serviceEndpoint.data.subscriptionId,$accessToken) if($responseObj) { $serviceId = $responseObj[2].Rows[0][4]; $projectID = $serviceConnEndPointDetail.serviceEndpoint.serviceEndpointProjectReferences.projectReference.id; $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $_.name; variableGroupID = $_.id; serviceID = $serviceId; projectName = $serviceConnEndPointDetail.serviceEndpoint.serviceEndpointProjectReferences.projectReference.name; projectID = $projectID; orgName = $this.OrgName } ) # add variable group mapping details in cache $this.AddMappinginfoInCache($this.OrgName,$projectID,$buildObj.id,$buildObj.name, $serviceId,$buildObj.createdDate,$_.id,$_.name,"VariableGroup","Build",(Get-date).AddDays($this.MappingExpirationLimit)); } } catch { } } } } catch { } } } } catch { } } } } } if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings for SecureFile",$null); try { if(($secureFiles | Measure-Object).Count -gt 0) { $secureFiles | ForEach-Object{ $_ | ForEach-Object{ $secureFile = $_; $secureFilesObj = $secureFileDetails | Where-Object {$_.Name -eq $secureFile -or $_.Id -eq $secureFile} $secFileExistinSt = $secureFileSTMapping.data | Where-Object -Property secureFileID -eq $secureFile if(!$secFileExistinSt) { if ($secureFilesObj) { if (!$buildSTData) { $buildSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $buildObj[0].id) -and ($_.projectName -eq $this.ProjectName) }; } if($buildSTData){ $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $secureFilesObj.name; secureFileID = $secureFilesObj.id; serviceID = $buildSTData.serviceID; projectName = $buildSTData.projectName; projectID = $buildSTData.projectID; orgName = $buildSTData.orgName } ) # add secure file mapping details in cache $this.AddMappinginfoInCache($buildSTData.orgName,$buildSTData.projectID,$buildObj.id,$buildObj.name, $buildSTData.serviceID,$buildObj.createdDate,$secureFilesObj.id,$secureFilesObj.name,"SecureFile","Build",(Get-date).AddDays($this.MappingExpirationLimit)); } } } } } } } catch { #eat exception } } } # find cached mappings for variable group/ secure file linked with build hidden [void] FindSTWithBuildForVGSecFileCache($buildObj, $secureFiles, $accessToken,$vgDetails, $secureFileDetails, $variableGroupSTMapping, $secureFileSTMapping) { if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if([Helpers]::CheckMember($buildObj[0],"variableGroups")) { $varGrps = @($buildObj[0].variableGroups) $varGrps | ForEach-Object{ $_ | ForEach-Object{ try { $varGroupExistinST = $variableGroupSTMapping.data | Where-Object -Property variableGroupID -eq $_ if(!$varGroupExistinST) { $cachedVGItem = $this.GetResourceDataFromCache("Build",$relDef.id,"VariableGroup", $_) $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $cachedVGItem.ResourceName; variableGroupID = $cachedVGItem.ResourceID; serviceID = $cachedVGItem.ServiceTreeID; projectName = $this.ProjectName; projectID = $cachedVGItem.ProjectID; orgName = $cachedVGItem.OrgName } ) } } catch { } } } } } if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { try { if(($secureFiles | Measure-Object).Count -gt 0) { $secureFiles | ForEach-Object{ $_ | ForEach-Object{ $secFileExistinST = $secureFileSTMapping.data | Where-Object -Property secureFileID -eq $_ if (!$secFileExistinST) { $cachedSecFileItem = $this.GetResourceDataFromCache("Build",$relDef.id,"SecureFile", $_) $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $cachedSecFileItem.name; secureFileID = $cachedSecFileItem.id; serviceID = $cachedSecFileItem.serviceID; projectName = $this.ProjectName; projectID = $cachedSecFileItem.ProjectID; orgName = $cachedSecFileItem.OrgName } ) } } } } } catch { #eat exception } } } # attribution of variable group/ secure file linked with release hidden [void] FindSTWithReleaseForVGSecFile($relDef, $varGrps,$secureFiles,$accessToken,$vgDetails, $secureFileDetails , $variableGroupSTMapping, $secureFileSTMapping) { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings for release",$null); if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if(($varGrps | Measure-Object).Count -gt 0){ $varGrps | ForEach-Object{ $_ | ForEach-Object{ try { $vg = $_; $varGrpObj = $vgDetails | Where-Object {$_.name -eq $vg -or $_.id -eq $vg} $varGroupExistinSt = $variableGroupSTMapping.data | Where-Object -Property variableGroupID -eq $vg $cachedVGItem = $this.GetResourceDataFromCache("Release",$relDef.id,"VariableGroup", $vg) if($this.resourceInCacheWithoutPipeline -eq $true){ $this.resourceInCacheWithoutPipeline = $false } else{ $mappingValid = $false if($cachedVGItem) { $mappingValid= $cachedVGItem.MappingExpiration -ge (Get-Date).ToUniversalTime().ToString('dd/MM/yyyy HH:mm:ss') -and $cachedVGItem.PipelineLastModified -ge $relDef.modifiedOn } if(!$varGroupExistinSt -and !$mappingValid) { if($varGrpObj) { $releaseSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $releaseObj[0].id) }; if($releaseSTData) { $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $varGrpObj.name; variableGroupID = $varGrpObj.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) # add variable group mapping details in cache $this.AddMappinginfoInCache($releaseSTData.orgName,$releaseSTData.projectID,$relDef.id,$relDef.name, $releaseSTData.serviceID,$relDef.modifiedOn,$varGrpObj.id,$varGrpObj.name,"VariableGroup","Release",(Get-date).AddDays($this.MappingExpirationLimit)); } else { if ($varGrpObj.Type -eq 'AzureKeyVault') { try { # get associated service connection id for variable group $servConnID = $varGrpObj[0].providerData.serviceEndpointId; # get azure subscription id from service connection $inputbody = "{'contributionIds':['ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider'],'dataProviderContext':{'properties':{'serviceEndpointId':'$($servConnID)','projectId':'$($this.projectId)','sourcePage':{'url':'$($sourcePageUrl)','routeId':'ms.vss-admin-web.project-admin-hub-route','routeValues':{'project':'$($this.ProjectName)','adminPivot':'adminservices','controller':'ContributedPage','action':'Execute'}}}}}" | ConvertFrom-Json $responseObj = [WebRequestHelper]::InvokePostWebRequest($apiURL, $inputbody); if ([Helpers]::CheckMember($responseObj, "dataProviders") -and $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider") { $serviceConnEndPointDetail = $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider" if ($serviceConnEndPointDetail.serviceEndpoint.type -eq "azurerm") { try { $responseObj = $this.GetServiceIdWithSubscrId($serviceConnEndPointDetail.serviceEndpoint.data.subscriptionId,$accessToken) if($responseObj) { $serviceId = $responseObj[2].Rows[0][4]; $projectID = $serviceConnEndPointDetail.serviceEndpoint.serviceEndpointProjectReferences.projectReference.id; $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $varGrpObj.name; variableGroupID = $varGrpObj.id; serviceID = $serviceId; projectName = $serviceConnEndPointDetail.serviceEndpoint.serviceEndpointProjectReferences.projectReference.name; projectID = $projectID; orgName = $this.OrgName } ) # add variable group mapping details in cache $this.AddMappinginfoInCache($this.OrgName,$projectID ,$relDef.id,$relDef.name, $serviceId,$relDef.modifiedOn,$varGrpObj.id,$varGrpObj.name,"VariableGroup","Release",(Get-date).AddDays($this.MappingExpirationLimit)); } } catch { } } } } catch { } } } } } } } catch { } } } } } if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { try { if(($secureFiles | Measure-Object).Count -gt 0) { $secureFiles | ForEach-Object{ $_ | ForEach-Object{ $secureFile = $_ $secureFilesObj = $secureFileDetails | Where-Object {$_.name -eq $secureFile -or $_.id -eq $secureFile} $secFileExistinSt = $secureFileSTMapping.data | Where-Object -Property secureFileID -eq $secureFile $cachedSecFileItem = $this.GetResourceDataFromCache("Release",$relDef.id,"SecureFile", $secureFile) if($this.resourceInCacheWithoutPipeline -eq $true){ $this.resourceInCacheWithoutPipeline = $false; } else { $mappingValid = $false if($cachedSecFileItem) { $mappingValid = $cachedSecFileItem.MappingExpiration -ge (Get-Date).ToUniversalTime().ToString('dd/MM/yyyy HH:mm:ss') -and $cachedSecFileItem.PipelineLastModified -ge $relDef.modifiedOn } if(!$secFileExistinSt -and !$mappingValid) { if ($secureFilesObj) { $releaseSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $relDef.id) }; if($releaseSTData){ $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $secureFilesObj.name; secureFileID = $secureFilesObj.id; serviceID = $releaseSTData.serviceID; projectName = $releaseSTData.projectName; projectID = $releaseSTData.projectID; orgName = $releaseSTData.orgName } ) # add secure file mapping details in cache $this.AddMappinginfoInCache($releaseSTData.orgName,$releaseSTData.projectID,$relDef.id,$relDef.name, $releaseSTData.serviceID,$relDef.modifiedOn,$secureFilesObj.id,$secureFilesObj.name,"SecureFile","Release",(Get-date).AddDays($this.MappingExpirationLimit)); } } } } } } } } catch { #eat exception } } } # find cached mappings for variable group/ secure file linked with release hidden [void] FindSTWithReleaseForVGSecFileCache($relDef, $varGrps,$secureFiles,$accessToken,$vgDetails, $secureFileDetails, $variableGroupSTMapping, $secureFileSTMapping) { if ($this.MappingType -eq "All" -or $this.MappingType -eq "VariableGroup") { if(($varGrps | Measure-Object).Count -gt 0) { $varGrps | ForEach-Object{ $_ | ForEach-Object{ try { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings from cache - variable group",$null); $varGroupExistinST = $variableGroupSTMapping.data | Where-Object -Property variableGroupID -eq $_ if(!$varGroupExistinST) { $cachedVGItem = $this.GetResourceDataFromCache("Release",$relDef.id,"VariableGroup", $_) $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $cachedVGItem.ResourceName; variableGroupID = $cachedVGItem.ResourceID; serviceID = $cachedVGItem.ServiceTreeID; projectName = $this.ProjectName; projectID = $cachedVGItem.ProjectID; orgName = $cachedVGItem.OrgName } ) } } catch { } } } } } if ($this.MappingType -eq "All" -or $this.MappingType -eq "SecureFile") { [ServiceMappingCacheHelper]::TelemetryLogging("FetchVarGrpSecureFileMapping - getting mappings from cache - securefile",$null); try { if(($secureFiles | Measure-Object).Count -gt 0) { $secureFiles | ForEach-Object{ $_ | ForEach-Object{ $secureFilesObj = $secureFileSTMapping.data | Where-Object -Property secureFileID -eq $_ if (!$secureFilesObj) { $cachedSecFileItem = $this.GetResourceDataFromCache("Release",$relDef.id,"SecureFile", $_) $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $secureFilesObj.name; secureFileID = $secureFilesObj.id; serviceID = $cachedSecFileItem.serviceID; projectName = $cachedSecFileItem.projectName; projectID = $cachedSecFileItem.projectID; orgName = $cachedSecFileItem.orgName } ) } } } } } catch { #eat exception } } } # log scan time duration for all resources hidden [void] SaveScanDuration($message,[switch] $finished) { $duration = [math]::Round($this.Stopwatch.Elapsed.TotalMinutes,0) $this.PublishCustomMessage("$($message) : $($duration)", [MessageType]::Info); if($finished) { $this.PublishCustomMessage("Total duration to finish the resource scan : $($duration - $this.lastDuration)", [MessageType]::Info); [ServiceMappingCacheHelper]::TelemetryLogging("Total duration to finish the resource scan : $($duration - $this.lastDuration)",$null); $this.lastDuration = $duration } } # method to fetch secure file mappings from cloudmine data hidden [void] FindSTForSecureFileWithIncremental($isSTMappingWorkFlow) { $secureFileDetails = @(); $secureFileSTMapping = @{ data = [System.Collections.Generic.List[PSCustomObject]]@(); }; #get all secure file details in one common object if (($secureFileDetails | Measure-Object).count -eq 0) { $secureFilesURL = "https://dev.azure.com/{0}/{1}/_apis/distributedtask/securefiles?api-version=6.1-preview.1" -f $this.OrgName, $this.projectId; $secureFileDetails = [WebRequestHelper]::InvokeGetWebRequest($secureFilesURL); } #either retrieve access token for the cluster or use a token via env variable (to be used to generate mappings when user doesn't have access to cluster and can use another authorized token) if ($env:AccessToken) { $accessToken = $env:AccessToken } else { $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) } if ($env:DataDuration) { $dataDuration = $env:DataDuration } else { $dataDuration = 30 } $header = @{ "Authorization" = "Bearer " + $accessToken } try { #add from build pipeline query $kustoQueryBody = '{"db":"AzureDevOps","csl":"{query}","properties":{"Options":{"servertimeout":"00:04:00","queryconsistency":"strongconsistency","query_language":"csl","request_readonly":false,"request_readonly_hardline":false}}}' $buildQueryInputBody = $kustoQueryBody.Replace("{query}",([KustoQueries]::SecureFileBuildQuery)) $this.AddSTDataInStorage($buildQueryInputBody,$dataDuration,$header,$false,$isSTMappingWorkFlow); #add from release pipeline query $releaseQueryInputBody = $kustoQueryBody.Replace("{query}",([KustoQueries]::SecureFileReleaseQuery)) $this.AddSTDataInStorage($releaseQueryInputBody,$dataDuration,$header,$false,$isSTMappingWorkFlow); #add from yaml query $yamlQueryInputBody = $kustoQueryBody.Replace("{query}",([KustoQueries]::SecureFileYAMLQuery)) $this.AddSTDataInStorage($yamlQueryInputBody,$dataDuration,$header,$true,$isSTMappingWorkFlow); if(!$isSTMappingWorkFlow){ return; } $storageData = $this.ServiceMappingCacheHelperObj.GetWorkItemByHashAzureTable("SecureFile", "", "", "", $this.projectId) #Create the ST mapping file from the storage table $progressCount =1; $storageData | foreach { if ($sw.Elapsed.TotalMilliseconds -ge 10000) { Write-Progress -Activity "Fetching service IDs for secure files... " -Status "Progress: " -PercentComplete ($progressCount / $storageData.Count * 100) $sw.Reset(); $sw.Start() } $progressCount++; $dateDiff = New-TimeSpan -Start ([datetime]$_.Timestamp) -End ([datetime]::UtcNow) #if the mapping has been added in the table recently, we need not find the mapping again as it has been already done above #if data is not added today, pipeline mapping might have been changed, hence get the mapping again $resourceObj = $_; if ($dateDiff.Days -gt 1) { if ($resourceObj.PipelineType -eq "Build") { $pipelineSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $resourceObj.PipelineID) } if($pipelineSTData){ $pipelineName = $pipelineSTData.buildDefinitionName } } else { $pipelineSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $resourceObj.PipelineID) } if($pipelineSTData){ $pipelineName = $pipelineSTData.releaseDefinitionName } } #if we have reached mapping expiration check if the secure file still exists or if the pipeline ST data exists if($resourceObj.MappingExpiration -ge (Get-Date).ToUniversalTime().ToString('dd/MM/yyyy HH:mm:ss')){ $secureFileObj = $secureFileDetails | Where-Object { $_.Id -eq $resourceObj.ResourceID } if (!$secureFileObj) { $this.ServiceMappingCacheHelperObj.DeleteDataFromTable($resourceObj.ProjectID, $resourceObj.ResourceID, $resourceObj.ResourceType) return; } } if ($pipelineSTData) { if($pipelineSTData.serviceID -ne $resourceObj.ServiceTreeID){ $this.AddMappinginfoInCache(($pipelineSTData.orgName).ToLower(), $pipelineSTData.projectID, $_.PipelineID,$pipelineName, $pipelineSTData.serviceID, $_.PipelineLastModified, $_.ResourceID, $_.ResourceName, "SecureFile", $_.PipelineType, (Get-date).AddDays($this.MappingExpirationLimit)); } $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $_.ResourceName; secureFileID = $_.ResourceID; serviceID = $pipelineSTData.serviceID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } else{ $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $_.ResourceName; secureFileID = $_.ResourceID; serviceID = $_.ServiceTreeID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } } else { $secureFileSTMapping.data += @([PSCustomObject] @{ secureFileName = $_.ResourceName; secureFileID = $_.ResourceID; serviceID = $_.ServiceTreeID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } } $this.PublishCustomMessage("Service mapping found: $(($secureFileSTMapping.data | Measure-Object).Count)", [MessageType]::Info) $this.ExportObjToJsonFile($secureFileSTMapping, 'SecureFileSTData.json'); $this.ExportObjToJsonFileUploadToBlob($secureFileSTMapping, 'SecureFileSTData.json'); } catch { $_ } } hidden [void] AddSTDataInStorage($inputbody,$dataDuration,$header, $isYAMLQuery, $isSTMappingWorkflow){ $apiURL = "https://1es.kusto.windows.net/v2/rest/query" $inputbody = $inputbody.Replace("{0}", $this.OrgName) $inputbody = $inputbody.Replace("{1}", $this.projectId) $inputbody = $inputbody.Replace("{2}", $dataDuration) $response = [WebRequestHelper]::InvokeWebRequest([Microsoft.PowerShell.Commands.WebRequestMethod]::Post, $apiURL, $header, $inputbody, "application/json; charset=UTF-8"); $response[2].Rows | foreach { $secureFileId = $_[0].ToString(); $pipelineId = $_[2].ToString(); $pipelineProcessDate = $_[1].ToString(); #if this is a response from yaml query, secure file column may contain secure file name or ID if($isYAMLQuery){ $pipelineType = 'Build' $secureFileObj = $secureFileDetails | Where-Object { $_.Id -eq $secureFileId -or $_.Name -eq $secureFileId} } else{ $pipelineType = $_[3].ToString(); $secureFileObj = $secureFileDetails | Where-Object { $_.Id -eq $secureFileId } } if ($secureFileObj) { $secureFileId = $secureFileObj.Id $secureFileName = $secureFileObj.Name } else { return; } $item = $this.GetResourceDataFromCache($pipelineType, $pipelineId, "SecureFile", $secureFileId) #if this is from inactive resources workflow, no need to find service ID if(!$isSTMappingWorkflow){ #if item is already present and pipelineLastModified is more than the current object, do nothing #else update the table with the current object details if ($item) { if ([datetime] $item.PipelineLastModified -gt $pipelineProcessDate) { return; } } $this.AddMappinginfoInCache(($this.OrgName).ToLower(), $this.projectId, $pipelineId, "", "", $pipelineProcessDate, $secureFileId, $secureFileName, "SecureFile", $pipelineType, ""); return; } if ($pipelineType -eq "Build") { $pipelineSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $pipelineId) } if($pipelineSTData){ $pipelineName = $pipelineSTData.buildDefinitionName } } else { $pipelineSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $pipelineId) } if($pipelineSTData){ $pipelineName = $pipelineSTData.releaseDefinitionName } } #if data is already in the storage, we had found the mapping previously. Update the mapping only if this pipeline was modified recently. if ($item) { if ([datetime] $item.PipelineLastModified -gt $pipelineProcessDate) { return; } #if the pipeline was modified recently but the mapping for this new pipeline doesnt exist, do not do anything if (!$pipelineSTData) { return; } } if ($pipelineSTData) { $this.AddMappinginfoInCache(($pipelineSTData.orgName).ToLower(), $pipelineSTData.projectID, $pipelineId, $pipelineName, $pipelineSTData.serviceID, $pipelineProcessDate, $secureFileId, $secureFileName, "SecureFile", $pipelineType, (Get-date).AddDays($this.MappingExpirationLimit)); } } } #method to fetch variable group mappings from cloudmin data hidden [void] FindSTForVGWithIncremental($isSTMappingWorkFlow) { $variableGroupDetails = @(); $variableGroupSTMapping = @{ data = [System.Collections.Generic.List[PSCustomObject]]@(); }; #get all variable group details in one object if (($variableGroupDetails | Measure-Object).count -eq 0) { $variableGroupsURL = "https://dev.azure.com/{0}/{1}/_apis/distributedtask/variablegroups?api-version=6.1-preview.1" -f $this.OrgName, $this.projectId; $variableGroupDetails = [WebRequestHelper]::InvokeGetWebRequest($variableGroupsURL); } #either retrieve access token for the cluster or use a token via env variable (to be used to generate mappings when user doesn't have access to cluster and can use another authorized token) if ($env:AccessToken) { $accessToken = $env:AccessToken } else { $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) } if ($env:DataDuration) { $dataDuration = $env:DataDuration } else { $dataDuration = 30 } $apiURL = "https://1es.kusto.windows.net/v2/rest/query" $kustoQueryBody = '{"db":"AzureDevOps","csl":"{query}","properties":{"Options":{"servertimeout":"00:04:00","queryconsistency":"strongconsistency","query_language":"csl","request_readonly":false,"request_readonly_hardline":false}}}' $inputbody = $kustoQueryBody.Replace("{query}",([KustoQueries]::VariableGroupQuery)) $inputbody = $inputbody.Replace("{0}", $this.OrgName) $inputbody = $inputbody.Replace("{1}", $this.projectId) $inputbody = $inputbody.Replace("{2}", $dataDuration) $header = @{ "Authorization" = "Bearer " + $accessToken } try { $response = [WebRequestHelper]::InvokeWebRequest([Microsoft.PowerShell.Commands.WebRequestMethod]::Post, $apiURL, $header, $inputbody, "application/json; charset=UTF-8"); $sw = [System.Diagnostics.Stopwatch]::StartNew() $progressCount = 1; $varGrpsDetailsFromCM = @() $response[2].Rows | foreach { if ($sw.Elapsed.TotalMilliseconds -ge 10000) { Write-Progress -Activity "Retrieving variable groups... " -Status "Progress: " -PercentComplete ($progressCount / $response[2].Rows.Count * 100) $sw.Reset(); $sw.Start() } $progressCount++; $variableGroupId = $_[0].ToString(); $pipelineId = $_[1].ToString(); $pipelineName = $_[4].ToString(); $pipelineProcessDate = $_[2].ToString(); $variableGroupObj = $variableGroupDetails | Where-Object { $_.Id -eq $variableGroupId } #check if variable group exists currently or is the data from a deleted variable group if ($variableGroupObj) { $variableGroupName = $variableGroupObj.Name } else { return; } $pipelineType = $_[3].ToString(); $detail = [PSCustomObject]@{ variableGroupId = $variableGroupId variableGroupName = $variableGroupName pipelineId = $pipelineId pipelineName = $pipelineName pipelineType = $pipelineType pipelineProcessDate = $pipelineProcessDate } $varGrpsDetailsFromCM+=$detail } $groups = $varGrpsDetailsFromCM | Group-Object "variableGroupId" $cachedObj = @{} #to cache build pipeline details $varGrpDetails = @() #to store final variable groups details $sw = [System.Diagnostics.Stopwatch]::StartNew() $progressCount = 1; foreach($group in $groups){ if ($sw.Elapsed.TotalMilliseconds -ge 10000) { Write-Progress -Activity "Grouping variable groups... " -Status "Progress: " -PercentComplete ($progressCount / $groups.Count * 100) $sw.Reset(); $sw.Start() } $progressCount++; [datetime] $maxLastActivity = 0 #max last activity for this variable group $pipelineObj = @() #contains var grp and pipeline details of the pipeline that most recently accessed the var grp #find the pipeline that last accessed the variable group $group.Group | foreach { [datetime] $lastActivity = 0 $varGrpObj = $_ [datetime] $createdDate = 0 [datetime] $queuedDate = 0 #if pipeline is Release, max time the pipeline used this variable is returned from cloudmine if($varGrpObj.pipelineType -eq "Release"){ $lastActivity = [datetime] ($varGrpObj.pipelineProcessDate) } #if pipeline is Build, max time may depend on the last build else{ #if pipeline details were found before, get the details from cache, else retrieve and add it if($cachedObj.ContainsKey(($varGrpObj.pipelineId))){ $createdDate = (Get-Date $cachedObj[($varGrpObj.pipelineId)].createdDate).ToUniversalTime() $queuedDate = (Get-Date $cachedObj[($varGrpObj.pipelineId)].queuedDate).ToUniversalTime() } else{ $url = ("https://dev.azure.com/{0}/{1}/_apis/build/definitions/{2}?&includeLatestBuilds=True&api-version=6.0" ) -f $($this.orgName), $this.projectName, ($varGrpObj.pipelineId); $buildObj = $null #get the latest build, will fall into catch if build definition doesn't exist try{ $buildObj = [WebRequestHelper]::InvokeGetWebRequest($url); } catch{ #eat exception } if($buildObj){ $createdDate = (Get-Date $buildObj.createdDate).ToUniversalTime() if([Helpers]::CheckMember($buildObj,"latestBuild")){ $queuedDate = (Get-Date $buildObj.latestBuild.queueTime).ToUniversalTime() } else{ $queuedDate = 0 } } else{ $createdDate = 0 $queuedDate = 0 } $cache = [PSCustomObject]@{ createdDate = $createdDate queuedDate = $queuedDate } $cachedObj[($varGrpObj.pipelineId)] = $cache } $lastActivity = (Get-Date $varGrpObj.pipelineProcessDate).ToUniversalTime() $pipelineProcessDate = (Get-Date $varGrpObj.pipelineProcessDate).ToUniversalTime() #if pipeline has been queued, determine last activity from queue time, else last activity is from cloudmine if($queuedDate -ne 0){ #if pipeline was queued after cloudmine date, check if pipeline was edited #if pipeline is not edited, createdDate and cloudmine date will be same, hence last activity becomes queue date #if pipeline is edited such that it does not use the variable group any more, last activity will be cloudmine date if($queuedDate -gt $varGrpObj.pipelineProcessDate){ if(($createdDate - $pipelineProcessDate).Days -eq 0){ $lastActivity = $queuedDate } } } } #check is last activity for the variable group in the pipeline is more than the previous pipeline #if yes, update max last activity, change the pipelineProcessDate for this variable group #store this var grp object in pipelineObj if($lastActivity -gt $maxLastActivity){ $maxLastActivity = $lastActivity $varGrpObj[0].pipelineProcessDate = $lastActivity.ToString("yyyy-MM-ddTHH:mm:ssZ") $pipelineObj = $varGrpObj } } #after all pipelines for the current var grp have been processed pipelineObj will have the pipeline details that most recently accessed the var grp $varGrpDetails +=$pipelineObj } $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) $sw = [System.Diagnostics.Stopwatch]::StartNew() $progressCount = 1; $varGrpDetails | foreach { if ($sw.Elapsed.TotalMilliseconds -ge 10000) { Write-Progress -Activity "Finding mappings for variable groups... " -Status "Progress: " -PercentComplete ($progressCount / $varGrpDetails.Count * 100) $sw.Reset(); $sw.Start() } $progressCount++ $varGrp = $_ #if called from inactive resource workflow, no need to fetch service ID, simply add details if(!$isSTMappingWorkFlow){ $this.AddMappinginfoInCache(($this.OrgName).ToLower(), $this.projectId, $varGrp.pipelineId, $varGrp.pipelineName, "", $varGrp.pipelineProcessDate, $varGrp.variableGroupId, $varGrp.variableGroupName, "VariableGroup", $varGrp.pipelineType, ""); return; } if ($varGrp.pipelineType -eq "Build") { $pipelineSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $varGrp.pipelineId) } if($pipelineSTData){ $pipelineName = $pipelineSTData.buildDefinitionName } } else { $pipelineSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $varGrp.pipelineId) } if($pipelineSTData){ $pipelineName = $pipelineSTData.releaseDefinitionName } } if ($pipelineSTData) { $this.AddMappinginfoInCache(($pipelineSTData.orgName).ToLower(), $pipelineSTData.projectID, $varGrp.pipelineId, $pipelineName, $pipelineSTData.serviceID, $varGrp.pipelineProcessDate, $varGrp.variableGroupId, $varGrp.variableGroupName, "VariableGroup", $varGrp.pipelineType, (Get-date).AddDays($this.MappingExpirationLimit)); } else { $variableGroupObj = $variableGroupDetails | Where-Object { $_.Id -eq $varGrp.variableGroupId } if ($variableGroupObj.Type -eq 'AzureKeyVault') { $apiURL = "https://{0}.visualstudio.com/_apis/Contribution/HierarchyQuery?api-version=5.0-preview.1" -f $this.OrgName $sourcePageUrl = "https://{0}.visualstudio.com/{1}/_settings/adminservices" -f $this.OrgName, $this.ProjectName; try { # get associated service connection id for variable group $servConnID = $variableGroupObj[0].providerData.serviceEndpointId; # get azure subscription id from service connection $inputbody = "{'contributionIds':['ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider'],'dataProviderContext':{'properties':{'serviceEndpointId':'$($servConnID)','projectId':'$($this.projectId)','sourcePage':{'url':'$($sourcePageUrl)','routeId':'ms.vss-admin-web.project-admin-hub-route','routeValues':{'project':'$($this.ProjectName)','adminPivot':'adminservices','controller':'ContributedPage','action':'Execute'}}}}}" | ConvertFrom-Json $responseObj = [WebRequestHelper]::InvokePostWebRequest($apiURL, $inputbody); if ([Helpers]::CheckMember($responseObj, "dataProviders") -and $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider") { $serviceConnEndPointDetail = $responseObj.dataProviders."ms.vss-serviceEndpoints-web.service-endpoints-details-data-provider" if ([Helpers]::CheckMember($serviceConnEndPointDetail.serviceEndpoint, "type") -and $serviceConnEndPointDetail.serviceEndpoint.type -eq "azurerm") { try { $responseObj = $this.GetServiceIdWithSubscrId($serviceConnEndPointDetail.serviceEndpoint.data.subscriptionId, $accessToken) if ($responseObj) { $serviceId = $responseObj[2].Rows[0][4]; $this.AddMappinginfoInCache(($this.OrgName).ToLower(), $this.projectId, $varGrp.pipelineId,$varGrp.pipelineName, $serviceId, $varGrp.pipelineProcessDate, $varGrp.variableGroupId, $varGrp.variableGroupName, "VariableGroup", $varGrp.pipelineType, (Get-date).AddDays($this.MappingExpirationLimit)); } else{ $accessToken = [ContextHelper]::GetDataExplorerAccessToken($true) } } catch { } } } } catch { } } } } if(!$isSTMappingWorkFlow){ return; } #after getting all mappings, create the ST mapping file $storageData = $this.ServiceMappingCacheHelperObj.GetWorkItemByHashAzureTable("VariableGroup", "", "", "", $this.projectId) $progressCount = 1; $storageData | foreach { if ($sw.Elapsed.TotalMilliseconds -ge 10000) { Write-Progress -Activity "Fetching service IDs for variable groups... " -Status "Progress: " -PercentComplete ($progressCount / $storageData.Count * 100) $sw.Reset(); $sw.Start() } $progressCount++; $dateDiff = New-TimeSpan -Start ([datetime]$_.Timestamp) -End ([datetime]::UtcNow) $resourceObj = $_; #if the mapping has been added in the table recently, we need not find the mapping again as it has been already done above if ($dateDiff.Days -gt 1) { if ($resourceObj.PipelineType -eq "Build") { $pipelineSTData = $this.BuildSTDetails.Data | Where-Object { ($_.buildDefinitionID -eq $resourceObj.PipelineID) } if($pipelineSTData){ $pipelineName = $pipelineSTData.buildDefinitionName } } else { $pipelineSTData = $this.ReleaseSTDetails.Data | Where-Object { ($_.releaseDefinitionID -eq $resourceObj.PipelineID) } if($pipelineSTData){ $pipelineName = $pipelineSTData.releaseDefinitionName } } #if we have reached mapping expiration check if the variable group still exists or if the pipeline ST data exists if($resourceObj.MappingExpiration -ge (Get-Date).ToUniversalTime().ToString('dd/MM/yyyy HH:mm:ss')){ $varGrpObj = $variableGroupDetails | Where-Object { $_.Id -eq $resourceObj.ResourceID } #if variable group no longer exists remove it if (!$varGrpObj) { $this.ServiceMappingCacheHelperObj.DeleteDataFromTable($resourceObj.ProjectID, $resourceObj.ResourceID, $resourceObj.ResourceType) return; } } if ($pipelineSTData) { if($pipelineSTData.serviceID -ne $resourceObj.ServiceTreeID){ $this.AddMappinginfoInCache(($pipelineSTData.orgName).ToLower(), $pipelineSTData.projectID, $_.PipelineID,$pipelineName, $pipelineSTData.serviceID, $_.PipelineLastModified, $_.ResourceID, $_.ResourceName, "VariableGroup", $_.PipelineType, (Get-date).AddDays($this.MappingExpirationLimit)); } $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $_.ResourceName; variableGroupID = $_.ResourceID; serviceID = $pipelineSTData.serviceID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } else{ $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $_.ResourceName; variableGroupID = $_.ResourceID; serviceID = $_.ServiceTreeID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } } else { $variableGroupSTMapping.data += @([PSCustomObject] @{ variableGroupName = $_.ResourceName; variableGroupID = $_.ResourceID; serviceID = $_.ServiceTreeID; projectName = $this.ProjectName; projectID = $_.projectID; orgName = $_.orgName } ) } } $this.PublishCustomMessage("Service mapping found: $(($variableGroupSTMapping.data | Measure-Object).Count)", [MessageType]::Info) $this.ExportObjToJsonFile($variableGroupSTMapping, 'VariableGroupSTData.json'); $this.ExportObjToJsonFileUploadToBlob($variableGroupSTMapping, 'VariableGroupSTData.json'); } catch { $_ } } } |