Public/Get-PSUADOVariableGroupInventory.ps1

function Get-PSUADOVariableGroupInventory {
    <#
    .SYNOPSIS
        Retrieves an inventory of Azure DevOps variable groups across projects.
 
    .DESCRIPTION
        This function connects to Azure DevOps using a Personal Access Token (PAT) and retrieves
        comprehensive metadata for variable groups across matching projects. It provides detailed
        inventory information including creation/modification metadata, variable counts, and
        optional export capabilities.
 
        Features:
        - Project filtering with wildcard support
        - Parallel processing using Microsoft ThreadJobs for improved performance
        - Comprehensive error handling and logging
        - Optional CSV export functionality
        - Pipeline support for batch processing
        - Detailed progress reporting
        - Type validation and parameter validation
 
        Array of variable group inventory objects with the following properties:
        - OrganizationName: The Azure DevOps organization name
        - ProjectName: The project containing the variable group
        - ProjectId: The unique project identifier
        - VariableGroupId: The unique variable group identifier
        - VariableGroupName: The display name of the variable group
        - VariableGroupType: The type of variable group (Vsts or AzureKeyVault)
        - Description: Variable group description (if any)
        - CreatedBy: Display name of the user who created the variable group
        - CreatedDate: Date and time when the variable group was created
        - ModifiedBy: Display name of the user who last modified the variable group
        - ModifiedDate: Date and time when the variable group was last modified
        - VariableCount: Total number of variables in the group
        - SecretVariableCount: Number of secret/encrypted variables
        - IsShared: Whether the variable group is shared across pipelines
        - PSTypeName: Custom type name for formatting
 
    .PARAMETER Organization
        The name of the Azure DevOps organization. This is the part that appears in your Azure DevOps URL
        (e.g., 'omg' in https://dev.azure.com/omg).
 
    .PARAMETER PAT
        Azure DevOps Personal Access Token with appropriate permissions. If not provided, the function
        will attempt to use the $env:ADO_PAT or $env:PAT environment variable. The PAT must have at least
        'Variable Groups (read)' and 'Project and Team (read)' permissions.
 
    .PARAMETER $project
        Optional wildcard filter for project names. Supports standard PowerShell wildcard patterns.
        Examples: '*Services*', 'Prod-*', '*API*'
        Default: '*' (all projects)
 
    .PARAMETER OutputFilePath
        Optional path to export the inventory as a CSV file. If specified, the results will be
        exported to this location in addition to being returned as objects.
 
    .PARAMETER IncludeVariableDetails
        Switch parameter to include additional variable-level details in the output. When enabled,
        provides information about variable types and counts by category.
 
    .PARAMETER ThrottleLimit
        Maximum number of concurrent threads when processing multiple projects using ThreadJobs.
        Default: 10 (recommended to balance performance with API throttling)
        Range: 1-20
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG'
 
        Retrieves variable group inventory for all projects in the 'OMG' organization.
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG' -Project @('ProjectA', 'ProjectB')
 
        Retrieves variable group inventory for specific projects only.
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG' -Project @('*-Prod', '*-Dev') -OutputFilePath 'C:\Reports\VarGroups.csv'
 
        Retrieves variable groups from projects matching wildcard patterns and exports to CSV.
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG' -Filter 'VariableGroupName -like "*prod*"'
 
        Retrieves variable groups with names containing 'prod'.
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG' -ThrottleLimit 15
 
        Retrieves variable group inventory with higher concurrency (15 parallel threads) for faster processing of large organizations.
 
    .EXAMPLE
        Get-PSUADOVariableGroupInventory -Organization 'OMG' -IncludeVariableDetails -Verbose
 
        Retrieves detailed variable group inventory with verbose logging and additional variable metadata.
 
    .INPUTS
        None
 
    .OUTPUTS
        [PSCustomObject[]]
 
    .NOTES
        Author: Lakshmanachari Panuganti
        Date: 2025-06-16
        Updated: 2025-07-24 - Added ThreadJobs for parallel processing
 
        Requirements:
        - PowerShell 5.1 or later
        - ThreadJob module (Install-Module ThreadJob) for parallel processing
        - Network access to dev.azure.com
        - Valid Azure DevOps PAT with appropriate permissions
 
        Performance Considerations:
        - Uses Microsoft ThreadJobs for parallel processing of projects
        - Configurable throttle limit to balance speed with API rate limits
        - Includes progress reporting for long-running operations
 
    .LINK
        https://docs.microsoft.com/en-us/rest/api/azure/devops/distributedtask/variablegroups
    #>


    [CmdletBinding(DefaultParameterSetName = 'Default')]
    [OutputType([PSCustomObject[]])]
    param (
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string]$Organization,

        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string]$PAT,

        [Parameter()]
        [ValidateNotNullOrEmpty()]
        [string[]]$Project = '*',

        [Parameter()]
        [ValidateScript({
                $directory = Split-Path $_ -Parent
                if ($directory -and -not (Test-Path $directory)) {
                    throw "Directory does not exist: $directory"
                }
                $extension = [System.IO.Path]::GetExtension($_)
                if ($extension -notin @('.csv', '.json', '.xml', 'xlsx')) {
                    throw "Output file must have .csv, .json, or .xml extension"
                }
                return $true
            })]
        [string]$OutputFilePath,

        [Parameter()]
        [switch]$IncludeVariableDetails,

        [Parameter()]
        [ValidateRange(1, 20)]
        [int]$ThrottleLimit = 10
    )

    begin {
        Write-Verbose "Starting Azure DevOps Variable Group inventory process"

        # Initialize PAT from environment if not provided
        if (-not $PAT) {
            $PAT = $env:ADO_PAT ?? $env:PAT
            if (-not $PAT) {
                throw "Personal Access Token is required. Provide via -PAT parameter or set `$env:ADO_PAT environment variable."
            }
            Write-Verbose "Using Personal Access Token from environment variable"
        }

        # Setup authentication headers
        $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$PAT"))
        $authHeaders = @{
            Authorization  = "Basic $base64AuthInfo"
            'Content-Type' = 'application/json'
            'Accept'       = 'application/json'
        }

        # Initialize results collection
        $variableGroupInventory = [System.Collections.Generic.List[PSCustomObject]]::new()

        Write-Verbose "Authentication configured for organization: $Organization"
        Write-Verbose "Project filter: $Project"
        Write-Verbose "Include variable details: $IncludeVariableDetails"
        Write-Verbose "Throttle limit: $ThrottleLimit"

        # Check if ThreadJob module is available
        $useThreadJobs = $false
        try {
            if (Get-Module -ListAvailable -Name ThreadJob) {
                Import-Module ThreadJob -Force -ErrorAction Stop
                $useThreadJobs = $true
                Write-Verbose "ThreadJob module loaded successfully - parallel processing enabled"
            }
            else {
                Write-Warning "ThreadJob module not found. Install it with: Install-Module ThreadJob"
                Write-Information "Falling back to sequential processing..." -InformationAction Continue
            }
        }
        catch {
            Write-Warning "Failed to load ThreadJob module: $($_.Exception.Message)"
            Write-Information "Falling back to sequential processing..." -InformationAction Continue
        }
    }

    process {
        try {
            # Get filtered projects
            Write-Verbose "Retrieving projects from Azure DevOps..."
            $projectsApiUrl = "https://dev.azure.com/$Organization/_apis/projects" +
            "?api-version=7.1-preview.4&`$top=1000&includeCapabilities=false"

            Write-Progress -Activity "Azure DevOps Variable Group Inventory" -Status "Retrieving projects..." -PercentComplete 10

            $projectsResponse = Invoke-RestMethod -Uri $projectsApiUrl -Method Get -Headers $authHeaders -ErrorAction Stop

            # Filter projects based on Project parameter
            if ($Project -and $Project.Count -gt 0) {
                $filteredProjects = @()
                foreach ($projectPattern in $Project) {
                    $matchingProjects = $projectsResponse.value | Where-Object { $_.name -like $projectPattern }
                    $filteredProjects += $matchingProjects
                }
                # Remove duplicates if any
                $filteredProjects = $filteredProjects | Sort-Object id -Unique
            }
            else {
                # Process all projects if no Project filter specified
                $filteredProjects = $projectsResponse.value
            }

            if (-not $filteredProjects) {
                Write-Warning "No projects matched the filter: '$Project'"
                Write-Warning "Available projects: $(($projectsResponse.value.name | Sort-Object) -join ', ')"
                return @()
            }

            Write-Verbose "Found $($filteredProjects.Count) matching projects"
            $processingMethod = if ($useThreadJobs -and $filteredProjects.Count -gt 1) { "parallel ($ThrottleLimit threads)" } else { "sequential" }
            Write-Information "Processing $($filteredProjects.Count) projects matching filter '$Project' using $processingMethod processing" -InformationAction Continue

            if ($useThreadJobs -and $filteredProjects.Count -gt 1) {
                # Parallel processing
                Write-Verbose "Starting parallel processing with ThreadJobs"

                # Create scriptblock for processing each project
                $scriptBlock = {
                    param($projectObj, $Organization, $AuthHeaders, $IncludeVariableDetails)

                    $results = [System.Collections.Generic.List[PSCustomObject]]::new()

                    try {
                        # Get variable groups for current project
                        $variableGroupsApiUrl = "https://dev.azure.com/$Organization/$($projectObj.name)/_apis/distributedtask/variablegroups?api-version=7.1-preview.2"

                        $variableGroupsResponse = Invoke-RestMethod -Uri $variableGroupsApiUrl -Method Get -Headers $AuthHeaders -ErrorAction Stop

                        if ($variableGroupsResponse.value -and $variableGroupsResponse.value.Count -gt 0) {
                            foreach ($variableGroup in $variableGroupsResponse.value) {
                                # Calculate variable counts - ensure single integer values
                                $totalVariables = 0
                                $secretVariables = 0

                                if ($variableGroup.variables -and $variableGroup.variables.PSObject.Properties) {
                                    # Get all variable properties and count them properly
                                    $variableProperties = @($variableGroup.variables.PSObject.Properties)
                                    $totalVariables = [int]$variableProperties.Count
                                    $secretProps = @($variableProperties | Where-Object {
                                            $_.Value -and $_.Value.PSObject.Properties['isSecret'] -and $_.Value.isSecret -eq $true
                                        })
                                    $secretVariables = [int]$secretProps.Count

                                }
                                $KeyVaultName = if ($variableGroup.providerData -and $variableGroup.providerData.vault) {
                                    $variableGroup.providerData.vault
                                }
                                elseif ($variableGroup.providerData -and $variableGroup.providerData.serviceEndpointId) {
                                    # Sometimes Key Vault name is in serviceEndpointId or requires additional API call
                                    "ServiceEndpoint:$($variableGroup.providerData.serviceEndpointId)"
                                }
                                else {
                                    $null
                                }
                                # Create inventory object
                                $inventoryItem = [PSCustomObject]@{
                                    OrganizationName    = $Organization
                                    ProjectName         = $projectObj.name
                                    ProjectId           = $projectObj.id
                                    VariableGroupId     = $variableGroup.id
                                    VariableGroupName   = $variableGroup.name
                                    VariableGroupType   = $variableGroup.type ?? 'Vsts'
                                    Description         = $variableGroup.description
                                    CreatedBy           = $variableGroup.createdBy.displayName
                                    CreatedDate         = if ($variableGroup.createdOn) { [datetime]$variableGroup.createdOn } else { $null }
                                    ModifiedBy          = $variableGroup.modifiedBy.displayName
                                    ModifiedDate        = if ($variableGroup.modifiedOn) { [datetime]$variableGroup.modifiedOn } else { $null }
                                    VariableCount       = $totalVariables
                                    SecretVariableCount = if ($IncludeVariableDetails) { $secretVariables } else { $null }
                                    IsShared            = $variableGroup.isShared ?? $false
                                    KeyVaultName        = $KeyVaultName
                                    PSTypeName          = 'PSU.ADO.VariableGroupInventory'
                                }
                                if ($IncludeVariableDetails) {
                                    # Collect variable name and value (omit secrets)
                                    $variableDetails = @()

                                    foreach ($property in $variableProperties) {
                                        $varName = $property.Name
                                        $varValue = if ($property.Value.isSecret -eq $true) {
                                            '********'
                                        }
                                        else {
                                            $property.Value.value
                                        }

                                        $variableDetails += [PSCustomObject]@{
                                            Project           = $projectObj.name
                                            VariableGroupName = $variableGroup.name
                                            VariableName      = $varName
                                            VariableValue     = $varValue
                                            IsSecret          = $property.Value.isSecret -eq $true
                                        }
                                    }
                                    $inventoryItem | Add-Member -MemberType NoteProperty -Name Variables -Value $variableDetails -Force
                                }
                                $results.Add($inventoryItem)
                            }
                        }

                        return @{
                            Success            = $true
                            ProjectName        = $projectObj.name
                            Results            = $results.ToArray()
                            VariableGroupCount = $results.Count
                            ErrorMessage       = $null
                        }
                    }
                    catch {
                        return @{
                            Success            = $false
                            ProjectName        = $projectObj.name
                            Results            = @()
                            VariableGroupCount = 0
                            ErrorMessage       = $_.Exception.Message
                        }
                    }
                }

                # Start ThreadJobs for all projects
                Write-Verbose "Starting $($filteredProjects.Count) ThreadJobs with throttle limit $ThrottleLimit"
                $jobs = @()

                foreach ($projectObj in $filteredProjects) {
                    $job = Start-ThreadJob -ScriptBlock $scriptBlock -ArgumentList $projectObj, $Organization, $authHeaders, $IncludeVariableDetails -ThrottleLimit $ThrottleLimit
                    $jobs += $job
                }

                # Monitor job progress and collect results
                $completedJobs = 0
                $totalJobs = $jobs.Count

                Write-Verbose "Monitoring $totalJobs ThreadJobs for completion..."

                while ($completedJobs -lt $totalJobs) {
                    Start-Sleep -Milliseconds 1000
                    $finishedJobs = $jobs | Where-Object { $_.State -in @('Completed', 'Failed', 'Stopped') }
                    $currentCompleted = $finishedJobs.Count

                    if ($currentCompleted -ne $completedJobs) {
                        $completedJobs = $currentCompleted
                        $percentComplete = [math]::Round(($completedJobs / $totalJobs) * 70) + 20

                        Write-Progress -Activity "Azure DevOps Variable Group Inventory" `
                            -Status "Processing projects in parallel: $completedJobs of $totalJobs completed" `
                            -PercentComplete $percentComplete

                        Write-Verbose "Progress: $completedJobs/$totalJobs jobs completed"
                    }
                }

                # Collect all results
                Write-Verbose "Collecting results from completed ThreadJobs"
                $successfulProjects = 0
                $failedProjects = 0

                foreach ($job in $jobs) {
                    try {
                        $jobResult = Receive-Job -Job $job -Wait -ErrorAction Stop

                        if ($jobResult.Success) {
                            Write-Verbose "Successfully processed project '$($jobResult.ProjectName)' - found $($jobResult.VariableGroupCount) variable groups"
                            if ($jobResult.Results -and $jobResult.Results.Count -gt 0) {
                                $variableGroupInventory.AddRange($jobResult.Results)
                            }
                            $successfulProjects++
                        }
                        else {
                            Write-Warning "Failed to process project '$($jobResult.ProjectName)': $($jobResult.ErrorMessage)"
                            $failedProjects++
                        }
                    }
                    catch {
                        Write-Warning "Error receiving job results for project: $($_.Exception.Message)"
                        $failedProjects++
                    }
                    finally {
                        Remove-Job -Job $job -Force -ErrorAction SilentlyContinue
                    }
                }

                Write-Verbose "ThreadJob processing completed. Successful: $successfulProjects, Failed: $failedProjects"
            }
            else {
                # Sequential processing
                Write-Verbose "Using sequential processing (ThreadJobs not available or single project)"

                $projectIndex = 0
                $totalProjects = $filteredProjects.Count

                foreach ($projectObj in $filteredProjects) {
                    $projectIndex++
                    $percentComplete = [math]::Round(($projectIndex / $totalProjects) * 70) + 20

                    Write-Progress -Activity "Azure DevOps Variable Group Inventory" `
                        -Status "Processing project: $($projectObj.name) ($projectIndex of $totalProjects)" `
                        -PercentComplete $percentComplete

                    Write-Verbose "Processing project: $($projectObj.name) (ID: $($projectObj.id))"

                    try {
                        # Get variable groups for current project
                        $variableGroupsApiUrl = "https://dev.azure.com/$Organization/$($projectObj.name)/_apis/distributedtask/variablegroups?api-version=7.1-preview.2"

                        $variableGroupsResponse = Invoke-RestMethod -Uri $variableGroupsApiUrl -Method Get -Headers $authHeaders -ErrorAction Stop

                        if ($variableGroupsResponse.value -and $variableGroupsResponse.value.Count -gt 0) {
                            Write-Verbose "Found $($variableGroupsResponse.value.Count) variable groups in project '$($projectObj.name)'"

                            foreach ($variableGroup in $variableGroupsResponse.value) {
                                # Calculate variable counts - ensure single integer values
                                $totalVariables = 0
                                $secretVariables = 0

                                if ($variableGroup.variables -and $variableGroup.variables.PSObject.Properties) {
                                    # Get all variable properties and count them properly
                                    $variableProperties = @($variableGroup.variables.PSObject.Properties)
                                    $totalVariables = [int]$variableProperties.Count

                                    $secretProps = @($variableProperties | Where-Object {
                                            $_.Value -and $_.Value.PSObject.Properties['isSecret'] -and $_.Value.isSecret -eq $true
                                        })
                                    $secretVariables = [int]$secretProps.Count

                                }

                                # Create inventory object
                                $inventoryItem = [PSCustomObject]@{
                                    OrganizationName    = $Organization
                                    ProjectName         = $projectObj.name
                                    ProjectId           = $projectObj.id
                                    VariableGroupId     = $variableGroup.id
                                    VariableGroupName   = $variableGroup.name
                                    VariableGroupType   = $variableGroup.type ?? 'Vsts'
                                    Description         = $variableGroup.description
                                    CreatedBy           = $variableGroup.createdBy.displayName
                                    CreatedDate         = if ($variableGroup.createdOn) { [datetime]$variableGroup.createdOn } else { $null }
                                    ModifiedBy          = $variableGroup.modifiedBy.displayName
                                    ModifiedDate        = if ($variableGroup.modifiedOn) { [datetime]$variableGroup.modifiedOn } else { $null }
                                    VariableCount       = $totalVariables
                                    SecretVariableCount = $secretVariables
                                    IsShared            = $variableGroup.isShared ?? $false
                                    KeyVaultName        = if ($variableGroup.providerData) { $variableGroup.providerData.vault } else { $null }
                                    PSTypeName          = 'PSU.ADO.VariableGroupInventory'
                                }

                                if ($IncludeVariableDetails) {
                                    # Collect variable name and value (omit secrets)
                                    $variableDetails = @()

                                    foreach ($property in $variableProperties) {
                                        $varName = $property.Name
                                        $varValue = if ($property.Value.isSecret -eq $true) {
                                            '********'
                                        }
                                        else {
                                            $property.Value.value
                                        }

                                        $variableDetails += [PSCustomObject]@{
                                            Project           = $projectObj.name
                                            VariableGroupName = $variableGroup.name
                                            VariableName      = $varName
                                            VariableValue     = $varValue
                                            IsSecret          = $property.Value.isSecret -eq $true
                                        }
                                    }
                                    $inventoryItem | Add-Member -MemberType NoteProperty -Name Variables -Value $variableDetails -Force
                                }

                                $variableGroupInventory.Add($inventoryItem)
                            }
                        }
                        else {
                            Write-Verbose "No variable groups found in project '$($projectObj.name)'"
                        }
                    }
                    catch {
                        $errorMessage = "Failed to retrieve variable groups for project '$($projectObj.name)': $($_.Exception.Message)"
                        Write-Warning $errorMessage
                        Write-Verbose "Full error details: $($_.Exception | Format-List * | Out-String)"
                        continue
                    }
                }
            }

            Write-Progress -Activity "Azure DevOps Variable Group Inventory" -Status "Generating summary..." -PercentComplete 95

            # Display summary
            if ($variableGroupInventory.Count -eq 0) {
                Write-Information "No variable groups found in the specified projects." -InformationAction Continue
                return @()
            }
            else {
                # Apply Filter if specified (similar to Get-ADUser -Filter)
                $finalResults = $variableGroupInventory

                # TODO: Integrate $Filter parameter to enable custom filter expressions

                # Calculate summary with proper error handling
                $totalVariables = 0
                foreach ($item in $finalResults) {
                    $count = $item.VariableCount
                    if ($count -is [array]) {
                        $totalVariables += [int]$count[0]
                    }
                    else {
                        $totalVariables += [int]$count
                    }
                }

                $summary = @{
                    TotalVariableGroups        = $finalResults.Count
                    ProjectsProcessed          = $filteredProjects.Count
                    ProjectsWithVariableGroups = ($finalResults | Select-Object -Unique ProjectName).Count
                    TotalVariables             = $totalVariables
                }

                Write-Information "Inventory Summary:" -InformationAction Continue
                Write-Information " - Total Variable Groups: $($summary.TotalVariableGroups)" -InformationAction Continue
                Write-Information " - Projects Processed: $($summary.ProjectsProcessed)" -InformationAction Continue
                Write-Information " - Projects with Variable Groups: $($summary.ProjectsWithVariableGroups)" -InformationAction Continue
                Write-Information " - Total Variables: $($summary.TotalVariables)" -InformationAction Continue

                # Export to file if specified
                if ($OutputFilePath) {
                    Write-Progress -Activity "Azure DevOps Variable Group Inventory" -Status "Exporting results..." -PercentComplete 98

                    $extension = [System.IO.Path]::GetExtension($OutputFilePath).ToLower()

                    try {
                        switch ($extension) {
                            '.csv' {
                                $finalResults | Export-Csv -Path $OutputFilePath -NoTypeInformation -Encoding UTF8
                            }
                            '.json' {
                                $finalResults | ConvertTo-Json -Depth 10 | Out-File -FilePath $OutputFilePath -Encoding UTF8
                            }
                            '.xml' {
                                $finalResults | Export-Clixml -Path $OutputFilePath -Encoding UTF8
                            }
                        }
                        Write-Information "Results exported to: $OutputFilePath" -InformationAction Continue
                    }
                    catch {
                        Write-Warning "Failed to export results to '$OutputFilePath': $($_.Exception.Message)"
                    }
                }

                Write-Progress -Activity "Azure DevOps Variable Group Inventory" -Status "Complete" -PercentComplete 100 -Completed

                # Return the inventory
                return $finalResults.ToArray()
            }
        }
        catch {
            Write-Progress -Activity "Azure DevOps Variable Group Inventory" -Status "Error occurred" -PercentComplete 100 -Completed
            $errorMessage = "Failed to retrieve variable group inventory: $($_.Exception.Message)"
            Write-Error $errorMessage
            throw
        }
    }

    end {
        Write-Verbose "Azure DevOps Variable Group inventory process completed"
    }
}