AzSKPreview.ADO

1.24.1

Framework/Helpers/IncrementalScanHelper.ps1

Set-StrictMode -Version Latest 

class IncrementalScanHelper
{
    hidden [string] $OrganizationName = $null;
    hidden [string] $ProjectName = $null;
    hidden [string] $ProjectId = $null;
    hidden $OrganizationContext = $null;
    [PSObject] $ControlSettings;
    hidden [string] $AzSKTempStatePath = (Join-Path $([Constants]::AzSKAppFolderPath) "IncrementalScan");
    hidden [string] $CAScanProgressSnapshotsContainerName = [Constants]::CAScanProgressSnapshotsContainerName;
    hidden [string] $ScanSource = $null;
    $StorageContext = $null;
    $ControlStateBlob = $null;
    $ContainerObject = $null;
    hidden [string] $IncrementalScanTimestampFile=$null;
    hidden [string] $CATempFile = $null;
    hidden [string] $MasterFilePath;
    hidden [PSObject] $ResourceTimestamps = $null;
    hidden [bool] $FirstScan = $false;
    hidden [datetime] $IncrementalDate = 0;
    hidden [datetime] $LastFullScan = 0;
    hidden [bool] $ShouldDiscardOldScan = $false;
    [bool] $UpdateTime = $true;
    hidden [datetime] $Timestamp = 0; 
    [bool] $isPartialScanActive = $false;
    [bool] $IsFullScanInProgress = $false;
    static [PSObject] $auditSchema = $null
    [bool] $isIncFileAlreadyAvailable = $false;
    
    IncrementalScanHelper([string] $organizationName, [string] $projectName, [datetime] $incrementalDate, [bool] $updateTimestamp, [datetime] $timestamp)
    {
        $this.OrganizationName = $organizationName
        $this.ProjectName = $projectName
        $this.IncrementalScanTimestampFile = $([Constants]::IncrementalScanTimeStampFile)
        $this.ScanSource = [AzSKSettings]::GetInstance().GetScanSource()
        $this.CATempFile = "CATempLocal.json" # temporary file to store Json Data to upload to container (in CA)
        $this.IncrementalDate = $incrementalDate
        $this.MasterFilePath = (Join-Path (Join-Path (Join-Path $this.AzSKTempStatePath $this.OrganizationName) $this.projectName) $this.IncrementalScanTimestampFile)
        $this.UpdateTime = $updateTimestamp
        $this.Timestamp = $timestamp
        $this.ControlSettings = [ConfigurationManager]::LoadServerConfigFile("ControlSettings.json");
        if($PSCmdlet.MyInvocation.BoundParameters.ContainsKey("UsePartialCommits")){
            [PartialScanManager] $partialScanMngr = [PartialScanManager]::GetInstance();
            if(($partialScanMngr.IsPartialScanInProgress($this.OrganizationName, $false) -eq [ActiveStatus]::Yes)){
                $this.isPartialScanActive = $true
            }
        }  
        if($null -eq [IncrementalScanHelper]::auditSchema){
            [IncrementalScanHelper]::auditSchema = [ConfigurationManager]::LoadServerConfigFile("IncrementalScanAudits.json")
        }      
    }
    IncrementalScanHelper($organizationContext, [string] $projectId,[string] $projectName, [datetime] $incrementalDate)
    {
        $this.OrganizationName = $organizationContext.OrganizationName
        $this.OrganizationContext = $organizationContext
        $this.ProjectId = $projectId
        $this.IncrementalScanTimestampFile = $([Constants]::IncrementalScanTimeStampFile)
        $this.ScanSource = [AzSKSettings]::GetInstance().GetScanSource()
        $this.CATempFile = "CATempLocal.json" # temporary file to store Json Data to upload to container (in CA)
        $this.IncrementalDate = $incrementalDate
        $this.ProjectName = $projectName 
        $this.MasterFilePath = (Join-Path (Join-Path (Join-Path $this.AzSKTempStatePath $this.OrganizationName) $this.projectName) $this.IncrementalScanTimestampFile)
        $this.ControlSettings = [ConfigurationManager]::LoadServerConfigFile("ControlSettings.json");
        if($PSCmdlet.MyInvocation.BoundParameters.ContainsKey("UsePartialCommits")){
            [PartialScanManager] $partialScanMngr = [PartialScanManager]::GetInstance();
            if(($partialScanMngr.IsPartialScanInProgress($this.OrganizationName, $false) -eq [ActiveStatus]::Yes)){
                $this.isPartialScanActive = $true
            }
        } 
               
    }
    hidden [datetime] GetThresholdTime([string] $resourceType)
    {
        # function to retrieve threshold time from storage, based on scan source.
        $latestScan = 0
        if($this.ScanSource -ne "CA" -and $this.ScanSource -ne "CICD")
        {
            if(![string]::isnullorwhitespace($this.OrganizationName))
            {
                if(Test-Path $this.MasterFilePath)    
                {
                    # File exists. Retrieve last timestamp.
                    $this.ResourceTimestamps = Get-Content $this.MasterFilePath | ConvertFrom-Json

                    if(-not ([Helpers]::CheckMember($this.ResourceTimestamps, $resourceType)) -or $null -eq $this.ResourceTimestamps.$resourceType -or [datetime]$this.ResourceTimestamps.$resourceType.LastScanTime -eq 0)
                    {
                        # Previous timestamp does not exist for this resource in the existing file.
                        $this.FirstScan = $true
                    }
                }
                else 
                {
                    #file does not exist
                    $this.FirstScan = $true
                }
            }
        }
        elseif ($this.ScanSource -eq 'CA') 
        {
            $this.MasterFilePath = (Join-Path (Join-Path (Join-Path $this.AzSKTempStatePath $this.OrganizationName) $this.ProjectName) $this.IncrementalScanTimestampFile)
            $tempPath = Join-Path $([Constants]::AzSKAppFolderPath) $this.CATempFile
            $blobPath = Join-Path (Join-Path (Join-Path "IncrementalScan" $this.OrganizationName) $this.ProjectName) $this.IncrementalScanTimestampFile
            try 
            {
                #Validate if Storage is found
                $keys = Get-AzStorageAccountKey -ResourceGroupName $env:StorageRG -Name $env:StorageName
                $this.StorageContext = New-AzStorageContext -StorageAccountName $env:StorageName -StorageAccountKey $keys[0].Value -Protocol Https
                $this.ContainerObject = Get-AzStorageContainer -Context $this.StorageContext -Name $this.CAScanProgressSnapshotsContainerName -ErrorAction SilentlyContinue 

                if($null -ne $this.ContainerObject)
                {
                    #container exists
                    $this.ControlStateBlob = Get-AzStorageBlob -Container $this.CAScanProgressSnapshotsContainerName -Context $this.StorageContext -Blob $blobPath -ErrorAction SilentlyContinue 
                    if($null -ne $this.ControlStateBlob)
                    {
                        # File exists. Copy existing timestamp file locally
                        Get-AzStorageBlobContent -CloudBlob $this.ControlStateBlob.ICloudBlob -Context $this.StorageContext -Destination $tempPath -Force                
                        $this.ResourceTimestamps  = Get-ChildItem -Path $tempPath -Force | Get-Content | ConvertFrom-Json
                        #Delete the local file
                        Remove-Item -Path $tempPath
                        if(-not ([Helpers]::CheckMember($this.ResourceTimestamps, $resourceType)) -or $null -eq $this.ResourceTimestamps.$resourceType -or [datetime]$this.ResourceTimestamps.$resourceType.LastScanTime -eq 0)
                        {
                            # Previous timestamp does not exist for current resource in existing file.
                            $this.FirstScan = $true
                        }
                    }
                    else 
                    {
                        # File does not exist.
                        $this.FirstScan = $true
                    }
                }
                else 
                {
                    # Container does not exist
                    $this.FirstScan = $true
                }
            }
            catch
            {
                write-host "Exception when trying to find/create incremental scan container: $_."
            }
        }
        elseif($this.ScanSource -eq 'CICD'){
            if (Test-Path env:incrementalScanURI)
            {
                #Uri is created in cicd task based on jobid
                $uri = $env:incrementalScanURI
            }
            else {
                $uri = [Constants]::StorageUri -f $this.OrgName, $this.OrgName, "IncrementalScanFile"
            }
            try {
                #check if file already in extension sotrage
                $webRequestResult = [WebRequestHelper]::InvokeGetWebRequest($uri)
                if($null -ne $webRequestResult){
                    $this.ResourceTimestamps = $webRequestResult | ConvertFrom-Json
                    if(-not ([Helpers]::CheckMember($this.ResourceTimestamps, $resourceType)) -or $null -eq $this.ResourceTimestamps.$resourceType -or [datetime]$this.ResourceTimestamps.$resourceType.LastScanTime -eq 0)
                    {
                        # Previous timestamp does not exist for this resource in the existing file.
                        $this.FirstScan = $true
                        $this.isIncFileAlreadyAvailable = $true;
                    }
                }
                else{
                    $this.FirstScan = $true
                    $this.isIncFileAlreadyAvailable = $false;
                }                
            }
            catch
            {
                $this.FirstScan = $true
                $this.isIncFileAlreadyAvailable = $false;
            }
        }
        if(-not $this.FirstScan)
        {
            if($this.isPartialScanActive){
                $latestScan = [datetime]$this.ResourceTimestamps.$resourceType.LastPartialTime
                #to check if full scan is currently in progress, if we dont check this and give -dt switch full scan wont work
                if($this.ResourceTimestamps.$resourceType.IsFullScanInProgress){
                    $this.IsFullScanInProgress = $true
                }
                else{
                    $this.IsFullScanInProgress = $false 
                }
            }
            else {
                $latestScan = [datetime]$this.ResourceTimestamps.$resourceType.LastScanTime  
                $this.IsFullScanInProgress = $false      
                
            }
            $this.LastFullScan = [datetime]$this.ResourceTimestamps.$resourceType.LastFullScanTime
            
        }
        if($this.IncrementalDate -ne 0)
        {
            # user input of incremental date to be used for scanning incrementally.
            $latestScan = $this.IncrementalDate
            if($this.ScanSource -eq 'CA'){
                $FromTimeZone = [System.TimeZoneInfo]::FindSystemTimeZoneById("Asia/Kolkata")
                $latestScan = [DateTime]::SpecifyKind((Get-Date $latestScan), [DateTimeKind]::Unspecified)
                $latestScan = [System.TimeZoneInfo]::ConvertTimeToUtc($latestScan, $FromTimeZone)

            }
        }
        return $latestScan
    }
    
    UpdateTimeStamp([string] $resourceType)
    {
        # Updates timestamp of current scan to storage, based on scan source.
        if($this.UpdateTime -ne $true)
        {
            return;
        }
        if($this.isPartialScanActive){
            return;
        }
        if($this.ScanSource -ne "CA" -and $this.ScanSource -ne "CICD")
        {
            if($this.FirstScan -eq $true)
            {
                # Check if file exists
                if((-not (Test-Path ($this.AzSKTempStatePath))) -or (-not (Test-Path (Join-Path $this.AzSKTempStatePath $this.OrganizationName))) -or (-not (Test-Path $this.MasterFilePath)))
                {
                    # Incremental Scan happening first time locally OR Incremental Scan happening first time for Org OR first time for current Project
                    New-Item -Type Directory -Path (Join-Path (Join-Path $this.AzSKTempStatePath $this.OrganizationName) $this.ProjectName) -ErrorAction Stop | Out-Null
                    $this.ResourceTimestamps = [IncrementalScanTimestamps]::new()
                    $resourceScanTimes = [IncrementalTimeStampsResources]@{
                        LastScanTime = $this.Timestamp;
                        LastFullScanTime = $this.Timestamp;
                        LastPartialTime = "0001-01-01T00:00:00.0000000";
                        IsFullScanInProgress = $false
                    }
                    $this.ResourceTimestamps.$resourceType = $resourceScanTimes                 
                    [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps) | Out-File $this.MasterFilePath -Force
                }
                else 
                {
                    # File exists for Organization and Project but first time scan for current resource type
                    $this.ResourceTimestamps = Get-ChildItem -Path $this.MasterFilePath -Force | Get-Content | ConvertFrom-Json
                    $resourceScanTimes = [IncrementalTimeStampsResources]@{
                        LastScanTime = $this.Timestamp;
                        LastFullScanTime = $this.Timestamp;
                        LastPartialTime = "0001-01-01T00:00:00.0000000";
                        IsFullScanInProgress = $false
                    }
                    $this.ResourceTimestamps.$resourceType = $resourceScanTimes
                                       
                    [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps) | Out-File $this.MasterFilePath -Force    
                }
            }
            else 
            {
                # Not a first time scan for the current resource
                $this.ResourceTimestamps = Get-ChildItem -Path $this.MasterFilePath -Force | Get-Content | ConvertFrom-Json
                $previousScanTime = $this.ResourceTimestamps.$resourceType.LastScanTime;
                $this.ResourceTimestamps.$resourceType.LastPartialTime= $previousScanTime
                if($this.IsFullScanInProgress -eq $false){
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $false
                }
                #if old scan, we trigger full scan, store full scan value, also reset upc scan time
                if($this.ShouldDiscardOldScan){
                    $this.ResourceTimestamps.$resourceType.LastFullScanTime = $this.Timestamp
                    $this.ResourceTimestamps.$resourceType.LastPartialTime = "0001-01-01T00:00:00.0000000";
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $true
                }   
                $this.ResourceTimestamps.$resourceType.LastScanTime = $this.Timestamp
                [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps) | Out-File $this.MasterFilePath -Force
            }
        }
        elseif ($this.ScanSource -eq 'CA') 
        {
            $tempPath = Join-Path $([Constants]::AzSKAppFolderPath) $this.CATempFile
            $blobPath = Join-Path (Join-Path (Join-Path "IncrementalScan" $this.OrganizationName) $this.ProjectName) $this.IncrementalScanTimestampFile
            if ($this.FirstScan -eq $true) 
            {
                # Check if container object does not exist
                if($null -eq $this.ContainerObject)
                {
                    # Container does not exist, create container.
                    $this.ContainerObject = New-AzStorageContainer -Name $this.CAScanProgressSnapshotsContainerName -Context $this.StorageContext -ErrorAction SilentlyContinue
                    if ($null -eq $this.ContainerObject )
                    {
                        $this.PublishCustomMessage("Could not find/create partial scan container in storage.", [MessageType]::Warning);
                    }
                    $this.ResourceTimestamps = [IncrementalScanTimestamps]::new()
                }
                if($null -eq $this.ControlStateBlob)
                {
                    $this.ResourceTimestamps = [IncrementalScanTimestamps]::new()
                }
                else 
                {
                    Get-AzStorageBlobContent -CloudBlob $this.ControlStateBlob.ICloudBlob -Context $this.StorageContext -Destination $tempPath -Force                
                    $this.ResourceTimestamps  = Get-ChildItem -Path $tempPath -Force | Get-Content | ConvertFrom-Json
                    #Delete the local file
                    Remove-Item -Path $tempPath

                }
                $resourceScanTimes = [IncrementalTimeStampsResources]@{
                    LastScanTime = $this.Timestamp;
                    LastFullScanTime = $this.Timestamp;
                    LastPartialTime = "0001-01-01T00:00:00.0000000";
                    IsFullScanInProgress = $false
                }
                $this.ResourceTimestamps.$resourceType = $resourceScanTimes             
                                 
                [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps) | Out-File $tempPath -Force
                Set-AzStorageBlobContent -File $tempPath -Container $this.ContainerObject.Name -Blob $blobPath -Context $this.StorageContext -Force
                Remove-Item -Path $tempPath
            }
            else 
            {
                Get-AzStorageBlobContent -CloudBlob $this.ControlStateBlob.ICloudBlob -Context $this.StorageContext -Destination $tempPath -Force                
                $this.ResourceTimestamps  = Get-ChildItem -Path $tempPath -Force | Get-Content | ConvertFrom-Json
                $previousScanTime = $this.ResourceTimestamps.$resourceType.LastScanTime;
                $this.ResourceTimestamps.$resourceType.LastPartialTime = $previousScanTime
                if($this.IsFullScanInProgress -eq $false){
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $false
                }            
                if($this.ShouldDiscardOldScan){
                    $this.ResourceTimestamps.$resourceType.LastFullScanTime = $this.Timestamp
                    $this.ResourceTimestamps.$resourceType.LastPartialTime  = "0001-01-01T00:00:00.0000000";
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $true
                }
                
                # Delete the local file
                Remove-Item -Path $tempPath
                $this.ResourceTimestamps.$resourceType.LastScanTime = $this.Timestamp
                [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps) | Out-File $tempPath -Force
                Set-AzStorageBlobContent -File $tempPath -Container $this.ContainerObject.Name -Blob $blobPath -Context $this.StorageContext -Force
                Remove-Item -Path $tempPath
            }
        }
        elseif($this.ScanSource -eq 'CICD'){
            $incrementalScanPayload = $null
            if($this.FirstScan -eq $true){
                #first scan for the pipeline for all resources
                if($this.isIncFileAlreadyAvailable -eq $false){
                    $this.ResourceTimestamps = [IncrementalScanTimestamps]::new()                  
                }           
                #will be called for both scenarios: first scan for the resource as well as for the entire pipeline
                $resourceScanTimes = [IncrementalTimeStampsResources]@{
                        LastScanTime = $this.Timestamp;
                        LastFullScanTime = $this.Timestamp;
                        LastPartialTime = "0001-01-01T00:00:00.0000000";
                        IsFullScanInProgress = $false
                    }
                $this.ResourceTimestamps.$resourceType = $resourceScanTimes                 
                $incrementalScanPayload = [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps)
            }
            #not a first scan
            else{
                $previousScanTime = $this.ResourceTimestamps.$resourceType.LastScanTime;
                $this.ResourceTimestamps.$resourceType.LastPartialTime= $previousScanTime
                if($this.IsFullScanInProgress -eq $false){
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $false
                }
                #if old scan, we trigger full scan, store full scan value, also reset upc scan time
                if($this.ShouldDiscardOldScan){
                    $this.ResourceTimestamps.$resourceType.LastFullScanTime = $this.Timestamp
                    $this.ResourceTimestamps.$resourceType.LastPartialTime = "0001-01-01T00:00:00.0000000";
                    $this.ResourceTimestamps.$resourceType.IsFullScanInProgress = $true
                }   
                $this.ResourceTimestamps.$resourceType.LastScanTime = $this.Timestamp
                $incrementalScanPayload = [JsonHelper]::ConvertToJsonCustom($this.ResourceTimestamps)
            }
            try{
                $rmContext = [ContextHelper]::GetCurrentContext();
                $user = "";
                $uri = "";
                $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $user,$rmContext.AccessToken)))               
                $body = "";
                if (Test-Path env:incrementalScanURI)
                {
                    $uri = $env:incrementalScanURI
                    $JobId ="";
                    $JobId = $uri.Replace('?','/').Split('/')[$JobId.Length -2]
                    #if the incremental scan is already present need to update the existing file
                    if ($this.FirstScan -eq $false -or $this.isIncFileAlreadyAvailable -eq $true){
                        $body = @{"id" = $Jobid; "__etag"=-1; "value"= $incrementalScanPayload;} | ConvertTo-Json
                    }
                    else{
                        $body = @{"id" = $Jobid; "value"= $incrementalScanPayload;} | ConvertTo-Json
                    }
                }
                else {
                    $uri = [Constants]::StorageUri -f $this.OrgName, $this.OrgName, "IncrementalScanFile"
                    if ($this.FirstScan -eq $false -or $this.isIncFileAlreadyAvailable -eq $true){
                        $body = @{"id" = "IncrementalScanFile";"__etag"=-1; "value"= $incrementalScanPayload;} | ConvertTo-Json
                    }
                    else{
                        $body = @{"id" = "IncrementalScanFile"; "value"= $incrementalScanPayload;} | ConvertTo-Json
                    }
                }
                $webRequestResult = Invoke-WebRequest -Uri $uri -Method Put -ContentType "application/json" -Headers @{Authorization = ("Basic {0}" -f $base64AuthInfo) } -Body $body 
                        
            }
            catch{  
                Write-Host "Error updating Incremental Scan file: $($_)"
            }
        }
    }

    [bool] IsIncScanOld($resourceType){
        $this.GetThresholdTime($resourceType)
        if($this.FirstScan){
            return $false;
        }        
        if($this.LastFullScan.AddDays($this.ControlSettings.IncrementalScan.IncrementalScanValidForDays) -lt [DateTime]::UtcNow){
            return $true;
        }     
     
        return $false;
    }

    [bool] ShouldDiscardOldIncScan($resourceType){
        $this.ShouldDiscardOldScan = $false
        if($this.IsIncScanOld($resourceType)){
            if($PSCmdlet.MyInvocation.BoundParameters.ContainsKey('Force')){
                $this.ShouldDiscardOldScan = $false
            }
            else{
                $this.ShouldDiscardOldScan = $true
            }
            
        }
        return $this.ShouldDiscardOldScan;
    }
    [System.Object[]] GetModifiedBuilds($buildDefnsObj)
    {       
        # Function to filter builds that have been modified after threshold time
        $latestBuildScan = $this.GetThresholdTime("Build")        
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0)
        {
            $this.UpdateTimeStamp("Build")
            return $buildDefnsObj
        }
        #if inc scan last time is 0 or if this is a full scan partial checkpoint, return all builds
        if($this.isPartialScanActive -and ($latestBuildScan -eq 0 -or $this.IsFullScanInProgress)){
            return $buildDefnsObj
        }
        #if scan is old and no upc file found, simply return all builds, update scan time for full scans and last scan
        if($this.ShouldDiscardOldIncScan('Build') -and -not($this.isPartialScanActive)){            
            $this.UpdateTimeStamp("Build")
            return $buildDefnsObj
        }

        $newBuildDefns = @()
        if ([datetime] $buildDefnsObj[0].createdDate -lt $latestBuildScan) 
        {
            # first resource is modified before the threshold time => all consequent are also modified before threshold
            # return empty list
            $this.UpdateTimeStamp("Build")
            return $newBuildDefns
        }
        #Binary search
        [int] $low = 0 # start index of array
        [int] $high = $buildDefnsObj.length - 1 # last index of array
        [int] $size = $buildDefnsObj.length # total length of array
        [int] $breakIndex = 0
        while($low -le $high)
        {
            [int] $mid = ($low + $high)/2 # seeking the middle of the array
            [datetime] $modifiedDate = [datetime]($buildDefnsObj[$mid].createdDate)
            if($modifiedDate -ge $latestBuildScan)
            {
                # modified date is after the threshold time
                if(($mid + 1) -eq $size)
                {
                    # all fetched build defs are modified after threshold time
                    # return unmodified
                    $this.UpdateTimeStamp("Build")
                    return $buildDefnsObj
                }
                else 
                {
                    # mid point is not the last build defn
                    if([datetime]($buildDefnsObj[$mid+1].createdDate) -lt $latestBuildScan)
                    {
                        # changing point found
                        $breakIndex = $mid
                        break
                    }
                    else 
                    {
                        # search on right half
                        $low = $mid + 1
                    }
                }
            }
            elseif ($modifiedDate -lt $latestBuildScan) 
            {
                if($mid -eq 0)
                {
                    # All fetched builds have been modified before the threshold
                    return $newBuildDefns
                }
                else 
                {
                    if([datetime]($buildDefnsObj[$mid - 1].createdDate)  -ge $latestBuildScan)
                    {
                        # changing point found
                        $breakIndex = $mid - 1
                        break
                    }    
                    else 
                    {
                        # search on left half
                        $high = $mid - 1
                    }
                }
            }
        }
        $newBuildDefns = @($buildDefnsObj[0..$breakIndex])
        $this.UpdateTimeStamp("Build")
        return $newBuildDefns
    }
    [System.Object[]] GetModifiedReleases($releaseDefnsObj)
    {
        $latestReleaseScan = $this.GetThresholdTime("Release")
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0)
        {
            $this.UpdateTimeStamp("Release")
            return $releaseDefnsObj
        }
        if($this.isPartialScanActive -and ($latestReleaseScan -eq 0 -or $this.IsFullScanInProgress)){
            return $releaseDefnsObj
        }
        
        if($this.ShouldDiscardOldIncScan('Release')){
            $this.UpdateTimeStamp("Release")
            return $releaseDefnsObj
        }
        $newReleaseDefns = @()
        # Searching Linearly
        foreach ($releaseDefn in $releaseDefnsObj)
        {
            if ([datetime]($releaseDefn.modifiedOn) -ge $latestReleaseScan) 
            {
                $newReleaseDefns += @($releaseDefn)    
            }
        }
        $this.UpdateTimeStamp("Release")
        return $newReleaseDefns                
    }

    #Get all resources attested after the latest scan
    [System.Object[]] GetAttestationAfterInc($projectName, $resourceType){
        $resourceIds = @();
        #if parameter not specified, wont be fetching these resources
        if(-not($PSCmdlet.MyInvocation.BoundParameters.ContainsKey('ScanAttestedResources'))){
            return $resourceIds
        }
        $latestResourceScan = $this.GetThresholdTime($resourceType)
        if($this.ScanSource -ne 'CA'){
            $latestResourceScan=$latestResourceScan.ToUniversalTime();
        }
        $latestResourceScan =Get-Date $latestResourceScan -Format s        
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0){
            return $resourceIds;   
        }
        [ControlStateExtension] $ControlStateExt = [ControlStateExtension]::new($this.OrganizationContext, $PSCmdlet.MyInvocation);
        $output = $ControlStateExt.RescanComputeControlStateIndexer($projectName, 'ADO.'+$resourceType);
        $output | ForEach-Object {
            if($_.AttestedDate -gt $latestResourceScan){
                try {                    
                    $resourceIds += ($_.ResourceId -split ($resourceType.ToLower() + "/"))[1]                                  
                
                }
                catch {

                }
            }
        }
        return $resourceIds
    }


    [System.Object[]] GetAuditTrailsForBuilds(){
        $latestBuildScan = $this.GetThresholdTime("Build")
        if($this.ScanSource -ne 'CA'){
            $latestBuildScan=$latestBuildScan.ToUniversalTime();
        }        
        $latestBuildScan =Get-Date $latestBuildScan -Format s
        $buildIds = @();
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0){
            return $buildIds;   
        }
        $auditUrl = "https://auditservice.dev.azure.com/{0}/_apis/audit/auditlog?startTime={1}&api-version=6.0-preview.1" -f $this.OrganizationName, $latestBuildScan
        try {
            $response = [WebRequestHelper]::InvokeGetWebRequest($auditUrl);
            $auditTrails = $response.decoratedAuditLogEntries;
            $modifiedBuilds = $auditTrails | Where-Object {$_.actionId  -eq 'Security.ModifyPermission' -and $_.data.NamespaceName -eq 'Build' -and $_.data.Token -match $this.ProjectId+"/" }
            $restrictedBroaderGroups = @{}
            $broaderGroups = $this.ControlSettings.Build.RestrictedBroaderGroupsForBuild
            $broaderGroups.psobject.properties | foreach { $restrictedBroaderGroups[$_.Name] = $_.Value }
            $modifiedBuilds | foreach {
                $group = ($_.data.SubjectDisplayName -split("\\"))[1]
                if($group -in $restrictedBroaderGroups.keys ){
                    if($_.data.ChangedPermission -in $restrictedBroaderGroups[$group]){
                        $buildIds += (($_.data.Token -split("/"))[-1])
                    }
                }
            }
            $buildIds = $buildIds | Select -Unique
        }
        catch {

        }
        return $buildIds;
    }
    
    [System.Object[]] GetModifiedBuildsFromAudit($buildIds, $projectName){
        $totalBuilds = $buildIds.Count
        $buildDefnObj =@()
        $newBuildDefns = @();
        $queryIdCount = 0;
        $currentbuildIds = ""
        $buildIds | foreach {
            
            if($totalBuilds -lt 100){
                $queryIdCount++;
                $currentbuildIds=$currentbuildIds+$_+","
                if($queryIdCount -eq $totalBuilds){
                    $buildDefnURL = "https://{0}.visualstudio.com/{1}/_apis/build/definitions?definitionIds={2}&api-version=6.0" -f $($this.OrganizationName), $projectName, $currentbuildIds;
                    try {
                        $buildDefnObj += ([WebRequestHelper]::InvokeGetWebRequest($buildDefnURL));
                    }
                    catch {

                    }
                }
            }
            else {
                $queryIdCount++;
                $currentbuildIds=$currentbuildIds+$_+",";
                if($queryIdCount -eq 100){
                    $buildDefnURL = "https://{0}.visualstudio.com/{1}/_apis/build/definitions?definitionIds={2}&api-version=6.0" -f $($this.OrganizationName), $projectName, $currentbuildIds;
                    try {
                        $buildDefnObj += ([WebRequestHelper]::InvokeGetWebRequest($buildDefnURL));
                        $queryIdCount =0;
                        $currentbuildIds="";
                        $totalBuilds -=100;                        
                    }
                    catch {

                    }
                }

            }
        }
        $latestBuildScan = $this.GetThresholdTime("Build");             
        foreach ($buildDefn in $buildDefnObj)
        {
            if ([Helpers]::CheckMember($buildDefn,'CreatedDate') -and [datetime]($buildDefn.CreatedDate) -lt $latestBuildScan) 
            {
                $newBuildDefns += @($buildDefn)    
            }
        }
     
        return $newBuildDefns;
    }

    [System.Object[]] GetAuditTrailsForReleases(){
        $latestReleaseScan = $this.GetThresholdTime("Release");
        if($this.ScanSource -ne 'CA'){
            $latestReleaseScan=$latestReleaseScan.ToUniversalTime();
        }
        $latestReleaseScan = Get-Date $latestReleaseScan -Format s
        $releaseIds = @();
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0){
            return $releaseIds;   
        }
        $auditUrl = "https://auditservice.dev.azure.com/{0}/_apis/audit/auditlog?startTime={1}&api-version=6.0-preview.1" -f $this.OrganizationName, $latestReleaseScan
        try {
            $response = [WebRequestHelper]::InvokeGetWebRequest($auditUrl);
            $auditTrails = $response.decoratedAuditLogEntries;
            $modifiedReleases = $auditTrails | Where-Object {$_.actionId  -eq 'Security.ModifyPermission' -and $_.data.NamespaceName -eq 'ReleaseManagement' -and $_.data.Token -match $this.ProjectId+"/" }
            $restrictedBroaderGroups = @{}
            $broaderGroups = $this.ControlSettings.Release.RestrictedBroaderGroupsForRelease
            $broaderGroups.psobject.properties | foreach { $restrictedBroaderGroups[$_.Name] = $_.Value }
            $modifiedReleases| foreach {
                $group = ($_.data.SubjectDisplayName -split("\\"))[1]
                if($group -in $restrictedBroaderGroups.keys ){
                    if($_.data.ChangedPermission -in $restrictedBroaderGroups[$group]){
                        $releaseIds += (($_.data.Token -split("/"))[-1])
                    }
                }
            }
            $releaseIds = $releaseIds | Select -Unique
        }
        catch {

        }
        return $releaseIds;
    }
    
    [System.Object[]] GetModifiedReleasesFromAudit($releaseIds, $projectName){
        $totalReleases = $releaseIds.Count
        $newReleaseDefns = @();
        $releaseDefnObj =@()
        $queryIdCount = 0;
        $currentReleaseIds = ""
        $releaseIds | foreach {
            
            if($totalReleases -lt 100){
                $queryIdCount++;
                $currentReleaseIds=$currentReleaseIds+$_+","
                if($queryIdCount -eq $totalReleases){
                    $releaseDefnURL = "https://vsrm.dev.azure.com/{0}/{1}/_apis/release/definitions?definitionIdFilter={2}&api-version=6.0" -f $($this.OrganizationName), $projectName, $currentReleaseIds;
                    try {
                        $releaseDefnObj += ([WebRequestHelper]::InvokeGetWebRequest($releaseDefnURL));
                    }
                    catch {

                    }
                }
            }
            else {
                $queryIdCount++;
                $currentReleaseIds=$currentReleaseIds+$_+",";
                if($queryIdCount -eq 100){
                    $releaseDefnURL = "https://vsrm.dev.azure.com/{0}/{1}/_apis/release/definitions?definitionIdFilter={2}&api-version=6.0" -f $($this.OrganizationName), $projectName, $currentReleaseIds;
                    try {
                        $releaseDefnObj += ([WebRequestHelper]::InvokeGetWebRequest($releaseDefnURL));
                        $queryIdCount =0;
                        $currentReleaseIds="";
                        $totalReleases -=100;                        
                    }
                    catch {

                    }
                }

            }
        }   
        $latestReleaseScan = $this.GetThresholdTime("Release");          
        foreach ($releaseDefn in $releaseDefnObj)
        {
            if ([Helpers]::CheckMember($releaseDefn,'modifiedOn') -and [datetime]($releaseDefn.modifiedOn) -lt $latestReleaseScan) 
            {
                $newReleaseDefns += @($releaseDefn)    
            }
        }       
      
        return $newReleaseDefns;
    }

    
    #common function to get modified resource ids from audits for common svts and variable group
    [System.Object[]] GetModifiedCommonSvtAuditTrails($resourceType){
        $resourceIds = @()
        #get last scan of the resources
        $latestScan = $this.GetThresholdTime($resourceType)
        if($this.ScanSource -ne 'CA'){
            $latestScan=$latestScan.ToUniversalTime();
        }        
        $latestScan = Get-Date $latestScan -Format s
        
        $auditUrl = "https://auditservice.dev.azure.com/{0}/_apis/audit/auditlog?startTime={1}&api-version=6.0-preview.1" -f $this.OrganizationName, $latestScan
        try {
            $response = [WebRequestHelper]::InvokeGetWebRequest($auditUrl);
            $auditTrails = $response.decoratedAuditLogEntries;
            #get modified resources from filter
            $modifiedResources = $this.GetModifiedResourcesFilter($resourceType,$auditTrails)                        
            $modifiedResources | foreach {
                #extract resource ids from modified resources
                $resourceIds+=($_.data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[1]) -split("/"))[-1]
                if($resourceType -eq "GitRepositories"){
                    #to handle events of permission changes on branches
                    $resourceIds+=(($_.data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[1]) -split("/refs"))[0]) -split("/")[-1]
                    #to handle events of new repository creation
                    $resourceIds+=($_.data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[1]) -split("\."))[-1]
                }
            }
            $resourceIds = $resourceIds | Select -Unique
        }
        catch {

        }
        return $resourceIds
    }

    #function to filter audits according to resource type
    [System.Object[]] GetModifiedResourcesFilter($resourceType,$auditTrails){
        $resourceTypeInFilter = $resourceType
        #in case of secure file and variable group the resource type in audits is library, for other resources the name is same
        if($resourceType -eq "SecureFile" -or $resourceType -eq "VariableGroup"){
            $resourceTypeInFilter = "Library"
        }
        if($resourceType -eq "GitRepositories"){
                $resourceTypeInFilter = "Git Repositories"
        }
        $modifiedResources = $auditTrails | Where-Object {$_.actionId  -in [IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.PSObject.Properties.Name -and  ([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[0] -eq $true -or( $_.Data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[0]) -eq $resourceTypeInFilter -or $_.Data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[0]) -eq "repository" -or $_.Data.([IncrementalScanHelper]::auditSchema.$resourceType.AuditEvents.($_.actionId)[0]) -eq $resourceType))}
        
        return $modifiedResources

    }

    #function to get modified resources
    [System.Object[]] GetModifiedCommonSvtFromAudit($resourceType,$response){
        $latestScan = $this.GetThresholdTime($resourceType)      
        $latestScan =Get-Date $latestScan -Format s
        #$response = [WebRequestHelper]::InvokeGetWebRequest($url);
        #if this a first scan return all resources
        if($this.FirstScan -eq $true -and $this.IncrementalDate -eq 0){    
            $this.UpdateTimeStamp($resourceType)        
            return $response   
        }
        #if partial scan is active and last scan is 0 or this is a full scan in progress return all resources
        if($this.isPartialScanActive -and ($latestScan -eq 0 -or $this.IsFullScanInProgress)){
            return $response
        }
        #if this is a old scan return all resources
        if($this.ShouldDiscardOldIncScan($resourceType)){
            $this.UpdateTimeStamp($resourceType)
            return $response
        }
        #get ids from above functions
        $modifiedResourceIds = @($this.GetModifiedCommonSvtAuditTrails($resourceType)); 
        if($resourceType -eq "GitRepositories"){
            $modifiedResourceIdsFromAttestation = @($this.GetAttestationAfterInc($this.ProjectName,"Repository"))
        }
        else{
            $modifiedResourceIdsFromAttestation = @($this.GetAttestationAfterInc($this.ProjectName,$resourceType))
        }
        $modifiedResourceIds = @($modifiedResourceIds + $modifiedResourceIdsFromAttestation | select -uniq)
        
        $modifiedResources = @()
        #if we get some ids from audit trails add them to modified resource obj
        if($modifiedResourceIds.Count -gt 0 -and $null -ne $modifiedResourceIds[0]){
            #filter all ids from audit trails in the api response
            $modifiedResources = @($response | Where-Object{$modifiedResourceIds -contains $_.id})
            #to capture events that dont come in audits but is reflected in api responses such as new resource created, properties of resources edited etc.
            if([Helpers]::CheckMember([IncrementalScanHelper]::auditSchema.$resourceType, "ApiResponseFilter")){
                $modifiedResources +=$response | Where-Object{$modifiedResourceIds -notcontains $_.id -and [datetime]($_.([IncrementalScanHelper]::auditSchema.$resourceType.ApiResponseFilter)) -gt $latestScan}
                
            }
        }
        #in case no ids were obtained from audits check from response for corresponding api response filtee if present
        else{
            if([Helpers]::CheckMember([IncrementalScanHelper]::auditSchema.$resourceType, "ApiResponseFilter")){
                $modifiedResources += $response | Where-Object{[datetime]($_.([IncrementalScanHelper]::auditSchema.$resourceType.ApiResponseFilter)) -gt $latestScan}
            }
        }
        $this.UpdateTimeStamp($resourceType)
        return $modifiedResources
    }

    [void] SetContext($projectId,$organizationContext){
        $this.ProjectId = $projectId
        $this.OrganizationContext = $organizationContext
    }

}