Courtesy Push Pipeline Improvement - Part#1 | Eliminate Job-4 build_all_tasks_for_deployments (#21417)

* update Job1 with Job4 functionalities

* job4 Removal

* Move GitHub Release to job $

* Modify Fixes

* Udpate#1 Review Comments

* Minor fixes

* Fix hotfix parameter passing

* Modify layout-version.txt handling

* Removal of unused parameters

* AB#2329703 Link Work-Item

Author: dassayantan24

.azure-pipelines/.vsts.release.yml

@@ -85,6 +85,8 @@ variables:
     value: 'true'
   ${{ else }}:
     value: 'false'
+- name: tasksBuildArtifact
+  value: 'allTasks'
 
 extends:
   template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
@@ -134,7 +136,7 @@ extends:
                   ne(variables['numTasks'], 0)
                 )
               PathtoPublish: _package/tasks.zip
-              ArtifactName: package
+              ArtifactName: $(tasksBuildArtifact)
               sbomBuildDropPath: $(Build.SourcesDirectory)/_package
         steps:
         - template: /ci/build-all-steps.yml@self
@@ -199,30 +201,26 @@ extends:
         steps:
         - checkout: self
         - template: /ci/courtesy-push.yml@self
-
-      - job: build_all_tasks_for_deployments
-        displayName: Build all tasks for deployments (Windows)
-        dependsOn:
-        - courtesy_push
-        - build_all_windows
-        condition: succeeded()
-        timeoutInMinutes: 360
+      
+      # GitHub Release
+      - job: github_release
+        displayName: Create GitHub Release
+        timeoutInMinutes: 60
         pool:
           name: 1ES-ABTT-Shared-Pool
           image: abtt-windows-2022
           os: windows
+        dependsOn:
+        - courtesy_push
         variables:
-          task_pattern: $[dependencies.build_all_windows.outputs['getTaskPattern.task_pattern']]
+          courtesy_push_pr_id: $[dependencies.courtesy_push.outputs['createCourtesyPushPR.PR_ID']]
+          courtesy_push_pr_link: $[dependencies.courtesy_push.outputs['createCourtesyPushPR.PR_LINK']]
+        condition: |
+          and(
+            succeeded(),
+            in(variables['build.reason'], 'Schedule', 'Manual')
+          )
         templateContext:
-          outputs:
-          - output: buildArtifacts
-            displayName: 'Publish package artifact'
-            PathtoPublish: _package/tasks.zip
-            sbomBuildDropPath: $(Build.SourcesDirectory)/_package
-            ArtifactName: allTasks
+          outputs: []
         steps:
-        - template: /ci/build-all-tasks.yml@self
-          parameters:
-            deploy_all_tasks: ${{ eq(parameters.task_deployment, 'Deploy all Tasks') }}
-            useSemverBuildConfig: ${{ parameters.useSemverBuildConfig }}
-      
+        - template: /ci/github-release.yml@self

ci/build-all-steps.yml

@@ -17,6 +17,12 @@ steps:
     displayName: Start collect diagnostics
     condition: and(succeeded(), eq(variables.collect_diagnostics, 'true'))
 
+# Use .NET SDK 8
+- task: UseDotNet@2
+  displayName: 'Install .NET Core SDK 8.x'
+  inputs:
+    version: 8.x
+
 # Use node 20, npm 9
 - task: NodeTool@0
   displayName: Use node 20
@@ -79,45 +85,6 @@ steps:
 - powershell: ./ci/set-sprint-variables.ps1
   displayName: Set currentSprint variables
 
-- template: /ci/generate-branch-name.yml@self
-  parameters:
-    prefix: releases
-
-- powershell: |
-    $releaseBranch = "$(branchName)"
-    $isDryRun = "$(isDryRun)"
-
-    # add config entry to avoid errors while pulling
-    git config --global user.email "$(username)@microsoft.com"
-    git config --global user.name "$(username)"
-
-    Write-Host 'Enabling verbose git tracing..'
-    git config --global http.verbose true
-    $env:GIT_TRACE = 1
-    $env:GIT_CURL_VERBOSE = 1
-
-    # Pull commits from remote and push branch to git
-    git checkout -b $releaseBranch
-    if( $isDryRun -eq 'true' ) {
-      Write-Host "DRY RUN: Skipping Git push operations to remote repository"
-    }else{
-        Write-Host "Trying to pull the remote branch.."
-        git pull https://$(GitHubPAT)@github.com/microsoft/azure-pipelines-tasks $releaseBranch
-        if (-not $?) {
-          Write-Host "Failed to pull the remote branch. This is expected if the remote branch doesn't exist."
-        }
-        Write-Host "Trying to push to the remote branch.."
-        git push https://$(GitHubPAT)@github.com/microsoft/azure-pipelines-tasks $releaseBranch
-    }
-  condition: |
-    and(
-      succeeded(),
-      in(variables['build.reason'], 'Schedule', 'Manual'),
-      eq(variables['COURTESY_PUSH'], 'true'),
-      eq(variables['Build.SourceBranch'], 'refs/heads/master')
-    )
-  displayName: Push release branch
-
 - script: node ./ci/check-downgrading.js --task "$(task_pattern_fordowngradingcheck)" --sprint $(currentSprint) --week $(currentSprintWeek)
   displayName: Check for downgrading tasks
   # remove SourceBranch condition after merging users/merlynop/node20merge-2 ; see https://github.com/microsoft/azure-pipelines-tasks/pull/20819
@@ -164,11 +131,27 @@ steps:
       ne(variables['numTasks'], 0)
     )
 
+# Test
+- script: node make.js test
+  displayName: Run tests
+  condition: and(succeeded(), ne(variables['numTasks'], 0), ne(variables['tasksSkipTests'], 'true'))
+
+- script: node make.js testLegacy --task "$(getTaskPattern.task_pattern)"
+  displayName: Legacy tests with node 6
+  condition: and(succeeded(), ne(variables['numTasks'], 0), ne(variables['tasksSkipTests'], 'true'))
+
+# Publish code coverage result
+- task: PublishCodeCoverageResults@1
+  displayName: Publish Code Coverage Results
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: $(System.DefaultWorkingDirectory)/_build/coverage/*coverage.xml
+
 # Only on Windows:
 - ${{ if eq(parameters.os, 'Windows_NT') }}:
 
   # Stage tasks individually into the package directory
-  - script: node ./ci/stage-package.js false individually
+  - script: node ./ci/stage-package.js true individually
     displayName: Stage tasks individually into the package directory
     condition: |
       and(
@@ -183,7 +166,7 @@ steps:
       layoutRoot: $(Build.SourcesDirectory)\_package\tasks-layout
 
   # Stage all the tasks into a single zip for upload
-  - script: node ./ci/stage-package.js false
+  - script: node ./ci/stage-package.js
     displayName: Stage all the tasks into a single zip for upload
     condition: |
       and(
@@ -192,21 +175,6 @@ steps:
         ne(variables['numTasks'], 0)
       )
 
-# Test
-- script: node make.js test
-  displayName: Run tests
-  condition: and(succeeded(), ne(variables['numTasks'], 0), ne(variables['tasksSkipTests'], 'true'))
-- script: node make.js testLegacy --task "$(getTaskPattern.task_pattern)"
-  displayName: Legacy tests with node 6
-  condition: and(succeeded(), ne(variables['numTasks'], 0), ne(variables['tasksSkipTests'], 'true'))
-
-# Publish code coverage result
-- task: PublishCodeCoverageResults@1
-  displayName: Publish Code Coverage Results
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: $(System.DefaultWorkingDirectory)/_build/coverage/*coverage.xml
-
 # Only when building on Windows:
 - ${{ if eq(parameters.os, 'Windows_NT') }}:
 
@@ -218,26 +186,4 @@ steps:
         always(),
         eq(variables.collect_diagnostics, 'true'),
         ne(variables['numTasks'], 0)
-      )
-
-  - powershell: |
-      $releaseBranch = "$(branchName)"
-      $isDryRun = "$(isDryRun)"
-
-      if (($(currentSprintWeek) -eq 3) -or ($isDryRun -eq 'true')) {
-        cd $(System.DefaultWorkingDirectory)/ci/ci-release-notes
-        npm ci
-        node release-notes.js --token $(GitHubPAT) --version $(currentSprint) --releaseBranch $releaseBranch --isDryRun $isDryRun
-      } else {
-        echo "Skipping since release notes generating on week 3"
-      }
-    condition: |
-      and(
-        succeeded(),
-        in(variables['build.reason'], 'Schedule', 'Manual'),
-        eq(variables['COURTESY_PUSH'], 'true'),
-        eq(variables['Build.SourceBranch'], 'refs/heads/master'),
-        eq(${{ parameters.generateReleaseNotes }}, true)
-      )
-    continueOnError: true
-    displayName: Create Release
+      )

ci/build-all-tasks.yml

@@ -27,15 +27,11 @@ steps:
 - template: /ci/get-AzDo-pat.yml@self
 
 - script: node $(Build.SourcesDirectory)/ci/courtesy-push/courtesy-push.js $(Build.SourcesDirectory)/IndividualNugetPackagesDownloaded/IndividualNugetPackages/unified_deps.xml
+  name: createCourtesyPushPR
   displayName: Update unified deps and create branch
   env:
     TOKEN: $(AzDo_PAT)
     BRANCH_NAME: $(branchName)
     DRYRUN: $(isDryRun)
     USERNAME: $(username)
-    
-- powershell: $(Build.SourcesDirectory)/ci/courtesy-push/send-notification.ps1
-  displayName: Send MS Teams notification
-  condition: or(eq(variables['build.reason'], 'Schedule'), eq(variables['build.reason'], 'Manual'))
-  env:
-    TEAMS_WEBHOOK: $(MSTeamsUri)
+

ci/courtesy-push.yml

@@ -173,8 +173,8 @@ Generated: ${new Date().toISOString()}`
         console.log(`Link to the PR: ${prLink}`);
 
         // Set Azure Pipeline variables
-        console.log(`##vso[task.setvariable variable=PR_ID]${PR.pullRequestId}`);
-        console.log(`##vso[task.setvariable variable=PR_LINK]${prLink}`);
+        console.log(`##vso[task.setvariable variable=PR_ID;isOutput=true]${PR.pullRequestId}`);
+        console.log(`##vso[task.setvariable variable=PR_LINK;isOutput=true]${prLink}`);
 
         return {
             prId: PR.pullRequestId,
@@ -194,8 +194,8 @@ Generated: ${new Date().toISOString()}`
         const dummyPrId = `DRYRUN-${timestamp}`;
         const dummyPrLink = `https://${orgUrl}/${project}/_git/${repo}/pullrequest/DRYRUN-${timestamp}`;
 
-        console.log(`##vso[task.setvariable variable=PR_ID]${dummyPrId}`);
-        console.log(`##vso[task.setvariable variable=PR_LINK]${dummyPrLink}`);
+        console.log(`##vso[task.setvariable variable=PR_ID;isOutput=true]${dummyPrId}`);
+        console.log(`##vso[task.setvariable variable=PR_LINK;isOutput=true]${dummyPrLink}`);
 
         console.log(`DRYRUN: Generated dummy PR_ID = ${dummyPrId}`);
         console.log(`DRYRUN: Generated dummy PR_LINK = ${dummyPrLink}`);

ci/courtesy-push/courtesy-push.js

@@ -3,7 +3,8 @@ $ErrorActionPreference = 'Stop'
 Add-Type -Assembly 'System.IO.Compression.FileSystem'
 try {
     # Download the package
-    $downloadUrl = "$($env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$env:SYSTEM_TEAMPROJECTID/_apis/build/builds/$env:BUILD_BUILDID/artifacts?artifactName=package&%24format=zip&api-version=3"
+    $tasksBuildArtifact = $env:TASKS_BUILD_ARTIFACT
+    $downloadUrl = "$($env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$env:SYSTEM_TEAMPROJECTID/_apis/build/builds/$env:BUILD_BUILDID/artifacts?artifactName=$tasksBuildArtifact&%24format=zip&api-version=3"
     $downloadTarget = "$env:SYSTEM_ARTIFACTSDIRECTORY\package.zip"
     Write-Host "Downloading: '$downloadUrl' to target '$downloadTarget'"
     $webClient = New-Object System.Net.WebClient