Sync eng/common directory with azure-sdk-tools for PR 17238 (#3973)

* Use scoped Copilot App token for hermetic Vally evals

* Reuse shared GitHub login for Copilot eval authentication

* Replace all Vally eval PATs with shared GitHub App authentication

* Remove PR guards from shared Copilot authentication

* Use one tested Copilot token environment variable

* Use default Azure owner for Copilot login

---------

Co-authored-by: helen229 <gaoh@microsoft.com>

eng/common/pipelines/live-eval.yml

@@ -22,8 +22,6 @@ parameters:
 variables:
   # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
   - template: /eng/pipelines/templates/variables/image.yml
-  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
-  - group: AzSDK_Eval_Variable_group
 
 extends:
   template: /eng/common/pipelines/templates/stages/archetype-eval.yml

eng/common/pipelines/skill-eval.yml

@@ -12,6 +12,9 @@ trigger:
       - eng/common/pipelines/skill-eval.yml
       - eng/common/pipelines/templates/jobs/**
       - eng/common/pipelines/templates/steps/eval-invoke.yml
+      - eng/common/pipelines/templates/steps/login-to-copilot.yml
+      - eng/common/pipelines/templates/steps/login-to-github.yml
+      - eng/common/scripts/login-to-github.ps1
       - eng/common/pipelines/templates/steps/eval-mcp-setup.yml
       - eng/common/pipelines/templates/steps/eval-publish-results.yml
       - eng/common/pipelines/templates/stages/archetype-eval.yml
@@ -28,8 +31,6 @@ parameters:
 variables:
   # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
   - template: /eng/pipelines/templates/variables/image.yml
-  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
-  - group: AzSDK_Eval_Variable_group
 
 extends:
   template: /eng/common/pipelines/templates/stages/archetype-eval.yml

eng/common/pipelines/templates/jobs/eval-shard.yml

@@ -111,6 +111,8 @@ jobs:
         env:
           EVAL_ARGS: $(evalArgs)
 
+      - template: /eng/common/pipelines/templates/steps/login-to-copilot.yml
+
       # Runs the eval via the invoke template (auth-conditional in the common default).
       - template: ${{ parameters.evalInvokeTemplate }}
         parameters:

eng/common/pipelines/templates/steps/eval-invoke.yml

@@ -1,7 +1,7 @@
 # Invokes `vally eval` for one shard. UseAzSdkAuthentication=false (default) runs the hermetic
 # node script directly. UseAzSdkAuthentication=true wraps the same run in AzureCLI@2 so a live
 # MCP's Azure DevOps calls inherit the service-connection identity.
-# copilot-sdk authenticates the model session via GITHUB_TOKEN (from the linked eval variable group).
+# The shard's login-to-copilot template supplies the executor and prompt-judge token.
 
 parameters:
   - name: vallyRoot
@@ -32,9 +32,8 @@ steps:
             --shard-name "$(shardName)" \
             --output-dir "$(Build.SourcesDirectory)/artifacts/vally-results/$(shardName)" \
             --threshold ${{ parameters.threshold }}
-      ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-        env:
-          GITHUB_TOKEN: $(azuresdk-copilot-github-pat)
+      env:
+        COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)
 
   - ${{ else }}:
     - script: |
@@ -45,6 +44,5 @@ steps:
           --threshold ${{ parameters.threshold }}
       displayName: 'vally eval $(shardName)'
       workingDirectory: ${{ parameters.vallyRoot }}
-      ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-        env:
-          GITHUB_TOKEN: $(azuresdk-copilot-github-pat)
+      env:
+        COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)

eng/common/pipelines/templates/steps/login-to-copilot.yml

@@ -0,0 +1,11 @@
+# Shared GitHub App authentication for Vally executors and prompt judges.
+steps:
+  - template: /eng/common/pipelines/templates/steps/login-to-github.yml
+    parameters:
+      VariableNamePrefix: COPILOT_GITHUB_TOKEN
+      TokenRequest:
+        repositories:
+          - ${{ split(variables['Build.Repository.Name'], '/')[1] }}
+        permissions:
+          organization_copilot_requests: read
+          metadata: read

eng/common/pipelines/templates/steps/login-to-github.yml

@@ -14,6 +14,10 @@ parameters:
 - name: ExportAsOutputVariable
   type: boolean 
   default: false
+- name: TokenRequest
+  # Optional GitHub access-token request body restricting repositories and permissions.
+  type: object
+  default: {}
 - name: ScriptDirectory
   default: eng/common/scripts
 - name: Condition
@@ -33,4 +37,6 @@ steps:
       -InstallationTokenOwners '${{ join(''',''', parameters.TokenOwners) }}'
       -VariableNamePrefix '${{ parameters.VariableNamePrefix }}'
       -AlwaysUseOwnerSuffix:$${{ parameters.AlwaysUseOwnerSuffix }}
-      -ExportAsOutputVariable:$${{ parameters.ExportAsOutputVariable }}
+      -ExportAsOutputVariable:$${{ parameters.ExportAsOutputVariable }}
+  env:
+    GITHUB_TOKEN_REQUEST: ${{ convertToJson(parameters.TokenRequest) }}

eng/common/pipelines/workflow-eval.yml

@@ -15,6 +15,9 @@ trigger:
       - eng/common/pipelines/workflow-eval.yml
       - eng/common/pipelines/templates/jobs/**
       - eng/common/pipelines/templates/steps/eval-invoke.yml
+      - eng/common/pipelines/templates/steps/login-to-copilot.yml
+      - eng/common/pipelines/templates/steps/login-to-github.yml
+      - eng/common/scripts/login-to-github.ps1
       - eng/common/pipelines/templates/steps/eval-mcp-setup.yml
       - eng/common/pipelines/templates/steps/eval-publish-results.yml
       - eng/common/pipelines/templates/stages/archetype-eval.yml
@@ -31,8 +34,6 @@ parameters:
 variables:
   # Defines LINUXPOOL + LINUXVMIMAGE used by the archetype's jobs.
   - template: /eng/pipelines/templates/variables/image.yml
-  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
-  - group: AzSDK_Eval_Variable_group
 
 extends:
   template: /eng/common/pipelines/templates/stages/archetype-eval.yml

eng/common/scripts/login-to-github.ps1

@@ -29,6 +29,10 @@
   When set in Azure DevOps, also exports the variable as an output variable
   (##vso[task.setvariable ...;isOutput=true]) for downstream jobs/stages.
 
+.PARAMETER TokenRequest
+  Optional JSON object restricting the installation token's repositories and permissions.
+  Defaults to GITHUB_TOKEN_REQUEST. When omitted, retains the installation's default scope.
+
 .OUTPUTS
   Sets environment variables in the current process and exports them to the CI system:
   - Azure DevOps: sets secret pipeline variables via ##vso logging commands
@@ -43,7 +47,8 @@ param(
   [string[]] $InstallationTokenOwners = @("Azure"),
   [string] $VariableNamePrefix = "GH_TOKEN",
   [switch] $AlwaysUseOwnerSuffix,
-  [switch] $ExportAsOutputVariable
+  [switch] $ExportAsOutputVariable,
+  [string] $TokenRequest = $env:GITHUB_TOKEN_REQUEST
 )
 
 $ErrorActionPreference = 'Stop'
@@ -210,16 +215,53 @@ function New-GitHubInstallationToken {
     [Parameter(Mandatory)] [string] $Jwt,
     [Parameter(Mandatory)] [string] $InstallationId,
     [Parameter(Mandatory)] [string] $ApiBase,
-    [Parameter(Mandatory)] [string] $ApiVersion
+    [Parameter(Mandatory)] [string] $ApiVersion,
+    [hashtable] $TokenRequest = @{}
   )
   $headers = Get-Headers -Jwt $Jwt -ApiVersion $ApiVersion
   $uri = "$ApiBase/app/installations/$InstallationId/access_tokens"
-  $resp = Invoke-RestMethod -Method Post -Headers $headers -Uri $uri -TimeoutSec 30 -MaximumRetryCount 3
-  if (!$resp.token) { throw "Failed to obtain installation access token for installation $InstallationId." }
-  return $resp.token
+  $request = @{
+    Method = 'Post'
+    Headers = $headers
+    Uri = $uri
+    TimeoutSec = 30
+    MaximumRetryCount = 3
+  }
+  if ($TokenRequest.Count -gt 0) {
+    $request.Body = ConvertTo-Json -InputObject $TokenRequest -Depth 5 -Compress
+    $request.ContentType = 'application/json'
+  }
+
+  $resp = Invoke-RestMethod @request
+  $token = Get-PropertyValue -InputObject $resp -PropertyName 'token'
+  if (!$token) { throw "Failed to obtain installation access token for installation $InstallationId." }
+
+  $requestedPermissions = Get-PropertyValue -InputObject $TokenRequest -PropertyName 'permissions'
+  if ($requestedPermissions) {
+    $grantedPermissions = Get-PropertyValue -InputObject $resp -PropertyName 'permissions'
+    foreach ($permission in $requestedPermissions.Keys) {
+      if ((Get-PropertyValue -InputObject $grantedPermissions -PropertyName $permission) -ne $requestedPermissions[$permission]) {
+        throw "Installation token did not grant requested permission '$permission': '$($requestedPermissions[$permission])'."
+      }
+    }
+  }
+  return $token
 }
 
 function Invoke-LoginToGitHub {
+  param(
+    [string] $TokenRequest,
+    [string] $VariableNamePrefix = 'GH_TOKEN'
+  )
+
+  $tokenRequestBody = @{}
+  if ($TokenRequest) {
+    $tokenRequestBody = ConvertFrom-Json -InputObject $TokenRequest -AsHashtable
+    if ($tokenRequestBody -isnot [System.Collections.IDictionary]) {
+      throw 'TokenRequest must be a JSON object.'
+    }
+  }
+
   Write-Host "Generating GitHub App JWT by signing via Azure Key Vault (no key export)..."
   $jwt = New-GitHubAppJwt -VaultName $KeyVaultName -KeyName $KeyName -AppId $GitHubAppId
 
@@ -232,7 +274,7 @@ function Invoke-LoginToGitHub {
     Write-Host "Installation ID resolved: $installationId"
 
     Write-Host "Exchanging JWT for installation access token..."
-    $installationToken = New-GitHubInstallationToken -Jwt $jwt -InstallationId $installationId -ApiBase $GitHubApiBaseUrl -ApiVersion $GitHubApiVersion
+    $installationToken = New-GitHubInstallationToken -Jwt $jwt -InstallationId $installationId -ApiBase $GitHubApiBaseUrl -ApiVersion $GitHubApiVersion -TokenRequest $tokenRequestBody
 
     $variableName = $VariableNamePrefix
     if ($AlwaysUseOwnerSuffix -or $InstallationTokenOwners.Count -gt 1) {
@@ -278,4 +320,4 @@ if ($env:PESTER_TEST_RUN -eq 'true') {
   return
 }
 
-Invoke-LoginToGitHub
+Invoke-LoginToGitHub -TokenRequest $TokenRequest -VariableNamePrefix $VariableNamePrefix