diff --git a/eng/common/pipelines/live-eval.yml b/eng/common/pipelines/live-eval.yml index b471748d2..e18bafa65 100644 --- a/eng/common/pipelines/live-eval.yml +++ b/eng/common/pipelines/live-eval.yml @@ -22,8 +22,6 @@ parameters: variables: # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local. - template: /eng/pipelines/templates/variables/image.yml - # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step. - - group: AzSDK_Eval_Variable_group extends: template: /eng/common/pipelines/templates/stages/archetype-eval.yml diff --git a/eng/common/pipelines/skill-eval.yml b/eng/common/pipelines/skill-eval.yml index 53b467039..2b639ae53 100644 --- a/eng/common/pipelines/skill-eval.yml +++ b/eng/common/pipelines/skill-eval.yml @@ -12,6 +12,9 @@ trigger: - eng/common/pipelines/skill-eval.yml - eng/common/pipelines/templates/jobs/** - eng/common/pipelines/templates/steps/eval-invoke.yml + - eng/common/pipelines/templates/steps/login-to-copilot.yml + - eng/common/pipelines/templates/steps/login-to-github.yml + - eng/common/scripts/login-to-github.ps1 - eng/common/pipelines/templates/steps/eval-mcp-setup.yml - eng/common/pipelines/templates/steps/eval-publish-results.yml - eng/common/pipelines/templates/stages/archetype-eval.yml @@ -28,8 +31,6 @@ parameters: variables: # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local. - template: /eng/pipelines/templates/variables/image.yml - # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step. - - group: AzSDK_Eval_Variable_group extends: template: /eng/common/pipelines/templates/stages/archetype-eval.yml diff --git a/eng/common/pipelines/templates/jobs/eval-shard.yml b/eng/common/pipelines/templates/jobs/eval-shard.yml index 126c0ade6..569f7d4fd 100644 --- a/eng/common/pipelines/templates/jobs/eval-shard.yml +++ b/eng/common/pipelines/templates/jobs/eval-shard.yml @@ -111,6 +111,8 @@ jobs: env: EVAL_ARGS: $(evalArgs) + - template: /eng/common/pipelines/templates/steps/login-to-copilot.yml + # Runs the eval via the invoke template (auth-conditional in the common default). - template: ${{ parameters.evalInvokeTemplate }} parameters: diff --git a/eng/common/pipelines/templates/steps/eval-invoke.yml b/eng/common/pipelines/templates/steps/eval-invoke.yml index 2ebb5133f..191765af3 100644 --- a/eng/common/pipelines/templates/steps/eval-invoke.yml +++ b/eng/common/pipelines/templates/steps/eval-invoke.yml @@ -1,7 +1,7 @@ # Invokes `vally eval` for one shard. UseAzSdkAuthentication=false (default) runs the hermetic # node script directly. UseAzSdkAuthentication=true wraps the same run in AzureCLI@2 so a live # MCP's Azure DevOps calls inherit the service-connection identity. -# copilot-sdk authenticates the model session via GITHUB_TOKEN (from the linked eval variable group). +# The shard's login-to-copilot template supplies the executor and prompt-judge token. parameters: - name: vallyRoot @@ -32,9 +32,8 @@ steps: --shard-name "$(shardName)" \ --output-dir "$(Build.SourcesDirectory)/artifacts/vally-results/$(shardName)" \ --threshold ${{ parameters.threshold }} - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - env: - GITHUB_TOKEN: $(azuresdk-copilot-github-pat) + env: + COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN) - ${{ else }}: - script: | @@ -45,6 +44,5 @@ steps: --threshold ${{ parameters.threshold }} displayName: 'vally eval $(shardName)' workingDirectory: ${{ parameters.vallyRoot }} - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - env: - GITHUB_TOKEN: $(azuresdk-copilot-github-pat) + env: + COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN) diff --git a/eng/common/pipelines/templates/steps/login-to-copilot.yml b/eng/common/pipelines/templates/steps/login-to-copilot.yml new file mode 100644 index 000000000..da1c12db9 --- /dev/null +++ b/eng/common/pipelines/templates/steps/login-to-copilot.yml @@ -0,0 +1,11 @@ +# Shared GitHub App authentication for Vally executors and prompt judges. +steps: + - template: /eng/common/pipelines/templates/steps/login-to-github.yml + parameters: + VariableNamePrefix: COPILOT_GITHUB_TOKEN + TokenRequest: + repositories: + - ${{ split(variables['Build.Repository.Name'], '/')[1] }} + permissions: + organization_copilot_requests: read + metadata: read \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/login-to-github.yml b/eng/common/pipelines/templates/steps/login-to-github.yml index 5814417bb..8da4155b8 100644 --- a/eng/common/pipelines/templates/steps/login-to-github.yml +++ b/eng/common/pipelines/templates/steps/login-to-github.yml @@ -14,6 +14,10 @@ parameters: - name: ExportAsOutputVariable type: boolean default: false +- name: TokenRequest + # Optional GitHub access-token request body restricting repositories and permissions. + type: object + default: {} - name: ScriptDirectory default: eng/common/scripts - name: Condition @@ -33,4 +37,6 @@ steps: -InstallationTokenOwners '${{ join(''',''', parameters.TokenOwners) }}' -VariableNamePrefix '${{ parameters.VariableNamePrefix }}' -AlwaysUseOwnerSuffix:$${{ parameters.AlwaysUseOwnerSuffix }} - -ExportAsOutputVariable:$${{ parameters.ExportAsOutputVariable }} \ No newline at end of file + -ExportAsOutputVariable:$${{ parameters.ExportAsOutputVariable }} + env: + GITHUB_TOKEN_REQUEST: ${{ convertToJson(parameters.TokenRequest) }} \ No newline at end of file diff --git a/eng/common/pipelines/workflow-eval.yml b/eng/common/pipelines/workflow-eval.yml index ac81e5781..01f5dbecb 100644 --- a/eng/common/pipelines/workflow-eval.yml +++ b/eng/common/pipelines/workflow-eval.yml @@ -15,6 +15,9 @@ trigger: - eng/common/pipelines/workflow-eval.yml - eng/common/pipelines/templates/jobs/** - eng/common/pipelines/templates/steps/eval-invoke.yml + - eng/common/pipelines/templates/steps/login-to-copilot.yml + - eng/common/pipelines/templates/steps/login-to-github.yml + - eng/common/scripts/login-to-github.ps1 - eng/common/pipelines/templates/steps/eval-mcp-setup.yml - eng/common/pipelines/templates/steps/eval-publish-results.yml - eng/common/pipelines/templates/stages/archetype-eval.yml @@ -31,8 +34,6 @@ parameters: variables: # Defines LINUXPOOL + LINUXVMIMAGE used by the archetype's jobs. - template: /eng/pipelines/templates/variables/image.yml - # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step. - - group: AzSDK_Eval_Variable_group extends: template: /eng/common/pipelines/templates/stages/archetype-eval.yml diff --git a/eng/common/scripts/login-to-github.ps1 b/eng/common/scripts/login-to-github.ps1 index c3553b91a..6680f89bf 100644 --- a/eng/common/scripts/login-to-github.ps1 +++ b/eng/common/scripts/login-to-github.ps1 @@ -29,6 +29,10 @@ When set in Azure DevOps, also exports the variable as an output variable (##vso[task.setvariable ...;isOutput=true]) for downstream jobs/stages. +.PARAMETER TokenRequest + Optional JSON object restricting the installation token's repositories and permissions. + Defaults to GITHUB_TOKEN_REQUEST. When omitted, retains the installation's default scope. + .OUTPUTS Sets environment variables in the current process and exports them to the CI system: - Azure DevOps: sets secret pipeline variables via ##vso logging commands @@ -43,7 +47,8 @@ param( [string[]] $InstallationTokenOwners = @("Azure"), [string] $VariableNamePrefix = "GH_TOKEN", [switch] $AlwaysUseOwnerSuffix, - [switch] $ExportAsOutputVariable + [switch] $ExportAsOutputVariable, + [string] $TokenRequest = $env:GITHUB_TOKEN_REQUEST ) $ErrorActionPreference = 'Stop' @@ -210,16 +215,53 @@ function New-GitHubInstallationToken { [Parameter(Mandatory)] [string] $Jwt, [Parameter(Mandatory)] [string] $InstallationId, [Parameter(Mandatory)] [string] $ApiBase, - [Parameter(Mandatory)] [string] $ApiVersion + [Parameter(Mandatory)] [string] $ApiVersion, + [hashtable] $TokenRequest = @{} ) $headers = Get-Headers -Jwt $Jwt -ApiVersion $ApiVersion $uri = "$ApiBase/app/installations/$InstallationId/access_tokens" - $resp = Invoke-RestMethod -Method Post -Headers $headers -Uri $uri -TimeoutSec 30 -MaximumRetryCount 3 - if (!$resp.token) { throw "Failed to obtain installation access token for installation $InstallationId." } - return $resp.token + $request = @{ + Method = 'Post' + Headers = $headers + Uri = $uri + TimeoutSec = 30 + MaximumRetryCount = 3 + } + if ($TokenRequest.Count -gt 0) { + $request.Body = ConvertTo-Json -InputObject $TokenRequest -Depth 5 -Compress + $request.ContentType = 'application/json' + } + + $resp = Invoke-RestMethod @request + $token = Get-PropertyValue -InputObject $resp -PropertyName 'token' + if (!$token) { throw "Failed to obtain installation access token for installation $InstallationId." } + + $requestedPermissions = Get-PropertyValue -InputObject $TokenRequest -PropertyName 'permissions' + if ($requestedPermissions) { + $grantedPermissions = Get-PropertyValue -InputObject $resp -PropertyName 'permissions' + foreach ($permission in $requestedPermissions.Keys) { + if ((Get-PropertyValue -InputObject $grantedPermissions -PropertyName $permission) -ne $requestedPermissions[$permission]) { + throw "Installation token did not grant requested permission '$permission': '$($requestedPermissions[$permission])'." + } + } + } + return $token } function Invoke-LoginToGitHub { + param( + [string] $TokenRequest, + [string] $VariableNamePrefix = 'GH_TOKEN' + ) + + $tokenRequestBody = @{} + if ($TokenRequest) { + $tokenRequestBody = ConvertFrom-Json -InputObject $TokenRequest -AsHashtable + if ($tokenRequestBody -isnot [System.Collections.IDictionary]) { + throw 'TokenRequest must be a JSON object.' + } + } + Write-Host "Generating GitHub App JWT by signing via Azure Key Vault (no key export)..." $jwt = New-GitHubAppJwt -VaultName $KeyVaultName -KeyName $KeyName -AppId $GitHubAppId @@ -232,7 +274,7 @@ function Invoke-LoginToGitHub { Write-Host "Installation ID resolved: $installationId" Write-Host "Exchanging JWT for installation access token..." - $installationToken = New-GitHubInstallationToken -Jwt $jwt -InstallationId $installationId -ApiBase $GitHubApiBaseUrl -ApiVersion $GitHubApiVersion + $installationToken = New-GitHubInstallationToken -Jwt $jwt -InstallationId $installationId -ApiBase $GitHubApiBaseUrl -ApiVersion $GitHubApiVersion -TokenRequest $tokenRequestBody $variableName = $VariableNamePrefix if ($AlwaysUseOwnerSuffix -or $InstallationTokenOwners.Count -gt 1) { @@ -278,4 +320,4 @@ if ($env:PESTER_TEST_RUN -eq 'true') { return } -Invoke-LoginToGitHub +Invoke-LoginToGitHub -TokenRequest $TokenRequest -VariableNamePrefix $VariableNamePrefix