Updates to Vally handling (#3383)

* Disable skills, disable MCP telemetry capture

* Add flag to New-BuildInfo to skip building all when common code is changed

* Validate changes

* Revert validation test change and update comment

.github/workflows/vally-eval.yml

@@ -67,7 +67,7 @@ jobs:
             Write-Host "Run type: manual run"
           }
 
-          ./eng/scripts/New-BuildInfo.ps1 -ServerName Azure.Mcp.Server
+          ./eng/scripts/New-BuildInfo.ps1 -ServerName Azure.Mcp.Server -CommonCodeBuildsAll $false
 
       - name: Build server code
         shell: pwsh

.vally.yaml

@@ -1,4 +1,5 @@
 paths:
+  skills: []
   evals: ["tools/Azure.Mcp.Tools.*/tests/", ".work/vally/evals/*/"]
   results: ".work/vally/vally-results/"
 
@@ -10,6 +11,8 @@ environments:
         command: ".work/build/Azure.Mcp.Server/linux-x64/azmcp"
         args: ["server", "start"]
         timeout: "2m"
+        env:
+          AZURE_MCP_COLLECT_TELEMETRY: 'false'
   windows:
     mcpServers:
       azmcp:
@@ -17,6 +20,8 @@ environments:
         command: ".work/build/Azure.Mcp.Server/windows-x64/azmcp"
         args: ["server", "start"]
         timeout: "2m"
+        env:
+          AZURE_MCP_COLLECT_TELEMETRY: 'false'
 
 suites:
   basic:

eng/scripts/New-BuildInfo.ps1

@@ -9,7 +9,8 @@ param(
     [string] $ServerName,
     [switch] $IncludeNative,
     [switch] $TestPipeline,
-    [switch] $CI
+    [switch] $CI,
+    [bool] $CommonCodeBuildsAll = $true
 )
 
 . "$PSScriptRoot/../common/scripts/common.ps1"
@@ -261,10 +262,10 @@ function Get-PathsToTest {
 
         # If we're in a pull request, use the set of changed files to narrow down the set of paths to test.
         $changedFiles = Get-ChangedFiles
-        # Track whether engineering, the Core libraries, or shared build changed. If so, build everything.
-        $coreChanged = ($changedFiles | Where-Object { $_ -match '^core/(Azure|Fabric|Microsoft).Mcp.Core/src/' }).Count -gt 0
-        $engChanged = ($changedFiles | Where-Object { $_ -match '^eng/' }).Count -gt 0
-        $sharedBuildChanged = ($changedFiles | Where-Object { $_ -match '^Directory.(Build|Packages).props' }).Count -gt 0
+        # When common code builds all, track whether engineering, the Core libraries, or shared build changed. If so, build everything.
+        $coreChanged = $CommonCodeBuildsAll -and ($changedFiles | Where-Object { $_ -match '^core/(Azure|Fabric|Microsoft).Mcp.Core/src/' }).Count -gt 0
+        $engChanged = $CommonCodeBuildsAll -and  ($changedFiles | Where-Object { $_ -match '^eng/' }).Count -gt 0
+        $sharedBuildChanged = $CommonCodeBuildsAll -and  ($changedFiles | Where-Object { $_ -match '^Directory.(Build|Packages).props' }).Count -gt 0
         if ($coreChanged -or $engChanged -or $sharedBuildChanged) {
             Write-Host "Core, engineering, or shared build changes detected. Building everything." -ForegroundColor Yellow
             $pathsToTest = @()