[main] Update common Docker engineering infrastructure with latest

eng/docker-tools/CHANGELOG.md

@@ -4,6 +4,31 @@ All breaking changes and new features in `eng/docker-tools` will be documented i
 
 ---
 
+## 2026-08-10: Pre-ImageBuilder build customization
+
+Build pipeline templates now accept `customPreImageBuilderBuildSteps`. These steps run after
+ImageBuilder is available but before repository content is copied into the Linux ImageBuilder
+image. Repositories can use the hook to stage files into Docker build contexts, such as shared
+`eng/common` content required by Dockerfiles.
+
+---
+
+## 2026-07-28: Publish stage artifacts consolidated
+
+The Publish stage now uploads `$(Build.ArtifactStagingDirectory)` once as
+`publish-attempt-$(System.JobAttempt)`. This replaces these separate artifacts:
+
+- `image-info-final-$(System.JobAttempt)`
+- `eol-annotation-data-$(System.JobAttempt)`
+- `annotation-digests-<registry>-$(System.JobAttempt)`
+- `source-build-id`
+
+Consumers of those artifact names must download the consolidated artifact instead. Files retain
+their staging-directory paths, including `imageInfo/`, `eol-annotation-data/`,
+`annotation-digests/`, and `sourceBuildId/source-build-id.txt`.
+
+---
+
 ## 2026-06-11: Configurable per-registry referrer-lookup rate limit
 
 - Issue: [#2141](https://github.com/dotnet/docker-tools/issues/2141)

eng/docker-tools/DEV-GUIDE.md

@@ -352,12 +352,10 @@ The system has built-in retry logic but requires manual intervention after repea
 
 Once you've fixed the underlying problem (Dockerfile change, test fix, etc.) and have a successful build:
 
-1. Navigate to the successful pipeline run in Azure DevOps
-2. Add the `autobuilder` label to that run
-3. This signals to the infrastructure that a successful build has occurred
-4. The system will resume automatic rebuilds for that image as needed
+1. Manually queue a build for the affected image paths
+2. After the build succeeds, the system will resume automatic rebuilds for that image as needed
 
-The `autobuilder` label is how the infrastructure tracks that the failure cycle has been broken and normal operations can resume.
+The infrastructure considers the three most recent pipeline runs, so any successful run breaks the failure cycle.
 
 ---
 
@@ -391,6 +389,22 @@ To force a rebuild regardless of cache state, set the `noCache` parameter to `tr
 
 ## Common Customization Patterns
 
+### Pattern: Staging Files into Docker Build Contexts
+
+Use `customPreImageBuilderBuildSteps` to modify repository content immediately before the
+repository is copied into the Linux ImageBuilder image. For example, a repository can stage
+shared `eng/common` files next to Dockerfiles whose build contexts cannot access the repository
+root:
+
+```yaml
+customPreImageBuilderBuildSteps:
+- powershell: ./eng/Stage-EngCommon.ps1
+  displayName: Stage eng/common in Docker Build Contexts
+```
+
+The steps run once per Linux build job, after ImageBuilder is available and before the
+`Dockerfile.WithRepo` image is built.
+
 ### Pattern: Adding Build Arguments
 
 Pass Dockerfile `ARG` values via ImageBuilder:

eng/docker-tools/Install-DotNetSdk.ps1

@@ -20,7 +20,7 @@ param(
     [string]
     $InstallPath,
     [string]
-    $Channel = "9.0"
+    $Channel = "10.0"
 )
 
 Set-StrictMode -Version Latest

eng/docker-tools/Update-ImageBuilder.ps1

@@ -0,0 +1,89 @@
+#!/usr/bin/env pwsh
+
+<#
+.SYNOPSIS
+Example script that updates the bundled docker-tools infrastructure in the current repo to a specific
+ImageBuilder image.
+
+.DESCRIPTION
+ImageBuilder ships a copy of the eng/docker-tools infrastructure and writes it back to disk via its
+'update' command. The reference that 'update' records into
+eng/docker-tools/templates/variables/docker-images.yml is supplied as an argument rather than being
+baked into the build, so the caller decides exactly which image the repo should pin to.
+
+This example resolves the multi-platform (manifest list / image index) digest of an ImageBuilder image
+(the published 'latest' tag by default) and passes that digest reference to the 'update' command, which
+runs inside the same image with the repository mounted so it can rewrite eng/docker-tools on disk.
+
+.PARAMETER ImageBuilderImage
+The ImageBuilder image to resolve and run. Defaults to the published 'latest' tag.
+
+.PARAMETER RepoRoot
+The root of the git repository to update. Defaults to the current directory.
+
+.NOTES
+To exercise an unpublished ImageBuilder (for example, the 'update' command before it is released),
+build the image, push it to a registry it can be pulled from, and pass its reference via
+-ImageBuilderImage. The digest is read from the registry, so the image must be pushed first.
+#>
+[CmdletBinding()]
+param(
+    [string]
+    $ImageBuilderImage = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest",
+
+    [string]
+    $RepoRoot = (Get-Location).Path
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Exec {
+    param ([string] $Cmd)
+
+    Write-Output "Executing: '$Cmd'"
+    Invoke-Expression $Cmd
+    if ($LASTEXITCODE -ne 0) {
+        throw "Failed: '$Cmd'"
+    }
+}
+
+# Strip any existing tag or digest so the resolved digest can be appended to the bare repository name.
+# A tag is a ':' within the final path segment; a registry host's ':port' precedes the last '/', so it
+# must not be mistaken for a tag.
+function Get-RepositoryName {
+    param ([string] $Reference)
+
+    $withoutDigest = $Reference.Split('@')[0]
+    $lastSlash = $withoutDigest.LastIndexOf('/')
+    $lastColon = $withoutDigest.LastIndexOf(':')
+    if ($lastColon -gt $lastSlash) {
+        return $withoutDigest.Substring(0, $lastColon)
+    }
+
+    return $withoutDigest
+}
+
+# Resolve the multi-platform digest. 'docker buildx imagetools inspect' reads the top-level manifest
+# straight from the registry, so for a multi-arch image this is the manifest list (image index) digest
+# rather than a single platform's digest. Pinning the index keeps the reference valid on every
+# platform the pipeline runs on.
+$digest = (docker buildx imagetools inspect $ImageBuilderImage --format '{{.Manifest.Digest}}')
+if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($digest)) {
+    throw "Unable to resolve a multi-platform digest for '$ImageBuilderImage'."
+}
+
+$repository = Get-RepositoryName $ImageBuilderImage
+$imageBuilderRef = "$repository@$($digest.Trim())"
+
+Write-Output "Resolved ImageBuilder reference: $imageBuilderRef"
+
+# Run 'update' from the resolved digest, mounting the repository so it can write eng/docker-tools to
+# disk. The command must run from the repository root, which is why $RepoRoot is the mounted working
+# directory. Running by the same digest that gets recorded keeps the writer and the pinned reference
+# identical.
+Exec ("docker run --rm " `
+    + "-v `"${RepoRoot}:/repo`" " `
+    + "-w /repo " `
+    + "$imageBuilderRef " `
+    + "update $imageBuilderRef")

eng/docker-tools/templates/1es.yml

@@ -54,7 +54,7 @@ resources:
   - repository: 1ESPipelineTemplates
     type: git
     name: 1ESPipelineTemplates/1ESPipelineTemplates
-    ref: refs/tags/release-3.12.2026-05-30-1
+    ref: refs/tags/release
 
 extends:
   template: /eng/docker-tools/templates/task-prefix-decorator.yml@self

eng/docker-tools/templates/jobs/build-images.yml

@@ -10,6 +10,8 @@ parameters:
   # Custom steps that run after ImageBuilder is set up but before the build starts.
   # Use for build-specific initialization (e.g., setting variables, additional setup).
   customBuildInitSteps: []
+  # Custom steps that modify repository content before it is copied into the ImageBuilder image.
+  customPreImageBuilderBuildSteps: []
   publishConfig: null
   versionsRepoRef: ""
   noCache: false
@@ -41,6 +43,7 @@ jobs:
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
       cleanupDocker: true
       customInitSteps: ${{ parameters.customInitSteps }}
+      customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }}
   - ${{ parameters.customBuildInitSteps }}
   - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self
     parameters:
@@ -140,4 +143,4 @@ jobs:
         displayName: Publish SBOM
         internalProjectName: ${{ parameters.internalProjectName }}
         publicProjectName: ${{ parameters.publicProjectName }}
-        condition: ne(variables['BuildImages.builtImages'], '')
+        condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))

eng/docker-tools/templates/jobs/cg-build-projects.yml

@@ -15,7 +15,7 @@ parameters:
 # See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values
 - name: dotnetVersionChannel
   type: string
-  default: '9.0'
+  default: '10.0'
   displayName: .NET Version
 # Additional steps to run before building projects (e.g. custom SDK installation).
 - name: initSteps

eng/docker-tools/templates/jobs/post-build.yml

@@ -11,7 +11,6 @@ jobs:
   variables:
     imageInfosSubDir: "/image-infos"
     imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)"
-    imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)"
     imageInfosOutputSubDir: "/output"
     sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom"
   steps:
@@ -73,11 +72,10 @@ jobs:
         exit 0
       }
 
-      New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force
       $(runImageBuilderCmd) mergeImageInfo `
         --manifest $(manifest) `
-        $(imageInfosContainerDir) `
-        $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json `
+        image-infos `
+        image-infos/output/image-info.json `
         $(manifestVariables)
     name: MergeImageInfoFiles
     displayName: Merge Image Info Files
@@ -89,7 +87,7 @@ jobs:
         condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
         args: >-
           createManifestList
-          '$(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json'
+          'image-infos/output/image-info.json'
           --repo-prefix '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}'
           --os-type '*'
           --architecture '*'

eng/docker-tools/templates/jobs/publish.yml

@@ -37,8 +37,6 @@ jobs:
       value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
   - name: imageInfoHostDir
     value: $(Build.ArtifactStagingDirectory)/imageInfo
-  - name: imageInfoContainerDir
-    value: $(artifactsPath)/imageInfo
   - name: sourceBuildIdOutputDir
     value: $(Build.ArtifactStagingDirectory)/sourceBuildId
   - name: commitOverrideArg
@@ -99,7 +97,7 @@ jobs:
 
   - script: >
       $(runImageBuilderCmd) trimUnchangedPlatforms
-      '$(imageInfoContainerDir)/image-info.json'
+      'imageInfo/image-info.json'
     displayName: Trim Unchanged Images
 
   - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
@@ -113,23 +111,15 @@ jobs:
         --os-type '*'
         --architecture '*'
         --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
-        --image-info '$(imageInfoContainerDir)/image-info.json'
+        --image-info 'imageInfo/image-info.json'
         $(dryRunArg)
         $(imageBuilder.pathArgs)
         $(imageBuilder.commonCmdArgs)
 
-  - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self
-    parameters:
-      path: $(imageInfoHostDir)
-      artifactName: image-info-final-$(System.JobAttempt)
-      displayName: Publish Image Info File Artifact
-      internalProjectName: ${{ parameters.internalProjectName }}
-      publicProjectName: ${{ parameters.publicProjectName }}
-
   - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self
     parameters:
       publishConfig: ${{ parameters.publishConfig }}
-      imageInfoPath: '$(imageinfoContainerDir)/image-info.json'
+      imageInfoPath: 'imageInfo/image-info.json'
       minQueueTime: $(imageQueueTime)
       dryRunArg: $(dryRunArg)
       condition: succeeded()
@@ -139,9 +129,6 @@ jobs:
       dryRunArg: $(dryRunArg)
       condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
 
-  - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
-    displayName: Create EOL Annotation Data Directory
-
   - script: |-
       cd $(versionsRepoRoot)
       git pull origin $(gitHubVersionsRepoInfo.branch)
@@ -155,13 +142,13 @@ jobs:
 
   - script: >
       $(runImageBuilderCmd) mergeImageInfo
-      $(imageInfoContainerDir)
-      $(imageInfoContainerDir)/full-image-info-new.json
+      imageInfo
+      imageInfo/full-image-info-new.json
       $(manifestVariables)
       $(dryRunArg)
       --manifest $(manifest)
       --publish
-      --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
+      --initial-image-info-path imageInfo/full-image-info-orig.json
       $(commitOverrideArg)
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
     displayName: Merge Image Info
@@ -178,7 +165,7 @@ jobs:
       condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true'))
       args: >-
         ingestKustoImageInfo
-        '$(imageInfoContainerDir)/image-info.json'
+        'imageInfo/image-info.json'
         '$(kusto.cluster)'
         '$(kusto.database)'
         '$(kusto.imageTable)'
@@ -197,29 +184,21 @@ jobs:
         generateEolAnnotationDataForPublish
         '${{ parameters.publishConfig.PublishRegistry.server }}'
         '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
-        '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
-        '$(imageInfoContainerDir)/full-image-info-orig.json'
-        '$(imageInfoContainerDir)/full-image-info-new.json'
+        'eol-annotation-data/eol-annotation-data.json'
+        'imageInfo/full-image-info-orig.json'
+        'imageInfo/full-image-info-new.json'
         $(generateEolAnnotationDataExtraOptions)
         $(dryRunArg)
 
-  - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self
-    parameters:
-      path: $(Build.ArtifactStagingDirectory)/eol-annotation-data
-      artifactName: eol-annotation-data-$(System.JobAttempt)
-      displayName: Publish EOL Annotation Data Artifact
-      internalProjectName: internal
-      publicProjectName: public
-      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
-
   - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self
     parameters:
       acr: ${{ parameters.publishConfig.PublishRegistry }}
-      dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
+      dataFile: eol-annotation-data/eol-annotation-data.json
+      skipArtifactPublish: true
 
   - script: >
       $(runImageBuilderCmd) publishImageInfo
-      '$(imageInfoContainerDir)/full-image-info-new.json'
+      'imageInfo/full-image-info-new.json'
       '$(gitHubVersionsRepoInfo.userName)'
       '$(gitHubVersionsRepoInfo.email)'
       $(gitHubVersionsRepoInfo.authArgs)
@@ -248,7 +227,7 @@ jobs:
       $(runImageBuilderCmd) postPublishNotification
       '$(publishNotificationRepoName)'
       '$(branchName)'
-      '$(imageInfoContainerDir)/image-info.json'
+      'imageInfo/image-info.json'
       $(Build.BuildId)
       '$(System.AccessToken)'
       '$(azdoOrgName)'
@@ -285,8 +264,10 @@ jobs:
 
   - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(sourceBuildIdOutputDir)
-      artifactName: source-build-id
-      displayName: Publish Source Build ID Artifact
+      path: $(Build.ArtifactStagingDirectory)
+      artifactName: publish-attempt-$(System.JobAttempt)
+      displayName: Publish Artifacts
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
+      # Always upload, even if the pipeline fails or is canceled.
+      condition: always()