diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index 0936f6fe..4b0b071b 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,6 +4,31 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-08-10: Pre-ImageBuilder build customization + +Build pipeline templates now accept `customPreImageBuilderBuildSteps`. These steps run after +ImageBuilder is available but before repository content is copied into the Linux ImageBuilder +image. Repositories can use the hook to stage files into Docker build contexts, such as shared +`eng/common` content required by Dockerfiles. + +--- + +## 2026-07-28: Publish stage artifacts consolidated + +The Publish stage now uploads `$(Build.ArtifactStagingDirectory)` once as +`publish-attempt-$(System.JobAttempt)`. This replaces these separate artifacts: + +- `image-info-final-$(System.JobAttempt)` +- `eol-annotation-data-$(System.JobAttempt)` +- `annotation-digests--$(System.JobAttempt)` +- `source-build-id` + +Consumers of those artifact names must download the consolidated artifact instead. Files retain +their staging-directory paths, including `imageInfo/`, `eol-annotation-data/`, +`annotation-digests/`, and `sourceBuildId/source-build-id.txt`. + +--- + ## 2026-06-11: Configurable per-registry referrer-lookup rate limit - Issue: [#2141](https://github.com/dotnet/docker-tools/issues/2141) diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index d47fb3eb..ba45f307 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -352,12 +352,10 @@ The system has built-in retry logic but requires manual intervention after repea Once you've fixed the underlying problem (Dockerfile change, test fix, etc.) and have a successful build: -1. Navigate to the successful pipeline run in Azure DevOps -2. Add the `autobuilder` label to that run -3. This signals to the infrastructure that a successful build has occurred -4. The system will resume automatic rebuilds for that image as needed +1. Manually queue a build for the affected image paths +2. After the build succeeds, the system will resume automatic rebuilds for that image as needed -The `autobuilder` label is how the infrastructure tracks that the failure cycle has been broken and normal operations can resume. +The infrastructure considers the three most recent pipeline runs, so any successful run breaks the failure cycle. --- @@ -391,6 +389,22 @@ To force a rebuild regardless of cache state, set the `noCache` parameter to `tr ## Common Customization Patterns +### Pattern: Staging Files into Docker Build Contexts + +Use `customPreImageBuilderBuildSteps` to modify repository content immediately before the +repository is copied into the Linux ImageBuilder image. For example, a repository can stage +shared `eng/common` files next to Dockerfiles whose build contexts cannot access the repository +root: + +```yaml +customPreImageBuilderBuildSteps: +- powershell: ./eng/Stage-EngCommon.ps1 + displayName: Stage eng/common in Docker Build Contexts +``` + +The steps run once per Linux build job, after ImageBuilder is available and before the +`Dockerfile.WithRepo` image is built. + ### Pattern: Adding Build Arguments Pass Dockerfile `ARG` values via ImageBuilder: diff --git a/eng/docker-tools/Install-DotNetSdk.ps1 b/eng/docker-tools/Install-DotNetSdk.ps1 index 35f6516e..07206c09 100644 --- a/eng/docker-tools/Install-DotNetSdk.ps1 +++ b/eng/docker-tools/Install-DotNetSdk.ps1 @@ -20,7 +20,7 @@ param( [string] $InstallPath, [string] - $Channel = "9.0" + $Channel = "10.0" ) Set-StrictMode -Version Latest diff --git a/eng/docker-tools/Update-ImageBuilder.ps1 b/eng/docker-tools/Update-ImageBuilder.ps1 new file mode 100644 index 00000000..88f0361c --- /dev/null +++ b/eng/docker-tools/Update-ImageBuilder.ps1 @@ -0,0 +1,89 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS +Example script that updates the bundled docker-tools infrastructure in the current repo to a specific +ImageBuilder image. + +.DESCRIPTION +ImageBuilder ships a copy of the eng/docker-tools infrastructure and writes it back to disk via its +'update' command. The reference that 'update' records into +eng/docker-tools/templates/variables/docker-images.yml is supplied as an argument rather than being +baked into the build, so the caller decides exactly which image the repo should pin to. + +This example resolves the multi-platform (manifest list / image index) digest of an ImageBuilder image +(the published 'latest' tag by default) and passes that digest reference to the 'update' command, which +runs inside the same image with the repository mounted so it can rewrite eng/docker-tools on disk. + +.PARAMETER ImageBuilderImage +The ImageBuilder image to resolve and run. Defaults to the published 'latest' tag. + +.PARAMETER RepoRoot +The root of the git repository to update. Defaults to the current directory. + +.NOTES +To exercise an unpublished ImageBuilder (for example, the 'update' command before it is released), +build the image, push it to a registry it can be pulled from, and pass its reference via +-ImageBuilderImage. The digest is read from the registry, so the image must be pushed first. +#> +[CmdletBinding()] +param( + [string] + $ImageBuilderImage = "mcr.microsoft.com/dotnet-buildtools/image-builder:latest", + + [string] + $RepoRoot = (Get-Location).Path +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Exec { + param ([string] $Cmd) + + Write-Output "Executing: '$Cmd'" + Invoke-Expression $Cmd + if ($LASTEXITCODE -ne 0) { + throw "Failed: '$Cmd'" + } +} + +# Strip any existing tag or digest so the resolved digest can be appended to the bare repository name. +# A tag is a ':' within the final path segment; a registry host's ':port' precedes the last '/', so it +# must not be mistaken for a tag. +function Get-RepositoryName { + param ([string] $Reference) + + $withoutDigest = $Reference.Split('@')[0] + $lastSlash = $withoutDigest.LastIndexOf('/') + $lastColon = $withoutDigest.LastIndexOf(':') + if ($lastColon -gt $lastSlash) { + return $withoutDigest.Substring(0, $lastColon) + } + + return $withoutDigest +} + +# Resolve the multi-platform digest. 'docker buildx imagetools inspect' reads the top-level manifest +# straight from the registry, so for a multi-arch image this is the manifest list (image index) digest +# rather than a single platform's digest. Pinning the index keeps the reference valid on every +# platform the pipeline runs on. +$digest = (docker buildx imagetools inspect $ImageBuilderImage --format '{{.Manifest.Digest}}') +if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($digest)) { + throw "Unable to resolve a multi-platform digest for '$ImageBuilderImage'." +} + +$repository = Get-RepositoryName $ImageBuilderImage +$imageBuilderRef = "$repository@$($digest.Trim())" + +Write-Output "Resolved ImageBuilder reference: $imageBuilderRef" + +# Run 'update' from the resolved digest, mounting the repository so it can write eng/docker-tools to +# disk. The command must run from the repository root, which is why $RepoRoot is the mounted working +# directory. Running by the same digest that gets recorded keeps the writer and the pinned reference +# identical. +Exec ("docker run --rm " ` + + "-v `"${RepoRoot}:/repo`" " ` + + "-w /repo " ` + + "$imageBuilderRef " ` + + "update $imageBuilderRef") diff --git a/eng/docker-tools/templates/1es.yml b/eng/docker-tools/templates/1es.yml index bf3c0fe5..4056610e 100644 --- a/eng/docker-tools/templates/1es.yml +++ b/eng/docker-tools/templates/1es.yml @@ -54,7 +54,7 @@ resources: - repository: 1ESPipelineTemplates type: git name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release-3.12.2026-05-30-1 + ref: refs/tags/release extends: template: /eng/docker-tools/templates/task-prefix-decorator.yml@self diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 7327b6d6..33b866f6 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -10,6 +10,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] publishConfig: null versionsRepoRef: "" noCache: false @@ -41,6 +43,7 @@ jobs: versionsRepoRef: ${{ parameters.versionsRepoRef }} cleanupDocker: true customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} - ${{ parameters.customBuildInitSteps }} - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self parameters: @@ -140,4 +143,4 @@ jobs: displayName: Publish SBOM internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - condition: ne(variables['BuildImages.builtImages'], '') + condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) diff --git a/eng/docker-tools/templates/jobs/cg-build-projects.yml b/eng/docker-tools/templates/jobs/cg-build-projects.yml index 10d50689..ade54b50 100644 --- a/eng/docker-tools/templates/jobs/cg-build-projects.yml +++ b/eng/docker-tools/templates/jobs/cg-build-projects.yml @@ -15,7 +15,7 @@ parameters: # See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values - name: dotnetVersionChannel type: string - default: '9.0' + default: '10.0' displayName: .NET Version # Additional steps to run before building projects (e.g. custom SDK installation). - name: initSteps diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml index 32b9c799..8a07759b 100644 --- a/eng/docker-tools/templates/jobs/post-build.yml +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -11,7 +11,6 @@ jobs: variables: imageInfosSubDir: "/image-infos" imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)" - imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)" imageInfosOutputSubDir: "/output" sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom" steps: @@ -73,11 +72,10 @@ jobs: exit 0 } - New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force $(runImageBuilderCmd) mergeImageInfo ` --manifest $(manifest) ` - $(imageInfosContainerDir) ` - $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json ` + image-infos ` + image-infos/output/image-info.json ` $(manifestVariables) name: MergeImageInfoFiles displayName: Merge Image Info Files @@ -89,7 +87,7 @@ jobs: condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'), ne(variables['Build.Reason'], 'PullRequest')) args: >- createManifestList - '$(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json' + 'image-infos/output/image-info.json' --repo-prefix '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' --os-type '*' --architecture '*' diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index 5839be2d..e78449c9 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -37,8 +37,6 @@ jobs: value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] - name: imageInfoHostDir value: $(Build.ArtifactStagingDirectory)/imageInfo - - name: imageInfoContainerDir - value: $(artifactsPath)/imageInfo - name: sourceBuildIdOutputDir value: $(Build.ArtifactStagingDirectory)/sourceBuildId - name: commitOverrideArg @@ -99,7 +97,7 @@ jobs: - script: > $(runImageBuilderCmd) trimUnchangedPlatforms - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' displayName: Trim Unchanged Images - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self @@ -113,23 +111,15 @@ jobs: --os-type '*' --architecture '*' --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' - --image-info '$(imageInfoContainerDir)/image-info.json' + --image-info 'imageInfo/image-info.json' $(dryRunArg) $(imageBuilder.pathArgs) $(imageBuilder.commonCmdArgs) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(imageInfoHostDir) - artifactName: image-info-final-$(System.JobAttempt) - displayName: Publish Image Info File Artifact - internalProjectName: ${{ parameters.internalProjectName }} - publicProjectName: ${{ parameters.publicProjectName }} - - template: /eng/docker-tools/templates/steps/wait-for-mcr-image-ingestion.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} - imageInfoPath: '$(imageinfoContainerDir)/image-info.json' + imageInfoPath: 'imageInfo/image-info.json' minQueueTime: $(imageQueueTime) dryRunArg: $(dryRunArg) condition: succeeded() @@ -139,9 +129,6 @@ jobs: dryRunArg: $(dryRunArg) condition: and(succeeded(), eq(variables['publishReadme'], 'true')) - - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data - displayName: Create EOL Annotation Data Directory - - script: |- cd $(versionsRepoRoot) git pull origin $(gitHubVersionsRepoInfo.branch) @@ -155,13 +142,13 @@ jobs: - script: > $(runImageBuilderCmd) mergeImageInfo - $(imageInfoContainerDir) - $(imageInfoContainerDir)/full-image-info-new.json + imageInfo + imageInfo/full-image-info-new.json $(manifestVariables) $(dryRunArg) --manifest $(manifest) --publish - --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json + --initial-image-info-path imageInfo/full-image-info-orig.json $(commitOverrideArg) condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) displayName: Merge Image Info @@ -178,7 +165,7 @@ jobs: condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) args: >- ingestKustoImageInfo - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' '$(kusto.cluster)' '$(kusto.database)' '$(kusto.imageTable)' @@ -197,29 +184,21 @@ jobs: generateEolAnnotationDataForPublish '${{ parameters.publishConfig.PublishRegistry.server }}' '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}' - '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' - '$(imageInfoContainerDir)/full-image-info-orig.json' - '$(imageInfoContainerDir)/full-image-info-new.json' + 'eol-annotation-data/eol-annotation-data.json' + 'imageInfo/full-image-info-orig.json' + 'imageInfo/full-image-info-new.json' $(generateEolAnnotationDataExtraOptions) $(dryRunArg) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/eol-annotation-data - artifactName: eol-annotation-data-$(System.JobAttempt) - displayName: Publish EOL Annotation Data Artifact - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self parameters: acr: ${{ parameters.publishConfig.PublishRegistry }} - dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json + dataFile: eol-annotation-data/eol-annotation-data.json + skipArtifactPublish: true - script: > $(runImageBuilderCmd) publishImageInfo - '$(imageInfoContainerDir)/full-image-info-new.json' + 'imageInfo/full-image-info-new.json' '$(gitHubVersionsRepoInfo.userName)' '$(gitHubVersionsRepoInfo.email)' $(gitHubVersionsRepoInfo.authArgs) @@ -248,7 +227,7 @@ jobs: $(runImageBuilderCmd) postPublishNotification '$(publishNotificationRepoName)' '$(branchName)' - '$(imageInfoContainerDir)/image-info.json' + 'imageInfo/image-info.json' $(Build.BuildId) '$(System.AccessToken)' '$(azdoOrgName)' @@ -285,8 +264,10 @@ jobs: - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self parameters: - path: $(sourceBuildIdOutputDir) - artifactName: source-build-id - displayName: Publish Source Build ID Artifact + path: $(Build.ArtifactStagingDirectory) + artifactName: publish-attempt-$(System.JobAttempt) + displayName: Publish Artifacts internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + # Always upload, even if the pipeline fails or is canceled. + condition: always() diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index a03804aa..6d602fe4 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -13,6 +13,8 @@ parameters: # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] + # Custom steps that modify repository content before it is copied into the ImageBuilder image. + customPreImageBuilderBuildSteps: [] customTestInitSteps: [] sourceBuildPipelineRunId: "" # When true, the Post-Build stage runs even if the Build stage failed (succeededOrFailed). @@ -113,6 +115,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -128,6 +131,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -143,6 +147,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -158,6 +163,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -173,6 +179,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -188,6 +195,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} @@ -203,6 +211,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} noCache: ${{ parameters.noCache }} publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml index 5b7e2337..81eee2c9 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-and-test.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-and-test.yml @@ -29,6 +29,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -58,6 +59,7 @@ stages: testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} customTestInitSteps: ${{ parameters.customTestInitSteps }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index 2c924ef0..9102d994 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -21,6 +21,7 @@ parameters: linuxArmBuildJobTimeout: 60 windowsAmdBuildJobTimeout: 60 customBuildInitSteps: [] + customPreImageBuilderBuildSteps: [] # Test parameters testMatrixType: platformVersionedOs @@ -60,6 +61,7 @@ stages: linuxArmBuildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }} windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} customBuildInitSteps: ${{ parameters.customBuildInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} customInitSteps: ${{ parameters.customInitSteps }} # Test sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/docker-tools/templates/stages/setup-service-connections.yml b/eng/docker-tools/templates/stages/setup-service-connections.yml deleted file mode 100644 index 405bc703..00000000 --- a/eng/docker-tools/templates/stages/setup-service-connections.yml +++ /dev/null @@ -1,68 +0,0 @@ -# This stage exists to tell Azure DevOps about all of the service connections -# that will be used in the pipeline. A service connection will not work unless -# it is declared in this stage's parameters, even if your pipeline has already -# been granted access to the service connection. This stage also does not need -# to complete before the service connection is used. -# -# There are two ways to specify service connections: -# - Pass `serviceConnections` directly (list of {name: string} objects) -# - Pass `publishConfig` + `registries` to look up auth from RegistryAuthentication -parameters: -- name: pool - type: object - default: - name: $(default1ESInternalPoolName) - image: $(default1ESInternalPoolImage) - os: linux - -# Explicit list of service connections to initialize -# Shape: [{ name: string }] -- name: serviceConnections - type: object - default: [] - -# List of registry servers that need authentication. These will be looked up in -# publishConfig.RegistryAuthentication. -# Make sure to provide the publishConfig parameter. -- name: usesRegistries - type: object - default: [] -# Look up service connections from publishConfig based on registries -# The publish configuration containing RegistryAuthentication entries. -- name: publishConfig - type: object - default: {} - -stages: -- stage: SetupServiceConnectionsStage - displayName: Setup service connections - jobs: - - - job: SetupServiceConnectionsJob - displayName: Setup service connections - pool: ${{ parameters.pool }} - steps: - - checkout: none - - # Direct service connections list - - ${{ each serviceConnection in parameters.serviceConnections }}: - - task: AzureCLI@2 - displayName: Setup ${{ serviceConnection.name }} - inputs: - azureSubscription: ${{ serviceConnection.name }} - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - az account show - - # Setup registry service connections - - ${{ if gt(length(parameters.usesRegistries), 0) }}: - - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: - - ${{ if containsValue(parameters.usesRegistries, auth.server) }}: - - task: AzureCLI@2 - displayName: Setup ${{ auth.serviceConnection.name }} - inputs: - azureSubscription: ${{ auth.serviceConnection.name }} - scriptType: pscore - scriptLocation: inlineScript - inlineScript: az account show diff --git a/eng/docker-tools/templates/steps/annotate-eol-digests.yml b/eng/docker-tools/templates/steps/annotate-eol-digests.yml index 8e2f7571..ded68285 100644 --- a/eng/docker-tools/templates/steps/annotate-eol-digests.yml +++ b/eng/docker-tools/templates/steps/annotate-eol-digests.yml @@ -4,10 +4,11 @@ parameters: # Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command - name: dataFile type: string +- name: skipArtifactPublish + type: boolean + default: false steps: - - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests - displayName: Create Annotation Digests Directory - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self parameters: displayName: Annotate EOL Images (${{ parameters.acr.server }}) @@ -18,16 +19,17 @@ steps: "${{ parameters.dataFile }}" "${{ parameters.acr.server }}" "${{ parameters.acr.repoPrefix }}" - $(artifactsPath)/annotation-digests/annotation-digests.txt + annotation-digests/annotation-digests.txt $(dryRunArg) - - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self - parameters: - path: $(Build.ArtifactStagingDirectory)/annotation-digests - artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) - displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) - internalProjectName: internal - publicProjectName: public - condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + - ${{ if not(parameters.skipArtifactPublish) }}: + - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/annotation-digests + artifactName: annotation-digests-${{ parameters.acr.server }}-$(System.JobAttempt) + displayName: Publish Annotation Digests List (${{ parameters.acr.server }}) + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self parameters: displayName: Wait for Annotation Ingestion (${{ parameters.acr.server }}) @@ -40,4 +42,4 @@ steps: condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'), eq(variables['waitForIngestionEnabled'], 'true')) args: >- waitForMarAnnotationIngestion - $(artifactsPath)/annotation-digests/annotation-digests.txt + annotation-digests/annotation-digests.txt diff --git a/eng/docker-tools/templates/steps/generate-appsettings.yml b/eng/docker-tools/templates/steps/generate-appsettings.yml index b1243e70..f1b5bf89 100644 --- a/eng/docker-tools/templates/steps/generate-appsettings.yml +++ b/eng/docker-tools/templates/steps/generate-appsettings.yml @@ -20,7 +20,17 @@ steps: - powershell: |- # Escape backslashes for JSON compatibility (Windows paths like D:\a\_work become D:\\a\\_work) $artifactStagingDirectory = "${{ parameters.artifactStagingDirectory }}" -replace '\\', '\\' - $appsettingsJsonContent = @" + + if ("$(Build.Reason)" -eq "PullRequest") { + $appsettingsJsonContent = @" + { + "BuildConfiguration": { + "ArtifactStagingDirectory": "$artifactStagingDirectory" + } + } + "@ + } else { + $appsettingsJsonContent = @" { "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }}, "BuildConfiguration": { @@ -28,7 +38,9 @@ steps: } } "@ + } + Set-Content -Path "appsettings.json" -Value $appsettingsJsonContent Get-Content -Path "appsettings.json" - displayName: Output publish configuration - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), ${{ parameters.condition }}) + displayName: Output ImageBuilder configuration + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index eda57261..25d22a7b 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -34,6 +34,11 @@ parameters: type: stepList default: [] +# Custom steps that modify repository content before it is copied into the ImageBuilder image. +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + # Registry and authentication configuration for publishing images. # Contains server URLs, repo prefixes, subscriptions, and resource groups. # When null, build/publish steps that require registry access will be skipped. @@ -245,3 +250,4 @@ steps: publishConfig: ${{ parameters.publishConfig }} condition: ${{ parameters.condition }} customInitSteps: ${{ parameters.customInitSteps }} + customPreImageBuilderBuildSteps: ${{ parameters.customPreImageBuilderBuildSteps }} diff --git a/eng/docker-tools/templates/steps/init-docker-linux.yml b/eng/docker-tools/templates/steps/init-docker-linux.yml deleted file mode 100644 index 3216afdb..00000000 --- a/eng/docker-tools/templates/steps/init-docker-linux.yml +++ /dev/null @@ -1,108 +0,0 @@ -parameters: -- name: setupImageBuilder - type: boolean - default: true -- name: setupTestRunner - type: boolean - default: false -# Whether existing Docker images will be deleted -- name: cleanupDocker - type: boolean - default: false -# Whether or not to run the steps in this template -- name: condition - type: string - default: "true" -- name: publishConfig - type: object - default: null - -steps: -- template: /eng/docker-tools/templates/steps/init-common.yml@self - parameters: - condition: ${{ parameters.condition }} -- script: echo "##vso[task.setvariable variable=artifactsPath]/artifacts" - displayName: Define Artifacts Path Variable - condition: and(succeeded(), ${{ parameters.condition }}) - - ################################################################################ - # Cleanup Docker Resources - ################################################################################ -- ${{ if eq(parameters.cleanupDocker, 'true') }}: - - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self - parameters: - condition: ${{ parameters.condition }} - - ################################################################################ - # Setup Image Builder (Optional) - ################################################################################ -- ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.imageBuilder) - displayName: Pull Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - - template: /eng/docker-tools/templates/steps/generate-appsettings.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - condition: ${{ parameters.condition }} - - - script: >- - docker build - -t $(imageNames.imageBuilder.withrepo) - --build-arg IMAGE=$(imageNames.imageBuilder) - -f $(engDockerToolsPath)/Dockerfile.WithRepo . - displayName: Build Image for Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - - task: PowerShell@2 - displayName: Define ImageBuilder Command Variables - condition: and(succeeded(), ${{ parameters.condition }}) - inputs: - targetType: 'inline' - script: | - $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" - Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" - - $dockerRunBaseCmd = @( - "docker run --rm" - ) - - $dockerRunArgs = @( - "-v /var/run/docker.sock:/var/run/docker.sock" - "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" - "-w /repo" - "$(imageBuilderDockerRunExtraOptions)" - "$(imageNames.imageBuilder.withrepo)" - ) - - $authedDockerRunArgs = @( - '-e' - 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' - '-e' - 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' - ) - - $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs - $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs - - $runImageBuilderCmd = $($dockerRunCmd -join ' ') - $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') - - Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" - Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" - - ################################################################################ - # Setup Test Runner (Optional) - ################################################################################ -- ${{ if eq(parameters.setupTestRunner, 'true') }}: - - powershell: $(engDockerToolsPath)/Pull-Image.ps1 $(imageNames.testrunner) - displayName: Pull Test Runner - condition: and(succeeded(), ${{ parameters.condition }}) - - script: > - docker build - -t $(imageNames.testRunner.withrepo) - --build-arg IMAGE=$(imageNames.testrunner) - -f $(engDockerToolsPath)/Dockerfile.WithRepo . - displayName: Build Test Runner Image - condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/docker-tools/templates/steps/init-docker-windows.yml b/eng/docker-tools/templates/steps/init-docker-windows.yml deleted file mode 100644 index 36bd8e6d..00000000 --- a/eng/docker-tools/templates/steps/init-docker-windows.yml +++ /dev/null @@ -1,55 +0,0 @@ -parameters: - setupImageBuilder: true - condition: "true" - publishConfig: null - -steps: -- template: /eng/docker-tools/templates/steps/init-common.yml@self - parameters: - condition: ${{ parameters.condition }} -- powershell: echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" - displayName: Define Artifacts Path Variable - condition: and(succeeded(), ${{ parameters.condition }}) - - ################################################################################ - # Cleanup Docker Resources - ################################################################################ -- template: /eng/docker-tools/templates/steps/cleanup-docker-windows.yml@self - parameters: - condition: ${{ parameters.condition }} - - ################################################################################ - # Setup Image Builder (Optional) - ################################################################################ -- ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - powershell: $(engDockerToolsPath)/Invoke-WithRetry.ps1 "docker pull $(imageNames.imageBuilder)" - displayName: Pull Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - script: docker create --name setupImageBuilder-$(Build.BuildId)-$(System.JobId) $(imageNames.imageBuilder) - displayName: Create Setup Container - condition: and(succeeded(), ${{ parameters.condition }}) - - script: > - docker cp - setupImageBuilder-$(Build.BuildId)-$(System.JobId):/image-builder - $(Build.BinariesDirectory)/.Microsoft.DotNet.ImageBuilder - displayName: Copy Image Builder - condition: and(succeeded(), ${{ parameters.condition }}) - - script: docker rm -f setupImageBuilder-$(Build.BuildId)-$(System.JobId) - displayName: Cleanup Setup Container - condition: and(always(), ${{ parameters.condition }}) - continueOnError: true - - - template: /eng/docker-tools/templates/steps/generate-appsettings.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - condition: ${{ parameters.condition }} - - - task: PowerShell@2 - displayName: Define runImageBuilderCmd Variables - condition: and(succeeded(), ${{ parameters.condition }}) - inputs: - targetType: 'inline' - script: | - $runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe" - Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" - Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runImageBuilderCmd" diff --git a/eng/docker-tools/templates/steps/init-imagebuilder.yml b/eng/docker-tools/templates/steps/init-imagebuilder.yml index b85c62b6..1e4f57a1 100644 --- a/eng/docker-tools/templates/steps/init-imagebuilder.yml +++ b/eng/docker-tools/templates/steps/init-imagebuilder.yml @@ -21,6 +21,10 @@ parameters: type: stepList default: [] +- name: customPreImageBuilderBuildSteps + type: stepList + default: [] + steps: # Custom ImageBuilder setup (e.g., bootstrap from source) - ${{ if gt(length(parameters.customInitSteps), 0) }}: @@ -67,6 +71,7 @@ steps: # The withrepo image layers the checked-out repository into the ImageBuilder # container at /repo, so ImageBuilder can access manifests and Dockerfiles - ${{ if eq(parameters.dockerClientOS, 'linux') }}: + - ${{ parameters.customPreImageBuilderBuildSteps }} - script: >- docker build -t $(imageNames.imageBuilder.withrepo) diff --git a/eng/docker-tools/templates/steps/init-matrix-build-publish.yml b/eng/docker-tools/templates/steps/init-matrix-build-publish.yml deleted file mode 100644 index e5e6bf19..00000000 --- a/eng/docker-tools/templates/steps/init-matrix-build-publish.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Initialize common variables used in -# - Generating build matrix -# - Building images -# - Running tests -# - Publishing images - -parameters: - publishConfig: null - versionsRepoRef: "" - versionsRepoPath: "versions" - -steps: -- checkout: self -- ${{ if ne(parameters.versionsRepoRef, '') }}: - - checkout: ${{ parameters.versionsRepoRef }} - path: s/${{ parameters.versionsRepoPath }} - persistCredentials: true - fetchDepth: 1 - condition: succeeded() -- powershell: | - $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" - if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.InternalMirrorRegistry.repoPrefix }} --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}" - } - - if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'" - } - - if ("${{ parameters.versionsRepoRef }}" -ne "") { - $versionsBasePath = "${{ parameters.versionsRepoPath }}/" - $pipelineDisabledCache = "false" - - $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/") - if ($pathSeparatorIndex -ge 0) { - $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1) - } - else { - $buildRepoName = "$(Build.Repository.Name)" - } - - $engDockerToolsPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engDockerToolsRelativePath)" - - $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng" - $manifest = "$buildRepoName/$(manifest)" - $testResultsDirectory = "$buildRepoName/$testResultsDirectory" - - if ("$(testScriptPath)") { - $testScriptPath = "$buildRepoName/$(testScriptPath)" - } - - echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName" - echo "##vso[task.setvariable variable=engDockerToolsPath]$engDockerToolsPath" - echo "##vso[task.setvariable variable=manifest]$manifest" - echo "##vso[task.setvariable variable=engPath]$engPath" - echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath" - echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory" - } - else { - $versionsBasePath = "" - $pipelineDisabledCache = "true" - } - - echo "##vso[task.setvariable variable=commonMatrixAndBuildOptions]$commonMatrixAndBuildOptions" - echo "##vso[task.setvariable variable=versionsBasePath]$versionsBasePath" - echo "##vso[task.setvariable variable=pipelineDisabledCache]$pipelineDisabledCache" - displayName: Set Common Variables for Matrix, Build, and Publish - -- ${{ if ne(parameters.versionsRepoRef, '') }}: - # Special logic is needed to copy the tsaoptions.json file to a well known location for the 1ES PT. - # This template has multiple checkouts and AzDO doesn't have support for dynamically determining the - # default repo path therefore the 1es-official logic can't calculate the repo's tsa config file path. - - task: CopyFiles@2 - displayName: Copy TSA Config - inputs: - SourceFolder: '$(Build.Repository.LocalPath)/$(buildRepoName)' - Contents: '.config/tsaoptions.json' - TargetFolder: '$(Build.SourcesDirectory)' diff --git a/eng/docker-tools/templates/steps/publish-artifact.yml b/eng/docker-tools/templates/steps/publish-artifact.yml index 72ce4700..6095a1dd 100644 --- a/eng/docker-tools/templates/steps/publish-artifact.yml +++ b/eng/docker-tools/templates/steps/publish-artifact.yml @@ -11,7 +11,7 @@ parameters: type: string - name: condition type: string - default: 'true' + default: succeeded() steps: - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: @@ -20,9 +20,9 @@ steps: path: ${{ parameters.path }} artifact: ${{ parameters.artifactName }} displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) + condition: ${{ parameters.condition }} - ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - publish: ${{ parameters.path }} artifact: ${{ parameters.artifactName }} displayName: ${{ parameters.displayName }} - condition: and(succeeded(), ${{ parameters.condition }}) + condition: ${{ parameters.condition }} diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 64f802e1..b57e3e26 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:3012807 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder@sha256:1b22952c28a1ce0579c10f9f3a772baf7b081e179394f91704e3fa8e5e7f4271 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner