diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md new file mode 100644 index 00000000..aba13609 --- /dev/null +++ b/eng/docker-tools/CHANGELOG.md @@ -0,0 +1,86 @@ +# Docker Tools / ImageBuilder Changelog + +All breaking changes and new features in `eng/docker-tools` will be documented in this file. + +--- + +## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable + +The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`. +This allows repos to independently control whether pre-build validation runs, without affecting functional tests. + +The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected. +Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. + +--- + +## 2026-02-19: Separate Registry Endpoints from Authentication + +- Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945) +- Issue: [#1914](https://github.com/dotnet/docker-tools/issues/1914) + +Authentication details (`serviceConnection`, `resourceGroup`, `subscription`) have been moved from individual registry endpoints into a centralized `RegistryAuthentication` list. +This fixes an issue where ACR authentication could fail when multiple service connections existed for the same registry. + +**Before:** Each registry endpoint embedded its own authentication: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(publishServiceConnectionName) + id: $(publishServiceConnection.id) + clientId: $(publishServiceConnection.clientId) + tenantId: $(tenant) +``` + +**After:** Registry endpoints only contain `server` and `repoPrefix`. Authentication is centralized: + +```yaml +publishConfig: + BuildRegistry: + server: $(acr.server) + repoPrefix: "my-prefix/" + PublishRegistry: + server: $(acr.server) + repoPrefix: "publish/" + RegistryAuthentication: + - server: $(acr.server) + resourceGroup: $(resourceGroup) + subscription: $(subscription) + serviceConnection: + name: $(serviceConnectionName) + id: $(serviceConnection.id) + clientId: $(serviceConnection.clientId) + tenantId: $(tenant) +``` + +How to update: +- Update any publishConfig parameters to match the new structure. + - Multiple registries can share authentication. If two registries use the same ACR server, only one entry is needed in `RegistryAuthentication`. + - The new structure should match [ImageBuilder's Configuration Model](https://github.com/dotnet/docker-tools/tree/a82572386854f15af441c50c6efa698a627e9f2b/src/ImageBuilder/Configuration). +- Update service connection setup (if using `setup-service-connections.yml`): + - The template now supports looking up service connections from `publishConfig.RegistryAuthentication` + - Use the new `usesRegistries` parameter to specify which registries need auth setup: + ```yaml + - template: eng/docker-tools/templates/stages/setup-service-connections.yml + parameters: + publishConfig: ${{ variables.publishConfig }} + usesRegistries: + - $(buildRegistry.server) + - $(publishRegistry.server) + ``` diff --git a/eng/docker-tools/DEV-GUIDE.md b/eng/docker-tools/DEV-GUIDE.md index ea0301f2..c1d48434 100644 --- a/eng/docker-tools/DEV-GUIDE.md +++ b/eng/docker-tools/DEV-GUIDE.md @@ -180,14 +180,16 @@ The `stages` variable is a comma-separated string that controls which pipeline s ```yaml variables: - name: stages - value: "build,test,publish" # Run all stages + value: "build,test,sign,publish" # Run all stages ``` Common patterns: -- `"build"` - Build only, no tests or publishing -- `"build,test"` - Build and test, but don't publish +- `"build"` - Build only, no tests, signing, or publishing +- `"build,test"` - Build and test, but don't sign or publish +- `"build,test,sign"` - Build, test, and sign, but don't publish +- `"sign"` - Sign only (when re-running failed signing from a previous build) - `"publish"` - Publish only (when re-running a failed publish from a previous build) -- `"build,test,publish"` - Full pipeline +- `"build,test,sign,publish"` - Full pipeline **Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs. @@ -372,11 +374,13 @@ Note: For simple retries of failed jobs, use the Azure Pipelines UI "Re-run fail | Scenario | stages | sourceBuildPipelineRunId | |----------|--------|--------------------------| -| Normal full build | `"build,test,publish"` | `$(Build.BuildId)` (default) | +| Normal full build | `"build,test,sign,publish"` | `$(Build.BuildId)` (default) | | Re-run publish after infra fix | `"publish"` | ID of the successful build run | | Re-test after infra fix | `"test"` | ID of the build run to test | +| Re-sign after infra fix | `"sign"` | ID of the build run to sign | | Build only (no publish) | `"build"` | `$(Build.BuildId)` (default) | | Test + publish (skip build) | `"test,publish"` | ID of the build run | +| Sign + publish (skip build/test) | `"sign,publish"` | ID of the build run | **In the Azure DevOps UI:** diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index db9b4f7f..025d3c8d 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -91,8 +91,6 @@ jobs: --architecture $(architecture) --retry --digests-out-var 'builtImages' - --acr-subscription '${{ parameters.publishConfig.BuildRegistry.subscription }}' - --acr-resource-group '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}' $(manifestVariables) $(imageBuilderBuildArgs) - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self diff --git a/eng/docker-tools/templates/jobs/post-build.yml b/eng/docker-tools/templates/jobs/post-build.yml index 3dfb84cf..8159872a 100644 --- a/eng/docker-tools/templates/jobs/post-build.yml +++ b/eng/docker-tools/templates/jobs/post-build.yml @@ -3,6 +3,7 @@ parameters: internalProjectName: null publicProjectName: null customInitSteps: [] + publishConfig: null jobs: - job: Build @@ -18,6 +19,7 @@ jobs: parameters: dockerClientOS: linux customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index 98e29f4a..b86ec1ee 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -97,8 +97,6 @@ jobs: internalProjectName: ${{ parameters.internalProjectName }} args: >- copyAcrImages - '${{ parameters.publishConfig.BuildRegistry.subscription }}' - '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}' '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}' '${{ parameters.publishConfig.BuildRegistry.server }}' --os-type '*' diff --git a/eng/docker-tools/templates/jobs/sign-images.yml b/eng/docker-tools/templates/jobs/sign-images.yml new file mode 100644 index 00000000..8162cf54 --- /dev/null +++ b/eng/docker-tools/templates/jobs/sign-images.yml @@ -0,0 +1,58 @@ +# Signs container images using ESRP/Notary v2. +# This job downloads the merged image-info artifact and signs all images listed in it. +parameters: + pool: {} + internalProjectName: null + publicProjectName: null + customInitSteps: [] + publishConfig: null + sourceBuildPipelineRunId: "" + +jobs: +- job: Sign + pool: ${{ parameters.pool }} + variables: + imageInfoDir: $(Build.ArtifactStagingDirectory)/image-info + steps: + + # Install MicroBuild signing plugin for ESRP container image signing + - template: /eng/docker-tools/templates/steps/init-signing-linux.yml@self + parameters: + signType: ${{ parameters.publishConfig.Signing.SignType }} + envFileVariableName: signingEnvFilePath + + # Setup docker and ImageBuilder + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux + setupImageBuilder: true + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + envFilePath: $(signingEnvFilePath) + + # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run) + - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self + parameters: + targetPath: $(imageInfoDir) + artifactName: image-info + pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: 🔏 Sign Container Images + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + signImages + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} + + - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: ✅ Verify Container Image Signatures + internalProjectName: ${{ parameters.internalProjectName }} + args: >- + verifySignatures + $(artifactsPath)/image-info/image-info.json + --registry-override ${{ parameters.publishConfig.BuildRegistry.server }} + --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} diff --git a/eng/docker-tools/templates/jobs/test-images-linux-client.yml b/eng/docker-tools/templates/jobs/test-images-linux-client.yml index 44e9e2fb..450e067f 100644 --- a/eng/docker-tools/templates/jobs/test-images-linux-client.yml +++ b/eng/docker-tools/templates/jobs/test-images-linux-client.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: $[ ${{ parameters.matrix }} ] ${{ if eq(parameters.preBuildValidation, 'true') }}: - condition: and(succeeded(), ne(variables.testScriptPath, '')) + condition: and(succeeded(), ne(variables.preBuildTestScriptPath, '')) pool: ${{ parameters.pool }} timeoutInMinutes: ${{ parameters.testJobTimeout }} steps: diff --git a/eng/docker-tools/templates/stages/build-and-test.yml b/eng/docker-tools/templates/stages/build-and-test.yml index 3f7118ab..d21e8de9 100644 --- a/eng/docker-tools/templates/stages/build-and-test.yml +++ b/eng/docker-tools/templates/stages/build-and-test.yml @@ -3,11 +3,13 @@ parameters: testMatrixType: platformVersionedOs buildMatrixCustomBuildLegGroupArgs: "" testMatrixCustomBuildLegGroupArgs: "" - customCopyBaseImagesInitSteps: [] - customGenerateMatrixInitSteps: [] # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source). # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull. customInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before copy-base-images runs. + customCopyBaseImagesInitSteps: [] + # Custom steps that run after ImageBuilder is set up but before matrix generation runs. + customGenerateMatrixInitSteps: [] # Custom steps that run after ImageBuilder is set up but before the build starts. # Use for build-specific initialization (e.g., setting variables, additional setup). customBuildInitSteps: [] @@ -218,6 +220,33 @@ stages: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + +################################################################################ +# Sign Images +################################################################################ +- ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - stage: Sign + dependsOn: Post_Build + condition: " + and( + ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'), + and( + contains(variables['stages'], 'sign'), + or( + and( + succeeded(), + contains(variables['stages'], 'build')), + not(contains(variables['stages'], 'build')))))" + jobs: + - template: /eng/docker-tools/templates/jobs/sign-images.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} + customInitSteps: ${{ parameters.customInitSteps }} + publishConfig: ${{ parameters.publishConfig }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} ################################################################################ # Test Images diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml index 424f3aaf..0ce87d4d 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml @@ -38,6 +38,11 @@ parameters: type: object default: {} +# Enable container image signing +- name: enableSigning + type: boolean + default: false + stages: - template: ${{ parameters.stagesTemplate }} @@ -53,35 +58,44 @@ stages: InternalMirrorRegistry: server: $(acr-staging-test.server) repoPrefix: $(internalMirrorRepoPrefix) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - serviceConnection: - name: $(internal-mirror-test.serviceConnectionName) - id: $(internal-mirror-test.serviceConnection.id) - clientId: $(internal-mirror-test.serviceConnection.clientId) - tenantId: $(testTenant) PublicMirrorRegistry: server: $(public-mirror.server) repoPrefix: $(publicMirrorRepoPrefix) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) BuildRegistry: server: $(acr-staging-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build-test.serviceConnectionName) - id: $(build-test.serviceConnection.id) - clientId: $(build-test.serviceConnection.clientId) - tenantId: $(testTenant) + + PublishRegistry: + server: $(acr-test.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) cleanServiceConnection: name: $(clean-test.serviceConnectionName) @@ -95,13 +109,11 @@ stages: clientId: $(test-nonprod.serviceConnection.clientId) tenantId: $(testTenant) - PublishRegistry: - server: $(acr-test.server) - resourceGroup: $(testResourceGroup) - subscription: $(testSubscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish-test.serviceConnectionName) - id: $(publish-test.serviceConnection.id) - clientId: $(publish-test.serviceConnection.clientId) - tenantId: $(testTenant) + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.testing) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.testing) + # Use signType 'real' even for non-prod to actually sign with the test certificate. + # The 'test' signType skips signing entirely on linux; the test keycode provides a non-production certificate. + SignType: real + TrustStoreName: test diff --git a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml index 7f9a4e00..f90e5e80 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish-config-prod.yml @@ -38,6 +38,11 @@ parameters: type: object default: {} +# Enable container image signing +- name: enableSigning + type: boolean + default: false + stages: - template: ${{ parameters.stagesTemplate }} @@ -53,35 +58,44 @@ stages: InternalMirrorRegistry: server: $(acr-staging.server) repoPrefix: $(internalMirrorRepoPrefix) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) - serviceConnection: - name: $(internal-mirror.serviceConnectionName) - id: $(internal-mirror.serviceConnection.id) - clientId: $(internal-mirror.serviceConnection.clientId) - tenantId: $(internal-mirror.serviceConnection.tenantId) PublicMirrorRegistry: server: $(public-mirror.server) repoPrefix: $(publicMirrorRepoPrefix) - resourceGroup: $(public-mirror.resourceGroup) - subscription: $(public-mirror.subscription) - serviceConnection: - name: $(public-mirror.serviceConnectionName) - id: $(public-mirror.serviceConnection.id) - tenantId: $(public-mirror.serviceConnection.tenantId) - clientId: $(public-mirror.serviceConnection.clientId) BuildRegistry: server: $(acr-staging.server) - resourceGroup: $(acr-staging.resourceGroup) - subscription: $(acr-staging.subscription) repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" - serviceConnection: - name: $(build.serviceConnectionName) - id: $(build.serviceConnection.id) - clientId: $(build.serviceConnection.clientId) - tenantId: $(build.serviceConnection.tenantId) + + PublishRegistry: + server: $(acr.server) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + + RegistryAuthentication: + - server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + - server: $(public-mirror.server) + resourceGroup: $(public-mirror.resourceGroup) + subscription: $(public-mirror.subscription) + serviceConnection: + name: $(public-mirror.serviceConnectionName) + id: $(public-mirror.serviceConnection.id) + tenantId: $(public-mirror.serviceConnection.tenantId) + clientId: $(public-mirror.serviceConnection.clientId) + - server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) cleanServiceConnection: name: $(clean.serviceConnectionName) @@ -95,13 +109,9 @@ stages: clientId: $(test.serviceConnection.clientId) tenantId: $(test.serviceConnection.tenantId) - PublishRegistry: - server: $(acr.server) - resourceGroup: $(acr.resourceGroup) - subscription: $(acr.subscription) - repoPrefix: "${{ parameters.publishRepoPrefix }}" - serviceConnection: - name: $(publish.serviceConnectionName) - id: $(publish.serviceConnection.id) - clientId: $(publish.serviceConnection.clientId) - tenantId: $(publish.serviceConnection.tenantId) + Signing: + Enabled: ${{ parameters.enableSigning }} + ImageSigningKeyCode: $(microBuildSigningKeyCode.containers) + ReferrerSigningKeyCode: $(microBuildSigningKeyCode.attestations) + SignType: real + TrustStoreName: supplychain diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml index b72d4fd9..f29c376c 100644 --- a/eng/docker-tools/templates/stages/publish.yml +++ b/eng/docker-tools/templates/stages/publish.yml @@ -33,39 +33,36 @@ stages: ${{ if eq(parameters.isStandalonePublish, true) }}: dependsOn: [] ${{ else }}: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build + dependsOn: + - ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}: + - Sign + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - Test + - ${{ else }}: + - Post_Build + # Run when all of the following are true: + # 1. The pipeline has not been canceled. + # 2. The stages variable includes 'publish'. + # 3. Either signing is not enabled, or the Sign stage succeeded. + # 4. Either the stages variable does not include 'build', or Post_Build succeeded. + # 5. Either the stages variable does not include 'test', or Test succeeded/was skipped. condition: " and( not(canceled()), - and( - contains(variables['stages'], 'publish'), - or( - or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" + contains(variables['stages'], 'publish'), + or( + ne(lower('${{ parameters.publishConfig.Signing.Enabled }}'), 'true'), + in(dependencies.Sign.result, 'Succeeded', 'SucceededWithIssues') + ), + or( + not(contains(variables['stages'], 'build')), + succeeded('Post_Build') + ), + or( + not(contains(variables['stages'], 'test')), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') + ) + )" jobs: - template: /eng/docker-tools/templates/jobs/publish.yml@self parameters: diff --git a/eng/docker-tools/templates/stages/setup-service-connections.yml b/eng/docker-tools/templates/stages/setup-service-connections.yml index 2ef74e90..405bc703 100644 --- a/eng/docker-tools/templates/stages/setup-service-connections.yml +++ b/eng/docker-tools/templates/stages/setup-service-connections.yml @@ -3,6 +3,10 @@ # it is declared in this stage's parameters, even if your pipeline has already # been granted access to the service connection. This stage also does not need # to complete before the service connection is used. +# +# There are two ways to specify service connections: +# - Pass `serviceConnections` directly (list of {name: string} objects) +# - Pass `publishConfig` + `registries` to look up auth from RegistryAuthentication parameters: - name: pool type: object @@ -10,14 +14,26 @@ parameters: name: $(default1ESInternalPoolName) image: $(default1ESInternalPoolImage) os: linux -# serviceConnections object shape: -# - name: string + +# Explicit list of service connections to initialize +# Shape: [{ name: string }] - name: serviceConnections type: object default: [] -stages: +# List of registry servers that need authentication. These will be looked up in +# publishConfig.RegistryAuthentication. +# Make sure to provide the publishConfig parameter. +- name: usesRegistries + type: object + default: [] +# Look up service connections from publishConfig based on registries +# The publish configuration containing RegistryAuthentication entries. +- name: publishConfig + type: object + default: {} +stages: - stage: SetupServiceConnectionsStage displayName: Setup service connections jobs: @@ -27,6 +43,8 @@ stages: pool: ${{ parameters.pool }} steps: - checkout: none + + # Direct service connections list - ${{ each serviceConnection in parameters.serviceConnections }}: - task: AzureCLI@2 displayName: Setup ${{ serviceConnection.name }} @@ -36,3 +54,15 @@ stages: scriptLocation: inlineScript inlineScript: | az account show + + # Setup registry service connections + - ${{ if gt(length(parameters.usesRegistries), 0) }}: + - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: + - ${{ if containsValue(parameters.usesRegistries, auth.server) }}: + - task: AzureCLI@2 + displayName: Setup ${{ auth.serviceConnection.name }} + inputs: + azureSubscription: ${{ auth.serviceConnection.name }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: az account show diff --git a/eng/docker-tools/templates/steps/clean-acr-images.yml b/eng/docker-tools/templates/steps/clean-acr-images.yml index 65b8ceff..2f4b53ad 100644 --- a/eng/docker-tools/templates/steps/clean-acr-images.yml +++ b/eng/docker-tools/templates/steps/clean-acr-images.yml @@ -5,7 +5,6 @@ parameters: age: null customArgs: "--dry-run" internalProjectName: null - publishConfig: null steps: - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self parameters: @@ -20,8 +19,6 @@ steps: args: >- cleanAcrImages ${{ parameters.repo }} - ${{ parameters.acr.subscription }} - ${{ parameters.acr.resourceGroup }} ${{ parameters.acr.server }} --action ${{ parameters.action }} --age ${{ parameters.age }} diff --git a/eng/docker-tools/templates/steps/copy-base-images.yml b/eng/docker-tools/templates/steps/copy-base-images.yml index 0e9e09f6..6664c8f9 100644 --- a/eng/docker-tools/templates/steps/copy-base-images.yml +++ b/eng/docker-tools/templates/steps/copy-base-images.yml @@ -3,8 +3,6 @@ parameters: type: object default: server: "" - subscription: "" - resourceGroup: "" repoPrefix: "" - name: additionalOptions type: string @@ -29,8 +27,6 @@ steps: # error args: >- copyBaseImages - '${{ parameters.acr.subscription }}' - '${{ parameters.acr.resourceGroup }}' $(dockerHubRegistryCreds) $(customCopyBaseImagesArgs) --repo-prefix '${{ parameters.acr.repoPrefix }}' diff --git a/eng/docker-tools/templates/steps/generate-appsettings.yml b/eng/docker-tools/templates/steps/generate-appsettings.yml index 5eaa5a14..b1243e70 100644 --- a/eng/docker-tools/templates/steps/generate-appsettings.yml +++ b/eng/docker-tools/templates/steps/generate-appsettings.yml @@ -1,17 +1,31 @@ # .NET Microsoft.Extensions.Configuration reads appsettings.json from the working directory # where ImageBuilder is run. Place it in the repo root so it will be available at runtime. parameters: +# See: +# - publish-config-prod.yml +# - publish-config-nonprod.yml +# - PublishConfiguration.cs - name: publishConfig type: object +# This should be the path to $(Build.ArtifactStagingDirectory). It is parameterized +# here since it is mounted into the ImageBuilder container at runtime. +- name: artifactStagingDirectory + type: string + default: "" - name: condition type: string default: "true" steps: - powershell: |- + # Escape backslashes for JSON compatibility (Windows paths like D:\a\_work become D:\\a\\_work) + $artifactStagingDirectory = "${{ parameters.artifactStagingDirectory }}" -replace '\\', '\\' $appsettingsJsonContent = @" { - "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }} + "PublishConfiguration": ${{ convertToJson(parameters.publishConfig) }}, + "BuildConfiguration": { + "ArtifactStagingDirectory": "$artifactStagingDirectory" + } } "@ Set-Content -Path "appsettings.json" -Value $appsettingsJsonContent diff --git a/eng/docker-tools/templates/steps/init-common.yml b/eng/docker-tools/templates/steps/init-common.yml index decad7b9..450ddc58 100644 --- a/eng/docker-tools/templates/steps/init-common.yml +++ b/eng/docker-tools/templates/steps/init-common.yml @@ -53,6 +53,12 @@ parameters: type: string default: "versions" +# Path to an env file for docker --env-file. +# Passed through to init-imagebuilder.yml. +- name: envFilePath + type: string + default: "" + steps: # Repository Checkout # Multi-repo checkout is used when a versions repository is needed for caching. @@ -197,14 +203,14 @@ steps: TargetFolder: '$(Build.SourcesDirectory)' # Artifacts Path Configuration -# Linux: Uses /artifacts inside the container, which is mounted to the host's -# Build.ArtifactStagingDirectory via docker run -v +# Linux: Uses Build.ArtifactStagingDirectory directly so container and host paths match +# through an identity volume mount # Windows: Uses Build.ArtifactStagingDirectory directly since ImageBuilder runs # as an extracted executable, not in a container - ${{ if eq(parameters.dockerClientOS, 'linux') }}: - script: | - echo "Setting artifactsPath to '/artifacts'" - echo "##vso[task.setvariable variable=artifactsPath]/artifacts" + echo "Setting artifactsPath to '$(Build.ArtifactStagingDirectory)'" + echo "##vso[task.setvariable variable=artifactsPath]$(Build.ArtifactStagingDirectory)" displayName: Define Artifacts Path Variable condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if eq(parameters.dockerClientOS, 'windows') }}: @@ -245,3 +251,4 @@ steps: publishConfig: ${{ parameters.publishConfig }} condition: ${{ parameters.condition }} customInitSteps: ${{ parameters.customInitSteps }} + envFilePath: ${{ parameters.envFilePath }} diff --git a/eng/docker-tools/templates/steps/init-imagebuilder.yml b/eng/docker-tools/templates/steps/init-imagebuilder.yml index 2c3f9485..a5488caa 100644 --- a/eng/docker-tools/templates/steps/init-imagebuilder.yml +++ b/eng/docker-tools/templates/steps/init-imagebuilder.yml @@ -21,6 +21,13 @@ parameters: type: stepList default: [] +# Path to an env file for docker --env-file. +# When set, --env-file is added to the docker run commands so the container +# receives the environment variables defined in the file. +- name: envFilePath + type: string + default: "" + steps: # Custom ImageBuilder setup (e.g., bootstrap from source) - ${{ if gt(length(parameters.customInitSteps), 0) }}: @@ -60,6 +67,7 @@ steps: - template: /eng/docker-tools/templates/steps/generate-appsettings.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} + artifactStagingDirectory: $(artifactsPath) condition: ${{ parameters.condition }} # On Linux, build the "withrepo" image that includes the repo's source code. @@ -96,18 +104,20 @@ steps: "-v $(Build.ArtifactStagingDirectory):$(artifactsPath)" "-w /repo" "$(imageBuilderDockerRunExtraOptions)" - "$(imageNames.imageBuilder.withrepo)" ) + $envFilePath = "${{ parameters.envFilePath }}" + if ($envFilePath) { + $dockerRunArgs += "--env-file $envFilePath" + } + $authedDockerRunArgs = @( - '-e' - 'SYSTEM_ACCESSTOKEN=$env:SYSTEM_ACCESSTOKEN' - '-e' - 'SYSTEM_OIDCREQUESTURI=$env:SYSTEM_OIDCREQUESTURI' + "-e", 'SYSTEM_ACCESSTOKEN' + "-e", 'SYSTEM_OIDCREQUESTURI' ) - $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs - $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs + @("$(imageNames.imageBuilder.withrepo)") + $authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs + @("$(imageNames.imageBuilder.withrepo)") $runImageBuilderCmd = $($dockerRunCmd -join ' ') $runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ') diff --git a/eng/docker-tools/templates/steps/init-signing-linux.yml b/eng/docker-tools/templates/steps/init-signing-linux.yml new file mode 100644 index 00000000..4513bc18 --- /dev/null +++ b/eng/docker-tools/templates/steps/init-signing-linux.yml @@ -0,0 +1,124 @@ +# Installs the MicroBuild signing plugin for ESRP container image signing. +# After installation, MBSIGN_APPFOLDER environment variable points to DDSignFiles.dll location. +parameters: +- name: signType + type: string + default: test + values: + - test + - real + +- name: condition + type: string + default: "true" + +- name: microBuildOutputFolder + type: string + default: $(Agent.TempDirectory)/MicroBuild + +# Name of the pipeline variable to set with the signing env file path. +# When set, a signing env file is written after plugin installation and +# the specified pipeline variable is set to its path via logging directive, +# allowing downstream steps to reference it as $(variableName). +# When empty, no env file is created (non-signing jobs). +- name: envFileVariableName + type: string + default: "" + +steps: +# Install .NET SDK on Linux - needed to download the MicroBuild plugin nupkgs when nuget.exe is unavailable. +# Install to an isolated location so the repo's global.json doesn't interfere. +- task: UseDotNet@2 + displayName: Install .NET SDK for MicroBuild Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + packageType: sdk + version: 8.0.x + installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet + +# Create a global.json in the MicroBuild folder that pins to the installed SDK. +# This prevents the repo's global.json from causing SDK resolution failures. +- script: | + mkdir -p ${{ parameters.microBuildOutputFolder }} + version=$(dotnet --version) + cat > ${{ parameters.microBuildOutputFolder }}/global.json << EOF + { + "sdk": { + "version": "$version" + } + } + EOF + displayName: Create global.json for MicroBuild + condition: and(succeeded(), ${{ parameters.condition }}) + +- task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild Signing Plugin + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + version: $(MicroBuildPluginVersion) + ${{ if eq(parameters.signType, 'test') }}: + signType: test + ${{ else }}: + signType: real + zipSources: false + feedSource: $(MicroBuildFeedSource) + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + workingDirectory: ${{ parameters.microBuildOutputFolder }} + env: + TeamName: $(TeamName) + MicroBuildOutputFolderOverride: $(Agent.TempDirectory)/MicroBuild + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +# Configure ImageBuilder docker run options and write env file for signing. +# Sets imageBuilderDockerRunExtraOptions with the plugin volume mount and, +# when envFileVariableName is provided, writes a signing env file and sets +# the named pipeline variable to its path via logging directive. +- ${{ if ne(parameters.envFileVariableName, '') }}: + - task: PowerShell@2 + displayName: Configure ImageBuilder Signing Options + condition: and(succeeded(), ${{ parameters.condition }}) + inputs: + targetType: 'inline' + script: | + # Mount the MicroBuild signing plugin directory (contains DDSignFiles.dll and esrpcli.dll). + $imageBuilderDockerRunExtraOptions = "-v $env:MBSIGN_APPFOLDER`:/microbuild" + Write-Host "MicroBuild signing enabled, mounting $env:MBSIGN_APPFOLDER to /microbuild" + Write-Host "##vso[task.setvariable variable=imageBuilderDockerRunExtraOptions]$imageBuilderDockerRunExtraOptions" + + # Write the signing env file for docker --env-file. + # Docker reads this file on the host before creating the container, + # so no volume mount is needed for the file itself. + $envFilePath = "$(Agent.TempDirectory)/imagebuilder-signing.env" + $envFileContent = @( + # MicroBuild plugin variables for DDSignFiles.dll + "MBSIGN_APPFOLDER=/microbuild" + "VSENGESRPSSL" + "USEESRPCLI" + "MBSIGN_CONNECTEDSERVICE" + + # Container-local temp/workspace paths (host paths aren't accessible inside the container) + "MBSIGNTEMPDIR=/tmp/MicroBuildSign" + "PIPELINE_WORKSPACE=$(Build.ArtifactStagingDirectory)" + "AGENT_TEMPDIRECTORY=/tmp" + + # Azure DevOps pipeline variables for ESRP bearer token auth (ESRPUtils.GetAccountInfo) + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI" + "BUILD_BUILDID" + "SYSTEM_TEAMPROJECT" + "BUILD_SOURCEBRANCH" + + # Azure DevOps pipeline variables for ESRP CLI federated token (ESRPCliDll.GetFederatedTokenData) + "SYSTEM_JOBID" + "SYSTEM_PLANID" + "SYSTEM_TEAMPROJECTID" + "SYSTEM_HOSTTYPE" + "SYSTEM_COLLECTIONURI" + + # Azure DevOps pipeline variables for DDSignFilesConfiguration + "BUILD_DEFINITIONNAME" + "BUILD_BUILDNUMBER" + ) + + $envFileContent | Set-Content -Path $envFilePath -Encoding utf8NoBOM + Write-Host "##vso[task.setvariable variable=${{ parameters.envFileVariableName }}]$envFilePath" diff --git a/eng/docker-tools/templates/variables/common-paths.yml b/eng/docker-tools/templates/variables/common-paths.yml index d8a52002..d676441b 100644 --- a/eng/docker-tools/templates/variables/common-paths.yml +++ b/eng/docker-tools/templates/variables/common-paths.yml @@ -3,3 +3,4 @@ variables: engDockerToolsPath: $(Build.Repository.LocalPath)/$(engDockerToolsRelativePath) engPath: $(Build.Repository.LocalPath)/eng testScriptPath: "" + preBuildTestScriptPath: $(testScriptPath) diff --git a/eng/docker-tools/templates/variables/common.yml b/eng/docker-tools/templates/variables/common.yml index 5cdd124e..680dc129 100644 --- a/eng/docker-tools/templates/variables/common.yml +++ b/eng/docker-tools/templates/variables/common.yml @@ -79,3 +79,11 @@ variables: value: 00000000-0000-0000-0000-000000000000 - name: acr-staging.subscription value: 00000000-0000-0000-0000-000000000000 + +# See https://devdiv.visualstudio.com/Engineering/_git/Sign?version=GBmain&path=/src/CertificateMappings.xml +- name: microBuildSigningKeyCode.containers + value: 4512 +- name: microBuildSigningKeyCode.attestations + value: 4571 +- name: microBuildSigningKeyCode.testing + value: 2151 diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index b9327035..293657be 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2887966 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2919324 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner diff --git a/eng/docker-tools/templates/variables/dotnet/common.yml b/eng/docker-tools/templates/variables/dotnet/common.yml index 1515bd75..1a032220 100644 --- a/eng/docker-tools/templates/variables/dotnet/common.yml +++ b/eng/docker-tools/templates/variables/dotnet/common.yml @@ -68,4 +68,15 @@ variables: - name: windowsServer2025PoolName value: Docker-2025-${{ variables['System.TeamProject'] }} +## Signing +# Team name required for MicroBuild signing plugin +- name: TeamName + value: DotNetCore +# MicroBuild signing plugin feed source +- name: MicroBuildFeedSource + value: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json +# MicroBuild signing plugin version +- name: MicroBuildPluginVersion + value: latest + - group: DotNet-Docker-Common-2