[main] Update common Docker engineering infrastructure with latest

eng/docker-tools/CHANGELOG.md

@@ -0,0 +1,86 @@
+# Docker Tools / ImageBuilder Changelog
+
+All breaking changes and new features in `eng/docker-tools` will be documented in this file.
+
+---
+
+## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable
+
+The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`.
+This allows repos to independently control whether pre-build validation runs, without affecting functional tests.
+
+The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected.
+Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely.
+
+---
+
+## 2026-02-19: Separate Registry Endpoints from Authentication
+
+- Pull request: [#1945](https://github.com/dotnet/docker-tools/pull/1945)
+- Issue: [#1914](https://github.com/dotnet/docker-tools/issues/1914)
+
+Authentication details (`serviceConnection`, `resourceGroup`, `subscription`) have been moved from individual registry endpoints into a centralized `RegistryAuthentication` list.
+This fixes an issue where ACR authentication could fail when multiple service connections existed for the same registry.
+
+**Before:** Each registry endpoint embedded its own authentication:
+
+```yaml
+publishConfig:
+  BuildRegistry:
+    server: $(acr.server)
+    repoPrefix: "my-prefix/"
+    resourceGroup: $(resourceGroup)
+    subscription: $(subscription)
+    serviceConnection:
+      name: $(serviceConnectionName)
+      id: $(serviceConnection.id)
+      clientId: $(serviceConnection.clientId)
+      tenantId: $(tenant)
+  PublishRegistry:
+    server: $(acr.server)
+    repoPrefix: "publish/"
+    resourceGroup: $(resourceGroup)
+    subscription: $(subscription)
+    serviceConnection:
+      name: $(publishServiceConnectionName)
+      id: $(publishServiceConnection.id)
+      clientId: $(publishServiceConnection.clientId)
+      tenantId: $(tenant)
+```
+
+**After:** Registry endpoints only contain `server` and `repoPrefix`. Authentication is centralized:
+
+```yaml
+publishConfig:
+  BuildRegistry:
+    server: $(acr.server)
+    repoPrefix: "my-prefix/"
+  PublishRegistry:
+    server: $(acr.server)
+    repoPrefix: "publish/"
+  RegistryAuthentication:
+    - server: $(acr.server)
+      resourceGroup: $(resourceGroup)
+      subscription: $(subscription)
+      serviceConnection:
+        name: $(serviceConnectionName)
+        id: $(serviceConnection.id)
+        clientId: $(serviceConnection.clientId)
+        tenantId: $(tenant)
+```
+
+How to update:
+- Update any publishConfig parameters to match the new structure.
+    - Multiple registries can share authentication. If two registries use the same ACR server, only one entry is needed in `RegistryAuthentication`.
+    - The new structure should match [ImageBuilder's Configuration Model](https://github.com/dotnet/docker-tools/tree/a82572386854f15af441c50c6efa698a627e9f2b/src/ImageBuilder/Configuration).
+- Update service connection setup (if using `setup-service-connections.yml`):
+   - The template now supports looking up service connections from `publishConfig.RegistryAuthentication`
+   - Use the new `usesRegistries` parameter to specify which registries need auth setup:
+     ```yaml
+     - template: eng/docker-tools/templates/stages/setup-service-connections.yml
+       parameters:
+         publishConfig: ${{ variables.publishConfig }}
+         usesRegistries:
+           - $(buildRegistry.server)
+           - $(publishRegistry.server)
+     ```

eng/docker-tools/DEV-GUIDE.md

@@ -180,14 +180,16 @@ The `stages` variable is a comma-separated string that controls which pipeline s
 ```yaml
 variables:
 - name: stages
-  value: "build,test,publish"    # Run all stages
+  value: "build,test,sign,publish"    # Run all stages
 ```
 
 Common patterns:
-- `"build"` - Build only, no tests or publishing
-- `"build,test"` - Build and test, but don't publish
+- `"build"` - Build only, no tests, signing, or publishing
+- `"build,test"` - Build and test, but don't sign or publish
+- `"build,test,sign"` - Build, test, and sign, but don't publish
+- `"sign"` - Sign only (when re-running failed signing from a previous build)
 - `"publish"` - Publish only (when re-running a failed publish from a previous build)
-- `"build,test,publish"` - Full pipeline
+- `"build,test,sign,publish"` - Full pipeline
 
 **Note:** The `Post_Build` stage is implicitly included whenever `build` is in the stages list. You don't need to specify it separately—it automatically runs after Build to merge image info files and consolidate SBOMs.
 
@@ -372,11 +374,13 @@ Note: For simple retries of failed jobs, use the Azure Pipelines UI "Re-run fail
 
 | Scenario | stages | sourceBuildPipelineRunId |
 |----------|--------|--------------------------|
-| Normal full build | `"build,test,publish"` | `$(Build.BuildId)` (default) |
+| Normal full build | `"build,test,sign,publish"` | `$(Build.BuildId)` (default) |
 | Re-run publish after infra fix | `"publish"` | ID of the successful build run |
 | Re-test after infra fix | `"test"` | ID of the build run to test |
+| Re-sign after infra fix | `"sign"` | ID of the build run to sign |
 | Build only (no publish) | `"build"` | `$(Build.BuildId)` (default) |
 | Test + publish (skip build) | `"test,publish"` | ID of the build run |
+| Sign + publish (skip build/test) | `"sign,publish"` | ID of the build run |
 
 **In the Azure DevOps UI:**
 

eng/docker-tools/templates/jobs/build-images.yml

@@ -91,8 +91,6 @@ jobs:
         --architecture $(architecture)
         --retry
         --digests-out-var 'builtImages'
-        --acr-subscription '${{ parameters.publishConfig.BuildRegistry.subscription }}'
-        --acr-resource-group '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self

eng/docker-tools/templates/jobs/post-build.yml

@@ -3,6 +3,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
   customInitSteps: []
+  publishConfig: null
 
 jobs:
 - job: Build
@@ -18,6 +19,7 @@ jobs:
     parameters:
       dockerClientOS: linux
       customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
   - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)

eng/docker-tools/templates/jobs/publish.yml

@@ -97,8 +97,6 @@ jobs:
       internalProjectName: ${{ parameters.internalProjectName }}
       args: >-
         copyAcrImages
-        '${{ parameters.publishConfig.BuildRegistry.subscription }}'
-        '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
         '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}'
         '${{ parameters.publishConfig.BuildRegistry.server }}'
         --os-type '*'

eng/docker-tools/templates/jobs/sign-images.yml

@@ -0,0 +1,58 @@
+# Signs container images using ESRP/Notary v2.
+# This job downloads the merged image-info artifact and signs all images listed in it.
+parameters:
+  pool: {}
+  internalProjectName: null
+  publicProjectName: null
+  customInitSteps: []
+  publishConfig: null
+  sourceBuildPipelineRunId: ""
+
+jobs:
+- job: Sign
+  pool: ${{ parameters.pool }}
+  variables:
+    imageInfoDir: $(Build.ArtifactStagingDirectory)/image-info
+  steps:
+
+  # Install MicroBuild signing plugin for ESRP container image signing
+  - template: /eng/docker-tools/templates/steps/init-signing-linux.yml@self
+    parameters:
+      signType: ${{ parameters.publishConfig.Signing.SignType }}
+      envFileVariableName: signingEnvFilePath
+
+  # Setup docker and ImageBuilder
+  - template: /eng/docker-tools/templates/steps/init-common.yml@self
+    parameters:
+      dockerClientOS: linux
+      setupImageBuilder: true
+      customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
+      envFilePath: $(signingEnvFilePath)
+
+  # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run)
+  - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self
+    parameters:
+      targetPath: $(imageInfoDir)
+      artifactName: image-info
+      pipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+
+  - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: 🔏 Sign Container Images
+      internalProjectName: ${{ parameters.internalProjectName }}
+      args: >-
+        signImages
+        $(artifactsPath)/image-info/image-info.json
+        --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}
+        --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }}
+
+  - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: ✅ Verify Container Image Signatures
+      internalProjectName: ${{ parameters.internalProjectName }}
+      args: >-
+        verifySignatures
+        $(artifactsPath)/image-info/image-info.json
+        --registry-override ${{ parameters.publishConfig.BuildRegistry.server }}
+        --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }}

eng/docker-tools/templates/jobs/test-images-linux-client.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix: $[ ${{ parameters.matrix }} ]
   ${{ if eq(parameters.preBuildValidation, 'true') }}:
-    condition: and(succeeded(), ne(variables.testScriptPath, ''))
+    condition: and(succeeded(), ne(variables.preBuildTestScriptPath, ''))
   pool: ${{ parameters.pool }}
   timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:

eng/docker-tools/templates/stages/build-and-test.yml

@@ -3,11 +3,13 @@ parameters:
   testMatrixType: platformVersionedOs
   buildMatrixCustomBuildLegGroupArgs: ""
   testMatrixCustomBuildLegGroupArgs: ""
-  customCopyBaseImagesInitSteps: []
-  customGenerateMatrixInitSteps: []
   # Custom steps to set up ImageBuilder instead of pulling from MCR (e.g., bootstrap from source).
   # Runs before ImageBuilder pull. If non-empty, skips the default ImageBuilder pull.
   customInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before copy-base-images runs.
+  customCopyBaseImagesInitSteps: []
+  # Custom steps that run after ImageBuilder is set up but before matrix generation runs.
+  customGenerateMatrixInitSteps: []
   # Custom steps that run after ImageBuilder is set up but before the build starts.
   # Use for build-specific initialization (e.g., setting variables, additional setup).
   customBuildInitSteps: []
@@ -218,6 +220,33 @@ stages:
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
       customInitSteps: ${{ parameters.customInitSteps }}
+      publishConfig: ${{ parameters.publishConfig }}
+
+################################################################################
+# Sign Images
+################################################################################
+- ${{ if eq(parameters.publishConfig.Signing.Enabled, true) }}:
+  - stage: Sign
+    dependsOn: Post_Build
+    condition: "
+      and(
+        ne(stageDependencies.Post_Build.outputs['Build.MergeImageInfoFiles.noImageInfos'], 'true'),
+        and(
+          contains(variables['stages'], 'sign'),
+          or(
+            and(
+              succeeded(),
+              contains(variables['stages'], 'build')),
+            not(contains(variables['stages'], 'build')))))"
+    jobs:
+    - template: /eng/docker-tools/templates/jobs/sign-images.yml@self
+      parameters:
+        pool: ${{ parameters.linuxAmd64Pool }}
+        internalProjectName: ${{ parameters.internalProjectName }}
+        publicProjectName: ${{ parameters.publicProjectName }}
+        customInitSteps: ${{ parameters.customInitSteps }}
+        publishConfig: ${{ parameters.publishConfig }}
+        sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
 ################################################################################
 # Test Images

eng/docker-tools/templates/stages/dotnet/publish-config-nonprod.yml

@@ -38,6 +38,11 @@ parameters:
   type: object
   default: {}
 
+# Enable container image signing
+- name: enableSigning
+  type: boolean
+  default: false
+
 
 stages:
 - template: ${{ parameters.stagesTemplate }}
@@ -53,35 +58,44 @@ stages:
       InternalMirrorRegistry:
         server: $(acr-staging-test.server)
         repoPrefix: $(internalMirrorRepoPrefix)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
-        serviceConnection:
-          name: $(internal-mirror-test.serviceConnectionName)
-          id: $(internal-mirror-test.serviceConnection.id)
-          clientId: $(internal-mirror-test.serviceConnection.clientId)
-          tenantId: $(testTenant)
 
       PublicMirrorRegistry:
         server: $(public-mirror.server)
         repoPrefix: $(publicMirrorRepoPrefix)
-        resourceGroup: $(public-mirror.resourceGroup)
-        subscription: $(public-mirror.subscription)
-        serviceConnection:
-          name: $(public-mirror.serviceConnectionName)
-          id: $(public-mirror.serviceConnection.id)
-          tenantId: $(public-mirror.serviceConnection.tenantId)
-          clientId: $(public-mirror.serviceConnection.clientId)
 
       BuildRegistry:
         server: $(acr-staging-test.server)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
         repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/"
-        serviceConnection:
-          name: $(build-test.serviceConnectionName)
-          id: $(build-test.serviceConnection.id)
-          clientId: $(build-test.serviceConnection.clientId)
-          tenantId: $(testTenant)
+
+      PublishRegistry:
+        server: $(acr-test.server)
+        repoPrefix: "${{ parameters.publishRepoPrefix }}"
+
+      RegistryAuthentication:
+        - server: $(acr-staging-test.server)
+          resourceGroup: $(testResourceGroup)
+          subscription: $(testSubscription)
+          serviceConnection:
+            name: $(build-test.serviceConnectionName)
+            id: $(build-test.serviceConnection.id)
+            clientId: $(build-test.serviceConnection.clientId)
+            tenantId: $(testTenant)
+        - server: $(public-mirror.server)
+          resourceGroup: $(public-mirror.resourceGroup)
+          subscription: $(public-mirror.subscription)
+          serviceConnection:
+            name: $(public-mirror.serviceConnectionName)
+            id: $(public-mirror.serviceConnection.id)
+            tenantId: $(public-mirror.serviceConnection.tenantId)
+            clientId: $(public-mirror.serviceConnection.clientId)
+        - server: $(acr-test.server)
+          resourceGroup: $(testResourceGroup)
+          subscription: $(testSubscription)
+          serviceConnection:
+            name: $(publish-test.serviceConnectionName)
+            id: $(publish-test.serviceConnection.id)
+            clientId: $(publish-test.serviceConnection.clientId)
+            tenantId: $(testTenant)
 
       cleanServiceConnection:
         name: $(clean-test.serviceConnectionName)
@@ -95,13 +109,11 @@ stages:
         clientId: $(test-nonprod.serviceConnection.clientId)
         tenantId: $(testTenant)
 
-      PublishRegistry:
-        server: $(acr-test.server)
-        resourceGroup: $(testResourceGroup)
-        subscription: $(testSubscription)
-        repoPrefix: "${{ parameters.publishRepoPrefix }}"
-        serviceConnection:
-          name: $(publish-test.serviceConnectionName)
-          id: $(publish-test.serviceConnection.id)
-          clientId: $(publish-test.serviceConnection.clientId)
-          tenantId: $(testTenant)
+      Signing:
+        Enabled: ${{ parameters.enableSigning }}
+        ImageSigningKeyCode: $(microBuildSigningKeyCode.testing)
+        ReferrerSigningKeyCode: $(microBuildSigningKeyCode.testing)
+        # Use signType 'real' even for non-prod to actually sign with the test certificate.
+        # The 'test' signType skips signing entirely on linux; the test keycode provides a non-production certificate.
+        SignType: real
+        TrustStoreName: test