Enable 1ES on Python CUDA Package Pipelines (#22560)

jchen351 · web-flow · commit 3ae7c3c0a652 · 2024-10-24T09:51:00.000-07:00
### Description These 3 following CUDA packaging pipeline shoud be enabled with 1ES after this pull request. • [Python-CUDA-Packaging-Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1299&view=runs) • [Python CUDA Alt Packaging Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1626) • [Python DML Packaging Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1625) This should also fix the issue where [Python packaging pipeline](https://aiinfra.visualstudio.com/Lotus/_build?definitionId=841&_a=summary) failed due to cannot find `publish_symbols` ### Motivation and Context
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml
@@ -1,5 +1,10 @@
 trigger: none
-
+resources:
+  repositories:
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
 parameters:
   - name: enable_linux_cuda
     type: boolean
@@ -17,11 +22,21 @@ parameters:
       - Release
       - RelWithDebInfo
       - MinSizeRel
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    # Update the pool with your team's 1ES hosted pool.
+    pool:
+      name: 'onnxruntime-Win-CPU-2022'  # Name of your hosted pool
+      os: windows  # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
 
-stages:
-  - template: stages/py-gpu-packaging-stage.yml
-    parameters:
-      enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
-      enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
-      cmake_build_type: ${{ parameters.cmake_build_type }}
-      cuda_version: '11.8'
+    stages:
+      - template: stages/py-gpu-packaging-stage.yml
+        parameters:
+          enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
+          enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
+          cmake_build_type: ${{ parameters.cmake_build_type }}
+          cuda_version: '11.8'
diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml
@@ -1,5 +1,10 @@
 trigger: none
-
+resources:
+  repositories:
+  - repository: 1esPipelines
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
 parameters:
   - name: cmake_build_type
     type: string
@@ -9,10 +14,19 @@ parameters:
       - Release
       - RelWithDebInfo
       - MinSizeRel
+extends:
+  # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
+  # For non-production pipelines, use "Unofficial" as defined below.
+  # For productions pipelines, use "Official".
+  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
+  parameters:
+    # Update the pool with your team's 1ES hosted pool.
+    pool:
+      name: 'onnxruntime-Win-CPU-2022'  # Name of your hosted pool
+      os: windows  # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
 
-stages:
-  - template: stages/py-gpu-packaging-stage.yml
-    parameters:
-      enable_windows_dml: true
-      cmake_build_type: ${{ parameters.cmake_build_type }}
-      publish_symbols: true
+    stages:
+      - template: stages/py-gpu-packaging-stage.yml
+        parameters:
+          enable_windows_dml: true
+          cmake_build_type: ${{ parameters.cmake_build_type }}
diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
@@ -77,5 +77,3 @@ stages:
     build_py_parameters: ${{ parameters.build_py_parameters }}
     cmake_build_type: ${{ parameters.cmake_build_type }}
     qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
-    publish_symbols: true
-
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml
@@ -48,10 +48,6 @@ parameters:
     - '3.12'
     - '3.13'
 
-- name: publish_symbols
-  type: boolean
-  default: false
-
 stages:
   - ${{ if eq(parameters.enable_windows_cuda, true) }}:
     - ${{ each python_version in parameters.PythonVersions }}:
@@ -89,5 +85,4 @@ stages:
           EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
           ENV_SETUP_SCRIPT: setup_env.bat
           EP_NAME: directml
-          publish_symbols: ${{ parameters.publish_symbols }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml
@@ -99,6 +99,7 @@ stages:
           mv $(Build.BinariesDirectory)/dist ./dist
           pushd dist
           find . -name \*.whl -exec unzip -qq -o {} \;
+          rm -r onnxruntime
           popd
           pushd ${{ parameters.cmake_build_type }} 
           find . -name \*.whl -exec unzip -qq -o {} \;
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
@@ -38,10 +38,6 @@ parameters:
    - RelWithDebInfo
    - MinSizeRel
 
-- name: publish_symbols
-  type: boolean
-  default: false
-
 stages:
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
     dependsOn: []
@@ -52,6 +48,22 @@ stages:
         clean: all
       pool:
         name: onnxruntime-Win-CPU-2022
+        os: windows
+      templateContext:
+        codeSignValidation:
+          enabled: true
+          break: true
+        psscriptanalyzer:
+          enabled: true
+        sdl:
+          binskim:
+            enabled: true
+            scanOutputDirectoryOnly: true
+            targetPathPattern: '+:file|*.dll;-:file|DirectML.dll'
+        outputs:
+        - output: pipelineArtifact
+          targetPath: $(Build.ArtifactStagingDirectory)
+          artifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
       variables:
         GRADLE_OPTS: '-Dorg.gradle.daemon=false'
         VSGenerator: 'Visual Studio 17 2022'
@@ -73,12 +85,6 @@ stages:
               addToPath: true
               architecture: 'x64'
 
-          - task: onebranch.pipeline.tsaoptions@1
-            displayName: 'OneBranch TSAOptions'
-            inputs:
-              tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
-              appendSourceBranchName: false
-
           - template: ../templates/download-deps.yml
 
           - ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
@@ -106,13 +112,6 @@ stages:
               arguments: --new_dir $(Build.BinariesDirectory)/deps
               workingDirectory: $(Build.BinariesDirectory)
 
-          - task: PowerShell@2
-            displayName: 'Install ONNX'
-            inputs:
-              filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-              workingDirectory: '$(Build.BinariesDirectory)'
-              arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
-
           - template: ../templates/set-nightly-build-option-variable-step.yml
 
           - task: PythonScript@0
@@ -126,19 +125,7 @@ stages:
                 --cmake_generator "$(VSGenerator)"
                 --enable_pybind
                 --enable_onnx_tests
-                --parallel --use_binskim_compliant_compile_flags --update
-                $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
-              workingDirectory: '$(Build.BinariesDirectory)'
-
-          # building with build.py so the parallelization parameters are added to the msbuild command
-          - task: PythonScript@0
-            displayName: 'Build'
-            inputs:
-              scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-              arguments: >
-                --config ${{ parameters.cmake_build_type }}
-                --build_dir $(Build.BinariesDirectory)
-                --parallel --build
+                --parallel --use_binskim_compliant_compile_flags --update --build
                 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
               workingDirectory: '$(Build.BinariesDirectory)'
 
@@ -164,48 +151,11 @@ stages:
               Contents: '*.whl'
               TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish Artifact: ONNXRuntime python wheel'
-            inputs:
-              ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
-
-          - ${{ if eq(parameters.publish_symbols, true) }}:
-            - task: PublishSymbols@2
-              displayName: 'Publish symbols'
-              condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
-              inputs:
-                SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
-                SearchPattern: |
-                   onnxruntime_pybind11_state.pdb
-                   onnxruntime_providers_shared.pdb
-                IndexSources: true
-                SymbolServerType: TeamServices
-                SymbolExpirationInDays: 3650
-                SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
-
           - script: |
               7z x *.whl
             workingDirectory: '$(Build.ArtifactStagingDirectory)'
             displayName: 'unzip the package'
 
-          - task: CredScan@3
-            displayName: 'Run CredScan'
-            inputs:
-              debugMode: false
-            continueOnError: true
-
-          - task: BinSkim@4
-            displayName: 'Run BinSkim'
-            inputs:
-              AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
-
-          - task: TSAUpload@2
-            displayName: 'TSA upload'
-            condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
-            inputs:
-              GdnPublishTsaOnboard: false
-              GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
-
           - template: ../templates/component-governance-component-detection-steps.yml
             parameters:
               condition: 'succeeded'
@@ -235,7 +185,7 @@ stages:
 
         - template: ../templates/flex-downloadPipelineArtifact.yml
           parameters:
-            ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
+            ArtifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
             StepName: 'Download Pipeline Artifact - Windows GPU Build'
             TargetPath: '$(Build.ArtifactStagingDirectory)'
 
