ci: improve and simplify accelerate and transformers workloads

dvrogozh · dvrogozh · commit 2aa85e8abd89 · 2025-09-18T16:14:00.000-07:00
* Fix torch package version checks
* Use `--dist loadfile` strategy in alignment with HF practice
* Group together `pip install` commands
* Add missing `pciutils` to accelerate workload
* Drop `apt-get` loops as runners are now isolated

test accelerate and transformers only
disable_build
disable_ut
disable_e2e
disable_distributed

Signed-off-by: Dmitry Rogozhkin &lt;dmitry.v.rogozhkin@intel.com&gt;
diff --git a/.github/workflows/_linux_accelerate.yml b/.github/workflows/_linux_accelerate.yml
@@ -44,6 +44,8 @@ permissions: read-all
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
+env:
+  TORCH_INDEX: '--pre --index-url https://download.pytorch.org/whl/nightly/xpu'
 
 defaults:
   run:
@@ -105,7 +107,10 @@ jobs:
         HF_HUB_DOWNLOAD_TIMEOUT: 120
         PARSE_JUNIT: ${{ github.workspace }}/torch-xpu-ops/.github/scripts/parse-junitxml.py
         AGENT_TOOLSDIRECTORY: /tmp/xpu-tool
-        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist worksteal ${{ needs.prepare.outputs.pytest_extra_args }}
+        # Usage of `--dist loadfile` is a must as HF tests has complex setups including
+        # setUpClass and @first_run clauses. So 'loadfile' stratagy allows to minimize
+        # race conditions scope.
+        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist loadfile ${{ needs.prepare.outputs.pytest_extra_args }}
     env:
       accelerate: ${{ inputs.accelerate != '' && inputs.accelerate || 'v1.6.0' }}
       transformers: ${{ inputs.transformers != '' && inputs.transformers || 'v4.51.3' }}
@@ -125,16 +130,28 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.python }}
+      - name: Prepare environment
+        run: |
+          sudo apt-get update
+          # pciutils is needed to report available GPUs (we use lspci)
+          # python3-dev is needed for torch inductor and extension compilations
+          sudo apt-get install -y --no-install-recommends pciutils python3-dev
       - name: Check python
         run: |
           which python && python -V
           which pip && pip list
           pip install -U pip wheel setuptools
       - name: Install pytorch and deps
         run: |
-          pip install junitparser
-          pip install transformers==${{ env.transformers }}
-          pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
+          pip install $TORCH_INDEX torch torchvision torchaudio
+          # Do NOT install HF transformers or accelerate before torch as we need
+          # very specific version of the torch and HF would bring its own.
+          pip install \
+            junitparser \
+            pytest \
+            pytest-timeout \
+            pytest-xdist \
+            transformers==${{ env.transformers }}
       - name: Prepare Accelerate
         run: |
           cd $WORK_DIR
@@ -155,8 +172,6 @@ jobs:
           xpu-smi discovery -y --json --dump -1
       - name: Sanity check installed packages
         run: |
-          # Use latest pytest
-          pip install -U pytest pytest-timeout pytest-xdist
           # These checks are to exit earlier if for any reason torch
           # packages were reinstalled back to CUDA versions (not expected).
           pip show torch | grep Version | grep xpu
diff --git a/.github/workflows/_linux_transformers.yml b/.github/workflows/_linux_transformers.yml
@@ -74,6 +74,7 @@ env:
     libswresample-dev
     libswscale-dev
     pciutils
+    python3-dev
   TORCH_INDEX: '--pre --index-url https://download.pytorch.org/whl/nightly/xpu'
   AGENT_TOOLSDIRECTORY: /tmp/xpu-tool
 
@@ -116,20 +117,36 @@ jobs:
       render_id: ${{ steps.runner-info.outputs.render_id }}
       hostname: ${{ steps.runner-info.outputs.hostname }}
       pytest_extra_args: ${{ steps.runner-info.outputs.pytest_extra_args }}
+    env:
+      VIRTUAL_ENV: ${{ github.workspace }}/.venv
     steps:
+      - name: Install uv and python-${{ env.python }}
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ env.python }}
+      - name: Prepare environment
+        run: |
+          rm -rf ${{ env.VIRTUAL_ENV }}
+          uv venv ${{ env.VIRTUAL_ENV }}
       - id: getver
         run: |
           # We can't just `pip index version...` and get the last available
           # version as pytorch packages may have tricky dependencies. Instead
-          # we dry run install packages and get versions which would be installed.
+          # we install packages and get versions which got installed. Note that
+          # trying to --dry-run is not actually reliable as it does not make
+          # the thorough check of package dependencies.
           # See: https://github.com/pytorch/pytorch/issues/154687
-          pip install --dry-run --ignore-installed $TORCH_INDEX \
+          uv pip install $TORCH_INDEX \
             torch torchvision torchaudio pytorch-triton-xpu >_log.txt
 
-          torch=$(cat _log.txt | grep "Would install" | sed -E "s/.*torch-([^ ]*).*/\1/")
-          torchvision=$(cat _log.txt | grep "Would install" | sed -E "s/.*torchvision-([^ ]*).*/\1/")
-          torchaudio=$(cat _log.txt | grep "Would install" | sed -E "s/.*torchaudio-([^ ]*).*/\1/")
-          triton=$(cat _log.txt | grep "Would install" | sed -E "s/.*pytorch-triton-xpu-([^ ]*).*/\1/")
+          torch=$(uv pip show torch | grep Version)
+          torchvision=$(uv pip show torchvision | grep Version)
+          torchaudio=$(uv pip show torchaudio | grep Version)
+          triton=$(uv pip show pytorch-triton-xpu | grep Version)
+          torch=${torch#Version: *}
+          torchvision=${torchvision#Version: *}
+          torchaudio=${torchaudio#Version: *}
+          triton=${triton#Version: *}
           echo "torch=$torch" | tee -a "$GITHUB_OUTPUT"
           echo "torchvision=$torchvision" | tee -a "$GITHUB_OUTPUT"
           echo "torchaudio=$torchaudio" | tee -a "$GITHUB_OUTPUT"
@@ -154,8 +171,12 @@ jobs:
       env:
         PYTORCH_DEBUG_XPU_FALLBACK: '1'
         TRANSFORMERS_TEST_DEVICE_SPEC: 'spec.py'
-        # enable pytest parallel run, and continue others if meets crash case such as segmentation fault
-        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist worksteal ${{ needs.prepare.outputs.pytest_extra_args }}
+        # Usage of `--dist loadfile` is a must as HF tests has complex setups including
+        # setUpClass and @first_run clauses. So 'loadfile' stratagy allows to minimize
+        # race conditions scope. Besides, that's how HF Transformers recommend to run
+        # tests and how they run them in their own CI.
+        # See: https://github.com/huggingface/transformers/blob/v4.56.2/CONTRIBUTING.md?plain=1#L312
+        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist loadfile ${{ needs.prepare.outputs.pytest_extra_args }}
     strategy:
       fail-fast: false
       max-parallel: 1
@@ -224,21 +245,9 @@ jobs:
           fi
       - name: Prepare OS environment
         run: |
-          # as jobs might run in parallel on the same system, apt-get might
-          # step into the lock hold by other job
-          start_time=$SECONDS
-          while ! sudo apt-get update; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
-          while ! sudo apt-get install -y $PACKAGES; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
-          while ! git lfs install; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
+          sudo apt-get update
+          sudo apt-get install -y $PACKAGES
+          git lfs install
       - name: Setup python-${{ env.python }}
         uses: actions/setup-python@v5
         with:
@@ -250,12 +259,17 @@ jobs:
           pip install -U pip wheel setuptools
       - name: Prepare pytorch and deps
         run: |
-          pip install junitparser
           pip install $TORCH_INDEX \
             torch==${{ needs.prepare.outputs.torch }} \
             torchvision==${{ needs.prepare.outputs.torchvision }} \
             torchaudio==${{ needs.prepare.outputs.torchaudio }} \
             pytorch-triton-xpu==${{needs.prepare.outputs.triton }}
+          pip install \
+            junitparser \
+            pytest \
+            pytest-timeout \
+            pytest-xdist \
+            pytest-shard
       - name: Prepare Transformers
         run: |
           pwd
@@ -281,8 +295,6 @@ jobs:
           xpu-smi discovery -y --json --dump -1
       - name: Sanity check installed packages
         run: |
-          # Use latest pytest
-          pip install -U pytest pytest-timeout pytest-xdist pytest-shard
           # These checks are to exit earlier if for any reason Transformers
           # reinstalled torch packages back to CUDA versions (not expected).
           pip show torch | grep Version | grep xpu
