ci: improve and simplify accelerate and transformers workloads

* Use `--dist loadfile` strategy in alignment with HF practice
* Group together `pip install` commands
* Add missing `pciutils` to accelerate workload
* Drop `apt-get` loops as runners are now isolated

test accelerate and transformers only
disable_build
disable_ut
disable_e2e
disable_distributed

Signed-off-by: Dmitry Rogozhkin <[email protected]>

Author: dvrogozh

.github/workflows/_linux_accelerate.yml

@@ -44,6 +44,8 @@ permissions: read-all
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
+env:
+  TORCH_INDEX: '--pre --index-url https://download.pytorch.org/whl/nightly/xpu'
 
 defaults:
   run:
@@ -105,7 +107,10 @@ jobs:
         HF_HUB_DOWNLOAD_TIMEOUT: 120
         PARSE_JUNIT: ${{ github.workspace }}/torch-xpu-ops/.github/scripts/parse-junitxml.py
         AGENT_TOOLSDIRECTORY: /tmp/xpu-tool
-        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist worksteal ${{ needs.prepare.outputs.pytest_extra_args }}
+        # Usage of `--dist loadfile` is a must as HF tests has complex setups including
+        # setUpClass and @first_run clauses. So 'loadfile' stratagy allows to minimize
+        # race conditions scope.
+        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist loadfile ${{ needs.prepare.outputs.pytest_extra_args }}
     env:
       accelerate: ${{ inputs.accelerate != '' && inputs.accelerate || 'v1.6.0' }}
       transformers: ${{ inputs.transformers != '' && inputs.transformers || 'v4.51.3' }}
@@ -125,16 +130,28 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.python }}
+      - name: Prepare environment
+        run: |
+          sudo apt-get update
+          # pciutils is needed to report available GPUs (we use lspci)
+          # python3-dev is needed for torch inductor and extension compilations
+          sudo apt-get install -y --no-install-recommends pciutils python3-dev
       - name: Check python
         run: |
           which python && python -V
           which pip && pip list
           pip install -U pip wheel setuptools
       - name: Install pytorch and deps
         run: |
-          pip install junitparser
-          pip install transformers==${{ env.transformers }}
-          pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
+          pip install $TORCH_INDEX torch torchvision torchaudio
+          # Do NOT install HF transformers or accelerate before torch as we need
+          # very specific version of the torch and HF would bring its own.
+          pip install \
+            junitparser \
+            pytest \
+            pytest-timeout \
+            pytest-xdist \
+            transformers==${{ env.transformers }}
       - name: Prepare Accelerate
         run: |
           cd $WORK_DIR
@@ -155,8 +172,6 @@ jobs:
           xpu-smi discovery -y --json --dump -1
       - name: Sanity check installed packages
         run: |
-          # Use latest pytest
-          pip install -U pytest pytest-timeout pytest-xdist
           # These checks are to exit earlier if for any reason torch
           # packages were reinstalled back to CUDA versions (not expected).
           pip show torch | grep Version | grep xpu

.github/workflows/_linux_transformers.yml

@@ -74,6 +74,7 @@ env:
     libswresample-dev
     libswscale-dev
     pciutils
+    python3-dev
   TORCH_INDEX: '--pre --index-url https://download.pytorch.org/whl/nightly/xpu'
   AGENT_TOOLSDIRECTORY: /tmp/xpu-tool
 
@@ -154,8 +155,12 @@ jobs:
       env:
         PYTORCH_DEBUG_XPU_FALLBACK: '1'
         TRANSFORMERS_TEST_DEVICE_SPEC: 'spec.py'
-        # enable pytest parallel run, and continue others if meets crash case such as segmentation fault
-        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist worksteal ${{ needs.prepare.outputs.pytest_extra_args }}
+        # Usage of `--dist loadfile` is a must as HF tests has complex setups including
+        # setUpClass and @first_run clauses. So 'loadfile' stratagy allows to minimize
+        # race conditions scope. Besides, that's how HF Transformers recommend to run
+        # tests and how they run them in their own CI.
+        # See: https://github.com/huggingface/transformers/blob/v4.56.2/CONTRIBUTING.md?plain=1#L312
+        PYTEST_ADDOPTS: -rsf --timeout 600 --timeout_method=thread --dist loadfile ${{ needs.prepare.outputs.pytest_extra_args }}
     strategy:
       fail-fast: false
       max-parallel: 1
@@ -224,21 +229,9 @@ jobs:
           fi
       - name: Prepare OS environment
         run: |
-          # as jobs might run in parallel on the same system, apt-get might
-          # step into the lock hold by other job
-          start_time=$SECONDS
-          while ! sudo apt-get update; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
-          while ! sudo apt-get install -y $PACKAGES; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
-          while ! git lfs install; do
-            sleep 1;
-            if (( $SECONDS - start_time > 60 )); then false; fi
-          done
+          sudo apt-get update
+          sudo apt-get install -y $PACKAGES
+          git lfs install
       - name: Setup python-${{ env.python }}
         uses: actions/setup-python@v5
         with:
@@ -250,12 +243,17 @@ jobs:
           pip install -U pip wheel setuptools
       - name: Prepare pytorch and deps
         run: |
-          pip install junitparser
           pip install $TORCH_INDEX \
             torch==${{ needs.prepare.outputs.torch }} \
             torchvision==${{ needs.prepare.outputs.torchvision }} \
             torchaudio==${{ needs.prepare.outputs.torchaudio }} \
             pytorch-triton-xpu==${{needs.prepare.outputs.triton }}
+          pip install \
+            junitparser \
+            pytest \
+            pytest-timeout \
+            pytest-xdist \
+            pytest-shard
       - name: Prepare Transformers
         run: |
           pwd
@@ -281,8 +279,6 @@ jobs:
           xpu-smi discovery -y --json --dump -1
       - name: Sanity check installed packages
         run: |
-          # Use latest pytest
-          pip install -U pytest pytest-timeout pytest-xdist pytest-shard
           # These checks are to exit earlier if for any reason Transformers
           # reinstalled torch packages back to CUDA versions (not expected).
           pip show torch | grep Version | grep xpu