LLVM and SPIRV-LLVM-Translator pulldown (WW39 2025) (#20230)

Author: sarnex

.ci/cache_lit_timing_files.py

@@ -0,0 +1,80 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Caches .lit_test_times.txt files between premerge invocations.
+
+.lit_test_times.txt files are used by lit to order tests to best take advantage
+of parallelism. Having them around and up to date can result in a ~15%
+improvement in test times. This script downloading cached test time files and
+uploading new versions to the GCS buckets used for caching.
+"""
+
+import sys
+import os
+import logging
+import multiprocessing.pool
+import pathlib
+import glob
+
+from google.cloud import storage
+
+GCS_PARALLELISM = 100
+
+
+def _maybe_upload_timing_file(bucket, timing_file_path):
+    if os.path.exists(timing_file_path):
+        timing_file_blob = bucket.blob("lit_timing/" + timing_file_path)
+        timing_file_blob.upload_from_filename(timing_file_path)
+
+
+def upload_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_path in glob.glob("**/.lit_test_times.txt", recursive=True):
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_upload_timing_file, (bucket, timing_file_path)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done uploading")
+
+
+def _maybe_download_timing_file(blob):
+    file_name = blob.name.removeprefix("lit_timing/")
+    pathlib.Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)
+    blob.download_to_filename(file_name)
+
+
+def download_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    blobs = bucket.list_blobs(prefix="lit_timing")
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_blob in blobs:
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_download_timing_file, (timing_file_blob,)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done downloading")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)
+    action = sys.argv[1]
+    storage_client = storage.Client()
+    bucket_name = os.environ["CACHE_GCS_BUCKET"]
+    if action == "download":
+        download_timing_files(storage_client, bucket_name)
+    elif action == "upload":
+        upload_timing_files(storage_client, bucket_name)
+    else:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)

.ci/compute_projects.py

@@ -49,8 +49,7 @@
         "flang",
     },
     "lld": {"bolt", "cross-project-tests"},
-    # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "cross-project-tests"},
+    "clang": {"clang-tools-extra", "cross-project-tests", "lldb"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     ".ci": {

.ci/compute_projects_test.py

@@ -83,11 +83,11 @@ def test_clang(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm",
+            "clang;clang-tools-extra;lld;lldb;llvm",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-tools",
+            "check-clang check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
@@ -158,11 +158,11 @@ def test_cir(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm;mlir",
+            "clang;clang-tools-extra;lld;lldb;llvm;mlir",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-cir check-clang-tools",
+            "check-clang check-clang-cir check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"

.github/workflows/libclang-python-tests.yml

@@ -4,7 +4,6 @@ permissions:
   contents: read
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - 'main'
@@ -13,29 +12,46 @@ on:
       - 'clang/tools/libclang/**'
       - 'clang/CMakeList.txt'
       - '.github/workflows/libclang-python-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
   pull_request:
     paths:
       - 'clang/bindings/python/**'
       - 'clang/tools/libclang/**'
       - 'clang/CMakeList.txt'
       - '.github/workflows/libclang-python-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
 
 jobs:
   check-clang-python:
     # Build libclang and then run the libclang Python binding's unit tests.
+    # There is an issue running on "windows-2019".
+    # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
     name: Build and run Python unit tests
     if: github.repository == 'llvm/llvm-project'
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.13"]
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-clang-python
-      projects: clang
-      # There is an issue running on "windows-2019".
-      # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
-      os_list: '["ubuntu-24.04"]'
-      python_version: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Setup Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
+        with:
+          max-size: 2G
+          key: spirv-ubuntu-24.04
+          variant: sccache
+      - name: Build and Test
+        run: |
+          mkdir build
+          cmake -GNinja \
+            -S llvm \
+            -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+            -DLLVM_ENABLE_PROJECTS=clang
+          ninja -C build check-clang-python