From 6101acef1cc66c8726603ea7bdf5aef4cfc2ca10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kami=C5=84ski?= Date: Wed, 24 Sep 2025 16:35:54 +0000 Subject: [PATCH 1/2] [Benchmarks] Use GitProject instead of utils methods --- devops/scripts/benchmarks/CONTRIB.md | 9 +- devops/scripts/benchmarks/benches/base.py | 2 +- devops/scripts/benchmarks/benches/benchdnn.py | 55 ++++++------ devops/scripts/benchmarks/benches/compute.py | 57 +++++-------- devops/scripts/benchmarks/benches/gromacs.py | 62 ++++++-------- devops/scripts/benchmarks/benches/llamacpp.py | 58 ++++++------- .../scripts/benchmarks/benches/syclbench.py | 56 ++++++------- devops/scripts/benchmarks/benches/test.py | 4 - devops/scripts/benchmarks/benches/velocity.py | 83 +++++++++++-------- devops/scripts/benchmarks/git_project.py | 2 +- devops/scripts/benchmarks/utils/flamegraph.py | 16 ++-- devops/scripts/benchmarks/utils/unitrace.py | 15 ++-- devops/scripts/benchmarks/utils/utils.py | 30 ------- 13 files changed, 200 insertions(+), 249 deletions(-) diff --git a/devops/scripts/benchmarks/CONTRIB.md b/devops/scripts/benchmarks/CONTRIB.md index 5f99c77f43301..ed11f007a7fc6 100644 --- a/devops/scripts/benchmarks/CONTRIB.md +++ b/devops/scripts/benchmarks/CONTRIB.md @@ -191,12 +191,17 @@ The benchmark suite generates an interactive HTML dashboard that visualizes `Res ## Utilities +* **`git_project.GitProject`:** Manages git repository cloning, building, and installation for benchmark suites: + * Automatically clones repositories to a specified directory and checks out specific commits/refs. + * Provides standardized directory structure with `src_dir`, `build_dir`, and `install_dir` properties. + * Handles incremental updates - only re-clones if the target commit has changed. + * Supports force rebuilds and custom directory naming via constructor options. + * Provides `configure()`, `build()`, and `install()` methods for CMake-based projects. + * Use this for benchmark suites that need to build from external git repositories (e.g., `ComputeBench`, `VelocityBench`). * **`utils.utils`:** Provides common helper functions: * `run()`: Executes shell commands with environment setup (SYCL paths, LD_LIBRARY_PATH). - * `git_clone()`: Clones/updates Git repositories. * `download()`: Downloads files via HTTP, checks checksums, optionally extracts tar/gz archives. * `prepare_workdir()`: Sets up the main working directory. - * `create_build_path()`: Creates a clean build directory. * **`utils.oneapi`:** Provides the `OneAPI` singleton class (`get_oneapi()`). Downloads and installs specified oneAPI components (oneDNN, oneMKL) into the working directory if needed, providing access to their paths (libs, includes, CMake configs). Use this if your benchmark depends on these components instead of requiring a system-wide install. * **`options.py`:** Defines and holds global configuration options, populated by `argparse` in `main.py`. Use options instead of defining your own global variables. * **`presets.py`:** Defines named sets of suites (`enabled_suites()`) used by the `--preset` argument. diff --git a/devops/scripts/benchmarks/benches/base.py b/devops/scripts/benchmarks/benches/base.py index 4dadb638dde50..0d11be2044602 100644 --- a/devops/scripts/benchmarks/benches/base.py +++ b/devops/scripts/benchmarks/benches/base.py @@ -84,8 +84,8 @@ def tracing_enabled(self, run_trace, force_trace, tr_type: TracingType): """Returns whether tracing is enabled for the given type.""" return (self.traceable(tr_type) or force_trace) and run_trace == tr_type - @abstractmethod def setup(self): + """Extra setup steps to be performed before running the benchmark.""" pass @abstractmethod diff --git a/devops/scripts/benchmarks/benches/benchdnn.py b/devops/scripts/benchmarks/benches/benchdnn.py index 71b9538c2ceae..129212334a368 100644 --- a/devops/scripts/benchmarks/benches/benchdnn.py +++ b/devops/scripts/benchmarks/benches/benchdnn.py @@ -8,19 +8,16 @@ from .base import Suite, Benchmark, TracingType from options import options -from utils.utils import git_clone, run, create_build_path from utils.result import Result from utils.oneapi import get_oneapi from utils.logger import log from .benchdnn_list import get_bench_dnn_list +from git_project import GitProject class OneDnnBench(Suite): def __init__(self, directory): - self.directory = Path(directory).resolve() - build_path = create_build_path(self.directory, "onednn-build") - self.build_dir = Path(build_path) - self.src_dir = self.directory / "onednn-repo" + self.project = None def git_url(self): return "https://github.com/uxlfoundation/oneDNN.git" @@ -62,36 +59,35 @@ def setup(self) -> None: if options.sycl is None: return - self.src_dir = git_clone( - self.directory, - "onednn-repo", - self.git_url(), - self.git_tag(), - ) - self.oneapi = get_oneapi() - cmake_args = [ - "cmake", - f"-S {self.src_dir}", - f"-B {self.build_dir}", + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_tag(), + Path(options.workdir), + "onednn", + force_rebuild=True, + ) + + extra_cmake_args = [ f"-DCMAKE_PREFIX_PATH={options.sycl}", "-DCMAKE_CXX_COMPILER=clang++", "-DCMAKE_C_COMPILER=clang", - "-DCMAKE_BUILD_TYPE=Release", "-DDNNL_BUILD_TESTS=ON", "-DDNNL_BUILD_EXAMPLES=OFF", "-DDNNL_CPU_RUNTIME=NONE", # Disable SYCL CPU support "-DDNNL_GPU_RUNTIME=SYCL", # Enable SYCL GPU support ] - run( - cmake_args, + self.project.configure( + extra_cmake_args, + install_prefix=False, add_sycl=True, ) - - run( - f"cmake --build {self.build_dir} --target benchdnn -j {options.build_jobs}", + self.project.build( + target="benchdnn", add_sycl=True, - ld_library=[str(self.build_dir) + "/src"] + self.oneapi.ld_libraries(), + ld_library=[str(self.project.build_dir / "src")] + + self.oneapi.ld_libraries(), timeout=60 * 20, ) @@ -113,7 +109,10 @@ def __init__(self, suite, bench_driver, bench_name, bench_args, syclgraph=True): self.bench_args += " --execution-mode=direct" self.bench_name += "-eager" self.bench_args += f" {bench_args}" - self.bench_bin = suite.build_dir / "tests" / "benchdnn" / "benchdnn" + + @property + def benchmark_bin(self) -> Path: + return self.suite.project.build_dir / "tests" / "benchdnn" / "benchdnn" def enabled(self): if options.sycl is None: @@ -129,8 +128,8 @@ def explicit_group(self) -> str: return self.exp_group def setup(self): - if not self.bench_bin.exists(): - raise FileNotFoundError(f"Benchmark binary not found: {self.bench_bin}") + if not self.benchmark_bin.exists(): + raise FileNotFoundError(f"Benchmark binary not found: {self.benchmark_bin}") def run( self, @@ -145,12 +144,12 @@ def run( extra_trace_opt = None command = [ - str(self.bench_bin), + str(self.benchmark_bin), *self.bench_args.split(), ] ld_library = self.suite.oneapi.ld_libraries() + [ - str(self.suite.build_dir / "src") + str(self.suite.project.build_dir / "src") ] env_vars = dict(env_vars) if env_vars else {} diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index dcb23e392dc35..220f4e88e6fd5 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -4,18 +4,18 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from itertools import product -import os import csv import io import copy import math from enum import Enum +from pathlib import Path -from utils.utils import run, git_clone, create_build_path from .base import Benchmark, Suite, TracingType from utils.result import BenchmarkMetadata, Result from .base import Benchmark, Suite from options import options +from git_project import GitProject class RUNTIMES(Enum): @@ -50,8 +50,8 @@ def runtime_to_tag_name(runtime: RUNTIMES) -> str: class ComputeBench(Suite): def __init__(self, directory): - self.directory = directory self.submit_graph_num_kernels = [4, 10, 32] + self.project = None def name(self) -> str: return "Compute Benchmarks" @@ -66,47 +66,36 @@ def setup(self) -> None: if options.sycl is None: return - repo_path = git_clone( - self.directory, - "compute-benchmarks-repo", - self.git_url(), - self.git_hash(), - ) - build_path = create_build_path(self.directory, "compute-benchmarks-build") + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_hash(), + Path(options.workdir), + "compute-benchmarks", + force_rebuild=True, + ) - configure_command = [ - "cmake", - f"-B {build_path}", - f"-S {repo_path}", - f"-DCMAKE_BUILD_TYPE=Release", + extra_args = [ f"-DBUILD_SYCL=ON", f"-DSYCL_COMPILER_ROOT={options.sycl}", f"-DALLOW_WARNINGS=ON", f"-DCMAKE_CXX_COMPILER=clang++", f"-DCMAKE_C_COMPILER=clang", ] - if options.ur_adapter == "cuda": - configure_command += [ + extra_args += [ "-DBUILD_SYCL_WITH_CUDA=ON", "-DBUILD_L0=OFF", "-DBUILD_OCL=OFF", ] - if options.ur is not None: - configure_command += [ + extra_args += [ f"-DBUILD_UR=ON", f"-Dunified-runtime_DIR={options.ur}/lib/cmake/unified-runtime", ] - run(configure_command, add_sycl=True) - - run( - f"cmake --build {build_path} -j {options.build_jobs}", - add_sycl=True, - ) - - self.built = True + self.project.configure(extra_args, install_prefix=False, add_sycl=True) + self.project.build(add_sycl=True) def additional_metadata(self) -> dict[str, BenchmarkMetadata]: metadata = { @@ -370,7 +359,7 @@ def __init__( runtime: RUNTIMES = None, profiler_type: PROFILERS = PROFILERS.TIMER, ): - super().__init__(bench.directory, bench) + super().__init__(options.workdir, bench) self.bench = bench self.bench_name = name self.test = test @@ -384,6 +373,11 @@ def __init__( self._validate_attr("iterations_regular") self._validate_attr("iterations_trace") + @property + def benchmark_bin(self) -> Path: + """Returns the path to the benchmark binary""" + return self.bench.project.build_dir / "bin" / self.bench_name + def get_iters(self, run_trace: TracingType): """Returns the number of iterations to run for the given tracing type.""" return ( @@ -437,11 +431,6 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]: def extra_env_vars(self) -> dict: return {} - def setup(self): - self.benchmark_bin = os.path.join( - self.bench.directory, "compute-benchmarks-build", "bin", self.bench_name - ) - def explicit_group(self): return "" @@ -455,7 +444,7 @@ def run( force_trace: bool = False, ) -> list[Result]: command = [ - f"{self.benchmark_bin}", + str(self.benchmark_bin), f"--test={self.test}", "--csv", "--noHeaders", diff --git a/devops/scripts/benchmarks/benches/gromacs.py b/devops/scripts/benchmarks/benches/gromacs.py index c19542ae8d834..56ac604bdc9ad 100644 --- a/devops/scripts/benchmarks/benches/gromacs.py +++ b/devops/scripts/benchmarks/benches/gromacs.py @@ -9,14 +9,21 @@ from .base import Suite, Benchmark, TracingType from options import options -from utils.utils import git_clone, download, run, create_build_path +from utils.utils import download, run from utils.result import Result from utils.oneapi import get_oneapi from utils.logger import log -from utils.unitrace import get_unitrace +from git_project import GitProject class GromacsBench(Suite): + def __init__(self, directory): + self.project = None + model_path = str(Path(options.workdir) / self.grappa_file()).replace( + ".tar.gz", "" + ) + self.grappa_dir = Path(model_path) + def git_url(self): return "https://gitlab.com/gromacs/gromacs.git" @@ -29,14 +36,6 @@ def grappa_url(self): def grappa_file(self): return Path(os.path.basename(self.grappa_url())) - def __init__(self, directory): - self.directory = Path(directory).resolve() - model_path = str(self.directory / self.grappa_file()).replace(".tar.gz", "") - self.grappa_dir = Path(model_path) - build_path = create_build_path(self.directory, "gromacs-build") - self.gromacs_build_path = Path(build_path) - self.gromacs_src = self.directory / "gromacs-repo" - def name(self): return "Gromacs Bench" @@ -52,12 +51,14 @@ def benchmarks(self) -> list[Benchmark]: ] def setup(self) -> None: - self.gromacs_src = git_clone( - self.directory, - "gromacs-repo", - self.git_url(), - self.git_tag(), - ) + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_tag(), + Path(options.workdir), + "gromacs", + force_rebuild=True, + ) # TODO: Detect the GPU architecture and set the appropriate flags @@ -65,11 +66,7 @@ def setup(self) -> None: self.oneapi = get_oneapi() - cmd_list = [ - "cmake", - f"-S {self.gromacs_src}", - f"-B {self.gromacs_build_path}", - "-DCMAKE_BUILD_TYPE=Release", + extra_args = [ "-DCMAKE_CXX_COMPILER=clang++", "-DCMAKE_C_COMPILER=clang", "-DGMX_GPU=SYCL", @@ -84,21 +81,14 @@ def setup(self) -> None: ] if options.unitrace: - cmd_list.append("-DGMX_USE_ITT=ON") + extra_args.append("-DGMX_USE_ITT=ON") - run( - cmd_list, - add_sycl=True, - ) - run( - f"cmake --build {self.gromacs_build_path} -j {options.build_jobs}", - add_sycl=True, - ld_library=self.oneapi.ld_libraries(), - ) + self.project.configure(extra_args, install_prefix=False, add_sycl=True) + self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries()) download( - self.directory, + options.workdir, self.grappa_url(), - self.directory / self.grappa_file(), + options.workdir / self.grappa_file(), checksum="cc02be35ba85c8b044e47d097661dffa8bea57cdb3db8b5da5d01cdbc94fe6c8902652cfe05fb9da7f2af0698be283a2", untar=True, ) @@ -114,9 +104,7 @@ def __init__(self, suite, model, type, option): self.type = type # The type of benchmark ("pme" or "rf") self.option = option # "graphs" or "eager" - self.gromacs_src = suite.gromacs_src self.grappa_dir = suite.grappa_dir - self.gmx_path = suite.gromacs_build_path / "bin" / "gmx" if self.type == "pme": self.extra_args = [ @@ -129,6 +117,10 @@ def __init__(self, suite, model, type, option): else: self.extra_args = [] + @property + def gmx_path(self) -> Path: + return self.suite.project.build_dir / "bin" / "gmx" + def name(self): return f"gromacs-{self.model}-{self.type}-{self.option}" diff --git a/devops/scripts/benchmarks/benches/llamacpp.py b/devops/scripts/benchmarks/benches/llamacpp.py index a0e3e8b6e719a..d83660bd11854 100644 --- a/devops/scripts/benchmarks/benches/llamacpp.py +++ b/devops/scripts/benchmarks/benches/llamacpp.py @@ -5,19 +5,20 @@ import csv import io +import os from pathlib import Path -from utils.utils import download, git_clone + +from utils.utils import download from .base import Benchmark, Suite, TracingType from utils.result import Result -from utils.utils import run, create_build_path from options import options from utils.oneapi import get_oneapi -import os +from git_project import GitProject class LlamaCppBench(Suite): def __init__(self, directory): - self.directory = directory + self.project = None def name(self) -> str: return "llama.cpp bench" @@ -32,18 +33,20 @@ def setup(self) -> None: if options.sycl is None: return - repo_path = git_clone( - self.directory, - "llamacpp-repo", - self.git_url(), - self.git_hash(), - ) + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_hash(), + Path(options.workdir), + "llamacpp", + force_rebuild=True, + ) - self.models_dir = os.path.join(self.directory, "models") - Path(self.models_dir).mkdir(parents=True, exist_ok=True) + models_dir = Path(options.workdir, "llamacpp-models") + models_dir.mkdir(parents=True, exist_ok=True) self.model = download( - self.models_dir, + models_dir, "https://huggingface.co/ggml-org/DeepSeek-R1-Distill-Qwen-1.5B-Q4_0-GGUF/resolve/main/deepseek-r1-distill-qwen-1.5b-q4_0.gguf", "deepseek-r1-distill-qwen-1.5b-q4_0.gguf", checksum="791f6091059b653a24924b9f2b9c3141c8f892ae13fff15725f77a2bf7f9b1b6b71c85718f1e9c0f26c2549aba44d191", @@ -51,13 +54,7 @@ def setup(self) -> None: self.oneapi = get_oneapi() - self.build_path = create_build_path(self.directory, "llamacpp-build") - - configure_command = [ - "cmake", - f"-B {self.build_path}", - f"-S {repo_path}", - f"-DCMAKE_BUILD_TYPE=Release", + extra_args = [ f"-DGGML_SYCL=ON", f"-DCMAKE_C_COMPILER=clang", f"-DCMAKE_CXX_COMPILER=clang++", @@ -67,14 +64,8 @@ def setup(self) -> None: f"-DSYCL_COMPILER=ON", f"-DMKL_DIR={self.oneapi.mkl_cmake()}", ] - - run(configure_command, add_sycl=True) - - run( - f"cmake --build {self.build_path} -j {options.build_jobs}", - add_sycl=True, - ld_library=self.oneapi.ld_libraries(), - ) + self.project.configure(extra_args, add_sycl=True) + self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries()) def benchmarks(self) -> list[Benchmark]: return [LlamaBench(self)] @@ -82,9 +73,13 @@ def benchmarks(self) -> list[Benchmark]: class LlamaBench(Benchmark): def __init__(self, bench): - super().__init__(bench.directory, bench) + super().__init__(options.workdir, bench) self.bench = bench + @property + def benchmark_bin(self) -> Path: + return self.bench.project.build_dir / "bin" / "llama-bench" + def enabled(self): if options.sycl is None: return False @@ -92,9 +87,6 @@ def enabled(self): return False return True - def setup(self): - self.benchmark_bin = os.path.join(self.bench.build_path, "bin", "llama-bench") - def model(self): return "DeepSeek-R1-Distill-Qwen-1.5B-Q4_0.gguf" @@ -122,7 +114,7 @@ def run( force_trace: bool = False, ) -> list[Result]: command = [ - f"{self.benchmark_bin}", + str(self.benchmark_bin), "--output", "csv", "-n", diff --git a/devops/scripts/benchmarks/benches/syclbench.py b/devops/scripts/benchmarks/benches/syclbench.py index b409f60a79f73..624f185ab7f8a 100644 --- a/devops/scripts/benchmarks/benches/syclbench.py +++ b/devops/scripts/benchmarks/benches/syclbench.py @@ -5,17 +5,17 @@ import os import csv -import io -from utils.utils import run, git_clone, create_build_path +from pathlib import Path + from .base import Benchmark, Suite, TracingType from utils.result import Result from options import options +from git_project import GitProject class SyclBench(Suite): def __init__(self, directory): - self.directory = directory - return + self.project = None def name(self) -> str: return "SYCL-Bench" @@ -30,38 +30,31 @@ def setup(self) -> None: if options.sycl is None: return - build_path = create_build_path(self.directory, "sycl-bench-build") - repo_path = git_clone( - self.directory, - "sycl-bench-repo", - self.git_url(), - self.git_hash(), - ) + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_hash(), + Path(options.workdir), + "sycl-bench", + force_rebuild=True, + ) - configure_command = [ - "cmake", - f"-B {build_path}", - f"-S {repo_path}", - f"-DCMAKE_BUILD_TYPE=Release", + extra_args = [ f"-DCMAKE_CXX_COMPILER={options.sycl}/bin/clang++", f"-DCMAKE_C_COMPILER={options.sycl}/bin/clang", f"-DSYCL_IMPL=dpcpp", ] - if options.ur_adapter == "cuda": - configure_command += [ + extra_args += [ f"-DCMAKE_CXX_FLAGS=-fsycl -fsycl-targets=nvptx64-nvidia-cuda" ] - if options.ur_adapter == "hip": - configure_command += [ + extra_args += [ f"-DCMAKE_CXX_FLAGS=-fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch={options.hip_arch}" ] - run(configure_command, add_sycl=True) - run(f"cmake --build {build_path} -j {options.build_jobs}", add_sycl=True) - - self.built = True + self.project.configure(extra_args, install_prefix=False, add_sycl=True) + self.project.build(add_sycl=True) def benchmarks(self) -> list[Benchmark]: return [ @@ -106,11 +99,15 @@ def benchmarks(self) -> list[Benchmark]: class SyclBenchmark(Benchmark): def __init__(self, bench, name, test): - super().__init__(bench.directory, bench) + super().__init__(options.workdir, bench) self.bench = bench self.bench_name = name self.test = test + @property + def benchmark_bin(self) -> Path: + return self.bench.project.build_dir / self.bench_name + def enabled(self) -> bool: return options.sycl is not None @@ -132,21 +129,16 @@ def get_tags(self): base_tags.append("latency") return base_tags - def setup(self): - self.benchmark_bin = os.path.join( - self.directory, "sycl-bench-build", self.bench_name - ) - def run( self, env_vars, run_trace: TracingType = TracingType.NONE, force_trace: bool = False, ) -> list[Result]: - self.outputfile = os.path.join(self.bench.directory, self.test + ".csv") + self.outputfile = os.path.join(options.workdir, self.test + ".csv") command = [ - f"{self.benchmark_bin}", + str(self.benchmark_bin), f"--warmup-run", f"--num-runs={options.iterations}", f"--output={self.outputfile}", diff --git a/devops/scripts/benchmarks/benches/test.py b/devops/scripts/benchmarks/benches/test.py index 99156eb49bcb2..586302a5af07e 100644 --- a/devops/scripts/benchmarks/benches/test.py +++ b/devops/scripts/benchmarks/benches/test.py @@ -4,12 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception import random -from utils.utils import git_clone from .base import Benchmark, Suite, TracingType from utils.result import BenchmarkMetadata, Result -from utils.utils import run, create_build_path -from options import options -import os class TestSuite(Suite): diff --git a/devops/scripts/benchmarks/benches/velocity.py b/devops/scripts/benchmarks/benches/velocity.py index f6213958dcf22..42f025c52af91 100644 --- a/devops/scripts/benchmarks/benches/velocity.py +++ b/devops/scripts/benchmarks/benches/velocity.py @@ -5,20 +5,20 @@ import re import shutil -from utils.utils import git_clone +import os +from pathlib import Path + from .base import Benchmark, Suite, TracingType from utils.result import Result -from utils.utils import run, create_build_path +from utils.utils import run from options import options from utils.oneapi import get_oneapi -import shutil - -import os +from git_project import GitProject class VelocityBench(Suite): - def __init__(self, directory): - self.directory = directory + def __init__(self, directory) -> None: + self.project = None def name(self) -> str: return "Velocity Bench" @@ -33,12 +33,13 @@ def setup(self) -> None: if options.sycl is None: return - self.repo_path = git_clone( - self.directory, - "velocity-bench-repo", - self.git_url(), - self.git_hash(), - ) + if self.project is None: + self.project = GitProject( + self.git_url(), + self.git_hash(), + Path(options.workdir), + "velocity-bench", + ) def benchmarks(self) -> list[Benchmark]: return [ @@ -56,12 +57,24 @@ def benchmarks(self) -> list[Benchmark]: class VelocityBase(Benchmark): def __init__(self, name: str, bin_name: str, vb: VelocityBench, unit: str): - super().__init__(vb.directory, vb) + super().__init__(options.workdir, vb) self.vb = vb self.bench_name = name self.bin_name = bin_name self.unit = unit + @property + def src_dir(self) -> Path: + return self.vb.project.src_dir / self.bench_name / "SYCL" + + @property + def build_dir(self) -> Path: + return self.vb.project.build_dir / self.bench_name + + @property + def benchmark_bin(self) -> Path: + return self.build_dir / self.bin_name + def enabled(self) -> bool: if options.sycl is None: return False @@ -92,25 +105,27 @@ def ld_libraries(self) -> list[str]: return [] def setup(self): - self.code_path = os.path.join(self.vb.repo_path, self.bench_name, "SYCL") self.download_deps() - self.benchmark_bin = os.path.join( - self.directory, self.bench_name, self.bin_name - ) + self.configure() + self.build() - build_path = create_build_path(self.directory, self.bench_name) + def configure(self) -> None: + if options.rebuild and self.build_dir.exists(): + shutil.rmtree(self.build_dir) + self.build_dir.mkdir(parents=True, exist_ok=True) - configure_command = [ + cmd = [ "cmake", - f"-B {build_path}", - f"-S {self.code_path}", - f"-DCMAKE_BUILD_TYPE=Release", + f"-S {self.src_dir}", + f"-B {self.build_dir}", + "-DCMAKE_BUILD_TYPE=Release", ] - configure_command += self.extra_cmake_args() + cmd += self.extra_cmake_args() + run(cmd, {"CC": "clang", "CXX": "clang++"}, add_sycl=True) - run(configure_command, {"CC": "clang", "CXX": "clang++"}, add_sycl=True) + def build(self) -> None: run( - f"cmake --build {build_path} -j {options.build_jobs}", + f"cmake --build {self.build_dir} -j {options.build_jobs}", add_sycl=True, ld_library=self.ld_libraries(), ) @@ -139,7 +154,7 @@ def run( env_vars.update(self.extra_env_vars()) command = [ - f"{self.benchmark_bin}", + str(self.benchmark_bin), ] command += self.bin_args() @@ -214,7 +229,9 @@ def description(self) -> str: ) def bin_args(self) -> list[str]: - self.data_path = os.path.join(self.vb.repo_path, "bitcracker", "hash_pass") + self.data_path = os.path.join( + self.vb.project.src_dir, "bitcracker", "hash_pass" + ) return [ "-f", @@ -323,7 +340,7 @@ def lower_is_better(self): def bin_args(self) -> list[str]: self.data_path = os.path.join( - self.vb.repo_path, "QuickSilver", "Examples", "AllScattering" + self.vb.project.src_dir, "QuickSilver", "Examples", "AllScattering" ) return ["-i", f"{self.data_path}/scatteringOnly.inp"] @@ -416,8 +433,8 @@ def __init__(self, vb: VelocityBench): super().__init__("cudaSift", "cudaSift", vb, "ms") def download_deps(self): - images = os.path.join(self.vb.repo_path, self.bench_name, "inputData") - dest = os.path.join(self.directory, "inputData") + images = os.path.join(self.vb.project.src_dir, self.bench_name, "inputData") + dest = os.path.join(options.workdir, "inputData") if not os.path.exists(dest): shutil.copytree(images, dest) @@ -615,8 +632,8 @@ def description(self) -> str: def bin_args(self): return [ - f"{self.code_path}/a9a", - f"{self.code_path}/a.m", + f"{self.src_dir}/a9a", + f"{self.src_dir}/a.m", ] def parse_output(self, stdout: str) -> float: diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py index 84fe722d2974c..6f45942945011 100644 --- a/devops/scripts/benchmarks/git_project.py +++ b/devops/scripts/benchmarks/git_project.py @@ -91,7 +91,7 @@ def needs_rebuild(self, check_build=False, check_install=False) -> bool: def configure( self, extra_args: list | None = None, - install_prefix=True, + install_prefix: bool = True, add_sycl: bool = False, ) -> None: """Configures the project.""" diff --git a/devops/scripts/benchmarks/utils/flamegraph.py b/devops/scripts/benchmarks/utils/flamegraph.py index aeeb9330fcc1e..ae51b0eae9d25 100644 --- a/devops/scripts/benchmarks/utils/flamegraph.py +++ b/devops/scripts/benchmarks/utils/flamegraph.py @@ -3,14 +3,13 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -import os import shutil from pathlib import Path from options import options -from utils.utils import run, git_clone, prune_old_files, remove_by_prefix +from utils.utils import run, prune_old_files, remove_by_prefix from utils.logger import log - +from git_project import GitProject from datetime import datetime, timezone @@ -29,14 +28,13 @@ def __init__(self): ) log.info("Downloading FlameGraph...") - repo_dir = git_clone( - options.workdir, - "flamegraph-repo", + self.project = GitProject( "https://github.com/brendangregg/FlameGraph.git", "41fee1f99f9276008b7cd112fca19dc3ea84ac32", + Path(options.workdir), + "flamegraph", ) log.info("FlameGraph tools ready.") - self.repo_dir = Path(repo_dir) if options.results_directory_override: self.flamegraphs_dir = ( @@ -130,7 +128,7 @@ def _convert_perf_to_folded(self, perf_data_file: Path, folded_file: Path): """Step 1: Convert perf script to folded format using a pipeline.""" log.debug(f"Converting perf data to folded format: {folded_file}") perf_script_cmd = ["perf", "script", "-i", str(perf_data_file)] - stackcollapse_cmd = [str(self.repo_dir / "stackcollapse-perf.pl")] + stackcollapse_cmd = [str(self.project.src_dir / "stackcollapse-perf.pl")] try: # Run perf script and capture its output @@ -150,7 +148,7 @@ def _generate_svg(self, folded_file: Path, svg_file: Path, bench_name: str): """Step 2: Generate flamegraph SVG from a folded file.""" log.debug(f"Generating flamegraph SVG: {svg_file}") flamegraph_cmd = [ - str(self.repo_dir / "flamegraph.pl"), + str(self.project.src_dir / "flamegraph.pl"), "--title", f"{options.save_name} - {bench_name}", "--width", diff --git a/devops/scripts/benchmarks/utils/unitrace.py b/devops/scripts/benchmarks/utils/unitrace.py index a61d3952f63dc..e37c02d8687cc 100644 --- a/devops/scripts/benchmarks/utils/unitrace.py +++ b/devops/scripts/benchmarks/utils/unitrace.py @@ -5,17 +5,18 @@ import os import shutil +from datetime import datetime, timezone +from pathlib import Path + from options import options from utils.utils import ( run, - git_clone, prune_old_files, remove_by_prefix, remove_by_extension, ) from utils.logger import log - -from datetime import datetime, timezone +from git_project import GitProject class Unitrace: @@ -29,14 +30,14 @@ def __init__(self): ) log.info("Downloading and building Unitrace...") - repo_dir = git_clone( - options.workdir, - "pti-gpu-repo", + self.project = GitProject( "https://github.com/intel/pti-gpu.git", "pti-0.12.4", + Path(options.workdir), + "pti-gpu", ) build_dir = os.path.join(options.workdir, "unitrace-build") - unitrace_src = os.path.join(repo_dir, "tools", "unitrace") + unitrace_src = self.project.src_dir / "tools" / "unitrace" os.makedirs(build_dir, exist_ok=True) unitrace_exe = os.path.join(build_dir, "unitrace") diff --git a/devops/scripts/benchmarks/utils/utils.py b/devops/scripts/benchmarks/utils/utils.py index be37cb7bfb363..afae989d4b8b9 100644 --- a/devops/scripts/benchmarks/utils/utils.py +++ b/devops/scripts/benchmarks/utils/utils.py @@ -92,25 +92,6 @@ def run( raise -def git_clone(dir, name, repo, commit): - repo_path = os.path.join(dir, name) - log.debug(f"Cloning {repo} into {repo_path} at commit {commit}") - - if os.path.isdir(repo_path) and os.path.isdir(os.path.join(repo_path, ".git")): - run("git fetch", cwd=repo_path) - run("git reset --hard", cwd=repo_path) - run(f"git checkout {commit}", cwd=repo_path) - elif not os.path.exists(repo_path): - run(f"git clone --recursive {repo} {repo_path}") - run(f"git checkout {commit}", cwd=repo_path) - else: - raise Exception( - f"The directory {repo_path} exists but is not a git repository." - ) - log.debug(f"Cloned {repo} into {repo_path} at commit {commit}") - return repo_path - - def prepare_bench_cwd(dir): # we need 2 deep to workaround a problem with a fixed relative paths in some velocity benchmarks options.benchmark_cwd = os.path.join(dir, "bcwd", "bcwd") @@ -146,17 +127,6 @@ def prepare_workdir(dir, version): version_file.write(version) -def create_build_path(directory, name): - build_path = os.path.join(directory, name) - - if options.rebuild and Path(build_path).exists(): - shutil.rmtree(build_path) - - Path(build_path).mkdir(parents=True, exist_ok=True) - - return build_path - - def calculate_checksum(file_path): sha_hash = hashlib.sha384() with open(file_path, "rb") as f: From 36247a09f0a6e65634f43caae4eeb512ef68c8e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kami=C5=84ski?= Date: Fri, 26 Sep 2025 08:16:45 +0000 Subject: [PATCH 2/2] Remove unused directory parameter --- devops/scripts/benchmarks/benches/base.py | 7 +++---- devops/scripts/benchmarks/benches/benchdnn.py | 2 +- devops/scripts/benchmarks/benches/compute.py | 4 ++-- devops/scripts/benchmarks/benches/gromacs.py | 2 +- devops/scripts/benchmarks/benches/llamacpp.py | 4 ++-- devops/scripts/benchmarks/benches/syclbench.py | 4 ++-- devops/scripts/benchmarks/benches/test.py | 2 +- devops/scripts/benchmarks/benches/umf.py | 5 +---- devops/scripts/benchmarks/benches/velocity.py | 4 ++-- 9 files changed, 15 insertions(+), 19 deletions(-) diff --git a/devops/scripts/benchmarks/benches/base.py b/devops/scripts/benchmarks/benches/base.py index 0d11be2044602..bbbedaf629bf5 100644 --- a/devops/scripts/benchmarks/benches/base.py +++ b/devops/scripts/benchmarks/benches/base.py @@ -49,8 +49,7 @@ class TracingType(Enum): class Benchmark(ABC): - def __init__(self, directory, suite): - self.directory = directory + def __init__(self, suite): self.suite = suite @abstractmethod @@ -205,9 +204,9 @@ def run_bench( def create_data_path(self, name, skip_data_dir=False): if skip_data_dir: - data_path = os.path.join(self.directory, name) + data_path = os.path.join(options.workdir, name) else: - data_path = os.path.join(self.directory, "data", name) + data_path = os.path.join(options.workdir, "data", name) if options.redownload and Path(data_path).exists(): shutil.rmtree(data_path) diff --git a/devops/scripts/benchmarks/benches/benchdnn.py b/devops/scripts/benchmarks/benches/benchdnn.py index 129212334a368..d06f473a3697c 100644 --- a/devops/scripts/benchmarks/benches/benchdnn.py +++ b/devops/scripts/benchmarks/benches/benchdnn.py @@ -16,7 +16,7 @@ class OneDnnBench(Suite): - def __init__(self, directory): + def __init__(self): self.project = None def git_url(self): diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 220f4e88e6fd5..532bdfb50d898 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -49,7 +49,7 @@ def runtime_to_tag_name(runtime: RUNTIMES) -> str: class ComputeBench(Suite): - def __init__(self, directory): + def __init__(self): self.submit_graph_num_kernels = [4, 10, 32] self.project = None @@ -359,7 +359,7 @@ def __init__( runtime: RUNTIMES = None, profiler_type: PROFILERS = PROFILERS.TIMER, ): - super().__init__(options.workdir, bench) + super().__init__(bench) self.bench = bench self.bench_name = name self.test = test diff --git a/devops/scripts/benchmarks/benches/gromacs.py b/devops/scripts/benchmarks/benches/gromacs.py index 56ac604bdc9ad..2a8ce37e338e7 100644 --- a/devops/scripts/benchmarks/benches/gromacs.py +++ b/devops/scripts/benchmarks/benches/gromacs.py @@ -17,7 +17,7 @@ class GromacsBench(Suite): - def __init__(self, directory): + def __init__(self): self.project = None model_path = str(Path(options.workdir) / self.grappa_file()).replace( ".tar.gz", "" diff --git a/devops/scripts/benchmarks/benches/llamacpp.py b/devops/scripts/benchmarks/benches/llamacpp.py index d83660bd11854..f93d040c246e0 100644 --- a/devops/scripts/benchmarks/benches/llamacpp.py +++ b/devops/scripts/benchmarks/benches/llamacpp.py @@ -17,7 +17,7 @@ class LlamaCppBench(Suite): - def __init__(self, directory): + def __init__(self): self.project = None def name(self) -> str: @@ -73,7 +73,7 @@ def benchmarks(self) -> list[Benchmark]: class LlamaBench(Benchmark): def __init__(self, bench): - super().__init__(options.workdir, bench) + super().__init__(bench) self.bench = bench @property diff --git a/devops/scripts/benchmarks/benches/syclbench.py b/devops/scripts/benchmarks/benches/syclbench.py index 624f185ab7f8a..36a0a6b27723e 100644 --- a/devops/scripts/benchmarks/benches/syclbench.py +++ b/devops/scripts/benchmarks/benches/syclbench.py @@ -14,7 +14,7 @@ class SyclBench(Suite): - def __init__(self, directory): + def __init__(self): self.project = None def name(self) -> str: @@ -99,7 +99,7 @@ def benchmarks(self) -> list[Benchmark]: class SyclBenchmark(Benchmark): def __init__(self, bench, name, test): - super().__init__(options.workdir, bench) + super().__init__(bench) self.bench = bench self.bench_name = name self.test = test diff --git a/devops/scripts/benchmarks/benches/test.py b/devops/scripts/benchmarks/benches/test.py index 586302a5af07e..e3dda9a3a1502 100644 --- a/devops/scripts/benchmarks/benches/test.py +++ b/devops/scripts/benchmarks/benches/test.py @@ -58,7 +58,7 @@ def additional_metadata(self) -> dict[str, BenchmarkMetadata]: class TestBench(Benchmark): def __init__(self, suite, name, value, diff, group="", notes=None, unstable=None): - super().__init__("", suite) + super().__init__(suite) self.bname = name self.value = value self.diff = diff diff --git a/devops/scripts/benchmarks/benches/umf.py b/devops/scripts/benchmarks/benches/umf.py index a47b363f08fea..07ea15607cfb1 100644 --- a/devops/scripts/benchmarks/benches/umf.py +++ b/devops/scripts/benchmarks/benches/umf.py @@ -20,9 +20,6 @@ def isUMFAvailable(): class UMFSuite(Suite): - def __init__(self, directory): - self.directory = directory - def name(self) -> str: return "UMF" @@ -44,7 +41,7 @@ def benchmarks(self) -> list[Benchmark]: class GBench(Benchmark): def __init__(self, bench): - super().__init__(bench.directory, bench) + super().__init__(bench) self.bench = bench self.bench_name = "umf-benchmark" diff --git a/devops/scripts/benchmarks/benches/velocity.py b/devops/scripts/benchmarks/benches/velocity.py index 42f025c52af91..ec76df8a9b3b6 100644 --- a/devops/scripts/benchmarks/benches/velocity.py +++ b/devops/scripts/benchmarks/benches/velocity.py @@ -17,7 +17,7 @@ class VelocityBench(Suite): - def __init__(self, directory) -> None: + def __init__(self) -> None: self.project = None def name(self) -> str: @@ -57,7 +57,7 @@ def benchmarks(self) -> list[Benchmark]: class VelocityBase(Benchmark): def __init__(self, name: str, bin_name: str, vb: VelocityBench, unit: str): - super().__init__(options.workdir, vb) + super().__init__(vb) self.vb = vb self.bench_name = name self.bin_name = bin_name