Skip to content

Commit b77e6c1

Browse files
alex-spacemitco-sevenCISC
authored
ggml: riscv: add riscv spacemit backend (#15288)
* ggml: add spacemit backend Change-Id: I249bdc043485d815a9c351867137bc1e27cc2e23 * add new line at end of file Change-Id: I889ed1c85fb45e62350ecde0c06f70450cadfbe2 * add riscv zba extension limit Change-Id: I321eb200f859751727afe5cae13074dfce2bb0ce * fixed for review comments, file renamed and format Change-Id: Ia20b6ec24a36638e62e0fe07cf100916a7cce3ce * fixed for code format, after clang-format Change-Id: I5dc33a0412da3d3f2d77075d8939185d3009eca2 * use _Float16 instead of __fp16 Change-Id: I039fb02bb95270e641bc4442204e658735859d43 * add ci for riscv64-spacemit-ime-native Change-Id: I711c1033061df1a289ea77891b2997599dfe8279 * update debian-13-riscv64-spacemit-ime-native ci label Change-Id: Ifb2b891e2fca57b5da604fce2ac255f27731179a * remove license comment for spacemit ime Change-Id: If0dc3ca30a958631ccca0a28b62e0b825f9fb0c3 * upgrade binutils for gcc ime Change-Id: Ibf2fa74c1064408974cb5b45f044d40987e5fb45 * add spacemit ime cross jobs Change-Id: I80d74909941d41cb9cd09e51d8baf01c985cbfc6 * remove native compile for riscv64-spacemit-ime Change-Id: I01920afafdc73fa7424014fd648d243f8ec9e25e * ci : add caching for spacemit ime cross toolchain Change-Id: Ic54a192019a2fd982bbd58225ce3bbc38f4053de * ci: bug fixed for cache path and env Change-Id: I28c42e10b6fff053bb6580926ca2353448cb042a * Update .github/workflows/build-linux-cross.yml for cache path Co-authored-by: Sigbjørn Skjæret <[email protected]> * bugfixed for build-linux-cross.yml, syntax error Co-authored-by: Sigbjørn Skjæret <[email protected]> --------- Co-authored-by: cailinxi <[email protected]> Co-authored-by: Sigbjørn Skjæret <[email protected]>
1 parent 2ddd3f2 commit b77e6c1

File tree

11 files changed

+4501
-0
lines changed

11 files changed

+4501
-0
lines changed

‎.github/workflows/build-linux-cross.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,3 +253,47 @@ jobs:
253253
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
254254
255255
cmake --build build --config Release -j $(nproc)
256+
257+
ubuntu-24-riscv64-cpu-spacemit-ime-cross:
258+
runs-on: ubuntu-24.04
259+
260+
env:
261+
SPACEMIT_IME_TOOLCHAIN_VERSION: "1.1.2"
262+
SPACEMIT_IME_TOOLCHAIN_PATH: "spacemit-toolchain-linux-glibc-x86_64"
263+
264+
steps:
265+
- uses: actions/checkout@v4
266+
267+
- name: Cache Toolchain
268+
uses: actions/cache@v4
269+
id: cache-spacemit-ime-cross-toolchain
270+
with:
271+
path: ./${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
272+
key: ${{ runner.os }}-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
273+
274+
- name: Setup Toolchain
275+
if: steps.cache-spacemit-ime-cross-toolchain.outputs.cache-hit != 'true'
276+
run: |
277+
wget --quiet --no-check-certificate https://archive.spacemit.com/toolchain/spacemit-toolchain-linux-glibc-x86_64-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}.tar.xz -O ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz
278+
rm -rf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
279+
mkdir -p ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
280+
tar xf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz -C ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }} --strip-components=1
281+
rm -rf ${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}.tar.xz
282+
283+
- name: Build
284+
run: |
285+
export RISCV_ROOT_PATH=${PWD}/${{ env.SPACEMIT_IME_TOOLCHAIN_PATH }}
286+
cmake -B build -DLLAMA_CURL=OFF \
287+
-DCMAKE_BUILD_TYPE=Release \
288+
-DGGML_OPENMP=OFF \
289+
-DLLAMA_BUILD_EXAMPLES=ON \
290+
-DLLAMA_BUILD_TOOLS=ON \
291+
-DLLAMA_BUILD_TESTS=OFF \
292+
-DGGML_CPU_RISCV64_SPACEMIT=ON \
293+
-DGGML_RVV=ON \
294+
-DGGML_RV_ZFH=ON \
295+
-DGGML_RV_ZICBOP=ON \
296+
-DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
297+
-DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake
298+
299+
cmake --build build --config Release -j $(nproc)

‎.github/workflows/build-riscv-native.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,63 @@ jobs:
5858
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
5959
6060
cmake --build build --config Release -j $(nproc)
61+
62+
# debian-13-riscv64-spacemit-ime-native: # Bianbu 2.2
63+
# runs-on: [self-hosted, RISCV64]
64+
65+
# steps:
66+
# - name: Install prerequisites
67+
# run: |
68+
# sudo apt-get update || true
69+
# sudo apt-get install -y libatomic1
70+
# - uses: actions/checkout@v4
71+
# - name: Setup Riscv
72+
# run: |
73+
# sudo apt-get update || true
74+
# sudo apt-get install -y --no-install-recommends \
75+
# build-essential \
76+
# gcc-14-riscv64-linux-gnu \
77+
# g++-14-riscv64-linux-gnu \
78+
# ccache \
79+
# cmake
80+
# sudo apt-get upgrade binutils -y
81+
82+
# - name: Setup ccache
83+
# run: |
84+
# mkdir -p $HOME/.ccache
85+
# ccache -M 5G -d $HOME/.ccache
86+
# export CCACHE_LOGFILE=/home/runneruser/ccache_debug/ccache.log
87+
# export CCACHE_DEBUGDIR="/home/runneruser/ccache_debug"
88+
# echo "$GITHUB_WORKSPACE"
89+
# echo "CCACHE_LOGFILE=$CCACHE_LOGFILE" >> $GITHUB_ENV
90+
# echo "CCACHE_DEBUGDIR=$CCACHE_DEBUGDIR" >> $GITHUB_ENV
91+
# echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV
92+
# echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
93+
94+
# - name: Build
95+
# run: |
96+
# cmake -B build \
97+
# -DLLAMA_CURL=OFF \
98+
# -DCMAKE_BUILD_TYPE=Release \
99+
# -DGGML_OPENMP=OFF \
100+
# -DLLAMA_BUILD_EXAMPLES=ON \
101+
# -DLLAMA_BUILD_TOOLS=ON \
102+
# -DLLAMA_BUILD_TESTS=OFF \
103+
# -DCMAKE_SYSTEM_NAME=Linux \
104+
# -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
105+
# -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
106+
# -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
107+
# -DCMAKE_C_COMPILER_LAUNCHER=ccache \
108+
# -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
109+
# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
110+
# -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
111+
# -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
112+
# -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
113+
# -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH \
114+
# -DGGML_RVV=ON \
115+
# -DGGML_RV_ZFH=ON \
116+
# -DGGML_RV_ZICBOP=ON \
117+
# -DGGML_CPU_RISCV64_SPACEMIT=ON \
118+
# -DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1
119+
120+
# cmake --build build --config Release -j $(nproc)

‎CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
/ggml/src/ggml-blas/ @slaren
5151
/ggml/src/ggml-common.h @ggerganov @slaren
5252
/ggml/src/ggml-cpu/ @ggerganov @slaren
53+
/ggml/src/ggml-cpu/spacemit/ @alex-spacemit
5354
/ggml/src/ggml-cuda/common.cuh @slaren
5455
/ggml/src/ggml-cuda/fattn* @JohannesGaessler
5556
/ggml/src/ggml-cuda/ggml-cuda.cu @slaren

‎cmake/riscv64-spacemit-linux-gnu-gcc.cmake‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
set(CMAKE_SYSTEM_NAME Linux)
2+
set(CMAKE_SYSTEM_PROCESSOR riscv64)
3+
set(CMAKE_SYSTEM_VERSION 1)
4+
5+
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(riscv)")
6+
message(STATUS "HOST SYSTEM ${CMAKE_HOST_SYSTEM_PROCESSOR}")
7+
else()
8+
set(GNU_MACHINE riscv64-unknown-linux-gnu CACHE STRING "GNU compiler triple")
9+
if (DEFINED ENV{RISCV_ROOT_PATH})
10+
file(TO_CMAKE_PATH $ENV{RISCV_ROOT_PATH} RISCV_ROOT_PATH)
11+
else()
12+
message(FATAL_ERROR "RISCV_ROOT_PATH env must be defined")
13+
endif()
14+
15+
set(RISCV_ROOT_PATH ${RISCV_ROOT_PATH} CACHE STRING "root path to riscv toolchain")
16+
set(CMAKE_C_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-gcc)
17+
set(CMAKE_CXX_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-g++)
18+
set(CMAKE_STRIP ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-strip)
19+
set(CMAKE_FIND_ROOT_PATH "${RISCV_ROOT_PATH}/riscv64-unknown-linux-gnu")
20+
set(CMAKE_SYSROOT "${RISCV_ROOT_PATH}/sysroot")
21+
endif()
22+
23+
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
24+
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
25+
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
26+
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
27+
set(CMAKE_C_FLAGS "-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CMAKE_C_FLAGS}")
28+
set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CXX_FLAGS}")
29+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -latomic")

‎docs/build-riscv64-spacemit.md‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
> [!IMPORTANT]
2+
> This build documentation is specific only to RISC-V SpacemiT SOCs.
3+
4+
## Build llama.cpp locally (for riscv64)
5+
6+
1. Prepare Toolchain For RISCV
7+
~~~
8+
wget https://archive.spacemit.com/toolchain/spacemit-toolchain-linux-glibc-x86_64-v1.1.2.tar.xz
9+
~~~
10+
11+
2. Build
12+
Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
13+
```bash
14+
15+
cmake -B build \
16+
-DCMAKE_BUILD_TYPE=Release \
17+
-DGGML_CPU_RISCV64_SPACEMIT=ON \
18+
-DLLAMA_CURL=OFF \
19+
-DGGML_RVV=ON \
20+
-DGGML_RV_ZFH=ON \
21+
-DGGML_RV_ZICBOP=ON \
22+
-DRISCV64_SPACEMIT_IME_SPEC=RISCV64_SPACEMIT_IME1 \
23+
-DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake \
24+
-DCMAKE_INSTALL_PREFIX=build/installed
25+
26+
cmake --build build --parallel $(nproc) --config Release
27+
28+
pushd build
29+
make install
30+
popd
31+
```
32+
33+
## Simulation
34+
You can use QEMU to perform emulation on non-RISC-V architectures.
35+
36+
1. Download QEMU
37+
~~~
38+
wget https://archive.spacemit.com/spacemit-ai/qemu/jdsk-qemu-v0.0.14.tar.gz
39+
~~~
40+
41+
2. Run Simulation
42+
After build your llama.cpp, you can run the executable file via QEMU for simulation, for example:
43+
~~~
44+
export QEMU_ROOT_PATH={your QEMU file path}
45+
export RISCV_ROOT_PATH_IME1={your RISC-V compiler path}
46+
47+
${QEMU_ROOT_PATH}/bin/qemu-riscv64 -L ${RISCV_ROOT_PATH_IME1}/sysroot -cpu max,vlen=256,elen=64,vext_spec=v1.0 ${PWD}/build/bin/llama-cli -m ${PWD}/models/Qwen2.5-0.5B-Instruct-Q4_0.gguf -t 1
48+
~~~
49+
## Performance
50+
#### Quantization Support For Matrix
51+
~~~
52+
model name : Spacemit(R) X60
53+
isa : rv64imafdcv_zicbom_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zfhmin_zca_zcd_zba_zbb_zbc_zbs_zkt_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt
54+
mmu : sv39
55+
uarch : spacemit,x60
56+
mvendorid : 0x710
57+
marchid : 0x8000000058000001
58+
~~~
59+
60+
Q4_0
61+
| Model | Size | Params | backend | threads | test | t/s |
62+
| -----------| -------- | ------ | ------- | ------- | ---- |------|
63+
Qwen2.5 0.5B |403.20 MiB|630.17 M| cpu | 4 | pp512|64.12 ± 0.26|
64+
Qwen2.5 0.5B |403.20 MiB|630.17 M| cpu | 4 | tg128|10.03 ± 0.01|
65+
Qwen2.5 1.5B |1011.16 MiB| 1.78 B | cpu | 4 | pp512|24.16 ± 0.02|
66+
Qwen2.5 1.5B |1011.16 MiB| 1.78 B | cpu | 4 | tg128|3.83 ± 0.06|
67+
Qwen2.5 3B | 1.86 GiB | 3.40 B | cpu | 4 | pp512|12.08 ± 0.02|
68+
Qwen2.5 3B | 1.86 GiB | 3.40 B | cpu | 4 | tg128|2.23 ± 0.02|
69+
70+
Q4_1
71+
| Model | Size | Params | backend | threads | test | t/s |
72+
| -----------| -------- | ------ | ------- | ------- | ---- |------|
73+
Qwen2.5 0.5B |351.50 MiB|494.03 M| cpu | 4 | pp512|62.07 ± 0.12|
74+
Qwen2.5 0.5B |351.50 MiB|494.03 M| cpu | 4 | tg128|9.91 ± 0.01|
75+
Qwen2.5 1.5B |964.06 MiB| 1.54 B | cpu | 4 | pp512|22.95 ± 0.25|
76+
Qwen2.5 1.5B |964.06 MiB| 1.54 B | cpu | 4 | tg128|4.01 ± 0.15|
77+
Qwen2.5 3B | 1.85 GiB | 3.09 B | cpu | 4 | pp512|11.55 ± 0.16|
78+
Qwen2.5 3B | 1.85 GiB | 3.09 B | cpu | 4 | tg128|2.25 ± 0.04|
79+
80+
81+
Q4_K
82+
| Model | Size | Params | backend | threads | test | t/s |
83+
| -----------| -------- | ------ | ------- | ------- | ---- |------|
84+
Qwen2.5 0.5B |462.96 MiB|630.17 M| cpu | 4 | pp512|9.29 ± 0.05|
85+
Qwen2.5 0.5B |462.96 MiB|630.17 M| cpu | 4 | tg128|5.67 ± 0.04|
86+
Qwen2.5 1.5B | 1.04 GiB | 1.78 B | cpu | 4 | pp512|10.38 ± 0.10|
87+
Qwen2.5 1.5B | 1.04 GiB | 1.78 B | cpu | 4 | tg128|3.17 ± 0.08|
88+
Qwen2.5 3B | 1.95 GiB | 3.40 B | cpu | 4 | pp512|4.23 ± 0.04|
89+
Qwen2.5 3B | 1.95 GiB | 3.40 B | cpu | 4 | tg128|1.73 ± 0.00|

‎ggml/src/ggml-cpu/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,15 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
439439
ggml-cpu/arch/riscv/quants.c
440440
ggml-cpu/arch/riscv/repack.cpp
441441
)
442+
if (GGML_CPU_RISCV64_SPACEMIT)
443+
target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC})
444+
list(APPEND GGML_CPU_SOURCES
445+
ggml-cpu/spacemit/ime.cpp
446+
ggml-cpu/spacemit/ime.h
447+
ggml-cpu/spacemit/ime1_kernels.cpp
448+
ggml-cpu/spacemit/ime_kernels.h
449+
)
450+
endif()
442451
set(MARCH_STR "rv64gc")
443452
if (GGML_RV_ZFH)
444453
string(APPEND MARCH_STR "_zfh")

‎ggml/src/ggml-cpu/ggml-cpu.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
# include "kleidiai/kleidiai.h"
1919
#endif
2020

21+
#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
22+
# include "spacemit/ime.h"
23+
#endif
24+
2125
#if defined(_WIN32)
2226
# define WIN32_LEAN_AND_MEAN
2327
# ifndef NOMINMAX
@@ -45,6 +49,12 @@ std::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffer_type
4549
}
4650
#endif
4751

52+
#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
53+
if (ggml_backend_cpu_riscv64_spacemit_buffer_type()) {
54+
bufts.push_back(ggml_backend_cpu_riscv64_spacemit_buffer_type());
55+
}
56+
#endif
57+
4858
#ifdef GGML_USE_CPU_KLEIDIAI
4959
if (ggml_backend_cpu_kleidiai_buffer_type()) {
5060
bufts.push_back(ggml_backend_cpu_kleidiai_buffer_type());

0 commit comments

Comments
 (0)