Migrate from Boost.Python to PyBind11 (#54)

Fixes #24

### Motivation

Remove the Boost.Python dependency so that on Windows there will be no DLL dependencies because PyBind11 is header only.

### Modifications

Since PyBind11 can perform type conversions between C++ types (STL, function, etc.) and Python types (list, dict, lambda, etc.), some wrapper classes are replaced with the classes in the Pulsar C++ library.

The only API changes are related to the `_pulsar` module, which should not be used directly. The authentication related classes were wrapper classes with constructors before, now they are created by the static `create` methods from Pulsar C++ API.

Fix the CMakeLists.txt and the workflows to build Python wheels on Linux, macOS and Windows. Finally add a workflow to build Windows wheels during a release.

Author: BewareMyPower

.github/workflows/ci-build-release-wheels.yaml

@@ -141,3 +141,64 @@ jobs:
           name: wheel-mac-py${{matrix.py.version}}
           path: dist/*.whl
 
+  windows-wheels:
+    name: Wheel Windows - Py ${{matrix.py.version}}
+    runs-on: windows-2019
+    env:
+      PULSAR_CPP_DIR: 'C:\\pulsar-cpp'
+    timeout-minutes: 300
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python:
+          - {version: '3.7'}
+          - {version: '3.8'}
+          - {version: '3.9'}
+          - {version: '3.10'}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python.version }}
+
+      - name: Download Pulsar C++ client on Windows
+        shell: bash
+        run: |
+          mkdir -p ${{ env.PULSAR_CPP_DIR }}
+          cd ${{ env.PULSAR_CPP_DIR }}
+          # TODO: switch to official releases
+          curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp/pulsar-client-cpp-3.1.0-candidate-1/x64-windows-static.tar.gz
+          tar zxf x64-windows-static.tar.gz
+          mv x64-windows-static/* .
+          ls -l ${{ env.PULSAR_CPP_DIR }}
+
+      - name: Configure CMake
+        shell: bash
+        run: |
+          pip3 install pyyaml
+          export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
+          curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
+          tar zxf v${PYBIND11_VERSION}.tar.gz
+          rm -rf pybind11
+          mv pybind11-${PYBIND11_VERSION} pybind11
+          cmake -B build -A x64 \
+            -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
+            -DLINK_STATIC=ON
+
+      - name: Build Python wheel
+        shell: bash
+        run: |
+          cmake --build build --config Release --target install
+          python -m pip install wheel
+          python setup.py bdist_wheel
+          python -m pip install ./dist/*.whl
+          python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()'
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheel-windows-py${{matrix.python.version}}
+          path: dist/*.whl

.github/workflows/ci-pr-validation.yaml

@@ -55,11 +55,8 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
 
-      - name: Install deps
-        run: sudo apt-get install -y libboost-python-dev
-
       - name: Install Pulsar C++ client
-        run: build-support/install-cpp-client.sh
+        run: build-support/install-dependencies.sh
 
       - name: CMake
         run: cmake .
@@ -176,21 +173,16 @@ jobs:
         run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
 
   windows-wheels:
-    name: "Python ${{ matrix.python.version }} Wheel on ${{ matrix.windows.name }}"
+    name: "Python ${{ matrix.python.version }} Wheel on Windows x64"
     needs: unit-tests
-    runs-on: ${{ matrix.windows.os }}
+    runs-on: windows-2019
     timeout-minutes: 120
 
     env:
       PULSAR_CPP_DIR: 'C:\\pulsar-cpp'
     strategy:
       fail-fast: false
       matrix:
-        windows:
-          - name: 'Windows x64'
-            os: windows-2022
-            arch: '-A x64'
-            triplet: 'x64-windows'
         python:
           - version: '3.7'
           - version: '3.8'
@@ -199,48 +191,33 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python.version }}
 
-      - name: Prepare vcpkg.json
-        shell: bash
-        run: |
-          python --version
-          cp -f vcpkg-${{ matrix.python.version }}.json vcpkg.json
-          cat vcpkg.json
-
       - name: Download Pulsar C++ client on Windows
         shell: bash
         run: |
           mkdir -p ${{ env.PULSAR_CPP_DIR }}
           cd ${{ env.PULSAR_CPP_DIR }}
           # TODO: switch to official releases
-          curl -O -L https://github.com/BewareMyPower/pulsar-client-cpp/releases/download/v3.1.0-rc-20221028/${{ matrix.windows.triplet }}-static.zip
-          unzip -q ${{ matrix.windows.triplet }}-static.zip
+          curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp/pulsar-client-cpp-3.1.0-candidate-1/x64-windows-static.tar.gz
+          tar zxf x64-windows-static.tar.gz
+          mv x64-windows-static/* .
           ls -l ${{ env.PULSAR_CPP_DIR }}
 
-      - name: Cache Vcpkg
-        uses: actions/cache@v3
-        id: cache-vcpkg
-        with:
-          path: build/vcpkg_installed
-          key: ${{ matrix.python.version }}-${{ hashFiles(format('vcpkg-{0}.json', matrix.python.version)) }}
-
-      - name: Install dependencies and configure CMake
+      - name: Configure CMake
         shell: bash
         run: |
-          COMMIT_ID=$(grep baseline vcpkg.json | sed 's/[",]//g' | awk '{print $2}')
-          cd vcpkg
-          echo "git fetch origin $COMMIT_ID"
-          git fetch origin $COMMIT_ID
-          cd -
-          cmake -B build ${{ matrix.windows.arch }} \
+          pip3 install pyyaml
+          export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
+          curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
+          tar zxf v${PYBIND11_VERSION}.tar.gz
+          rm -rf pybind11
+          mv pybind11-${PYBIND11_VERSION} pybind11
+          cmake -B build -A x64 \
             -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \
-            -DUSE_VCPKG=ON \
             -DLINK_STATIC=ON
 
       - name: Build Python wheel
@@ -250,9 +227,6 @@ jobs:
           python -m pip install wheel
           python setup.py bdist_wheel
           python -m pip install ./dist/*.whl
-          cp ./build/Release/boost_python*.dll .
-          echo "The extra DLLs:"
-          ls -l *.dll
           python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()'
 
   check-completion:

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "vcpkg"]
-	path = vcpkg
-	url = https://github.com/microsoft/vcpkg.git
+[submodule "pybind11"]
+	path = pybind11
+	url = https://github.com/pybind/pybind11.git

CMakeLists.txt

@@ -20,12 +20,8 @@
 cmake_minimum_required(VERSION 3.18)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
-option(USE_VCPKG "Use Vcpkg to install dependencies" OFF)
-if (USE_VCPKG)
-    set(CMAKE_TOOLCHAIN_FILE "${CMAKE_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake"
-        CACHE STRING "Vcpkg toolchain file")
-endif ()
 project (pulsar-client-python)
+set(CMAKE_PREFIX_PATH ${PROJECT_SOURCE_DIR}/pybind11/include ${CMAKE_PREFIX_PATH})
 option(LINK_STATIC "Link against static libraries" OFF)
 MESSAGE(STATUS "LINK_STATIC:  " ${LINK_STATIC})
 
@@ -58,43 +54,15 @@ SET(CMAKE_CXX_STANDARD 11)
 find_package (Python3 REQUIRED COMPONENTS Development.Module)
 MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS})
 
-find_package(Boost REQUIRED ${Boost_INCLUDE_DIRS})
-message(STATUS "Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
-
-SET(Boost_USE_STATIC_LIBS   ${LINK_STATIC})
-
-set(BOOST_PYTHON_NAME_LIST python3 python310 python39 python38 python37)
-foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
-    find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
-    if (${Boost_FOUND})
-        set(BOOST_PYTHON_COMPONENT_FOUND ${BOOST_PYTHON_NAME})
-        message(STATUS "Found Boost COMPONENTS " ${BOOST_PYTHON_COMPONENT_FOUND})
-        break ()
-    endif ()
-endforeach ()
-if (NOT BOOST_PYTHON_COMPONENT_FOUND)
-    message(FATAL_ERROR "Could not find Boost Python library")
-endif ()
+find_path(PYBIND11_INCLUDE_DIRS NAMES "pybind11/pybind11.h")
+message(STATUS "PYBIND11_INCLUDE_DIRS: " ${PYBIND11_INCLUDE_DIRS})
 
 ########################################################################################################################
 
-INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")
-
-ADD_LIBRARY(_pulsar SHARED src/pulsar.cc
-                           src/producer.cc
-                           src/consumer.cc
-                           src/config.cc
-                           src/enums.cc
-                           src/client.cc
-                           src/message.cc
-                           src/authentication.cc
-                           src/reader.cc
-                           src/schema.cc
-                           src/cryptoKeyReader.cc
-                           src/exceptions.cc
-                           src/utils.cc
-        )
+INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} ${PYBIND11_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS})
 
+file(GLOB SOURCES src/*.cc)
+ADD_LIBRARY(_pulsar SHARED ${SOURCES})
 if (MSVC)
     set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd)
 else ()
@@ -109,10 +77,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
   set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -Qunused-arguments -undefined dynamic_lookup")
 endif()
 
-# Try all possible boost-python variable namings
 set(PYTHON_WRAPPER_LIBS
     ${PULSAR_LIBRARY}
-    Boost::${BOOST_PYTHON_COMPONENT_FOUND}
 )
 if (MSVC)
     set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module)

README.md

@@ -27,49 +27,43 @@
 - A C++ compiler that supports C++11
 - CMake >= 3.18
 - [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp)
-- [Boost.Python](https://github.com/boostorg/python)
+- [PyBind11](https://github.com/pybind/pybind11)
 
-## Install the Python wheel
-
-### Windows (with Vcpkg)
+PyBind11 is a header-only library and a submodule, so you can simply download the submodule so that CMake can find this dependency.
 
-First, install the dependencies via [Vcpkg](https://github.com/microsoft/vcpkg).
-
-```PowerShell
-vcpkg install --feature-flags=manifests --triplet x64-windows
+```bash
+git submodule update --init
 ```
 
-> NOTE: For Windows 32-bit library, change `x64-windows` to `x86-windows`, see [here](https://github.com/microsoft/vcpkg/tree/master/triplets) for all available triplets.
+You can also download the pybind11 directly like:
 
-Then, build and install the Python wheel.
-
-```PowerShell
-# Assuming the Pulsar C++ client has been installed under the `PULSAR_CPP` directory.
-cmake -B build -DUSE_VCPKG=ON -DCMAKE_PREFIX_PATH="$env:PULSAR_CPP" -DLINK_STATIC=ON
-cmake --build build --config Release
-cmake --install build
-py setup.py bdist_wheel
-py -m pip install ./dist/pulsar_client-*.whl
+```bash
+pip3 install pyyaml
+export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
+curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
+tar zxf v${PYBIND11_VERSION}.tar.gz
+mv pybind11-${PYBIND11_VERSION} pybind11
 ```
 
-Since the Python client links to Boost.Python dynamically, you have to copy the dll (e.g. `boost_python310-vc142-mt-x64-1_80.dll`) into the system path (the `PATH` environment variable). If the `-DLINK_STATIC=ON` option is not specified, you have to copy the `pulsar.dll` into the system path as well.
+After that, you only need to install the Pulsar C++ client dependency into the system path. You can [install the pre-built binaries](https://pulsar.apache.org/docs/next/client-libraries-cpp/#installation) or [build from source](https://github.com/apache/pulsar-client-cpp#compilation).
 
-### Linux or macOS
+## Install the Python wheel
 
-Assuming the Pulsar C++ client and Boost.Python have been installed under the system path.
+Make sure the PyBind11 submodule has been downloaded and the Pulsar C++ client has been installed. Then run the following commands:
 
 ```bash
 cmake -B build
-cmake --build build -j8
+cmake --build build 
 cmake --install build
-./setup.py bdist_wheel
-pip3 install dist/pulsar_client-*.whl --force-reinstall
+python3 ./setup.py bdist_wheel
+python3 -m pip install dist/pulsar_client-*.whl --force-reinstall
 ```
 
 > **NOTE**
 >
 > 1. Here a separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` is under the project directory.
 > 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before.
+> 3. On Windows, the Python command is `py` instead of `python3`.
 
 ## Running examples
 

build-support/copy-deps-versionfile.sh

@@ -25,6 +25,5 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 for dir in manylinux2014 manylinux_musl; do
   mkdir -p pkg/$dir/.build
   cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
-  cp $ROOT_DIR/pulsar-client-cpp-version.txt pkg/$dir/.build
   cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
 done

build-support/dep-version.py

@@ -20,5 +20,15 @@
 
 import yaml, sys
 
-deps = yaml.safe_load(open("dependencies.yaml"))
+if len(sys.argv) < 2:
+    print(f'''Usage: {sys.argv[0]} dependency-name [dependency-file]
+
+  The dependency file is "dependencies.yaml" by default.''')
+    sys.exit(1)
+
+if len(sys.argv) > 2:
+    dependency_file = sys.argv[2]
+else:
+    dependency_file = 'dependencies.yaml'
+deps = yaml.safe_load(open(dependency_file))
 print(deps[sys.argv[1]])

build-support/install-cpp-client.sh renamed to build-support/install-dependencies.sh

@@ -20,8 +20,10 @@
 
 set -e -x
 
-ROOT_DIR=$(dirname $(dirname $0))
-CPP_CLIENT_VERSION=$(cat $ROOT_DIR/pulsar-client-cpp-version.txt | xargs)
+cd `dirname $0`
+
+CPP_CLIENT_VERSION=$(./dep-version.py pulsar-cpp ../dependencies.yaml)
+PYBIND11_VERSION=$(./dep-version.py pybind11 ../dependencies.yaml)
 
 if [ $USER != "root" ]; then
   SUDO="sudo"
@@ -62,5 +64,7 @@ else
   exit 1
 fi
 
-
-
+curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
+tar zxf v${PYBIND11_VERSION}.tar.gz
+$SUDO cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 /usr/include/
+rm -rf pybind11-${PYBIND11_VERSION} v${PYBIND11_VERSION}.tar.gz

dependencies.yaml

@@ -17,8 +17,10 @@
 # under the License.
 #
 
-boost: 1.80.0
 cmake: 3.24.2
+pulsar-cpp: 3.0.0
+pybind11: 2.10.1
+boost: 1.80.0
 protobuf: 3.20.0
 zlib: 1.2.13
 zstd: 1.5.2

pkg/build-wheel-inside-docker.sh

@@ -22,6 +22,12 @@ set -e -x
 
 cd /pulsar-client-python
 
+PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
+curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
+tar zxf v${PYBIND11_VERSION}.tar.gz
+rm -rf pybind11
+mv pybind11-${PYBIND11_VERSION} pybind11
+
 rm -f CMakeCache.txt CMakeFiles
 
 cmake . \