feat: replace pip with uv and add use_python parameter (#730)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Author: aaronsteers

README.md

@@ -18,6 +18,47 @@ Watch this [Getting Started Loom video](https://www.loom.com/share/3de81ca3ce914
 * [GitHub](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Github_Incremental_Demo.ipynb)
 * [Postgres (cache)](https://github.com/airbytehq/quickstarts/blob/main/pyairbyte_notebooks/PyAirbyte_Postgres_Custom_Cache_Demo.ipynb)
 
+## Connector Installation
+
+### Declarative Source Installation
+
+For Declarative Sources defined in YAML, the installation process will is to simply download the yaml file from the `connectors.airbyte.com` public URLs, and to run them directly as YAML.
+
+Declarative sources have the fastest download times, due to the simplicity and each of install.
+
+In some cases, you may get better stability by using `docker_image=True` in `get_source()`/`get_destination()`, due to the fact that all dependencies are locked within the docker image.
+
+### Python Installation
+
+Generally, when Python-based installation is possible, it will be performed automatically given a Python-based connector name.
+
+In some cases, you may get better stability by using `docker_image=True` in `get_source()`/`get_destination()`, due to the fact that all dependencies are locked within the docker image.
+
+#### Installing Connectors with `uv`
+
+By default, beginning with version `0.29.0`, PyAirbyte defaults to [`uv`](https://docs.astral.sh/uv) instead of `pip` for Python connector installation. Compared with `pip`, `uv` is much faster. It also provides the unique ability of specifying different versions of Python than PyAirbyte is using, and even Python versions which are not already pre-installed on the local workstation.
+
+If you prefer to fall back to the prior `pip`-based installation methods, set the env var `AIRBYTE_NO_UV=true`.
+
+#### Installing Connectors With a Custom Python Version
+
+In both `get_source()` and `get_destination()`, you can provide a `use_python` input arg that is equal to the desired version of Python that you with to use for the given connector. This can be helpful if an older connector doesn't support the version of Python that you are using for PyAirbyte itself.
+
+For example, assuming PyAirbyte is running on Python 3.12, you can install a connector using Python 3.10.13 with the following code snippet:
+
+```py
+import airbyte as ab
+
+source = ab.get_source(
+    "source-faker",
+    use_python="3.10.17",
+)
+```
+
+### Installing Connectors with Docker
+
+For any connector (`get_source()`/`get_destination()`), you can specify the `docker_image` argument to `True` to prefer Docker over other default installation methods or `docker_image=MY_IMAGE` to leverage a specific docker image tag for the execution.
+
 ## Contributing
 
 To learn how you can contribute to PyAirbyte, please see our [PyAirbyte Contributors Guide](./docs/CONTRIBUTING.md).

airbyte/_executors/python.py

@@ -17,6 +17,7 @@
 from airbyte._util.meta import is_windows
 from airbyte._util.telemetry import EventState, log_install_state
 from airbyte._util.venv_util import get_bin_dir
+from airbyte.constants import NO_UV
 
 
 if TYPE_CHECKING:
@@ -32,6 +33,7 @@ def __init__(
         target_version: str | None = None,
         pip_url: str | None = None,
         install_root: Path | None = None,
+        use_python: bool | Path | str | None = None,
     ) -> None:
         """Initialize a connector executor that runs a connector in a virtual environment.
 
@@ -42,6 +44,11 @@ def __init__(
             pip_url: (Optional.) The pip URL of the connector to install.
             install_root: (Optional.) The root directory where the virtual environment will be
                 created. If not provided, the current working directory will be used.
+            use_python: (Optional.) Python interpreter specification:
+                - True: Use current Python interpreter
+                - False: Use Docker instead (handled by factory)
+                - Path: Use interpreter at this path or interpreter name/command
+                - str: Use uv-managed Python version (semver patterns like "3.12", "3.11.5")
         """
         super().__init__(name=name, metadata=metadata, target_version=target_version)
 
@@ -59,6 +66,7 @@ def __init__(
             else f"airbyte-{self.name}"
         )
         self.install_root = install_root or Path.cwd()
+        self.use_python = use_python
 
     def _get_venv_name(self) -> str:
         return f".venv-{self.name}"
@@ -106,20 +114,62 @@ def install(self) -> None:
 
         After installation, the installed version will be stored in self.reported_version.
         """
-        self._run_subprocess_and_raise_on_failure(
-            [sys.executable, "-m", "venv", str(self._get_venv_path())]
-        )
+        if not (
+            self.use_python is None
+            or self.use_python is True
+            or self.use_python is False
+            or isinstance(self.use_python, (str, Path))
+        ):
+            raise exc.PyAirbyteInputError(
+                message="Invalid use_python parameter type",
+                input_value=str(self.use_python),
+            )
+
+        python_override: str | None = None
+        if not NO_UV and isinstance(self.use_python, Path):
+            python_override = str(self.use_python.absolute())
+
+        elif not NO_UV and isinstance(self.use_python, str):
+            python_override = self.use_python
+
+        uv_cmd_prefix = ["uv"] if not NO_UV else []
+        python_clause: list[str] = ["--python", python_override] if python_override else []
 
-        pip_path = str(get_bin_dir(self._get_venv_path()) / "pip")
+        venv_cmd: list[str] = [
+            *(uv_cmd_prefix or [sys.executable, "-m"]),
+            "venv",
+            str(self._get_venv_path()),
+            *python_clause,
+        ]
         print(
-            f"Installing '{self.name}' into virtual environment '{self._get_venv_path()!s}'.\n"
-            f"Running 'pip install {self.pip_url}'...\n",
+            f"Creating '{self.name}' virtual environment with command '{' '.join(venv_cmd)}'",
+            file=sys.stderr,
+        )
+        self._run_subprocess_and_raise_on_failure(venv_cmd)
+
+        install_cmd = (
+            [
+                "uv",
+                "pip",
+                "install",
+                "--python",  # uv requires --python after the subcommand
+                str(self.interpreter_path),
+            ]
+            if not NO_UV
+            else [
+                "pip",
+                "--python",  # pip requires --python before the subcommand
+                str(self.interpreter_path),
+                "install",
+            ]
+        ) + shlex.split(self.pip_url)
+        print(
+            f"Installing '{self.name}' into virtual environment '{self._get_venv_path()!s}' with "
+            f"command '{' '.join(install_cmd)}'...\n",
             file=sys.stderr,
         )
         try:
-            self._run_subprocess_and_raise_on_failure(
-                args=[pip_path, "install", *shlex.split(self.pip_url)]
-            )
+            self._run_subprocess_and_raise_on_failure(install_cmd)
         except exc.AirbyteSubprocessFailedError as ex:
             # If the installation failed, remove the virtual environment
             # Otherwise, the connector will be considered as installed and the user may not be able

airbyte/_executors/util.py

@@ -166,6 +166,7 @@ def get_connector_executor(  # noqa: PLR0912, PLR0913, PLR0914, PLR0915, C901 #
     source_manifest: bool | dict | Path | str | None = None,
     install_if_missing: bool = True,
     install_root: Path | None = None,
+    use_python: bool | Path | str | None = None,
 ) -> Executor:
     """This factory function creates an executor for a connector.
 
@@ -175,11 +176,18 @@ def get_connector_executor(  # noqa: PLR0912, PLR0913, PLR0914, PLR0915, C901 #
         [
             bool(local_executable),
             bool(docker_image),
-            bool(pip_url),
+            bool(pip_url) or bool(use_python),
             bool(source_manifest),
         ]
     )
 
+    if use_python is False:
+        docker_image = True
+
+    if use_python is None and pip_url is not None:
+        # If pip_url is set, we assume the user wants to use Python.
+        use_python = True
+
     if version and pip_url:
         raise exc.PyAirbyteInputError(
             message=(
@@ -341,6 +349,7 @@ def get_connector_executor(  # noqa: PLR0912, PLR0913, PLR0914, PLR0915, C901 #
             target_version=version,
             pip_url=pip_url,
             install_root=install_root,
+            use_python=use_python,
         )
         if install_if_missing:
             executor.ensure_installation()