From 659e726ff1594c06786233e0891ebecb1edd2744 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:25:09 +0200 Subject: [PATCH 1/5] Remove pip, use uv for creating espidf venv --- builder/frameworks/espidf.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index dc132c06b..2097040db 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -44,7 +44,6 @@ from platformio.compat import IS_WINDOWS from platformio.proc import exec_command from platformio.builder.tools.piolib import ProjectAsLibBuilder -from platformio.project.config import ProjectConfig from platformio.package.version import get_original_version, pepver_to_semver @@ -83,7 +82,7 @@ if mcu in ("esp32", "esp32s2", "esp32s3") else "toolchain-riscv32-esp" ) - +PLATFORMIO_DIR = env.subst("$PROJECT_CORE_DIR") assert os.path.isdir(FRAMEWORK_DIR) assert os.path.isdir(TOOLCHAIN_DIR) @@ -351,7 +350,7 @@ def HandleCOMPONENTsettings(env): if flag_custom_sdkonfig == True and "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK"): HandleArduinoIDFsettings(env) - LIB_SOURCE = os.path.join(ProjectConfig.get_instance().get("platformio", "platforms_dir"), "espressif32", "builder", "build_lib") + LIB_SOURCE = os.path.join(PLATFORMIO_DIR, "platforms", "espressif32", "builder", "build_lib") if not bool(os.path.exists(os.path.join(PROJECT_DIR, ".dummy"))): shutil.copytree(LIB_SOURCE, os.path.join(PROJECT_DIR, ".dummy")) PROJECT_SRC_DIR = os.path.join(PROJECT_DIR, ".dummy") @@ -1607,11 +1606,13 @@ def _is_venv_outdated(venv_data_file): return True def _create_venv(venv_dir): - pip_path = os.path.join( - venv_dir, - "Scripts" if IS_WINDOWS else "bin", - "pip" + (".exe" if IS_WINDOWS else ""), - ) + penv_setup_path = os.path.join(PLATFORMIO_DIR, "platforms", "espressif32", "builder") + sys.path.insert(0, penv_setup_path) + + from penv_setup import get_executable_path + + penv_dir = os.path.join(PLATFORMIO_DIR, "penv") + uv_path = get_executable_path(penv_dir, "uv") if os.path.isdir(venv_dir): try: @@ -1624,17 +1625,20 @@ def _create_venv(venv_dir): ) env.Exit(1) - # Use the built-in PlatformIO Python to create a standalone IDF virtual env + # Use uv to create a standalone IDF virtual env env.Execute( env.VerboseAction( - '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir, - "Creating a new virtual environment for IDF Python dependencies", + '"%s" venv --clear --quiet "%s"' % (uv_path, venv_dir), + "Creating a new virtual environment for IDF Python dependencies using uv", ) ) + # Verify that the venv was created successfully by checking for Python executable + python_path = get_executable_path(venv_dir, "python") + assert os.path.isfile( - pip_path - ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!" + python_path + ), "Error: Failed to create a proper virtual environment. Missing the Python executable!" venv_dir = get_idf_venv_dir() venv_data_file = os.path.join(venv_dir, "pio-idf-venv.json") From caed5978586bdf0441f386dd1599404943f8c1bc Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:47:40 +0200 Subject: [PATCH 2/5] Update library source path for platform compatibility --- builder/frameworks/espidf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 2097040db..74739c20d 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -350,7 +350,7 @@ def HandleCOMPONENTsettings(env): if flag_custom_sdkonfig == True and "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK"): HandleArduinoIDFsettings(env) - LIB_SOURCE = os.path.join(PLATFORMIO_DIR, "platforms", "espressif32", "builder", "build_lib") + LIB_SOURCE = os.path.join(platform.get_dir(), "builder", "build_lib") if not bool(os.path.exists(os.path.join(PROJECT_DIR, ".dummy"))): shutil.copytree(LIB_SOURCE, os.path.join(PROJECT_DIR, ".dummy")) PROJECT_SRC_DIR = os.path.join(PROJECT_DIR, ".dummy") @@ -1606,7 +1606,7 @@ def _is_venv_outdated(venv_data_file): return True def _create_venv(venv_dir): - penv_setup_path = os.path.join(PLATFORMIO_DIR, "platforms", "espressif32", "builder") + penv_setup_path = os.path.join(platform.get_dir(), "builder") sys.path.insert(0, penv_setup_path) from penv_setup import get_executable_path From 28843fe13acb258b5756fb314f1d4ee68ca0ce0d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:54:07 +0200 Subject: [PATCH 3/5] ensure penv uv is used for creating idf venv --- builder/frameworks/espidf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 74739c20d..4e68f29a8 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1628,7 +1628,7 @@ def _create_venv(venv_dir): # Use uv to create a standalone IDF virtual env env.Execute( env.VerboseAction( - '"%s" venv --clear --quiet "%s"' % (uv_path, venv_dir), + '"%s" venv --clear --quiet --python "%s" "%s"' % (uv_path, env.subst("$PYTHONEXE"), venv_dir), "Creating a new virtual environment for IDF Python dependencies using uv", ) ) From 04044e2025a1b9901d8f81d13ca850e59094e0b2 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 25 Aug 2025 17:19:34 +0200 Subject: [PATCH 4/5] Refactor install_python_deps for virtual environment Refactor Python dependency installation to use a virtual environment and update UV executable path retrieval. --- builder/frameworks/espidf.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 4e68f29a8..01954e9ea 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1488,8 +1488,12 @@ def generate_mbedtls_bundle(sdk_config): def install_python_deps(): - PYTHON_EXE = env.subst("$PYTHONEXE") - UV_EXE = os.path.join(os.path.dirname(PYTHON_EXE), "uv" + (".exe" if IS_WINDOWS else "")) + penv_setup_path = os.path.join(platform.get_dir(), "builder") + sys.path.insert(0, penv_setup_path) + from penv_setup import get_executable_path + + penv_dir = os.path.join(PLATFORMIO_DIR, "penv") + UV_EXE = get_executable_path(penv_dir, "uv") def _get_installed_uv_packages(python_exe_path): result = {} try: @@ -1511,7 +1515,6 @@ def _get_installed_uv_packages(python_exe_path): return deps = { - "uv": ">=0.1.0", # https://github.com/platformio/platformio-core/issues/4614 "urllib3": "<2", # https://github.com/platformio/platform-espressif32/issues/635 @@ -1562,9 +1565,7 @@ def get_idf_venv_dir(): # as an IDF component requires a different version of the IDF package and # hence a different set of Python deps or their versions idf_version = get_framework_version() - return os.path.join( - env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version - ) + return os.path.join(PLATFORMIO_DIR, "penv", f".espidf-{idf_version}") def ensure_python_venv_available(): From caa2a7017b03d8d3bc8e23d2944bbf5cc5126368 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:44:15 +0200 Subject: [PATCH 5/5] Use importlib.util --- builder/frameworks/espidf.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 01954e9ea..9902237c1 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -21,6 +21,7 @@ """ import copy +import importlib.util import json import subprocess import sys @@ -49,6 +50,13 @@ env = DefaultEnvironment() env.SConscript("_embed_files.py", exports="env") +platform = env.PioPlatform() + +_penv_setup_file = os.path.join(platform.get_dir(), "builder", "penv_setup.py") +_spec = importlib.util.spec_from_file_location("penv_setup", _penv_setup_file) +_penv_setup = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_penv_setup) # type: ignore[attr-defined] +get_executable_path = _penv_setup.get_executable_path # remove maybe existing old map file in project root map_file = os.path.join(env.subst("$PROJECT_DIR"), env.subst("$PROGNAME") + ".map") @@ -58,7 +66,6 @@ # Allow changes in folders of managed components os.environ["IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS"] = "1" -platform = env.PioPlatform() config = env.GetProjectConfig() board = env.BoardConfig() mcu = board.get("build.mcu", "esp32") @@ -87,6 +94,7 @@ assert os.path.isdir(FRAMEWORK_DIR) assert os.path.isdir(TOOLCHAIN_DIR) + def create_silent_action(action_func): """Create a silent SCons action that suppresses output""" silent_action = env.Action(action_func) @@ -1487,13 +1495,13 @@ def generate_mbedtls_bundle(sdk_config): ) +def _get_uv_exe(): + return get_executable_path(os.path.join(PLATFORMIO_DIR, "penv"), "uv") + + def install_python_deps(): - penv_setup_path = os.path.join(platform.get_dir(), "builder") - sys.path.insert(0, penv_setup_path) - from penv_setup import get_executable_path + UV_EXE = _get_uv_exe() - penv_dir = os.path.join(PLATFORMIO_DIR, "penv") - UV_EXE = get_executable_path(penv_dir, "uv") def _get_installed_uv_packages(python_exe_path): result = {} try: @@ -1607,13 +1615,7 @@ def _is_venv_outdated(venv_data_file): return True def _create_venv(venv_dir): - penv_setup_path = os.path.join(platform.get_dir(), "builder") - sys.path.insert(0, penv_setup_path) - - from penv_setup import get_executable_path - - penv_dir = os.path.join(PLATFORMIO_DIR, "penv") - uv_path = get_executable_path(penv_dir, "uv") + uv_path = _get_uv_exe() if os.path.isdir(venv_dir): try: