diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bb86e38a3..da0c1b8ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,10 @@ jobs: test: strategy: fail-fast: false + matrix: + dep-type: + - latest + - frozen runs-on: ${{ inputs.os }} steps: - name: Checkout @@ -27,9 +31,16 @@ jobs: python-version: ${{ inputs.pyVersion }} - name: Run tests - run: make dev install test + shell: bash + run: | + if [ "${{ matrix.dep-type }}" = "latest" ]; then + make dev-latest test + else + make dev test + fi - name: Publish test coverage + if: ${{ matrix.dep-type == 'frozen' }} uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index ac1aa7945..0e22f8ec7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.venv +.venv* *. build dist @@ -10,4 +10,5 @@ __pycache__ databricks-sdk-py.iml .databricks .coverage -htmlcov \ No newline at end of file +htmlcov +.python-version diff --git a/MANIFEST.in b/MANIFEST.in index a6f944eed..591e2fa89 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,6 +16,7 @@ exclude NEXT_CHANGELOG.md exclude tagging.py exclude .package.json exclude .release_metadata.json +exclude requirements-dev-*.txt recursive-exclude docs * recursive-exclude examples * diff --git a/Makefile b/Makefile index c147f4074..20141f661 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,52 @@ -dev: - python3 -m venv .venv +# Detect Python version from the system +PYTHON_VERSION := $(shell python3 -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")') + +# Generate requirements filename based on Python version +REQUIREMENTS_FILE = requirements-dev-py$(PYTHON_VERSION).txt + +# Testing workflow: +# 1. GitHub Actions runs tests for each Python version (3.8-3.12) on multiple OSes +# 2. For each Python version, tests run in two modes: +# - 'latest': Uses dependencies directly from pyproject.toml +# - 'frozen': Uses version-specific requirements-dev-pyXY.txt +# 3. Each Python version gets its own virtual environment (.venvXY) +# 4. Tests are only blocking in `frozen` mode. Tests run in `latest` mode are +# allowed to fail and serve as a warning that there may have been a breaking +# change in a dependency. +# 5. To run tests locally: +# - make dev test # uses frozen dependencies +# - make dev-latest test # uses latest dependencies +# +# To update dependencies for all Python versions using pyenv: +# rm -rf .venv* requirements-dev-py*.txt +# for v in 3.8 3.9 3.10 3.11 3.12 3.13; do +# pyenv local $v +# make dev-env update-dev-dep-lockfile +# done + +dev-env: + python -m venv .venv$(PYTHON_VERSION) ifeq ($(OS), Windows_NT) - .venv\Scripts\activate + .venv$(PYTHON_VERSION)\Scripts\activate else - . .venv/bin/activate + . .venv$(PYTHON_VERSION)/bin/activate endif + +dev: dev-env + # Install all dependencies from the version-specific requirements file + # Regenerate this file with `make update-dev-dep-lockfile PYTHON_VERSION=X.Y` + pip install -r $(REQUIREMENTS_FILE) + +dev-latest: dev-env + # Install all dependencies from the pyproject.toml file pip install '.[dev]' +install-pip-tools: + pip install pip-tools + +update-dev-dep-lockfile: install-pip-tools + pip-compile pyproject.toml --extra dev --output-file $(REQUIREMENTS_FILE) + install: pip install . diff --git a/pyproject.toml b/pyproject.toml index 60c33f0e6..f4fffe91d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,6 @@ dev = [ "ipywidgets", "requests-mock", "pyfakefs", - "databricks-connect", "pytest-rerunfailures", "openai", 'langchain-openai; python_version > "3.7"', diff --git a/requirements-dev-py310.txt b/requirements-dev-py310.txt new file mode 100644 index 000000000..bee9998ac --- /dev/null +++ b/requirements-dev-py310.txt @@ -0,0 +1,231 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py310.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +black==25.1.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.9.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +exceptiongroup==1.3.0 + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +ipython==8.37.0 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==6.0.1 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.10.0 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.3.65 + # via langchain-openai +langchain-openai==0.3.22 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.3.45 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.18 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # langsmith + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +platformdirs==4.3.8 + # via black +pluggy==1.6.0 + # via + # pytest + # pytest-cov +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.13.0 + # via databricks-sdk (pyproject.toml) +pydantic==2.11.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.33.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.3.2 + # via autoflake +pygments==2.19.1 + # via + # ipython + # pytest +pyproject-hooks==1.2.0 + # via build +pytest==8.4.0 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==6.2.1 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==15.1 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.7.0 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==9.1.2 + # via langchain-core +tiktoken==0.9.0 + # via langchain-openai +tomli==2.2.1 + # via + # autoflake + # black + # build + # coverage + # pytest +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.14.0 + # via + # anyio + # black + # exceptiongroup + # ipython + # langchain-core + # openai + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +urllib3==2.4.0 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zstandard==0.23.0 + # via langsmith diff --git a/requirements-dev-py311.txt b/requirements-dev-py311.txt new file mode 100644 index 000000000..74f5133bc --- /dev/null +++ b/requirements-dev-py311.txt @@ -0,0 +1,220 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py311.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +black==25.1.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.9.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +ipython==9.3.0 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipython-pygments-lexers==1.1.1 + # via ipython +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==6.0.1 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.10.0 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.3.65 + # via langchain-openai +langchain-openai==0.3.22 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.3.45 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.18 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # langsmith + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +platformdirs==4.3.8 + # via black +pluggy==1.6.0 + # via + # pytest + # pytest-cov +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.13.0 + # via databricks-sdk (pyproject.toml) +pydantic==2.11.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.33.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.3.2 + # via autoflake +pygments==2.19.1 + # via + # ipython + # ipython-pygments-lexers + # pytest +pyproject-hooks==1.2.0 + # via build +pytest==8.4.0 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==6.2.1 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==15.1 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.7.0 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==9.1.2 + # via langchain-core +tiktoken==0.9.0 + # via langchain-openai +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.14.0 + # via + # anyio + # ipython + # langchain-core + # openai + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +urllib3==2.4.0 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zstandard==0.23.0 + # via langsmith diff --git a/requirements-dev-py312.txt b/requirements-dev-py312.txt new file mode 100644 index 000000000..90eac2fa9 --- /dev/null +++ b/requirements-dev-py312.txt @@ -0,0 +1,219 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py312.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +black==25.1.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.9.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +ipython==9.3.0 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipython-pygments-lexers==1.1.1 + # via ipython +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==6.0.1 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.10.0 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.3.65 + # via langchain-openai +langchain-openai==0.3.22 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.3.45 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.18 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # langsmith + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +platformdirs==4.3.8 + # via black +pluggy==1.6.0 + # via + # pytest + # pytest-cov +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.13.0 + # via databricks-sdk (pyproject.toml) +pydantic==2.11.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.33.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.3.2 + # via autoflake +pygments==2.19.1 + # via + # ipython + # ipython-pygments-lexers + # pytest +pyproject-hooks==1.2.0 + # via build +pytest==8.4.0 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==6.2.1 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==15.1 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.7.0 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==9.1.2 + # via langchain-core +tiktoken==0.9.0 + # via langchain-openai +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.14.0 + # via + # anyio + # langchain-core + # openai + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +urllib3==2.4.0 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zstandard==0.23.0 + # via langsmith diff --git a/requirements-dev-py313.txt b/requirements-dev-py313.txt new file mode 100644 index 000000000..999a533d7 --- /dev/null +++ b/requirements-dev-py313.txt @@ -0,0 +1,218 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py313.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +black==25.1.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.9.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +ipython==9.3.0 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipython-pygments-lexers==1.1.1 + # via ipython +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==6.0.1 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.10.0 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.3.65 + # via langchain-openai +langchain-openai==0.3.22 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.3.45 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.18 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # langsmith + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +platformdirs==4.3.8 + # via black +pluggy==1.6.0 + # via + # pytest + # pytest-cov +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.13.0 + # via databricks-sdk (pyproject.toml) +pydantic==2.11.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.33.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.3.2 + # via autoflake +pygments==2.19.1 + # via + # ipython + # ipython-pygments-lexers + # pytest +pyproject-hooks==1.2.0 + # via build +pytest==8.4.0 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==6.2.1 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==15.1 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.7.0 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==9.1.2 + # via langchain-core +tiktoken==0.9.0 + # via langchain-openai +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.14.0 + # via + # langchain-core + # openai + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +urllib3==2.4.0 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zstandard==0.23.0 + # via langsmith diff --git a/requirements-dev-py38.txt b/requirements-dev-py38.txt new file mode 100644 index 000000000..a629ecaad --- /dev/null +++ b/requirements-dev-py38.txt @@ -0,0 +1,229 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py38.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.5.2 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +backcall==0.2.0 + # via ipython +black==24.8.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.1.8 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.6.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +exceptiongroup==1.3.0 + # via + # anyio + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +importlib-metadata==8.5.0 + # via build +iniconfig==2.1.0 + # via pytest +ipython==8.12.3 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==5.13.2 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.9.1 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.2.43 + # via langchain-openai +langchain-openai==0.1.25 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.1.147 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.15 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +pickleshare==0.7.5 + # via ipython +platformdirs==4.3.6 + # via black +pluggy==1.5.0 + # via pytest +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.12.1 + # via databricks-sdk (pyproject.toml) +pydantic==2.10.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.27.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.2.0 + # via autoflake +pygments==2.19.1 + # via ipython +pyproject-hooks==1.2.0 + # via build +pytest==8.3.5 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==5.0.0 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==14.0 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.6.1 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==8.5.0 + # via langchain-core +tiktoken==0.7.0 + # via langchain-openai +tomli==2.2.1 + # via + # autoflake + # black + # build + # coverage + # pytest +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.13.2 + # via + # annotated-types + # anyio + # black + # exceptiongroup + # ipython + # langchain-core + # openai + # pydantic + # pydantic-core +urllib3==2.2.3 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zipp==3.20.2 + # via importlib-metadata diff --git a/requirements-dev-py39.txt b/requirements-dev-py39.txt new file mode 100644 index 000000000..49368a7d1 --- /dev/null +++ b/requirements-dev-py39.txt @@ -0,0 +1,235 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev-py39.txt pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # httpx + # openai +asttokens==3.0.0 + # via stack-data +autoflake==2.3.1 + # via databricks-sdk (pyproject.toml) +black==25.1.0 + # via databricks-sdk (pyproject.toml) +build==1.2.2.post1 + # via databricks-sdk (pyproject.toml) +cachetools==5.5.2 + # via google-auth +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.1.8 + # via black +comm==0.2.2 + # via ipywidgets +coverage[toml]==7.9.1 + # via pytest-cov +decorator==5.2.1 + # via ipython +distro==1.9.0 + # via openai +exceptiongroup==1.3.0 + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.2.0 + # via stack-data +google-auth==2.40.3 + # via databricks-sdk (pyproject.toml) +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # openai +idna==3.10 + # via + # anyio + # httpx + # requests +importlib-metadata==8.7.0 + # via build +iniconfig==2.1.0 + # via pytest +ipython==8.18.1 + # via + # databricks-sdk (pyproject.toml) + # ipywidgets +ipywidgets==8.1.7 + # via databricks-sdk (pyproject.toml) +isort==6.0.1 + # via databricks-sdk (pyproject.toml) +jedi==0.19.2 + # via ipython +jiter==0.10.0 + # via openai +jsonpatch==1.33 + # via langchain-core +jsonpointer==3.0.0 + # via jsonpatch +jupyterlab-widgets==3.0.15 + # via ipywidgets +langchain-core==0.3.65 + # via langchain-openai +langchain-openai==0.3.22 ; python_version > "3.7" + # via databricks-sdk (pyproject.toml) +langsmith==0.3.45 + # via langchain-core +matplotlib-inline==0.1.7 + # via ipython +mypy-extensions==1.1.0 + # via black +openai==1.86.0 + # via + # databricks-sdk (pyproject.toml) + # langchain-openai +orjson==3.10.18 + # via langsmith +packaging==24.2 + # via + # black + # build + # langchain-core + # langsmith + # pytest + # pytest-rerunfailures +parso==0.8.4 + # via jedi +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +platformdirs==4.3.8 + # via black +pluggy==1.6.0 + # via + # pytest + # pytest-cov +prompt-toolkit==3.0.51 + # via ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycodestyle==2.13.0 + # via databricks-sdk (pyproject.toml) +pydantic==2.11.6 + # via + # langchain-core + # langsmith + # openai +pydantic-core==2.33.2 + # via pydantic +pyfakefs==5.8.0 + # via databricks-sdk (pyproject.toml) +pyflakes==3.3.2 + # via autoflake +pygments==2.19.1 + # via + # ipython + # pytest +pyproject-hooks==1.2.0 + # via build +pytest==8.4.0 + # via + # databricks-sdk (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-rerunfailures + # pytest-xdist +pytest-cov==6.2.1 + # via databricks-sdk (pyproject.toml) +pytest-mock==3.14.1 + # via databricks-sdk (pyproject.toml) +pytest-rerunfailures==15.1 + # via databricks-sdk (pyproject.toml) +pytest-xdist==3.7.0 + # via databricks-sdk (pyproject.toml) +pyyaml==6.0.2 + # via langchain-core +regex==2024.11.6 + # via tiktoken +requests==2.32.4 + # via + # databricks-sdk (pyproject.toml) + # langsmith + # requests-mock + # requests-toolbelt + # tiktoken +requests-mock==1.12.1 + # via databricks-sdk (pyproject.toml) +requests-toolbelt==1.0.0 + # via langsmith +rsa==4.9.1 + # via google-auth +sniffio==1.3.1 + # via + # anyio + # openai +stack-data==0.6.3 + # via ipython +tenacity==9.1.2 + # via langchain-core +tiktoken==0.9.0 + # via langchain-openai +tomli==2.2.1 + # via + # autoflake + # black + # build + # coverage + # pytest +tqdm==4.67.1 + # via openai +traitlets==5.14.3 + # via + # comm + # ipython + # ipywidgets + # matplotlib-inline +typing-extensions==4.14.0 + # via + # anyio + # black + # exceptiongroup + # ipython + # langchain-core + # openai + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +urllib3==2.4.0 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via databricks-sdk (pyproject.toml) +widgetsnbextension==4.0.14 + # via ipywidgets +zipp==3.23.0 + # via importlib-metadata +zstandard==0.23.0 + # via langsmith diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 55114bd84..0c936f2b5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -38,13 +38,11 @@ def pytest_configure(config): def pytest_collection_modifyitems(items): - # safer to refer to fixture fns instead of strings - client_fixtures = [x.__name__ for x in [a, w, ucws, ucacct]] + # All tests defined in tests/integration are considered integration tests. for item in items: - current_fixtures = getattr(item, "fixturenames", ()) - for requires_client in client_fixtures: - if requires_client in current_fixtures: - item.add_marker("integration") + directory = item.path.parent + if directory.name == "integration" and directory.parent.name == "tests": + item.add_marker("integration") @pytest.fixture(scope="session") diff --git a/tests/integration/test_dbconnect.py b/tests/integration/test_dbconnect.py deleted file mode 100644 index ecc82855b..000000000 --- a/tests/integration/test_dbconnect.py +++ /dev/null @@ -1,73 +0,0 @@ -import pytest - -DBCONNECT_DBR_CLIENT = { - "13.3": "13.3.3", - "14.3": "14.3.1", -} - - -def reload_modules(name: str): - """ - Reloads the specified module. This is useful when testing Databricks Connect, since both - the `databricks.connect` and `databricks.sdk.runtime` modules are stateful, and we need - to reload these modules to reset the state cache between test runs. - """ - - import importlib - import sys - - v = sys.modules.get(name) - if v is None: - return - try: - print(f"Reloading {name}") - importlib.reload(v) - except Exception as e: - print(f"Failed to reload {name}: {e}") - - -@pytest.fixture(params=list(DBCONNECT_DBR_CLIENT.keys())) -def setup_dbconnect_test(request, env_or_skip, restorable_env): - dbr = request.param - assert ( - dbr in DBCONNECT_DBR_CLIENT - ), f"Unsupported Databricks Runtime version {dbr}. Please update DBCONNECT_DBR_CLIENT." - - import os - - os.environ["DATABRICKS_CLUSTER_ID"] = env_or_skip(f"TEST_DBR_{dbr.replace('.', '_')}_DBCONNECT_CLUSTER_ID") - - import subprocess - import sys - - lib = f"databricks-connect=={DBCONNECT_DBR_CLIENT[dbr]}" - subprocess.check_call([sys.executable, "-m", "pip", "install", lib]) - - yield - - subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", "databricks-connect"]) - - -@pytest.mark.xdist_group(name="databricks-connect") -def test_dbconnect_initialisation(w, setup_dbconnect_test): - reload_modules("databricks.connect") - from databricks.connect import DatabricksSession - - spark = DatabricksSession.builder.getOrCreate() - assert spark.sql("SELECT 1").collect()[0][0] == 1 - - -@pytest.mark.xdist_group(name="databricks-connect") -def test_dbconnect_runtime_import(w, setup_dbconnect_test): - reload_modules("databricks.sdk.runtime") - from databricks.sdk.runtime import spark - - assert spark.sql("SELECT 1").collect()[0][0] == 1 - - -@pytest.mark.xdist_group(name="databricks-connect") -def test_dbconnect_runtime_import_no_error_if_doesnt_exist(w): - reload_modules("databricks.sdk.runtime") - from databricks.sdk.runtime import spark - - assert spark is None diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index feafac00a..fbc4d152c 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -212,3 +212,9 @@ def test_secrets(w, random): assert all_secrets[key_for_string] == random_value assert all_secrets[key_for_bytes] == random_value + + +def test_dbutils_spark_import_no_error_if_doesnt_exist(): + from databricks.sdk.runtime import spark + + assert spark is None