diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41862573..986099db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,13 +1,12 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package +name: Build on: push: - branches: [ master ] + branches: + - master pull_request: - branches: [ master ] + branches: + - master jobs: build: @@ -15,13 +14,31 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy3] + include: + - python-version: 3.7 + env: + TOXENV: docs + - python-version: 3.8 + env: + TOXENV: flake8 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.8 + env: + TOXENV: security steps: - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Greet - run: python -c "print('hello there')" + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install --upgrade pip + pip install --upgrade tox + tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..9390e788 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +name: Publish + +on: + push: + tags: + - v* + +jobs: + publish: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade pip + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..c47af5f8 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + tests: + + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy3] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install --upgrade pip + pip install --upgrade tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.gitignore b/.gitignore index 3fe67fd1..a279e1af 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ dist docs/_build _trial_temp .coverage -.cache \ No newline at end of file +coverage.xml +.cache diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 41c72504..00000000 --- a/.travis.yml +++ /dev/null @@ -1,60 +0,0 @@ -language: python -sudo: false -matrix: - include: - - python: 3.8 - env: TOXENV=security - - python: 3.8 - env: TOXENV=flake8 - - python: 3.8 - env: TOXENV=pylint - - python: 2.7 - env: TOXENV=py27 - - python: pypy - env: PYPY_VERSION=2.7-v7.0.0 TOXENV=pypy - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8 - env: TOXENV=py38 - - python: pypy3 - env: TOXENV=pypy3 - - python: 3.7 - env: TOXENV=docs - -install: - # https://bitbucket.org/pypy/pypy/issues/2389/different-behavior-of-bytesdecode-utf8 - # The bug above causes Python 2 PyPy 7.1.0+ to fail one test, so below we - # install 7.0.0. This can be removed once the Python 2 PyPy version in Travis - # includes a fix. - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov -script: tox -after_success: - - codecov - -deploy: - provider: pypi - distributions: sdist bdist_wheel - user: scrapy - password: - secure: gIMpTo+g15n4fmR2Aw4X6GqCpkhd2NyLK0E1OzJVQqDMFD2e296I2MEWJQ5etKFYP+xABllMao9sO+pBKZLXvVilhHa1PCO5v9Mo3i9TDvszW+o9iW4ArrMRkfZR2+NV47vff1KUkBHFuqRkzOks/NS9wMG6A3MXAa4EFkH9UhI= - on: - tags: true - all_branches: true - repo: scrapy/w3lib - condition: "$TOXENV == py37" - -cache: - directories: - - $HOME/.cache/pip diff --git a/tests/test_url.py b/tests/test_url.py index 8b07c005..07695500 100644 --- a/tests/test_url.py +++ b/tests/test_url.py @@ -2,11 +2,14 @@ from __future__ import absolute_import import os import unittest + +import pytest +from six.moves.urllib.parse import urlparse + from w3lib.url import (is_url, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, file_uri_to_path, parse_data_uri, path_to_file_uri, any_to_uri, urljoin_rfc, canonicalize_url, parse_url, add_or_replace_parameters) -from six.moves.urllib.parse import urlparse class UrlTests(unittest.TestCase): @@ -76,17 +79,16 @@ def test_safe_url_string_remove_ascii_tab_and_newlines(self): def test_safe_url_string_unsafe_chars(self): safeurl = safe_url_string(r"http://localhost:8001/unwise{,},|,\,^,[,],`?|=[]&[]=|") self.assertEqual(safeurl, r"http://localhost:8001/unwise%7B,%7D,|,%5C,%5E,[,],%60?|=[]&[]=|") - + def test_safe_url_string_quote_path(self): safeurl = safe_url_string(u'http://google.com/"hello"', quote_path=True) self.assertEqual(safeurl, u'http://google.com/%22hello%22') - + safeurl = safe_url_string(u'http://google.com/"hello"', quote_path=False) self.assertEqual(safeurl, u'http://google.com/"hello"') - + safeurl = safe_url_string(u'http://google.com/"hello"') self.assertEqual(safeurl, u'http://google.com/%22hello%22') - def test_safe_url_string_with_query(self): safeurl = safe_url_string(u"http://www.example.com/£?unit=µ") @@ -310,10 +312,6 @@ def test_add_or_replace_parameter(self): self.assertEqual(add_or_replace_parameter(url, 'arg3', 'nv3'), 'http://domain/test?arg1=v1&arg2=v2&arg3=nv3') - url = 'http://domain/test?arg1=v1;arg2=v2' - self.assertEqual(add_or_replace_parameter(url, 'arg1', 'v3'), - 'http://domain/test?arg1=v3&arg2=v2') - self.assertEqual(add_or_replace_parameter("http://domain/moreInfo.asp?prodID=", 'prodID', '20'), 'http://domain/moreInfo.asp?prodID=20') url = 'http://rmc-offers.co.uk/productlist.asp?BCat=2%2C60&CatID=60' @@ -338,6 +336,13 @@ def test_add_or_replace_parameter(self): self.assertEqual(add_or_replace_parameter(url, 'arg1', 'v3'), 'http://domain/test?arg1=v3&arg2=v2') + @pytest.mark.xfail(reason="https://github.com/scrapy/w3lib/issues/164") + def test_add_or_replace_parameter_fail(self): + self.assertEqual( + add_or_replace_parameter('http://domain/test?arg1=v1;arg2=v2', 'arg1', 'v3'), + 'http://domain/test?arg1=v3&arg2=v2' + ) + def test_add_or_replace_parameters(self): url = 'http://domain/test' self.assertEqual(add_or_replace_parameters(url, {'arg': 'v'}), @@ -767,4 +772,3 @@ def test_scheme_case_insensitive(self): if __name__ == "__main__": unittest.main() - diff --git a/tox.ini b/tox.ini index 61d2fec9..24398ac8 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = commands = py.test \ --doctest-modules \ - --cov=w3lib --cov-report=term \ + --cov=w3lib --cov-report=term --cov-report=xml \ {posargs:w3lib tests} [testenv:security]