flake8-scrapy → scrapy-lint (#63)

Author: Gallaecio

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
         - python-version: "3.11"
         - python-version: "3.12"
         - python-version: "3.13"
+        - python-version: "3.14.0-rc.1"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -39,7 +40,8 @@ jobs:
         include:
         - python-version: "3.9"
           toxenv: "mypy"
-        - toxenv: "twinecheck"
+        - toxenv: "pylint"
+        - toxenv: "twine"
         - toxenv: "docs"
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -2,16 +2,21 @@ repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.12.5
   hooks:
-    - id: ruff-check
-      args: [ --fix ]
-    - id: ruff-format
+  - id: ruff-check
+    args: [ --fix ]
+  - id: ruff-format
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0
   hooks:
+  - id: end-of-file-fixer
   - id: trailing-whitespace
 - repo: https://github.com/adamchainz/blacken-docs
   rev: 1.19.1
   hooks:
   - id: blacken-docs
     additional_dependencies:
     - black==25.1.0
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v1.0.0
+  hooks:
+  - id: sphinx-lint

.pre-commit-hooks.yaml

@@ -0,0 +1,4 @@
+- id: scrapy-lint
+  name: scrapy-lint
+  entry: scrapy-lint
+  language: python

CHANGES.rst

@@ -5,6 +5,9 @@ Release notes
 0.1.0 (unreleased)
 ==================
 
+-   Switched from being a `flake8 <https://flake8.pycqa.org/en/latest/>`_
+    plugin to being a standalone tool, to support rules on non-Python files.
+
 -   Dropped Python 3.8 support, added Python 3.9+ support.
 
 -   Requires Scrapy 2.0.1+.
@@ -50,11 +53,10 @@ Release notes
     | :ref:`scp41`
     | :ref:`scp42`
     | :ref:`scp43`
-    | :ref:`scp44`
     | :ref:`scp45`
     | :ref:`scp46`
 
--   Added `documentation <https://flake8-scrapy.readthedocs.io/en/latest/>`_.
+-   Added `documentation <https://scrapy-lint.readthedocs.io/en/latest/>`_.
 
 -   Improved CI and achieved full test coverage.
 
@@ -63,4 +65,4 @@ Earlier releases
 ================
 
 Find the earlier commit history `at GitHub
-<https://github.com/scrapy/flake8-scrapy/commits/4be77a75b5a88f58f20b49067afda8e7e7a9bf6d>`_.
+<https://github.com/scrapy/scrapy-lint/commits/4be77a75b5a88f58f20b49067afda8e7e7a9bf6d>`_.

README.rst

@@ -1,37 +1,53 @@
-=============
-flake8-scrapy
-=============
+===========
+scrapy-lint
+===========
 
 |version| |python_version| |ci|
 
-.. |version| image:: https://img.shields.io/pypi/v/flake8-scrapy.svg
-   :target: https://pypi.org/pypi/flake8-scrapy
+.. |version| image:: https://img.shields.io/pypi/v/scrapy-lint.svg
+   :target: https://pypi.org/pypi/scrapy-lint
    :alt: PyPI version
 
-.. |python_version| image:: https://img.shields.io/pypi/pyversions/flake8-scrapy.svg
-   :target: https://pypi.org/pypi/flake8-scrapy
+.. |python_version| image:: https://img.shields.io/pypi/pyversions/scrapy-lint.svg
+   :target: https://pypi.org/pypi/scrapy-lint
    :alt: Supported Python versions
 
-.. |ci| image:: https://github.com/scrapy/flake8-scrapy/workflows/CI/badge.svg
-   :target: https://github.com/scrapy/flake8-scrapy/actions?query=workflow%3ACI
+.. |ci| image:: https://github.com/scrapy/scrapy-lint/workflows/CI/badge.svg
+   :target: https://github.com/scrapy/scrapy-lint/actions?query=workflow%3ACI
    :alt: CI
 
-.. intro-start
+.. readme-start
+
+**scrapy-lint** is a linter for `Scrapy <https://scrapy.org/>`_ projects.
+
+To install::
 
-A Flake8_ plugin to catch common issues in Scrapy projects.
+    pip install scrapy-lint
 
-.. _Flake8: https://flake8.pycqa.org/en/latest/
+To run::
 
-Install::
+    scrapy-lint
 
-    pip install flake8-scrapy
+To use with `pre-commit <https://pre-commit.com/>`__, add the following to your
+``.pre-commit-config.yaml``:
 
-And run Flake8_::
+.. code-block:: yaml
 
-    flake8
+    - repo: https://github.com/scrapy/scrapy-lint
+      rev: "0.0.2"
+      hooks:
+      - id: scrapy-lint
 
-.. intro-end
+Can be combined with `ruff <https://docs.astral.sh/ruff/>`_,
+`mypy <https://mypy.readthedocs.io/en/stable/>`_,
+`pylint <https://pylint.readthedocs.io/en/stable/>`_ and
+`flake8-requirements <https://pypi.org/project/flake8-requirements/>`_.
+
+.. readme-end
+
+Documentation
+=============
 
 See the documentation_ for more.
 
-.. _documentation: https://flake8-scrapy.readthedocs.io/en/latest/
+.. _documentation: https://scrapy-lint.readthedocs.io/en/latest/

docs/changes.rst

@@ -1 +1 @@
-.. include:: ../CHANGES.rst
+.. include:: ../CHANGES.rst

docs/conf.py

@@ -1,4 +1,4 @@
-project = "flake8-scrapy"
+project = "scrapy-lint"
 project_copyright = "Valdir Stumm Junior"
 author = "Valdir Stumm Junior"
 
@@ -12,9 +12,6 @@
 html_theme = "sphinx_rtd_theme"
 default_dark_mode = False
 
-intersphinx_mapping = {
-    "flake8": ("https://flake8.pycqa.org/en/latest/", None),
-}
 scrapy_intersphinx_enable = [
     "shub",
     "scrapy-zyte-api",

docs/index.rst

@@ -1,26 +1,16 @@
-===========================
-flake8-scrapy documentation
-===========================
+=========================
+scrapy-lint documentation
+=========================
 
 .. include:: ../README.rst
-   :start-after: intro-start
-   :end-before: intro-end
-
-.. toctree::
-    :maxdepth: 1
-    :caption: Configuration
-    :hidden:
-
-    pre-commit
-    tools
-    requirements
-    zyte
-    options
+   :start-after: readme-start
+   :end-before: readme-end
 
 .. toctree::
     :maxdepth: 1
     :caption: Reference
     :hidden:
 
     rules/index
-    changes
+    options
+    changes

docs/options.rst

@@ -4,38 +4,73 @@
 Options
 =======
 
-This is a list of options of flake8-scrapy.
+Options that you can set in ``pyproject.toml``, under ``[tool.scrapy-lint]``.
 
-See :ref:`configuration` for information on how to configure these options,
-disable rules, etc.
+.. _ignore:
 
-.. _scrapy-known-settings:
+ignore
+======
 
-scrapy_known_settings
-=====================
+:ref:`rules` to ignore for all files.
 
-A comma-separated list of additional setting names to treat as known (i.e. not
-trigger :ref:`SCP27`):
+For example:
 
-.. code-block:: ini
+.. code-block:: toml
 
-    [flake8]
-    scrapy_known_settings = FOO,BAR,MY_CUSTOM_SETTING
+    [tool.scrapy-lint]
+    ignore = ["SCP46"]
 
-Use this to silence false positives for custom or project-specific settings.
 
+.. _known-settings:
 
-.. _scrapy-requirements-file:
+known-settings
+==============
 
-scrapy_requirements_file
-========================
+Setting names that must not trigger :ref:`SCP27`.
+
+For example:
+
+.. code-block:: toml
+
+    [tool.scrapy-lint]
+    known-settings = [
+        "FOO",
+        "BAR",
+    ]
+
+
+.. _per-file-ignores:
+
+per-file-ignores
+================
+
+:ref:`rules` to ignore for specific files.
+
+For example:
+
+.. code-block:: toml
+
+    [tool.scrapy-lint.per-file-ignores]
+    "spiders/toscrape_com.py" = ["SCP46"]
+
+
+.. _requirements-file:
+
+requirements-file
+=================
 
 The path to the requirements file of the Scrapy project:
 
-.. code-block:: ini
+.. code-block:: toml
+
+    [tool.scrapy-lint]
+    requirements-file = "path/to/my-requirements.txt"
+
+If not specified, a requirements file is looked up as follows:
 
-    [flake8]
-    scrapy_requirements_file = path/to/my-requirements.txt
+#.  The requirements file specified in ``scrapinghub.yml`` if such file exists
+    and contains a root ``requirements`` key with a ``file`` value pointing to
+    an existing file (path interpreted relative to the project root).
 
-Only necessary if the :ref:`default lookup <requirements>` does not find the
-right file.
+#.  The ``requirements.txt`` file in the project root directory, i.e. where
+    ``scrapy.cfg`` lives.