feat: Add possibility to use None as no proxy in tiered proxies (#760)

Using None in input lists for tiered proxy configuration will be
interpreted as no proxy. This allows to have for example lowest tier
without proxy.

Add tests for it.
Update docs.

- Closes: #687

Author: Pijukatel

docs/guides/code/proxy_management/tiers_bs_example.py

@@ -8,6 +8,8 @@ async def main() -> None:
     # Create a ProxyConfiguration object and pass it to the crawler.
     proxy_configuration = ProxyConfiguration(
         tiered_proxy_urls=[
+            # No proxy tier. (Not needed, but optional in case you do not want to use any proxy on lowest tier.)
+            [None],
             # lower tier, cheaper, preferred as long as they work
             ['http://cheap-datacenter-proxy-1.com/', 'http://cheap-datacenter-proxy-2.com/'],
             # higher tier, more expensive, used as a fallback

docs/guides/code/proxy_management/tiers_pw_example.py

@@ -8,6 +8,8 @@ async def main() -> None:
     # Create a ProxyConfiguration object and pass it to the crawler.
     proxy_configuration = ProxyConfiguration(
         tiered_proxy_urls=[
+            # No proxy tier. (Not needed, but optional in case you do not want to use any proxy on lowest tier.)
+            [None],
             # lower tier, cheaper, preferred as long as they work
             ['http://cheap-datacenter-proxy-1.com/', 'http://cheap-datacenter-proxy-2.com/'],
             # higher tier, more expensive, used as a fallback

src/crawlee/proxy_configuration.py

@@ -3,7 +3,7 @@
 import inspect
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union
 
 from more_itertools import flatten
 from pydantic import AnyHttpUrl, TypeAdapter
@@ -68,9 +68,9 @@ class ProxyConfiguration:
     def __init__(
         self,
         *,
-        proxy_urls: list[str] | None = None,
+        proxy_urls: list[str | None] | None = None,
         new_url_function: _NewUrlFunction | None = None,
-        tiered_proxy_urls: list[list[str]] | None = None,
+        tiered_proxy_urls: list[list[str | None]] | None = None,
     ) -> None:
         """A default constructor.
 
@@ -85,7 +85,7 @@ def __init__(
                 the proxy selection mechanism.
         """
         self._next_custom_url_index = 0
-        self._used_proxy_urls = dict[str, URL]()
+        self._used_proxy_urls = dict[str, Union[URL, None]]()
         self._url_validator = TypeAdapter(AnyHttpUrl)
 
         # Validation
@@ -95,18 +95,22 @@ def __init__(
                 'must be specified (and non-empty).'
             )
 
-        self._proxy_urls = (
-            [URL(url) for url in proxy_urls if self._url_validator.validate_python(url)] if proxy_urls else []
-        )
+        self._proxy_urls = [self._create_url(url) for url in proxy_urls] if proxy_urls else []
         self._proxy_tier_tracker = (
-            _ProxyTierTracker(
-                [[URL(url) for url in tier if self._url_validator.validate_python(url)] for tier in tiered_proxy_urls]
-            )
+            _ProxyTierTracker([[self._create_url(url) for url in tier] for tier in tiered_proxy_urls])
             if tiered_proxy_urls
             else None
         )
         self._new_url_function = new_url_function
 
+    def _create_url(self, url: str | None) -> URL | None:
+        """Create URL from input string. None means that intentionally no proxy should be used."""
+        if url is None:
+            return None
+
+        self._url_validator.validate_python(url)
+        return URL(url)
+
     async def new_proxy_info(
         self, session_id: str | None, request: Request | None, proxy_tier: int | None
     ) -> ProxyInfo | None:
@@ -208,16 +212,16 @@ async def _pick_url(
 class _ProxyTierTracker:
     """Tracks the state of currently used proxy tiers and their error frequency for individual crawled domains."""
 
-    def __init__(self, tiered_proxy_urls: list[list[URL]]) -> None:
+    def __init__(self, tiered_proxy_urls: list[list[URL | None]]) -> None:
         self._tiered_proxy_urls = tiered_proxy_urls
         self._histogram_by_domain = defaultdict[str, list[int]](lambda: [0 for _tier in tiered_proxy_urls])
         self._current_tier_by_domain = defaultdict[str, int](lambda: 0)
 
     @property
-    def all_urls(self) -> Sequence[URL]:
+    def all_urls(self) -> Sequence[URL | None]:
         return list(flatten(self._tiered_proxy_urls))
 
-    def get_tier_urls(self, tier_number: int) -> Sequence[URL]:
+    def get_tier_urls(self, tier_number: int) -> Sequence[URL | None]:
         return self._tiered_proxy_urls[tier_number]
 
     def add_error(self, domain: str, tier: int) -> None:

tests/unit/proxy_configuration/test_new_proxy_info.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from itertools import cycle
 from typing import TYPE_CHECKING
 
 import pytest
@@ -11,10 +12,13 @@
 
 
 async def test_returns_proxy_info() -> None:
-    config = ProxyConfiguration(proxy_urls=['http://proxy.com:1111'])
+    """Test that proxy_urls can return contain both string and None."""
+    config = ProxyConfiguration(proxy_urls=[None, 'http://proxy.com:1111'])
 
     proxy_info = await config.new_proxy_info(None, None, None)
+    assert proxy_info is None
 
+    proxy_info = await config.new_proxy_info(None, None, None)
     assert proxy_info is not None
     assert proxy_info.url == 'http://proxy.com:1111'
     assert proxy_info.hostname == 'proxy.com'
@@ -33,10 +37,15 @@ async def test_throws_on_invalid_new_url_function() -> None:
 
 
 async def test_returns_proxy_info_with_new_url_function() -> None:
-    config = ProxyConfiguration(new_url_function=lambda session_id=None, request=None: 'http://proxy.com:1111')  # noqa: ARG005
+    """Test that new_url_function can return string and None."""
+    proxy_iterator = cycle([None, 'http://proxy.com:1111'])
+
+    config = ProxyConfiguration(new_url_function=lambda session_id=None, request=None: next(proxy_iterator))  # noqa: ARG005
 
     proxy_info = await config.new_proxy_info(None, None, None)
+    assert proxy_info is None
 
+    proxy_info = await config.new_proxy_info(None, None, None)
     assert proxy_info is not None
     assert proxy_info.url == 'http://proxy.com:1111'
     assert proxy_info.hostname == 'proxy.com'
@@ -62,7 +71,7 @@ async def new_url(session_id: str | None = None, request: Request | None = None)
 
 
 async def test_rotates_proxies() -> None:
-    proxy_urls = ['http://proxy:1111', 'http://proxy:2222', 'http://proxy:3333']
+    proxy_urls: list[str | None] = ['http://proxy:1111', 'http://proxy:2222', 'http://proxy:3333']
     config = ProxyConfiguration(proxy_urls=proxy_urls)
 
     info = await config.new_proxy_info(None, None, None)
@@ -79,7 +88,7 @@ async def test_rotates_proxies() -> None:
 
 
 async def test_rotates_proxies_with_sessions() -> None:
-    proxy_urls = ['http://proxy:1111', 'http://proxy:2222', 'http://proxy:3333']
+    proxy_urls: list[str | None] = ['http://proxy:1111', 'http://proxy:2222', 'http://proxy:3333']
     sessions = [f'session_{i}' for i in range(6)]
 
     config = ProxyConfiguration(proxy_urls=proxy_urls)

tests/unit/proxy_configuration/test_tiers.py

@@ -5,7 +5,7 @@
 
 
 async def test_rotates_proxies_uniformly_with_no_request() -> None:
-    tiered_proxy_urls = [
+    tiered_proxy_urls: list[list[str | None]] = [
         ['http://proxy:1111', 'http://proxy:2222'],
         ['http://proxy:3333', 'http://proxy:4444'],
     ]
@@ -34,7 +34,7 @@ async def test_rotates_proxies_uniformly_with_no_request() -> None:
 
 
 async def test_retrying_request_makes_tier_go_up() -> None:
-    tiered_proxy_urls = [
+    tiered_proxy_urls: list[list[str | None]] = [
         ['http://proxy:1111'],
         ['http://proxy:2222'],
         ['http://proxy:3333'],
@@ -71,7 +71,7 @@ async def test_successful_request_makes_tier_go_down() -> None:
     ProxyConfiguration assumes those are retries. Then, requesting a proxy for different requests to the same domain
     will cause the tier to drop back down."""
 
-    tiered_proxy_urls = [
+    tiered_proxy_urls: list[list[str | None]] = [
         ['http://proxy:1111'],
         ['http://proxy:2222'],
         ['http://proxy:3333'],
@@ -94,3 +94,49 @@ async def test_successful_request_makes_tier_go_down() -> None:
 
     assert info is not None
     assert info.url == tiered_proxy_urls[0][0]
+
+
+async def test_none_proxy_retrying_request_makes_tier_go_up() -> None:
+    tiered_proxy_urls: list[list[str | None]] = [
+        [None],
+        ['http://proxy:1111'],
+    ]
+
+    config = ProxyConfiguration(tiered_proxy_urls=tiered_proxy_urls)
+
+    # Calling `new_proxy_info` with the same request most probably means it's being retried
+    request_1 = Request(url='http://some.domain/abc', unique_key='1', id='1')
+
+    # No proxy used.
+    info = await config.new_proxy_info(None, request_1, None)
+    assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.'
+
+    # Proxy should go up one tier for same request that was already sent before.
+    info = await config.new_proxy_info(None, request_1, None)
+    assert info is not None, (
+        'config.new_proxy_info is expected to generate non-none proxy info from non-none ' 'tiered_proxy_urls.'
+    )
+    assert info.url == tiered_proxy_urls[1][0]
+
+
+async def test_none_proxy_rotates_proxies_uniformly_with_no_request() -> None:
+    tiered_proxy_urls = [
+        [None, 'http://proxy:1111'],
+    ]
+
+    config = ProxyConfiguration(tiered_proxy_urls=tiered_proxy_urls)
+
+    # No proxy used.
+    info = await config.new_proxy_info(None, None, None)
+    assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.'
+
+    # Proxy should be rotated on the same proxy tier for a new request.
+    info = await config.new_proxy_info(None, None, None)
+    assert info is not None, (
+        'config.new_proxy_info is expected to generate non-none proxy info from non-none ' 'tiered_proxy_urls.'
+    )
+    assert info.url == tiered_proxy_urls[0][1]
+
+    # Proxy rotation starts from the beginning of the proxy list after last proxy in tier was used. No proxy used again.
+    info = await config.new_proxy_info(None, None, None)
+    assert info is None, 'First entry in tired_proxy_urls is None. config.new_proxy_info is expected to generate None.'