refactor!: Change method HttpResponse.read from sync to async (#1296)

### Description

- Makes the `read` method for `HttpResponse` asynchronous. This is more
consistent in a fully asynchronous framework. More flexible than a
synchronous method

Author: Mantisus

docs/examples/code_examples/fill_and_submit_web_form_crawler.py

@@ -12,7 +12,7 @@ async def main() -> None:
     @crawler.router.default_handler
     async def request_handler(context: HttpCrawlingContext) -> None:
         context.log.info(f'Processing {context.request.url} ...')
-        response = context.http_response.read().decode('utf-8')
+        response = (await context.http_response.read()).decode('utf-8')
         context.log.info(f'Response: {response}')  # To see the response in the logs.
 
     # Prepare a POST request to the form endpoint.

docs/guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py

@@ -8,10 +8,10 @@
 from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
 
 
-def archive_response(context: ParselCrawlingContext, writer: WARCWriter) -> None:
+async def archive_response(context: ParselCrawlingContext, writer: WARCWriter) -> None:
     """Helper function for archiving response in WARC format."""
     # Create WARC records for response
-    response_body = context.http_response.read()
+    response_body = await context.http_response.read()
     response_payload_stream = io.BytesIO(response_body)
 
     response_headers = StatusAndHeaders(
@@ -51,7 +51,7 @@ async def main() -> None:
         @crawler.router.default_handler
         async def request_handler(context: ParselCrawlingContext) -> None:
             context.log.info(f'Archiving {context.request.url} ...')
-            archive_response(context=context, writer=writer)
+            await archive_response(context=context, writer=writer)
             await context.enqueue_links(strategy='same-domain')
 
         await crawler.run(['https://crawlee.dev/'])

docs/guides/code_examples/error_handling/change_handle_error_status.py

@@ -30,7 +30,7 @@ async def default_handler(context: HttpCrawlingContext) -> None:
             response = await context.send_request(
                 'https://placeholder.org/refresh', headers=headers
             )
-            data = json.loads(response.read())
+            data = json.loads(await response.read())
             # Add the new token to our `Request` headers
             new_headers = {
                 **context.request.headers,

docs/guides/code_examples/login_crawler/http_login.py

@@ -46,7 +46,7 @@ async def login_handler(context: HttpCrawlingContext) -> None:
             raise RuntimeError('Session not found')
 
         # Parse the API response containing authentication tokens and user data
-        data = json.loads(context.http_response.read())
+        data = json.loads(await context.http_response.read())
 
         # Extract authentication data from the response
         token = data['token']

docs/guides/code_examples/session_management/sm_basic.py

@@ -30,7 +30,7 @@ async def default_handler(context: BasicCrawlingContext) -> None:
         # and `context.proxy_info`.
         response = await context.send_request(context.request.url)
 
-        page_content = response.read().decode()
+        page_content = (await response.read()).decode()
         title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
 
         if context.session and (title := title_match.group(1) if title_match else None):

docs/guides/code_examples/session_management/sm_http.py

@@ -26,7 +26,7 @@ async def main() -> None:
     # based on the response content and potential blocking
     @crawler.router.default_handler
     async def default_handler(context: HttpCrawlingContext) -> None:
-        page_content = context.http_response.read().decode()
+        page_content = (await context.http_response.read()).decode()
         title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
 
         if context.session and (title := title_match.group(1) if title_match else None):

docs/upgrading/upgrading_to_v1.md

@@ -121,6 +121,8 @@ dataset = await Dataset.open(
 The `persist_storage` and `persist_metadata` fields have been removed from the `Configuration` class.
 Persistence is now determined solely by the storage client class you use.
 
+The `read` method for `HttpResponse` has been changed from synchronous to asynchronous.
+
 ### Storage client instance behavior
 
 Instance caching is implemented for the storage open methods: `Dataset.open()`, `KeyValueStore.open()`,

src/crawlee/_utils/robots.py

@@ -57,7 +57,9 @@ async def load(cls, url: str, http_client: HttpClient, proxy_info: ProxyInfo | N
             proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
         """
         response = await http_client.send_request(url, proxy_info=proxy_info)
-        body = b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else response.read()
+        body = (
+            b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else await response.read()
+        )
 
         robots = Protego.parse(body.decode('utf-8'))
 

src/crawlee/crawlers/_abstract_http/_http_crawling_context.py

@@ -26,7 +26,7 @@ def from_basic_crawling_context(cls, context: BasicCrawlingContext, http_respons
 
     async def get_snapshot(self) -> PageSnapshot:
         """Get snapshot of crawled page."""
-        return PageSnapshot(html=self.http_response.read().decode('utf-8'))
+        return PageSnapshot(html=(await self.http_response.read()).decode('utf-8'))
 
 
 @dataclass(frozen=True)

src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py

@@ -21,7 +21,7 @@ def __init__(self, parser: BeautifulSoupParserType = 'lxml') -> None:
 
     @override
     async def parse(self, response: HttpResponse) -> BeautifulSoup:
-        return BeautifulSoup(response.read(), features=self._parser)
+        return BeautifulSoup(await response.read(), features=self._parser)
 
     @override
     async def parse_text(self, text: str) -> BeautifulSoup: