fix: Fix the order in which cookies are saved to the SessionCookies and the handler is executed for PlaywrightCrawler (#1163)

### Description

- For `PlaywrightCrawler`, cookies should only be saved to the session
store when the handler is fully executed. This is because the browser
may continue to set cookies while the handler is being executed

### Testing

- Add a test simulating the installation of a cookie in the browser
during the `default_handler` execution process
- Update the `test_isolation_cookies` test

Author: Mantisus

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -232,10 +232,6 @@ async def _navigate(
             # Set the loaded URL to the actual URL after redirection.
             context.request.loaded_url = context.page.url
 
-            if context.session:
-                pw_cookies = await self._get_cookies(context.page)
-                context.session.cookies.set_cookies_from_playwright_format(pw_cookies)
-
             extract_links = self._create_extract_links_function(context)
 
             error = yield PlaywrightCrawlingContext(
@@ -256,6 +252,10 @@ async def _navigate(
                 block_requests=partial(block_requests, page=context.page),
             )
 
+            if context.session:
+                pw_cookies = await self._get_cookies(context.page)
+                context.session.cookies.set_cookies_from_playwright_format(pw_cookies)
+
             # Collect data in case of errors, before the page object is closed.
             if error:
                 await self.statistics.error_tracker.add(error=error, context=context, early=True)

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -22,7 +22,7 @@
 from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values
 from crawlee.fingerprint_suite._consts import BROWSER_TYPE_HEADER_KEYWORD
 from crawlee.proxy_configuration import ProxyConfiguration
-from crawlee.sessions import SessionPool
+from crawlee.sessions import Session, SessionPool
 from crawlee.statistics import Statistics
 from crawlee.statistics._error_snapshotter import ErrorSnapshotter
 from tests.unit.server_endpoints import GENERIC_RESPONSE, HELLO_WORLD
@@ -304,6 +304,7 @@ async def some_hook(context: PlaywrightPreNavCrawlingContext) -> None:
 )
 async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) -> None:
     sessions_ids: list[str] = []
+    sessions: dict[str, Session] = {}
     sessions_cookies: dict[str, dict[str, str]] = {}
     response_cookies: dict[str, dict[str, str]] = {}
 
@@ -319,13 +320,11 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
             return
 
         sessions_ids.append(context.session.id)
+        sessions[context.session.id] = context.session
 
         if context.request.unique_key not in {'1', '2'}:
             return
 
-        sessions_cookies[context.session.id] = {
-            cookie['name']: cookie['value'] for cookie in context.session.cookies.get_cookies_as_dicts()
-        }
         response_data = json.loads(await context.response.text())
         response_cookies[context.session.id] = response_data.get('cookies')
 
@@ -343,11 +342,20 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
         ]
     )
 
-    assert len(sessions_cookies) == 2
     assert len(response_cookies) == 2
+    assert len(sessions) == 2
 
     assert sessions_ids[0] == sessions_ids[1]
 
+    sessions_cookies = {
+        sessions_id: {
+            cookie['name']: cookie['value'] for cookie in sessions[sessions_id].cookies.get_cookies_as_dicts()
+        }
+        for sessions_id in sessions_ids
+    }
+
+    assert len(sessions_cookies) == 2
+
     cookie_session_id = sessions_ids[0]
     clean_session_id = sessions_ids[2]
 
@@ -372,6 +380,33 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
         assert sessions_cookies[clean_session_id] == response_cookies[clean_session_id] == {'a': '1'}
 
 
+async def test_save_cookies_after_handler_processing(server_url: URL) -> None:
+    """Test that cookies are saved correctly."""
+    async with SessionPool(max_pool_size=1) as session_pool:
+        crawler = PlaywrightCrawler(session_pool=session_pool)
+
+        session_ids = []
+
+        @crawler.router.default_handler
+        async def request_handler(context: PlaywrightCrawlingContext) -> None:
+            # Simulate cookies installed from an external source in the browser
+            await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])
+
+            if context.session:
+                session_ids.append(context.session.id)
+
+        await crawler.run([str(server_url)])
+
+        assert len(session_ids) == 1
+
+        check_session = await session_pool.get_session()
+
+        assert check_session.id == session_ids[0]
+        session_cookies = {cookie['name']: cookie['value'] for cookie in check_session.cookies.get_cookies_as_dicts()}
+
+        assert session_cookies == {'check': 'test'}
+
+
 async def test_custom_fingerprint_uses_generator_options(server_url: URL) -> None:
     min_width = 300
     max_width = 600