feat: Integrate browserforge fingerprints

pyproject.toml

@@ -44,6 +44,7 @@ keywords = [
 python = "^3.9"
 apify = { version = ">=2.0.0", optional = true }
 beautifulsoup4 = { version = ">=4.12.0", optional = true }
+browserforge = { version = "*", optional = true }
 colorama = ">=0.4.0"
 cookiecutter = ">=2.6.0"
 curl-cffi = { version = ">=0.7.2", optional = true }
@@ -94,10 +95,10 @@ types-psutil = "~5.9.5.20240205"
 types-python-dateutil = "~2.9.0.20240316"
 
 [tool.poetry.extras]
-all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright"]
+all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright", "browserforge"]
 beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]
 curl-impersonate = ["curl-cffi"]
-playwright = ["playwright"]
+playwright = ["playwright", "browserforge"]
 parsel = ["parsel"]
 
 [tool.poetry.scripts]

src/crawlee/browsers/_browser_pool.py

@@ -51,6 +51,8 @@ def __init__(
         browser_inactive_threshold: timedelta = timedelta(seconds=10),
         identify_inactive_browsers_interval: timedelta = timedelta(seconds=20),
         close_inactive_browsers_interval: timedelta = timedelta(seconds=30),
+        use_fingerprints: bool = True,
+        fingerprint_generator_options: dict[str, Any] | None = None,
     ) -> None:
         """A default constructor.
 
@@ -66,6 +68,8 @@ def __init__(
             close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers
                 and closes them. The browser is considered as inactive if it has no active pages and has been idle
                 for the specified period.
+            use_fingerprints: Inject generated fingerprints to page.
+            fingerprint_generator_options: Override generated fingerprints with these specific values, if possible.
         """
         self._plugins = plugins or [PlaywrightBrowserPlugin()]
         self._operation_timeout = operation_timeout
@@ -95,6 +99,9 @@ def __init__(
         # Flag to indicate the context state.
         self._active = False
 
+        self._use_fingerprints = use_fingerprints
+        self._fingerprint_generator_options = fingerprint_generator_options
+
     @classmethod
     def with_default_plugin(
         cls,
@@ -103,6 +110,8 @@ def with_default_plugin(
         browser_options: Mapping[str, Any] | None = None,
         page_options: Mapping[str, Any] | None = None,
         headless: bool | None = None,
+        use_fingerprints: bool = False,
+        fingerprint_generator_options: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> BrowserPool:
         """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
@@ -116,6 +125,8 @@ def with_default_plugin(
                 Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
                 https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
             headless: Whether to run the browser in headless mode.
+            use_fingerprints: Inject generated fingerprints to page.
+            fingerprint_generator_options: Override generated fingerprints with these specific values, if possible.
             kwargs: Additional arguments for default constructor.
         """
         plugin_options: dict = defaultdict(dict)
@@ -128,7 +139,11 @@ def with_default_plugin(
         if browser_type:
             plugin_options['browser_type'] = browser_type
 
-        plugin = PlaywrightBrowserPlugin(**plugin_options)
+        plugin = PlaywrightBrowserPlugin(
+            **plugin_options,
+            use_fingerprints=use_fingerprints,
+            fingerprint_generator_options=fingerprint_generator_options,
+        )
         return cls(plugins=[plugin], **kwargs)
 
     @property

src/crawlee/browsers/_playwright_browser_controller.py

@@ -5,6 +5,7 @@
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
+from browserforge.injectors.playwright import AsyncNewContext
 from playwright.async_api import BrowserContext, Page, ProxySettings
 from typing_extensions import override
 
@@ -38,6 +39,8 @@ def __init__(
         *,
         max_open_pages_per_browser: int = 20,
         header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR,
+        use_fingerprints: bool = True,
+        fingerprint_generator_options: dict[str, Any] | None = None,
     ) -> None:
         """A default constructor.
 
@@ -47,6 +50,8 @@ def __init__(
             header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for
                 requests made by the browser. By default, a predefined header generator is used. Set to `None` to
                 disable automatic header modifications.
+            use_fingerprints: Inject generated fingerprints to page.
+            fingerprint_generator_options: Override generated fingerprints with these specific values, if possible.
         """
         self._browser = browser
         self._max_open_pages_per_browser = max_open_pages_per_browser
@@ -56,6 +61,9 @@ def __init__(
         self._pages = list[Page]()
         self._last_page_opened_at = datetime.now(timezone.utc)
 
+        self._use_fingerprints = use_fingerprints
+        self._fingerprint_generator_options = fingerprint_generator_options
+
     @property
     @override
     def pages(self) -> list[Page]:
@@ -98,13 +106,15 @@ async def new_page(
         proxy_info: ProxyInfo | None = None,
     ) -> Page:
         if not self._browser_context:
-            self._browser_context = await self._create_browser_context(proxy_info)
+            await self._set_browser_context(
+                fingerprint_options=self._fingerprint_generator_options, proxy_info=proxy_info
+            )
 
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
         page_options = dict(page_options) if page_options else {}
-        page = await self._browser_context.new_page(**page_options)
+        page = await self._get_browser_context().new_page(**page_options)
 
         # Handle page close event
         page.on(event='close', f=self._on_page_close)
@@ -115,33 +125,14 @@ async def new_page(
 
         return page
 
-    @override
-    async def close(self, *, force: bool = False) -> None:
-        if force:
-            for page in self._pages:
-                await page.close()
-
-        if self.pages_count > 0:
-            raise ValueError('Cannot close the browser while there are open pages.')
-
-        await self._browser.close()
-
-    def _on_page_close(self, page: Page) -> None:
-        """Handle actions after a page is closed."""
-        self._pages.remove(page)
-
-    async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> BrowserContext:
-        """Create a new browser context with the specified proxy settings."""
-        if self._header_generator:
-            common_headers = self._header_generator.get_common_headers()
-            sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
-            user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
-            extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
-            user_agent = user_agent_header.get('User-Agent')
-        else:
-            extra_http_headers = None
-            user_agent = None
+    async def _set_browser_context(
+        self, proxy_info: ProxyInfo | None = None, fingerprint_options: dict | None = None
+    ) -> None:
+        """Set browser context.
 
+        Create context using `browserforge`  if `_use_fingerprints` is True.
+        Create context without fingerprints with headers based header generator if available.
+        """
         proxy = (
             ProxySettings(
                 server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}',
@@ -152,8 +143,45 @@ async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) ->
             else None
         )
 
-        return await self._browser.new_context(
+        if self._use_fingerprints:
+            self._browser_context = await AsyncNewContext(
+                browser=self._browser, fingerprint_options=(fingerprint_options or {}), proxy=proxy
+            )
+            return
+
+        if self._header_generator:
+            common_headers = self._header_generator.get_common_headers()
+            sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
+            user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
+            headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
+            extra_http_headers = headers
+            user_agent = headers.get('User-Agent')
+        else:
+            extra_http_headers = None
+            user_agent = None
+
+        self._browser_context = await self._browser.new_context(
             user_agent=user_agent,
             extra_http_headers=extra_http_headers,
             proxy=proxy,
         )
+
+    def _get_browser_context(self) -> BrowserContext:
+        if not self._browser_context:
+            raise RuntimeError('Browser context was not set yet.')
+        return self._browser_context
+
+    @override
+    async def close(self, *, force: bool = False) -> None:
+        if force:
+            for page in self._pages:
+                await page.close()
+
+        if self.pages_count > 0:
+            raise ValueError('Cannot close the browser while there are open pages.')
+
+        await self._browser.close()
+
+    def _on_page_close(self, page: Page) -> None:
+        """Handle actions after a page is closed."""
+        self._pages.remove(page)

src/crawlee/browsers/_playwright_browser_plugin.py

@@ -38,6 +38,8 @@ def __init__(
         browser_options: Mapping[str, Any] | None = None,
         page_options: Mapping[str, Any] | None = None,
         max_open_pages_per_browser: int = 20,
+        use_fingerprints: bool = True,
+        fingerprint_generator_options: dict[str, Any] | None = None,
     ) -> None:
         """A default constructor.
 
@@ -51,6 +53,9 @@ def __init__(
                 https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
             max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance.
                 Once reached, a new browser instance will be launched to handle the excess.
+            use_fingerprints: Inject generated fingerprints to page.
+            fingerprint_generator_options: Override generated fingerprints with these specific values, if possible.
+
         """
         self._browser_type = browser_type
         self._browser_options = browser_options or {}
@@ -63,6 +68,9 @@ def __init__(
         # Flag to indicate the context state.
         self._active = False
 
+        self._use_fingerprints = use_fingerprints
+        self._fingerprint_generator_options = fingerprint_generator_options
+
     @property
     @override
     def active(self) -> bool:
@@ -128,4 +136,6 @@ async def new_browser(self) -> PlaywrightBrowserController:
         return PlaywrightBrowserController(
             browser,
             max_open_pages_per_browser=self._max_open_pages_per_browser,
+            use_fingerprints=self._use_fingerprints,
+            fingerprint_generator_options=self._fingerprint_generator_options,
         )

src/crawlee/playwright_crawler/_playwright_crawler.py

@@ -69,7 +69,9 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
 
     def __init__(
         self,
+        *,
         browser_pool: BrowserPool | None = None,
+        browser_pool_options: Mapping[str, Any] | None = None,
         browser_type: BrowserType | None = None,
         browser_options: Mapping[str, Any] | None = None,
         page_options: Mapping[str, Any] | None = None,
@@ -80,6 +82,7 @@ def __init__(
 
         Args:
             browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages.
+            browser_pool_options: Arguments passed to `BrowserPool`.
             browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
                 This option should not be used if `browser_pool` is provided.
             browser_options: Keyword arguments to pass to the browser launch method. These options are provided
@@ -109,6 +112,7 @@ def __init__(
                 browser_type=browser_type,
                 browser_options=browser_options,
                 page_options=page_options,
+                **(browser_pool_options or {}),
             )
 
         self._browser_pool = browser_pool
@@ -168,9 +172,7 @@ async def _navigate(
         async with context.page:
             if context.request.headers:
                 await context.page.set_extra_http_headers(context.request.headers.model_dump())
-            # Navigate to the URL and get response.
             response = await context.page.goto(context.request.url)
-
             if response is None:
                 raise SessionError(f'Failed to load the URL: {context.request.url}')