feat: Integrate browserforge fingerprints

pyproject.toml

@@ -44,6 +44,7 @@ keywords = [
 python = "^3.9"
 apify = { version = ">=2.0.0", optional = true }
 beautifulsoup4 = { version = ">=4.12.0", optional = true }
+browserforge = { version = "*", optional = true }
 colorama = ">=0.4.0"
 cookiecutter = ">=2.6.0"
 curl-cffi = { version = ">=0.7.2", optional = true }
@@ -92,10 +93,10 @@ types-psutil = "~5.9.5.20240205"
 types-python-dateutil = "~2.9.0.20240316"
 
 [tool.poetry.extras]
-all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright"]
+all = ["beautifulsoup4", "curl-cffi", "lxml", "html5lib", "parsel", "playwright", "browserforge"]
 beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]
 curl-impersonate = ["curl-cffi"]
-playwright = ["playwright"]
+playwright = ["playwright", "browserforge"]
 parsel = ["parsel"]
 
 [tool.poetry.scripts]

src/crawlee/browsers/_browser_pool.py

@@ -23,6 +23,7 @@
     from types import TracebackType
 
     from crawlee.browsers._base_browser_plugin import BaseBrowserPlugin
+    from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator
     from crawlee.proxy_configuration import ProxyInfo
 
 logger = getLogger(__name__)
@@ -103,6 +104,7 @@ def with_default_plugin(
         browser_launch_options: Mapping[str, Any] | None = None,
         browser_new_context_options: Mapping[str, Any] | None = None,
         headless: bool | None = None,
+        fingerprint_generator: FingerprintGenerator | None = None,
         **kwargs: Any,
     ) -> BrowserPool:
         """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
@@ -116,6 +118,8 @@ def with_default_plugin(
                 are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
                 Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             headless: Whether to run the browser in headless mode.
+            fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used
+                to generate browser fingerprints together with consistent headers.
             kwargs: Additional arguments for default constructor.
         """
         plugin_options: dict = defaultdict(dict)
@@ -128,7 +132,10 @@ def with_default_plugin(
         if browser_type:
             plugin_options['browser_type'] = browser_type
 
-        plugin = PlaywrightBrowserPlugin(**plugin_options)
+        plugin = PlaywrightBrowserPlugin(
+            **plugin_options,
+            fingerprint_generator=fingerprint_generator,
+        )
         return cls(plugins=[plugin], **kwargs)
 
     @property

src/crawlee/browsers/_playwright_browser_controller.py

@@ -5,6 +5,7 @@
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
+from browserforge.injectors.playwright import AsyncNewContext
 from playwright.async_api import BrowserContext, Page, ProxySettings
 from typing_extensions import override
 
@@ -18,6 +19,7 @@
 
     from playwright.async_api import Browser
 
+    from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator
     from crawlee.proxy_configuration import ProxyInfo
 
 from logging import getLogger
@@ -42,6 +44,7 @@ def __init__(
         *,
         max_open_pages_per_browser: int = 20,
         header_generator: HeaderGenerator | None = _DEFAULT_HEADER_GENERATOR,
+        fingerprint_generator: FingerprintGenerator | None = None,
     ) -> None:
         """A default constructor.
 
@@ -51,10 +54,18 @@ def __init__(
             header_generator: An optional `HeaderGenerator` instance used to generate and manage HTTP headers for
                 requests made by the browser. By default, a predefined header generator is used. Set to `None` to
                 disable automatic header modifications.
+            fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used
+                to generate browser fingerprints together with consistent headers.
         """
+        if fingerprint_generator and header_generator is not self._DEFAULT_HEADER_GENERATOR:
+            raise ValueError(
+                'Do not use `header_generator` and `fingerprint_generator` arguments at the same time. '
+                'Choose only one. `fingerprint_generator` generates headers as well.'
+            )
         self._browser = browser
         self._max_open_pages_per_browser = max_open_pages_per_browser
         self._header_generator = header_generator
+        self._fingerprint_generator = fingerprint_generator
 
         self._browser_context: BrowserContext | None = None
         self._pages = list[Page]()
@@ -116,7 +127,10 @@ async def new_page(
             ValueError: If the browser has reached the maximum number of open pages.
         """
         if not self._browser_context:
-            self._browser_context = await self._create_browser_context(browser_new_context_options, proxy_info)
+            self._browser_context = await self._create_browser_context(
+                browser_new_context_options=browser_new_context_options,
+                proxy_info=proxy_info,
+            )
 
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
@@ -154,21 +168,18 @@ def _on_page_close(self, page: Page) -> None:
         self._pages.remove(page)
 
     async def _create_browser_context(
-        self, browser_new_context_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None
+        self,
+        browser_new_context_options: Mapping[str, Any] | None = None,
+        proxy_info: ProxyInfo | None = None,
     ) -> BrowserContext:
-        """Create a new browser context with the specified proxy settings."""
-        if self._header_generator:
-            common_headers = self._header_generator.get_common_headers()
-            sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
-            user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
-            extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
-        else:
-            extra_http_headers = None
+        """Create a new browser context with the specified proxy settings.
 
+        Create context with fingerprints and headers using with `self._fingerprint_generator` if available.
+        Create context without fingerprints, but with headers based on `self._header_generator` if available.
+        Create context without headers and without fingerprints if neither `self._header_generator` nor
+        `self._fingerprint_generator` is available.
+        """
         browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
-        browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
-            'extra_http_headers', extra_http_headers
-        )
 
         if proxy_info:
             if browser_new_context_options.get('proxy'):
@@ -180,4 +191,22 @@ async def _create_browser_context(
                 password=proxy_info.password,
             )
 
+        if self._fingerprint_generator:
+            return await AsyncNewContext(
+                browser=self._browser, fingerprint=self._fingerprint_generator.generate(), **browser_new_context_options
+            )
+
+        if self._header_generator:
+            common_headers = self._header_generator.get_common_headers()
+            sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
+            user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
+            headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
+            extra_http_headers = headers
+        else:
+            extra_http_headers = None
+
+        browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
+            'extra_http_headers', extra_http_headers
+        )
+
         return await self._browser.new_context(**browser_new_context_options)

src/crawlee/browsers/_playwright_browser_plugin.py

@@ -19,6 +19,7 @@
     from types import TracebackType
 
     from crawlee.browsers._types import BrowserType
+    from crawlee.fingerprint_suite._fingerprint_generator import FingerprintGenerator
 
 logger = getLogger(__name__)
 
@@ -43,6 +44,7 @@ def __init__(
         browser_launch_options: dict[str, Any] | None = None,
         browser_new_context_options: dict[str, Any] | None = None,
         max_open_pages_per_browser: int = 20,
+        fingerprint_generator: FingerprintGenerator | None = None,
     ) -> None:
         """A default constructor.
 
@@ -56,6 +58,8 @@ def __init__(
                 Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
             max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance.
                 Once reached, a new browser instance will be launched to handle the excess.
+            fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used
+                to generate browser fingerprints together with consistent headers.
         """
         config = service_locator.get_configuration()
 
@@ -77,6 +81,8 @@ def __init__(
         # Flag to indicate the context state.
         self._active = False
 
+        self._fingerprint_generator = fingerprint_generator
+
     @property
     @override
     def active(self) -> bool:
@@ -154,4 +160,5 @@ async def new_browser(self) -> PlaywrightBrowserController:
         return PlaywrightBrowserController(
             browser,
             max_open_pages_per_browser=self._max_open_pages_per_browser,
+            fingerprint_generator=self._fingerprint_generator,
         )

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -25,6 +25,7 @@
 
     from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs
     from crawlee.browsers._types import BrowserType
+    from crawlee.fingerprint_suite import FingerprintGenerator
 
 
 @docs_group('Classes')
@@ -70,10 +71,12 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
 
     def __init__(
         self,
+        *,
         browser_pool: BrowserPool | None = None,
         browser_type: BrowserType | None = None,
         browser_launch_options: Mapping[str, Any] | None = None,
         browser_new_context_options: Mapping[str, Any] | None = None,
+        fingerprint_generator: FingerprintGenerator | None = None,
         headless: bool | None = None,
         **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext]],
     ) -> None:
@@ -91,6 +94,8 @@ def __init__(
                 are provided directly to Playwright's `browser.new_context` method. For more details, refer to the
                 [Playwright documentation](https://playwright.dev/python/docs/api/class-browser#browser-new-context).
                 This option should not be used if `browser_pool` is provided.
+            fingerprint_generator: An optional instance of implementation of `FingerprintGenerator` that is used
+                to generate browser fingerprints together with consistent headers.
             headless: Whether to run the browser in headless mode.
                 This option should not be used if `browser_pool` is provided.
             kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`.
@@ -99,11 +104,18 @@ def __init__(
             # Raise an exception if browser_pool is provided together with other browser-related arguments.
             if any(
                 param is not None
-                for param in (headless, browser_type, browser_launch_options, browser_new_context_options)
+                for param in (
+                    headless,
+                    browser_type,
+                    browser_launch_options,
+                    browser_new_context_options,
+                    fingerprint_generator,
+                )
             ):
                 raise ValueError(
-                    'You cannot provide `headless`, `browser_type`, `browser_launch_options` or '
-                    '`browser_new_context_options` arguments when `browser_pool` is provided.'
+                    'You cannot provide `headless`, `browser_type`, `browser_launch_options`, '
+                    '`browser_new_context_options` or `fingerprint_generator` arguments when `browser_pool` '
+                    'is provided.'
                 )
 
         # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments.
@@ -113,6 +125,7 @@ def __init__(
                 browser_type=browser_type,
                 browser_launch_options=browser_launch_options,
                 browser_new_context_options=browser_new_context_options,
+                fingerprint_generator=fingerprint_generator,
             )
 
         self._browser_pool = browser_pool

src/crawlee/fingerprint_suite/__init__.py

@@ -1 +1,4 @@
+from ._browserforge_adapter import BrowserforgeFingerprintGenerator as DefaultFingerprintGenerator
+from ._fingerprint_generator import FingerprintGenerator
 from ._header_generator import HeaderGenerator
+from ._types import HeaderGeneratorOptions, ScreenOptions

src/crawlee/fingerprint_suite/_browserforge_adapter.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any
+
+from browserforge.fingerprints import Fingerprint as bf_Fingerprint
+from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator
+from browserforge.fingerprints import Screen
+from typing_extensions import override
+
+from ._fingerprint_generator import FingerprintGenerator
+
+if TYPE_CHECKING:
+    from ._types import HeaderGeneratorOptions, ScreenOptions
+
+
+class BrowserforgeFingerprintGenerator(FingerprintGenerator):
+    def __init__(
+        self,
+        *,
+        header_options: HeaderGeneratorOptions | None = None,
+        screen_options: ScreenOptions | None = None,
+        mock_web_rtc: bool | None = None,
+        slim: bool | None = None,
+    ) -> None:
+        """A default constructor.
+
+        All generator options are optional. If any value is not specified, then `None` is set in the options.
+        Default values for options set to `None` are implementation detail of used fingerprint generator.
+        Specific default values should not be relied upon. Use explicit values if it matters for your use case.
+
+        Args:
+            header_options: Collection of header related attributes that can be used by the fingerprint generator.
+            screen_options: Defines the screen constrains for the fingerprint generator.
+            mock_web_rtc: Whether to mock WebRTC when injecting the fingerprint.
+            slim: Disables performance-heavy evasions when injecting the fingerprint.
+            strict: If set to `True`, it will raise error if it is not possible to generate fingerprints based on the
+                `options`. Default behavior is relaxation of `options` until it is possible to generate a fingerprint.
+        """
+        bf_options: dict[str, Any] = {'mock_webrtc': mock_web_rtc, 'slim': slim}
+
+        if header_options is None:
+            bf_header_options = {}
+        else:
+            bf_header_options = deepcopy(header_options.model_dump())
+            bf_header_options['browser'] = bf_header_options.pop('browsers', None)
+            bf_header_options['os'] = bf_header_options.pop('operating_systems', None)
+            bf_header_options['device'] = bf_header_options.pop('devices', None)
+            bf_header_options['locale'] = bf_header_options.pop('locales', None)
+
+        if screen_options is None:
+            bf_options['screen'] = Screen()
+        else:
+            bf_options['screen'] = Screen(**screen_options.model_dump())
+
+        self._options = {**bf_options, **bf_header_options}
+
+    @override
+    def generate(self) -> bf_Fingerprint:
+        return bf_FingerprintGenerator().generate(**self._options)

src/crawlee/fingerprint_suite/_fingerprint_generator.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from browserforge.fingerprints import Fingerprint
+
+
+class FingerprintGenerator(ABC):
+    @abstractmethod
+    def generate(self) -> Fingerprint:
+        """Method that is capable of generating fingerprints.
+
+        This is experimental feature.
+        Return type is temporarily set to `Fingerprint` from `browserforge`. This is subject to change and most likely
+        it will change to custom `Fingerprint` class defined in this repo later.
+        """
+        ...

src/crawlee/fingerprint_suite/_header_generator.py

@@ -18,7 +18,7 @@
 )
 
 if TYPE_CHECKING:
-    from crawlee.browsers._types import BrowserType
+    from crawlee.fingerprint_suite._types import SupportedBrowserType
 
 
 @docs_group('Classes')
@@ -45,7 +45,7 @@ def get_random_user_agent_header(self) -> HttpHeaders:
     def get_user_agent_header(
         self,
         *,
-        browser_type: BrowserType = 'chromium',
+        browser_type: SupportedBrowserType = 'chromium',
     ) -> HttpHeaders:
         """Get the User-Agent header based on the browser type."""
         headers = dict[str, str]()
@@ -67,7 +67,7 @@ def get_user_agent_header(
     def get_sec_ch_ua_headers(
         self,
         *,
-        browser_type: BrowserType = 'chromium',
+        browser_type: SupportedBrowserType = 'chromium',
     ) -> HttpHeaders:
         """Get the Sec-Ch-Ua headers based on the browser type."""
         headers = dict[str, str]()