feat: Add AdaptivePlaywrightCrawler

src/crawlee/_types.py

@@ -298,7 +298,6 @@ class UseStateFunction(Protocol):
 
     def __call__(
         self,
-        key: str,
         default_value: dict[str, JsonSerializable] | None = None,
     ) -> Coroutine[None, None, dict[str, JsonSerializable]]: ...
 

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING, Any, Callable, Generic
 
 from pydantic import ValidationError
-from typing_extensions import NotRequired, TypeVar
+from typing_extensions import NotRequired, TypedDict, TypeVar
 
 from crawlee import EnqueueStrategy
 from crawlee._request import BaseRequestData
@@ -30,18 +30,21 @@
 
 
 @docs_group('Data structures')
-class HttpCrawlerOptions(Generic[TCrawlingContext], BasicCrawlerOptions[TCrawlingContext]):
-    """Arguments for the `AbstractHttpCrawler` constructor.
-
-    It is intended for typing forwarded `__init__` arguments in the subclasses.
-    """
-
+class _HttpCrawlerOptions(Generic[TCrawlingContext], TypedDict):
     additional_http_error_status_codes: NotRequired[Iterable[int]]
     """Additional HTTP status codes to treat as errors, triggering automatic retries when encountered."""
 
     ignore_http_error_status_codes: NotRequired[Iterable[int]]
     """HTTP status codes typically considered errors but to be treated as successful responses."""
 
+@docs_group('Data structures')
+class HttpCrawlerOptions(Generic[TCrawlingContext],_HttpCrawlerOptions, BasicCrawlerOptions[TCrawlingContext]):
+    """Arguments for the `AbstractHttpCrawler` constructor.
+
+    It is intended for typing forwarded `__init__` arguments in the subclasses.
+    """
+
+
 
 @docs_group('Abstract classes')
 class AbstractHttpCrawler(Generic[TCrawlingContext, TParseResult], BasicCrawler[TCrawlingContext], ABC):

src/crawlee/crawlers/_adaptive_playwright/__init__.py

@@ -0,0 +1,14 @@
+from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawler import AdaptivePlaywrightCrawler
+from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import (
+    AdaptivePlaywrightCrawlingContext,
+)
+
+__all__ = [
+    'AdaptivePlaywrightCrawler',
+    'AdaptivePlaywrightCrawlingContext',
+    'HttpCrawlerOptions',
+    'ParsedHttpCrawlingContext',
+]
+
+
+

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import TYPE_CHECKING, Annotated, Any, cast
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import override
+
+from crawlee._utils.docs import docs_group
+from crawlee.statistics import Statistics, StatisticsState
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from typing_extensions import Self
+
+    from crawlee.storages import KeyValueStore
+
+
+@docs_group('Data structures')
+class PredictorState(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, ser_json_inf_nan='constants')
+
+    http_only_request_handler_runs: Annotated[int, Field(alias='http_only_request_handler_runs')] = 0
+    browser_request_handler_runs: Annotated[int, Field(alias='browser_request_handler_runs')] = 0
+    rendering_type_mispredictions: Annotated[int, Field(alias='rendering_type_mispredictions')] = 0
+
+@docs_group('Classes')
+class AdaptivePlaywrightCrawlerStatistics(Statistics):
+
+
+    def __init__(self,*,
+        persistence_enabled: bool = False,
+        persist_state_kvs_name: str = 'default',
+        persist_state_key: str | None = None,
+        key_value_store: KeyValueStore | None = None,
+        log_message: str = 'Statistics',
+        periodic_message_logger: Logger | None = None,
+        log_interval: timedelta = timedelta(minutes=1),
+        state_model: type[StatisticsState] = StatisticsState) -> None:
+        self.predictor_state = PredictorState()
+        super().__init__(persistence_enabled=persistence_enabled,
+                         persist_state_kvs_name=persist_state_kvs_name,
+                         persist_state_key=persist_state_key,
+                         key_value_store=key_value_store,
+                         log_message=log_message,
+                         periodic_message_logger=periodic_message_logger,
+                         log_interval=log_interval,
+                         state_model=state_model)
+        self._persist_predictor_state_key = self._persist_state_key + '_PREDICTOR'
+
+    @classmethod
+    def from_statistics(cls, statistics: Statistics) -> Self:
+        return cls(persistence_enabled=statistics._persistence_enabled,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         persist_state_kvs_name=statistics._persist_state_kvs_name,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         persist_state_key=statistics._persist_state_key,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         key_value_store=statistics._key_value_store,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         log_message=statistics._log_message,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         periodic_message_logger=statistics._periodic_message_logger,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         log_interval=statistics._log_interval,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                         state_model=statistics._state_model,  # noqa:SLF001  # Accessing private member to create copy like-object.
+                   )
+
+    def track_http_only_request_handler_runs(self) -> None:
+        self.predictor_state.http_only_request_handler_runs += 1
+
+    def track_browser_request_handler_runs(self) -> None:
+        self.predictor_state.browser_request_handler_runs += 1
+
+    def track_rendering_type_mispredictions(self) -> None:
+        self.predictor_state.rendering_type_mispredictions += 1
+
+    @override
+    async def _persist_other_statistics(self, key_value_store: KeyValueStore) -> None:
+        """Persist state of predictor."""
+        await key_value_store.set_value(
+            self._persist_predictor_state_key,
+            self.predictor_state.model_dump(mode='json', by_alias=True),
+            'application/json',
+        )
+
+
+    @override
+    async def _load_other_statistics(self, key_value_store: KeyValueStore) -> None:
+        """Load state of predictor."""
+        stored_state = await key_value_store.get_value(self._persist_predictor_state_key, cast(Any, {}))
+        self.predictor_state = self.predictor_state.__class__.model_validate(stored_state)
+

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, fields
+from typing import TYPE_CHECKING
+
+from bs4 import BeautifulSoup
+
+from crawlee import HttpHeaders
+from crawlee._utils.docs import docs_group
+from crawlee.crawlers import BeautifulSoupCrawlingContext, BeautifulSoupParserType, PlaywrightCrawlingContext
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+    from playwright.async_api import Page, Response
+    from typing_extensions import Self
+
+class AdaptiveContextError(RuntimeError):
+    pass
+
+
+
+@dataclass(frozen=True)
+@docs_group('Data structures')
+class AdaptivePlaywrightCrawlingContext(BeautifulSoupCrawlingContext):
+    _response: Response | None = None
+    _infinite_scroll: Callable[[], Awaitable[None]] | None = None
+    _page : Page | None = None
+
+    @property
+    def page(self) -> Page:
+        if not self._page:
+            raise AdaptiveContextError('Page was not crawled with PlaywrightCrawler.')
+        return self._page
+
+    @property
+    def infinite_scroll(self) -> Callable[[], Awaitable[None]]:
+        if not self._infinite_scroll:
+            raise AdaptiveContextError('Page was not crawled with PlaywrightCrawler.')
+        return self._infinite_scroll
+
+    @property
+    def response(self) -> Response:
+        if not self._response:
+            raise AdaptiveContextError('Page was not crawled with PlaywrightCrawler.')
+        return self._response
+
+    @classmethod
+    def from_beautifulsoup_crawling_context(cls, context: BeautifulSoupCrawlingContext) -> Self:
+        """Convenience constructor that creates new context from existing `BeautifulSoupCrawlingContext`."""
+        return cls(**{field.name: getattr(context, field.name) for field in fields(context)})
+
+    @classmethod
+    async def from_playwright_crawling_context(cls, context: PlaywrightCrawlingContext,
+                                            beautiful_soup_parser_type: BeautifulSoupParserType | None) -> Self:
+        """Convenience constructor that creates new context from existing `PlaywrightCrawlingContext`."""
+        context_kwargs = {field.name: getattr(context, field.name) for field in fields(context)}
+        # Remove playwright specific attributes and pass them as private instead to be available as property.
+        context_kwargs['_response'] = context_kwargs.pop('response')
+        context_kwargs['_page'] = context_kwargs.pop('page')
+        context_kwargs['_infinite_scroll'] = context_kwargs.pop('infinite_scroll')
+        # This might be always available.
+        protocol_guess = await context_kwargs['_page'].evaluate('() => performance.getEntries()[0].nextHopProtocol')
+        http_response = await _HttpResponse.from_playwright_response(response = context.response,
+                                                                     protocol = protocol_guess or '')
+        return cls(parsed_content= BeautifulSoup(http_response.read(), features=beautiful_soup_parser_type),
+                   http_response = http_response,
+                   **context_kwargs)
+
+
+@dataclass(frozen=True)
+class _HttpResponse:
+    http_version : str
+    status_code : int
+    headers: HttpHeaders
+    _content: bytes
+
+    def read(self) -> bytes:
+        return self._content
+
+    @classmethod
+    async def from_playwright_response(cls, response: Response, protocol: str) -> Self:
+        headers = HttpHeaders(response.headers)
+        status_code = response.status
+        # Can't find this anywhere in PlayWright, but some headers can include information about protocol.
+        # In firefox for example: 'x-firefox-spdy'
+        # Might be also obtained by executing JS code in browser: performance.getEntries()[0].nextHopProtocol
+        # Response headers capitalization not respecting http1.1 Pascal case. Always lower case in PlayWright.
+        http_version = protocol
+        _content = await response.body()
+
+        return cls(http_version=http_version, status_code=status_code, headers=headers, _content=_content)

src/crawlee/crawlers/_adaptive_playwright/_example.py

@@ -0,0 +1,45 @@
+import asyncio
+import logging
+from logging import getLogger
+
+from crawlee._types import BasicCrawlingContext
+from crawlee.crawlers import PlaywrightPreNavCrawlingContext
+from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawler import AdaptivePlaywrightCrawler
+from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import (
+    AdaptivePlaywrightCrawlingContext,
+)
+
+
+async def main() ->None:
+    # TODO: remove in review. Move this to documentation examples instead.
+    top_logger = getLogger(__name__)
+    top_logger.setLevel(logging.DEBUG)
+    i=0
+
+    crawler = AdaptivePlaywrightCrawler(max_requests_per_crawl=10,
+                                        _logger=top_logger,
+                                        playwright_crawler_args={'headless':False})
+
+    @crawler.router.default_handler
+    async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
+        nonlocal i
+        i = i+1
+        context.log.info(f'Processing with Top adaptive_crawler: {context.request.url} ...')
+        await context.enqueue_links()
+        await context.push_data({'Top crwaler Url': context.request.url})
+        await context.use_state({'bla':i})
+
+    @crawler.pre_navigation_hook_bs
+    async def bs_hook(context: BasicCrawlingContext) -> None:
+        context.log.info(f'BS pre navigation hook for: {context.request.url} ...')
+
+    @crawler.pre_navigation_hook_pw
+    async def pw_hook(context: PlaywrightPreNavCrawlingContext) -> None:
+        context.log.info(f'PW pre navigation hook for: {context.request.url} ...')
+
+    # Run the crawler with the initial list of URLs.
+    await crawler.run(['https://crawlee.dev/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from random import choice
+from typing import Literal
+
+from typing_extensions import override
+
+RenderingType = Literal['static', 'client only']
+
+@dataclass(frozen=True)
+class RenderingTypePrediction:
+    rendering_type: RenderingType
+    detection_probability_recommendation: float
+
+
+
+class RenderingTypePredictor(ABC):
+
+    @abstractmethod
+    def predict(self, url: str, label: str | None) -> RenderingTypePrediction:
+        ...
+
+    @abstractmethod
+    def store_result(self, url: str, label: str | None, crawl_type: RenderingType) -> None:
+        ...
+
+
+class DefaultRenderingTypePredictor(RenderingTypePredictor):
+    #Dummy version of predictor. Proper version will be implemented in another change.
+
+    @override
+    def predict(self, url: str, label: str | None) -> RenderingTypePrediction: # Will be implemented later
+        return RenderingTypePrediction(choice(['static', 'client only']), 0.1)
+
+    @override
+    def store_result(self, url: str, label: str | None, crawl_type: RenderingType) -> None:
+        pass

src/crawlee/crawlers/_adaptive_playwright/_result_comparator.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from crawlee._types import RequestHandlerRunResult
+
+
+@dataclass(frozen=True)
+class SubCrawlerRun:
+    result: RequestHandlerRunResult | None= None
+    exception: Exception | None= None
+
+
+
+def create_comparator(result_checker: Callable[[RequestHandlerRunResult], bool] | None
+                      ) -> Callable[[RequestHandlerRunResult, RequestHandlerRunResult], bool]:
+    """Factory for creating comparator function."""
+    if result_checker:
+        # Fallback comparator if only user-specific checker is defined.
+        return lambda result_1, result_2: result_checker(result_1) and result_checker(result_2)
+    # Fallback default comparator.
+    return push_data_only_comparator
+
+
+def full_result_comparator(result_1: RequestHandlerRunResult, result_2: RequestHandlerRunResult) -> bool:
+    """Compare results by comparing all their parts."""
+    # PlayWright can produce links with extra arguments compared to pure BS. Default comparator ignores this.
+    # Maybe full comparator should have flag about taking into account only urls without parameters.
+    # https://sdk.apify.com/docs/guides/getting-started
+    # https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.1735494277124&__hstc=7591405.e2b9302ed00c5bfaee3a870166792181.1735494277124.1735494277124.1735494277124.1
+
+    return (
+        (result_1.push_data_calls == result_2.push_data_calls) and
+        (result_1.add_requests_calls == result_2.add_requests_calls) and
+        (result_1.key_value_store_changes == result_2.key_value_store_changes))
+
+def push_data_only_comparator(result_1: RequestHandlerRunResult, result_2: RequestHandlerRunResult) -> bool:
+    """Compare results by comparing their push dara calls. Ignore other parts of results in comparison."""
+    return result_1.push_data_calls == result_2.push_data_calls