Merge pull request #27 from scrapinghub/handle_urls_mvp

kmike · web-flow · commit 831754b301ba · 2022-04-11T00:46:56.000+05:00
introduce PageObjectRegistry with @hande_urls annotations
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -5,6 +5,8 @@ Changelog
 TBR
 ------------------
 
+* added a ``PageObjectRegistry`` class which has the  ``handle_urls`` decorator
+  to conveniently declare and collect ``OverrideRule``.
 * removed support for Python 3.6
 * added support for Python 3.10
 * Backward Incompatible Change:
@@ -13,7 +15,6 @@ TBR
       specific attribute types like ``HttpResponseBody`` and
       ``HttpResponseHeaders``.
 
-
 0.1.1 (2021-06-02)
 ------------------
 
diff --git a/docs/api_reference.rst b/docs/api_reference.rst
@@ -1,3 +1,5 @@
+.. _`api-reference`:
+
 =============
 API Reference
 =============
@@ -45,3 +47,18 @@ Mixins
 .. autoclass:: web_poet.mixins.ResponseShortcutsMixin
    :members:
    :no-special-members:
+
+
+.. _`api-overrides`:
+
+Overrides
+=========
+
+See the tutorial section on :ref:`intro-overrides` for more context about its
+use cases and some examples.
+
+.. autofunction:: web_poet.handle_urls
+
+.. automodule:: web_poet.overrides
+   :members:
+   :exclude-members: handle_urls
diff --git a/docs/conf.py b/docs/conf.py
@@ -192,5 +192,6 @@
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None, ),
     'scrapy': ('https://docs.scrapy.org/en/latest', None, ),
+    'url-matcher': ('https://url-matcher.readthedocs.io/en/stable/', None, ),
     'parsel': ('https://parsel.readthedocs.io/en/latest/', None, ),
 }
diff --git a/docs/index.rst b/docs/index.rst
@@ -33,6 +33,7 @@ and the motivation behind ``web-poet``, start with :ref:`from-ground-up`.
 
    intro/tutorial
    intro/from-ground-up
+   intro/overrides
 
 .. toctree::
    :caption: Reference
diff --git a/docs/intro/overrides.rst b/docs/intro/overrides.rst
diff --git a/setup.py b/setup.py
@@ -22,6 +22,7 @@
     install_requires=[
         'attrs >= 21.3.0',
         'parsel',
+        'url-matcher',
         'multidict',
         'w3lib >= 1.22.0',
     ],
diff --git a/tests/po_lib/__init__.py b/tests/po_lib/__init__.py
@@ -0,0 +1,39 @@
+"""
+This package is just for overrides testing purposes.
+"""
+from typing import Dict, Any, Callable
+
+from url_matcher import Patterns
+
+from .. import po_lib_sub # NOTE: this module contains a PO with @handle_rules
+from web_poet import handle_urls, PageObjectRegistry
+
+
+class POBase:
+    expected_overrides: Callable
+    expected_patterns: Patterns
+    expected_meta: Dict[str, Any]
+
+
+class POTopLevelOverriden1:
+    ...
+
+
+class POTopLevelOverriden2:
+    ...
+
+
+# This first annotation is ignored. A single annotation per registry is allowed
+@handle_urls("example.com", overrides=POTopLevelOverriden1)
+@handle_urls("example.com", overrides=POTopLevelOverriden1, exclude="/*.jpg|", priority=300)
+class POTopLevel1(POBase):
+    expected_overrides = POTopLevelOverriden1
+    expected_patterns = Patterns(["example.com"], ["/*.jpg|"], priority=300)
+    expected_meta = {}  # type: ignore
+
+
+@handle_urls("example.com", overrides=POTopLevelOverriden2)
+class POTopLevel2(POBase):
+    expected_overrides = POTopLevelOverriden2
+    expected_patterns = Patterns(["example.com"])
+    expected_meta = {}  # type: ignore
diff --git a/tests/po_lib/a_module.py b/tests/po_lib/a_module.py
@@ -0,0 +1,16 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase
+from web_poet import handle_urls
+
+
+class POModuleOverriden:
+    ...
+
+
+@handle_urls("example.com", overrides=POModuleOverriden, extra_arg="foo")
+class POModule(POBase):
+    expected_overrides = POModuleOverriden
+    expected_patterns = Patterns(["example.com"])
+    expected_meta = {"extra_arg": "foo"}  # type: ignore
+
diff --git a/tests/po_lib/an_empty_module.py b/tests/po_lib/an_empty_module.py
diff --git a/tests/po_lib/an_empty_package/__init__.py b/tests/po_lib/an_empty_package/__init__.py
diff --git a/tests/po_lib/nested_package/__init__.py b/tests/po_lib/nested_package/__init__.py
@@ -0,0 +1,15 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase
+from web_poet import handle_urls
+
+
+class PONestedPkgOverriden:
+    ...
+
+
+@handle_urls(include=["example.com", "example.org"], exclude=["/*.jpg|"], overrides=PONestedPkgOverriden)
+class PONestedPkg(POBase):
+    expected_overrides = PONestedPkgOverriden
+    expected_patterns = Patterns(["example.com", "example.org"], ["/*.jpg|"])
+    expected_meta = {}  # type: ignore
diff --git a/tests/po_lib/nested_package/a_nested_module.py b/tests/po_lib/nested_package/a_nested_module.py
@@ -0,0 +1,15 @@
+from url_matcher import Patterns
+
+from tests.po_lib import POBase
+from web_poet import handle_urls
+
+
+class PONestedModuleOverriden:
+    ...
+
+
+@handle_urls(include=["example.com", "example.org"], exclude=["/*.jpg|"], overrides=PONestedModuleOverriden)
+class PONestedModule(POBase):
+    expected_overrides = PONestedModuleOverriden
+    expected_patterns = Patterns(include=["example.com", "example.org"], exclude=["/*.jpg|"])
+    expected_meta = {}  # type: ignore
diff --git a/tests/po_lib_sub/__init__.py b/tests/po_lib_sub/__init__.py
@@ -0,0 +1,25 @@
+"""This package is being used by tests/po_lib to validate some behaviors on
+external depedencies.
+"""
+from typing import Dict, Any, Callable
+
+from url_matcher import Patterns
+
+from web_poet import handle_urls
+
+
+class POBase:
+    expected_overrides: Callable
+    expected_patterns: Patterns
+    expected_meta: Dict[str, Any]
+
+
+class POLibSubOverriden:
+    ...
+
+
+@handle_urls("sub_example.com", overrides=POLibSubOverriden)
+class POLibSub(POBase):
+    expected_overrides = POLibSubOverriden
+    expected_patterns = Patterns(["sub_example.com"])
+    expected_meta = {}  # type: ignore
diff --git a/tests/test_overrides.py b/tests/test_overrides.py
@@ -0,0 +1,112 @@
+import pytest
+from url_matcher import Patterns
+
+from tests.po_lib_sub import POLibSub
+from tests.po_lib import (
+    POTopLevel1,
+    POTopLevel2,
+    POTopLevelOverriden2,
+)
+from tests.po_lib.a_module import POModule, POModuleOverriden
+from tests.po_lib.nested_package import PONestedPkg
+from tests.po_lib.nested_package.a_nested_module import PONestedModule
+from web_poet import (
+    default_registry,
+    consume_modules,
+    OverrideRule,
+    PageObjectRegistry,
+)
+
+
+POS = {POTopLevel1, POTopLevel2, POModule, PONestedPkg, PONestedModule}
+
+
+def test_override_rule_uniqueness():
+    """The same instance of an OverrideRule with the same attribute values should
+    have the same hash identity.
+    """
+
+    patterns = Patterns(include=["example.com"], exclude=["example.com/blog"])
+
+    rule1 = OverrideRule(
+        for_patterns=patterns,
+        use=POTopLevel1,
+        instead_of=POTopLevelOverriden2,
+        meta={"key_1": 1},
+    )
+    rule2 = OverrideRule(
+        for_patterns=patterns,
+        use=POTopLevel1,
+        instead_of=POTopLevelOverriden2,
+        meta={"key_2": 2},
+    )
+
+    assert hash(rule1) == hash(rule2)
+
+
+def test_list_page_objects_all():
+    rules = default_registry.get_overrides()
+    page_objects = {po.use for po in rules}
+
+    # Note that the 'tests_extra.po_lib_sub_not_imported.POLibSubNotImported'
+    # Page Object is not included here since it was never imported anywhere in
+    # our test package. It would only be included if we run any of the following
+    # below. (Note that they should run before `get_overrides` is called.)
+    #   - from tests_extra import po_lib_sub_not_imported
+    #   - import tests_extra.po_lib_sub_not_imported
+    #   - web_poet.consume_modules("tests_extra")
+    # Merely having `import tests_extra` won't work since the subpackages and
+    # modules needs to be traversed and imported as well.
+    assert all(["po_lib_sub_not_imported" not in po.__module__ for po in page_objects])
+
+    # Ensure that ALL Override Rules are returned as long as the given
+    # registry's @handle_urls annotation was used.
+    assert page_objects == POS.union({POLibSub})
+    for rule in rules:
+        assert rule.instead_of == rule.use.expected_overrides, rule.use
+        assert rule.for_patterns == rule.use.expected_patterns, rule.use
+        assert rule.meta == rule.use.expected_meta, rule.use
+
+
+def test_consume_module_not_existing():
+    with pytest.raises(ImportError):
+        consume_modules("this_does_not_exist")
+
+
+def test_list_page_objects_all_consume():
+    """A test similar to the one above but calls ``consume_modules()`` to properly
+    load the @handle_urls annotations from other modules/packages.
+    """
+    consume_modules("tests_extra")
+    rules = default_registry.get_overrides()
+    page_objects = {po.use for po in rules}
+    assert any(["po_lib_sub_not_imported" in po.__module__ for po in page_objects])
+
+
+def test_registry_search_overrides():
+    rules = default_registry.search_overrides(use=POTopLevel2)
+    assert len(rules) == 1
+    assert rules[0].use == POTopLevel2
+
+    rules = default_registry.search_overrides(instead_of=POTopLevelOverriden2)
+    assert len(rules) == 1
+    assert rules[0].instead_of == POTopLevelOverriden2
+
+    # Such rules doesn't exist
+    rules = default_registry.search_overrides(use=POModuleOverriden)
+    assert len(rules) == 0
+
+
+def test_from_override_rules():
+    rules = [
+        OverrideRule(
+            for_patterns=Patterns(include=["sample.com"]),
+            use=POTopLevel1,
+            instead_of=POTopLevelOverriden2,
+        )
+    ]
+
+    registry = PageObjectRegistry.from_override_rules(rules)
+
+    assert registry.get_overrides() == rules
+    assert default_registry.get_overrides() != rules
diff --git a/tests_extra/__init__.py b/tests_extra/__init__.py
@@ -0,0 +1,5 @@
+"""
+This test package was created separately to see the behavior of retrieving the
+Override rules declared on a registry where @handle_urls is defined on another
+package.
+"""
diff --git a/tests_extra/po_lib_sub_not_imported/__init__.py b/tests_extra/po_lib_sub_not_imported/__init__.py
@@ -0,0 +1,28 @@
+"""
+This package quite is similar to tests/po_lib_sub in terms of code contents.
+
+What we're ultimately trying to test here is to see if the `default_registry`
+captures the rules annotated in this module if it was not imported.
+"""
+from typing import Dict, Any, Callable
+
+from url_matcher import Patterns
+
+from web_poet import handle_urls
+
+
+class POBase:
+    expected_overrides: Callable
+    expected_patterns: Patterns
+    expected_meta: Dict[str, Any]
+
+
+class POLibSubOverridenNotImported:
+    ...
+
+
+@handle_urls("sub_example_not_imported.com", overrides=POLibSubOverridenNotImported)
+class POLibSubNotImported(POBase):
+    expected_overrides = POLibSubOverridenNotImported
+    expected_patterns = Patterns(["sub_example_not_imported.com"])
+    expected_meta = {}  # type: ignore
diff --git a/web_poet/__init__.py b/web_poet/__init__.py
@@ -1,2 +1,6 @@
 from .pages import WebPage, ItemPage, ItemWebPage, Injectable
 from .page_inputs import HttpResponse, HttpResponseBody, HttpResponseHeaders
+from .overrides import PageObjectRegistry, consume_modules, OverrideRule
+
+default_registry = PageObjectRegistry()
+handle_urls = default_registry.handle_urls
diff --git a/web_poet/overrides.py b/web_poet/overrides.py
diff --git a/web_poet/utils.py b/web_poet/utils.py

Original file line number	Diff line number	Diff line change
`@@ -192,5 +192,6 @@`
`192`	`192`	`intersphinx_mapping = {`
`193`	`193`	`'python': ('https://docs.python.org/3', None, ),`
`194`	`194`	`'scrapy': ('https://docs.scrapy.org/en/latest', None, ),`
	`195`	`+ 'url-matcher': ('https://url-matcher.readthedocs.io/en/stable/', None, ),`
`195`	`196`	`'parsel': ('https://parsel.readthedocs.io/en/latest/', None, ),`
`196`	`197`	`}`