Merge pull request #72 from dlt-hub/fix/fix_paginator_rendering

correct behavior for paginators with unknown total_path

Author: sh-rp

dlt_openapi/detector/default/__init__.py

@@ -400,7 +400,9 @@ def detect_pagination(self, endpoint: Endpoint) -> Optional[Pagination]:
             # 20 should be safe for most APIs
             limit_initial = to_int(limit_param.maximum) if limit_param.maximum else to_int(limit_param.default)
         total_prop = (
-            response_schema.nested_properties.find_property(RE_TOTAL_PROPERTY, require_type="integer")
+            response_schema.nested_properties.find_property(
+                RE_TOTAL_PROPERTY, require_type="integer", allow_unknown_types=True
+            )
             if response_schema
             else None
         )
@@ -415,6 +417,7 @@ def detect_pagination(self, endpoint: Endpoint) -> Optional[Pagination]:
             if total_prop:
                 pagination_config["total_path"] = total_prop.json_path
             else:
+                pagination_config["total_path"] = ""
                 pagination_config["maximum_offset"] = DEFAULT_MAXIMUM_PAGINATOR_OFFSET
 
             return Pagination(
@@ -440,6 +443,7 @@ def detect_pagination(self, endpoint: Endpoint) -> Optional[Pagination]:
             if total_prop:
                 pagination_config["total_path"] = total_prop.json_path
             else:
+                pagination_config["total_path"] = ""
                 pagination_config["maximum_page"] = DEFAULT_MAXIMUM_PAGINATOR_OFFSET
 
             return Pagination(

dlt_openapi/detector/default/const.py

@@ -18,7 +18,7 @@
 RE_TOTAL_PAGE_PROPERTY = re.compile(r"(?i)(total|count)", re.IGNORECASE)
 RE_OFFSET_PARAM = re.compile(r"(?i)(start|offset|skip)", re.IGNORECASE)
 RE_LIMIT_PARAM = re.compile(r"(?i)(limit|per_page|page_size|size)", re.IGNORECASE)
-RE_TOTAL_PROPERTY = re.compile(r"(?i)(total|count)", re.IGNORECASE)
+RE_TOTAL_PROPERTY = re.compile(r"(?i)(total|count|total_count)", re.IGNORECASE)
 RE_CURSOR_PARAM = re.compile(r"(?i)(cursor|after|since)", re.IGNORECASE)
 RE_CURSOR_PROP = re.compile(r"(?i)(cursor|next_cursor)", re.IGNORECASE)
 RE_NEXT_PROPERTY = re.compile(r"(?i)(next|next_url|more)", re.IGNORECASE)

tests/integration/basics/test_paginator.py

@@ -30,6 +30,7 @@ def test_offset_limit_pagination_no_count_1(paginators: Dict[str, Any]) -> None:
         "offset_param": "offset",
         "type": "offset",
         "maximum_offset": 20,
+        "total_path": "",
     }
 
 
@@ -49,6 +50,7 @@ def test_page_number_paginator_no_count(paginators: Dict[str, Any]) -> None:
         "type": "page_number",
         "page_param": "page",
         "maximum_page": 20,
+        "total_path": "",
     }
 
 

tests/integration/basics/test_transformer.py

@@ -39,7 +39,6 @@ def test_simple_transformer_with_deselected_parent() -> None:
         config=Config(name_resources_by_operation=True, endpoint_filter=lambda _c: {"single_collection"}),
     )
 
-    print(resources.keys())
     assert len(resources) == 2
 
     assert resources["collections"] == {

tests/integration/extracted/test_dotastats_cases.py

@@ -12,6 +12,7 @@ def test_simple_dotastats_load() -> None:
         "maximum_page": 20,
         "page_param": "page",
         "type": "page_number",
+        "total_path": "",
     }
 
     # use page query parameter

tests/integration/extracted/test_museums_cases.py

@@ -12,6 +12,7 @@ def test_simple_museums_load() -> None:
         "maximum_page": 20,
         "page_param": "page",
         "type": "page_number",
+        "total_path": "",
     }
 
     assert source["resources"][0] == {
@@ -34,6 +35,7 @@ def test_simple_museums_pagination() -> None:
         "maximum_page": 20,
         "page_param": "page",
         "type": "page_number",
+        "total_path": "",
     }
 
     assert source["resources"][0] == {

tests/integration/extracted/test_newspilot_cases.py

@@ -13,6 +13,7 @@ def test_newspilot_load() -> None:
         "maximum_page": 20,
         "page_param": "page",
         "type": "page_number",
+        "total_path": "",
     }
 
     # FIXME: they use page and articles_per_page query parameters

tests/integration/extracted/test_openartnft_cases.py

@@ -14,6 +14,7 @@ def test_simple_openartnft_load() -> None:
         "maximum_offset": 20,
         "offset_param": "skip",
         "type": "offset",
+        "total_path": "",
     }
 
     assert source["resources"][0] == {

tests/integration/extracted/test_quotesapi_cases.py

@@ -16,6 +16,7 @@ def test_simple_quotesapi_load() -> None:
         "maximum_offset": 20,
         "offset_param": "start",
         "type": "offset",
+        "total_path": "",
     }
 
     assert source["resources"][0] == {

tests/integration/extracted/test_teamgpt_cases.py

@@ -12,6 +12,7 @@ def test_teamgpt_load() -> None:
         "maximum_page": 20,
         "page_param": "page",
         "type": "page_number",
+        "total_path": "",
     }
 
     # FIXME: they use page and page_size parameters