Properly handle pagination in the REST backend

pwclient/api.py

@@ -19,6 +19,8 @@
 from . import __version__
 from .xmlrpc import xmlrpclib
 
+from requests.utils import parse_header_links
+
 
 class API(metaclass=abc.ABCMeta):
     @abc.abstractmethod
@@ -594,8 +596,23 @@ def _list(
             url = f'{url}{resource_id}/{subresource_type}/'
         if params:
             url = f'{url}?{urllib.parse.urlencode(params)}'
-        data, _ = self._get(url)
-        return json.loads(data)
+
+        while url:
+            data, headers = self._get(url)
+            ret = json.loads(data)
+
+            yield from ret
+
+            try:
+                link = next(value for name, value in headers if name == 'Link')
+                url = next(
+                    a['url']
+                    for a in parse_header_links(link)
+                    if a['rel'] == "next"
+                )
+
+            except StopIteration:
+                url = None
 
     # project
 
@@ -627,7 +644,7 @@ def project_list(self, search_str=None, max_count=0):
             )
 
         projects = self._list('projects')
-        return [self._project_to_dict(project) for project in projects]
+        return (self._project_to_dict(project) for project in projects)
 
     def project_get(self, project_id):
         project = self._detail('projects', project_id)
@@ -662,7 +679,7 @@ def person_list(self, search_str=None, max_count=0):
             )
 
         people = self._list('people')
-        return [self._person_to_dict(person) for person in people]
+        return (self._person_to_dict(person) for person in people)
 
     def person_get(self, person_id):
         person = self._detail('people', person_id)
@@ -747,11 +764,14 @@ def patch_list(
         if archived is not None:
             filters['archived'] = archived
 
+        if submitter is not None:
+            filters['submitter'] = submitter
+
         if delegate is not None:
             filters['delegate'] = delegate
 
         patches = self._list('patches', params=filters)
-        return [self._patch_to_dict(patch) for patch in patches]
+        return (self._patch_to_dict(patch) for patch in patches)
 
     def patch_get(self, patch_id):
         patch = self._detail('patches', patch_id)

pwclient/patches.py

@@ -6,6 +6,7 @@
 
 import io
 import itertools
+import operator
 import os
 import re
 import subprocess
@@ -56,6 +57,10 @@ def patch_field(matchobj):
             )
 
 
+def sortgroupby(iterable, key=None):
+    return itertools.groupby(sorted(iterable, key=key), key=key)
+
+
 def action_list(
     api,
     project=None,
@@ -89,10 +94,9 @@ def action_list(
         filters['submitter'] = submitter
 
         patches = api.patch_list(**filters)
-        patches.sort(key=lambda x: x['submitter'])
 
-        for person, person_patches in itertools.groupby(
-            patches, key=lambda x: x['submitter']
+        for person, person_patches in sortgroupby(
+            patches, key=operator.itemgetter('submitter')
         ):
             print(f'Patches submitted by {person}:')
             _list_patches(list(person_patches), format_str)
@@ -103,10 +107,9 @@ def action_list(
         filters['delegate'] = delegate
 
         patches = api.patch_list(**filters)
-        patches.sort(key=lambda x: x['delegate'])
 
-        for delegate, delegate_patches in itertools.groupby(
-            patches, key=lambda x: x['delegate']
+        for delegate, delegate_patches in sortgroupby(
+            patches, key=operator.itemgetter('delegate')
         ):
             print(f'Patches delegated to {delegate}:')
             _list_patches(list(delegate_patches), format_str)

requirements.txt

@@ -0,0 +1,2 @@
+requests
+types-requests