feat(gitlab): implemented keyset pagination for gitlab #8529

backend/plugins/gitlab/tasks/account_collector.go

@@ -36,6 +36,7 @@ func init() {
 }
 
 const RAW_USER_TABLE = "gitlab_api_users"
+const KEYSET_MIN_VERSION = "v16.5.0"
 
 var CollectAccountsMeta = plugin.SubTaskMeta{
 	Name:             "Collect Users",
@@ -62,13 +63,36 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 		urlTemplate = "/users"
 	}
 
+	apiVersion := data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string)
+
+	useKeyset := false
+	if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(apiVersion, KEYSET_MIN_VERSION) >= 0 {
+		useKeyset = true
+	} else if urlTemplate == "/users" && !semver.IsValid(apiVersion) {
+		// If version unknown, be conservative for CE 11–16.4: default to offset
+		logger.Debug("GitLab version is unknown/invalid; falling back to offset pagination for /users")
+	}
+
+	var lastID int
+
 	collector, err := api.NewApiCollector(api.ApiCollectorArgs{
 		RawDataSubTaskArgs: *rawDataSubTaskArgs,
 		ApiClient:          data.ApiClient,
 		UrlTemplate:        urlTemplate,
 		PageSize:           100,
 		Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
 			query := url.Values{}
+			// Use keyset only when gated true and only on /users endpoint
+			if useKeyset && urlTemplate == "/users" {
+				query.Set("pagination", "keyset")
+				query.Set("order_by", "id")
+				query.Set("sort", "asc")
+				query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+				if lastID > 0 {
+					query.Set("id_after", fmt.Sprintf("%d", lastID))
+				}
+				return query, nil
+			}
 			query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
 			query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
 			return query, nil
@@ -80,6 +104,15 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
 			if err != nil {
 				return nil, err
 			}
+			if len(items) > 0 && urlTemplate == "/users" {
+				var tail struct {
+					ID int `json:"id"`
+				}
+				_ = json.Unmarshal(items[len(items)-1], &tail)
+				if tail.ID > 0 {
+					lastID = tail.ID
+				}
+			}
 			return items, nil
 		},
 	})