fix(crawler): skip forbidden crawlers in list and surface non-JSON API errors (#267)

Author: LorrisSaintGenez

api/crawler/client.go

@@ -3,10 +3,10 @@ package crawler
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 )
 
 const (
@@ -60,25 +60,31 @@ func (c *Client) request(
 	}
 
 	if resp.StatusCode >= 400 {
-		var errResp ErrResponse
-		if err := unmarshalTo(resp, &errResp); err != nil {
-			return err
-		}
+		raw, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
 
-		if errResp.Err.Errors != nil {
-			var errs []string
-			for _, e := range errResp.Err.Errors {
-				errs = append(errs, e.Message)
+		message := strings.TrimSpace(string(raw))
+
+		var errResp ErrResponse
+		if json.Unmarshal(raw, &errResp) == nil {
+			if errResp.Err.Errors != nil {
+				var errs []string
+				for _, e := range errResp.Err.Errors {
+					errs = append(errs, e.Message)
+				}
+				message = fmt.Sprintf("[%s] %s", errResp.Err.Code, errs)
+			} else if errResp.Err.Message != "" {
+				message = fmt.Sprintf("[%s] %s", errResp.Err.Code, errResp.Err.Message)
+			} else if errResp.Err.Code != "" {
+				message = errResp.Err.Code
 			}
-			return fmt.Errorf("[%s] %s", errResp.Err.Code, errs)
 		}
 
-		// Message might be empty
-		if errResp.Err.Message == "" {
-			return errors.New(errResp.Err.Code)
-		} else {
-			return fmt.Errorf("[%s] %s", errResp.Err.Code, errResp.Err.Message)
+		if message == "" {
+			message = http.StatusText(resp.StatusCode)
 		}
+
+		return &APIError{StatusCode: resp.StatusCode, Message: message}
 	}
 
 	if res != nil {

api/crawler/client_test.go

@@ -0,0 +1,44 @@
+package crawler
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+func newTestClient(fn roundTripFunc) *Client {
+	return NewClientWithHTTPClient("user", "key", &http.Client{Transport: fn})
+}
+
+func TestRequest_PlainTextError(t *testing.T) {
+	client := newTestClient(func(_ *http.Request) (*http.Response, error) {
+		return &http.Response{
+			StatusCode: http.StatusForbidden,
+			Header:     http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
+			Body:       io.NopCloser(strings.NewReader("Forbidden")),
+		}, nil
+	})
+
+	_, err := client.Get("some-id", true)
+	if err == nil {
+		t.Fatal("expected an error, got nil")
+	}
+
+	apiErr, ok := err.(*APIError)
+	if !ok {
+		t.Fatalf("expected *APIError, got %T: %v", err, err)
+	}
+	if apiErr.StatusCode != http.StatusForbidden {
+		t.Errorf("expected status 403, got %d", apiErr.StatusCode)
+	}
+	if apiErr.Error() != "Forbidden" {
+		t.Errorf("expected message %q, got %q", "Forbidden", apiErr.Error())
+	}
+}

api/crawler/types.go

@@ -6,6 +6,16 @@ import (
 	"github.com/algolia/algoliasearch-client-go/v4/algolia/search"
 )
 
+// APIError is an error returned by the Crawler API, carrying the HTTP status.
+type APIError struct {
+	StatusCode int
+	Message    string
+}
+
+func (e *APIError) Error() string {
+	return e.Message
+}
+
 // ErrResponse is a Crawler API error response.
 type ErrResponse struct {
 	Err Err `json:"error"`

pkg/cmd/crawler/list/list.go

@@ -1,7 +1,9 @@
 package list
 
 import (
+	"errors"
 	"fmt"
+	"net/http"
 	"sort"
 
 	"github.com/MakeNowJust/heredoc"
@@ -88,6 +90,10 @@ func runListCmd(opts *ListOptions) error {
 		opts.IO.UpdateProgressIndicatorLabel(fmt.Sprintf("Fetching Crawler %s details", item.ID))
 		c, err := client.Get(item.ID, true)
 		if err != nil {
+			var apiErr *crawler.APIError
+			if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusForbidden {
+				continue
+			}
 			opts.IO.StopProgressIndicator()
 			return err
 		}

pkg/cmd/crawler/list/list_test.go

@@ -0,0 +1,81 @@
+package list
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/algolia/cli/api/crawler"
+	"github.com/algolia/cli/pkg/cmdutil"
+	"github.com/algolia/cli/pkg/iostreams"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+func jsonResponse(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: status,
+		Header:     http.Header{"Content-Type": []string{"application/json"}},
+		Body:       io.NopCloser(strings.NewReader(body)),
+	}
+}
+
+func Test_runListCmd_skipsForbiddenCrawlers(t *testing.T) {
+	transport := roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/crawlers"):
+			return jsonResponse(200, `{
+				"items": [
+					{"id": "ok-id", "name": "accessible"},
+					{"id": "forbidden-id", "name": "not-mine"}
+				],
+				"page": 1, "itemsPerPage": 20, "total": 2
+			}`), nil
+		case strings.Contains(r.URL.Path, "forbidden-id"):
+			return &http.Response{
+				StatusCode: http.StatusForbidden,
+				Header:     http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
+				Body:       io.NopCloser(strings.NewReader("Forbidden")),
+			}, nil
+		case strings.Contains(r.URL.Path, "ok-id"):
+			return jsonResponse(200, `{
+				"name": "accessible",
+				"createdAt": "2026-01-01T00:00:00.000Z",
+				"updatedAt": "2026-01-02T00:00:00.000Z",
+				"running": false,
+				"blocked": false,
+				"config": {"appId": "APP_ID"}
+			}`), nil
+		default:
+			t.Fatalf("unexpected request: %s", r.URL.Path)
+			return nil, nil
+		}
+	})
+
+	client := crawler.NewClientWithHTTPClient("user", "key", &http.Client{Transport: transport})
+
+	ios, _, stdout, _ := iostreams.Test()
+	printFlags := cmdutil.NewPrintFlags()
+	printFlags.OutputFlagSpecified = func() bool { return false }
+	opts := &ListOptions{
+		IO: ios,
+		CrawlerClient: func() (*crawler.Client, error) {
+			return client, nil
+		},
+		PrintFlags: printFlags,
+	}
+
+	err := runListCmd(opts)
+	require.NoError(t, err)
+
+	assert.Contains(t, stdout.String(), "accessible")
+	assert.NotContains(t, stdout.String(), "not-mine")
+}