Skip to content

Commit 2dcf13a

Browse files
vincentkocvincentkoc
authored
fix(api): retry transient Notion failures (#21)
1 parent 091db7b commit 2dcf13a

2 files changed

Lines changed: 156 additions & 39 deletions

File tree

‎internal/notionapi/api.go‎

Lines changed: 107 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717

1818
const SourceName = "api"
1919

20+
const maxAPIAttempts = 4
21+
2022
type Client struct {
2123
BaseURL string
2224
Version string
@@ -448,56 +450,50 @@ func (c Client) ingestComments(ctx context.Context, st *store.Store, pageID, spa
448450
}
449451

450452
func (c Client) do(ctx context.Context, method, path string, body any, out any) error {
451-
var reader io.Reader
453+
var bodyBytes []byte
452454
if body != nil {
453455
b, err := json.Marshal(body)
454456
if err != nil {
455457
return err
456458
}
457-
reader = bytes.NewReader(b)
458-
}
459-
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(c.BaseURL, "/")+path, reader)
460-
if err != nil {
461-
return err
459+
bodyBytes = b
462460
}
463-
req.Header.Set("Authorization", "Bearer "+c.Token)
464-
req.Header.Set("Notion-Version", c.Version)
465-
req.Header.Set("Accept", "application/json")
466-
if body != nil {
467-
req.Header.Set("Content-Type", "application/json")
468-
}
469-
resp, err := c.HTTP.Do(req)
470-
if err != nil {
471-
return err
472-
}
473-
defer resp.Body.Close()
474-
if resp.StatusCode == http.StatusTooManyRequests {
475-
if wait, err := time.ParseDuration(resp.Header.Get("Retry-After") + "s"); err == nil && wait > 0 {
476-
timer := time.NewTimer(wait)
477-
select {
478-
case <-ctx.Done():
479-
timer.Stop()
480-
return ctx.Err()
481-
case <-timer.C:
482-
}
483-
return c.do(ctx, method, path, body, out)
461+
for attempt := 1; attempt <= maxAPIAttempts; attempt++ {
462+
var reader io.Reader
463+
if bodyBytes != nil {
464+
reader = bytes.NewReader(bodyBytes)
484465
}
485-
}
486-
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
487-
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
488-
bodyText := strings.TrimSpace(string(b))
489-
apiErr := notionAPIError{Method: method, Path: path, Status: resp.Status, StatusCode: resp.StatusCode, Body: bodyText}
490-
var payload struct {
491-
Code string `json:"code"`
492-
Message string `json:"message"`
466+
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(c.BaseURL, "/")+path, reader)
467+
if err != nil {
468+
return err
493469
}
494-
if err := json.Unmarshal(b, &payload); err == nil {
495-
apiErr.Code = payload.Code
496-
apiErr.Message = payload.Message
470+
req.Header.Set("Authorization", "Bearer "+c.Token)
471+
req.Header.Set("Notion-Version", c.Version)
472+
req.Header.Set("Accept", "application/json")
473+
if body != nil {
474+
req.Header.Set("Content-Type", "application/json")
475+
}
476+
resp, err := c.HTTP.Do(req)
477+
if err != nil {
478+
return err
479+
}
480+
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
481+
defer resp.Body.Close()
482+
return json.NewDecoder(resp.Body).Decode(out)
483+
}
484+
485+
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
486+
resp.Body.Close()
487+
apiErr := apiErrorFromResponse(method, path, resp, b)
488+
if attempt < maxAPIAttempts && shouldRetry(apiErr) {
489+
if err := waitBeforeRetry(ctx, apiErr.RetryAfter); err != nil {
490+
return err
491+
}
492+
continue
497493
}
498494
return apiErr
499495
}
500-
return json.NewDecoder(resp.Body).Decode(out)
496+
return nil
501497
}
502498

503499
type notionAPIError struct {
@@ -508,6 +504,8 @@ type notionAPIError struct {
508504
Code string
509505
Message string
510506
Body string
507+
RetryAfter time.Duration
508+
Retryable bool
511509
}
512510

513511
func (e notionAPIError) Error() string {
@@ -517,6 +515,76 @@ func (e notionAPIError) Error() string {
517515
return fmt.Sprintf("notion api %s %s: %s: %s", e.Method, e.Path, e.Status, e.Body)
518516
}
519517

518+
func apiErrorFromResponse(method, path string, resp *http.Response, body []byte) notionAPIError {
519+
bodyText := strings.TrimSpace(string(body))
520+
apiErr := notionAPIError{
521+
Method: method,
522+
Path: path,
523+
Status: resp.Status,
524+
StatusCode: resp.StatusCode,
525+
Body: bodyText,
526+
RetryAfter: retryAfter(resp.Header.Get("Retry-After"), body),
527+
}
528+
var payload struct {
529+
Code string `json:"code"`
530+
Message string `json:"message"`
531+
Retryable bool `json:"retryable"`
532+
RetryAfter float64 `json:"retry_after"`
533+
}
534+
if err := json.Unmarshal(body, &payload); err == nil {
535+
apiErr.Code = payload.Code
536+
apiErr.Message = payload.Message
537+
apiErr.Retryable = payload.Retryable
538+
if payload.RetryAfter > 0 && apiErr.RetryAfter == 0 {
539+
apiErr.RetryAfter = time.Duration(payload.RetryAfter * float64(time.Second))
540+
}
541+
}
542+
return apiErr
543+
}
544+
545+
func shouldRetry(err notionAPIError) bool {
546+
if err.StatusCode == http.StatusTooManyRequests || err.Retryable {
547+
return true
548+
}
549+
return err.StatusCode == http.StatusBadGateway ||
550+
err.StatusCode == http.StatusServiceUnavailable ||
551+
err.StatusCode == http.StatusGatewayTimeout
552+
}
553+
554+
func retryAfter(header string, body []byte) time.Duration {
555+
if header != "" {
556+
if seconds, err := time.ParseDuration(header + "s"); err == nil && seconds > 0 {
557+
return seconds
558+
}
559+
if when, err := http.ParseTime(header); err == nil {
560+
if wait := time.Until(when); wait > 0 {
561+
return wait
562+
}
563+
}
564+
}
565+
var payload struct {
566+
RetryAfter float64 `json:"retry_after"`
567+
}
568+
if err := json.Unmarshal(body, &payload); err == nil && payload.RetryAfter > 0 {
569+
return time.Duration(payload.RetryAfter * float64(time.Second))
570+
}
571+
return 0
572+
}
573+
574+
func waitBeforeRetry(ctx context.Context, wait time.Duration) error {
575+
if wait <= 0 {
576+
return nil
577+
}
578+
timer := time.NewTimer(wait)
579+
defer timer.Stop()
580+
select {
581+
case <-ctx.Done():
582+
return ctx.Err()
583+
case <-timer.C:
584+
return nil
585+
}
586+
}
587+
520588
func isIgnoredCommentError(err error) bool {
521589
apiErr, ok := err.(notionAPIError)
522590
if !ok {

‎internal/notionapi/api_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,3 +213,52 @@ func TestIngestCommentsSkipsRestrictedResource(t *testing.T) {
213213
t.Fatalf("unexpected comment count: %d", count)
214214
}
215215
}
216+
217+
func TestIngestCommentsRetriesTransientGatewayError(t *testing.T) {
218+
attempts := 0
219+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
220+
w.Header().Set("Content-Type", "application/json")
221+
if r.URL.Path != "/comments" {
222+
t.Fatalf("unexpected request: %s %s", r.Method, r.URL.String())
223+
}
224+
attempts++
225+
if attempts == 1 {
226+
w.WriteHeader(http.StatusBadGateway)
227+
_, _ = w.Write([]byte(`{"retryable":true,"retry_after":0}`))
228+
return
229+
}
230+
_, _ = w.Write([]byte(`{
231+
"object":"list",
232+
"results":[{
233+
"id":"comment1",
234+
"rich_text":[{"type":"text","plain_text":"Looks good","text":{"content":"Looks good"}}],
235+
"created_by":{"id":"user1"},
236+
"created_time":"2026-01-01T00:00:00Z",
237+
"last_edited_time":"2026-01-01T00:00:00Z"
238+
}],
239+
"has_more":false
240+
}`))
241+
}))
242+
defer server.Close()
243+
244+
st, err := store.Open(filepath.Join(t.TempDir(), "notcrawl.db"))
245+
if err != nil {
246+
t.Fatal(err)
247+
}
248+
defer st.Close()
249+
250+
count, err := (Client{BaseURL: server.URL, Version: "2026-03-11", Token: "secret", HTTP: http.DefaultClient}).ingestComments(context.Background(), st, "page1", "space1")
251+
if err != nil {
252+
t.Fatal(err)
253+
}
254+
if count != 1 || attempts != 2 {
255+
t.Fatalf("unexpected count/attempts: count=%d attempts=%d", count, attempts)
256+
}
257+
comments, err := st.PageComments(context.Background(), "page1")
258+
if err != nil {
259+
t.Fatal(err)
260+
}
261+
if len(comments) != 1 || comments[0].Text != "Looks good" {
262+
t.Fatalf("unexpected comments: %+v", comments)
263+
}
264+
}

0 commit comments

Comments
 (0)