Skip to content

Commit 875dd62

Browse files
authored
GT-209 Retry mechanism for leadership challenge is ongoing case in tests (#451)
1 parent 0bdaffa commit 875dd62

File tree

5 files changed

+114
-7
lines changed

5 files changed

+114
-7
lines changed

go.mod

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
module github.com/arangodb/go-driver
22

3-
go 1.16
3+
go 1.17
44

55
require (
66
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e
7-
github.com/coreos/go-iptables v0.4.3
7+
github.com/coreos/go-iptables v0.6.0
88
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
99
github.com/golang-jwt/jwt v3.2.2+incompatible
1010
github.com/google/uuid v1.1.1
@@ -14,3 +14,9 @@ require (
1414
golang.org/x/net v0.0.0-20200625001655-4c5254603344
1515
golang.org/x/text v0.3.0
1616
)
17+
18+
require (
19+
github.com/davecgh/go-spew v1.1.0 // indirect
20+
github.com/pmezard/go-difflib v1.0.0 // indirect
21+
gopkg.in/yaml.v2 v2.2.2 // indirect
22+
)

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2
22
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
33
github.com/coreos/go-iptables v0.4.3 h1:jJg1aFuhCqWbgBl1VTqgTHG5faPM60A5JDMjQ2HYv+A=
44
github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
5+
github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
6+
github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
57
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
68
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
79
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

test/client_test.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import (
4141
driver "github.com/arangodb/go-driver"
4242
"github.com/arangodb/go-driver/http"
4343
"github.com/arangodb/go-driver/jwt"
44+
"github.com/arangodb/go-driver/util/connection/wrappers"
4445
"github.com/arangodb/go-driver/vst"
4546
"github.com/arangodb/go-driver/vst/protocol"
4647
)
@@ -212,7 +213,10 @@ func createConnection(t testEnv, disallowUnknownFields bool) driver.Connection {
212213
t.Fatalf("Failed to create new vst connection: %s", describe(err))
213214
}
214215
if disallowUnknownFields {
215-
return http.NewConnectionDebugWrapper(conn, driver.ContentTypeVelocypack)
216+
conn = http.NewConnectionDebugWrapper(conn, driver.ContentTypeVelocypack)
217+
}
218+
if getTestMode() == testModeResilientSingle {
219+
conn = wrappers.NewActiveFailoverWrapper(t, conn)
216220
}
217221
return conn
218222

@@ -227,7 +231,10 @@ func createConnection(t testEnv, disallowUnknownFields bool) driver.Connection {
227231
t.Fatalf("Failed to create new http connection: %s", describe(err))
228232
}
229233
if disallowUnknownFields {
230-
return http.NewConnectionDebugWrapper(conn, config.ContentType)
234+
conn = http.NewConnectionDebugWrapper(conn, config.ContentType)
235+
}
236+
if getTestMode() == testModeResilientSingle {
237+
conn = wrappers.NewActiveFailoverWrapper(t, conn)
231238
}
232239
return conn
233240

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
20+
package wrappers
21+
22+
import (
23+
"context"
24+
"fmt"
25+
"time"
26+
27+
"github.com/arangodb/go-driver"
28+
)
29+
30+
const (
31+
timeout = time.Second * 60
32+
interval = time.Second * 2
33+
)
34+
35+
type activeFailoverWrapper struct {
36+
testEnv
37+
driver.Connection
38+
}
39+
40+
type testEnv interface {
41+
Error(message ...interface{})
42+
Errorf(format string, args ...interface{})
43+
Fatal(message ...interface{})
44+
Fatalf(format string, args ...interface{})
45+
Log(message ...interface{})
46+
Logf(format string, args ...interface{})
47+
Name() string
48+
FailNow()
49+
}
50+
51+
// NewActiveFailoverWrapper is meant for use by tests generated by 'go test' to cover AF failing cases.
52+
// It wraps the given connection and retries requests that fail with an error indicating that there is no leader.
53+
func NewActiveFailoverWrapper(t testEnv, conn driver.Connection) driver.Connection {
54+
return &activeFailoverWrapper{t, conn}
55+
}
56+
57+
// TODO: with Go 1.18 we can extract this method to utils using generics
58+
func (af *activeFailoverWrapper) Do(ctx context.Context, req driver.Request) (driver.Response, error) {
59+
timeoutT := time.NewTimer(timeout)
60+
defer timeoutT.Stop()
61+
62+
intervalT := time.NewTicker(interval)
63+
defer intervalT.Stop()
64+
65+
for {
66+
resp, err := af.Connection.Do(ctx, req)
67+
if err != nil {
68+
if driver.IsNoLeaderOrOngoing(err) {
69+
af.Logf("RETRYING (ERROR) - there is no Leader or Leader change is ongoing: %v", err)
70+
} else {
71+
return resp, err
72+
}
73+
} else {
74+
if errBody := resp.CheckStatus(); errBody != nil && driver.IsNoLeaderOrOngoing(errBody) {
75+
af.Logf("RETRYING (ERROR IN BODY CASE) - there is no Leader or Leader change is ongoing: %v", errBody)
76+
} else {
77+
return resp, err
78+
}
79+
}
80+
81+
select {
82+
case <-timeoutT.C:
83+
return nil, fmt.Errorf("activeFailoverWrapper function time out (waiting for the Leader)")
84+
case <-intervalT.C:
85+
continue
86+
}
87+
}
88+
}

v2/go.mod

+7-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
11
module github.com/arangodb/go-driver/v2
22

3-
go 1.16
3+
go 1.17
44

55
require (
66
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e
7-
github.com/davecgh/go-spew v1.1.1 // indirect
87
github.com/golang-jwt/jwt v3.2.2+incompatible
98
github.com/google/uuid v1.1.1
10-
github.com/kr/pretty v0.2.0 // indirect
119
github.com/pkg/errors v0.9.1
1210
github.com/rs/zerolog v1.19.0
1311
github.com/stretchr/testify v1.5.1
1412
golang.org/x/net v0.0.0-20190620200207-3b0461eec859
1513
golang.org/x/text v0.3.0
14+
)
15+
16+
require (
17+
github.com/davecgh/go-spew v1.1.1 // indirect
18+
github.com/kr/pretty v0.2.0 // indirect
19+
github.com/pmezard/go-difflib v1.0.0 // indirect
1620
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
1721
gopkg.in/yaml.v2 v2.2.8 // indirect
1822
)

0 commit comments

Comments
 (0)