forked from arangodb/arangodb
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtest-definitions.txt
170 lines (132 loc) · 10.5 KB
/
test-definitions.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
# test_suite [flags ...] -- [additional-params...]
# for possible flags and params see
# https://github.com/arangodb/oskar#testing
# Single Server only Tests
BackupAuthNoSysTests,BackupAuthSysTests,BackupNoAuthSysTests,arangosh single name=single_server_only
arangobench,permissions,paths_server,version single name=single_server_only_part2
# Single Server Replication Tests
replication_fuzz priority=500 parallelity=4 size=small single -- --extraArgs:log.level replication=trace
replication_ongoing priority=500 parallelity=3 size=small single -- --extraArgs:log.level replication=trace
replication_ongoing_frompresent parallelity=3 size=small single -- --extraArgs:log.level replication=trace
replication_ongoing_global priority=500 parallelity=3 size=small single -- --extraArgs:log.level replication=trace
replication_ongoing_global_spec parallelity=3 size=small single -- --extraArgs:log.level replication=trace
# one run for each test...
replication_static priority=500 parallelity=3 size=small single buckets=2 -- --extraArgs:log.level replication=trace --extraArgs:experimental-vector-index true
# TODO - we could actually use 4 buckets instead of 2, but ATM this causes issues with other tests on Jenkins
# For CircleCI we have a special case in the generate_config script to override this. Once we can change the buckets
# here, make sure to remove this special case!
replication_sync priority=8000 buckets=9 parallelity=3 size=medium single -- --extraArgs:log.level replication=trace
shell_replication parallelity=3 size=small single
http_replication priority=500 parallelity=3 size=small single
# Misc single server tests
agency,agency-restart priority=350 parallelity=3 size=small single -- --dumpAgencyOnError true
authentication priority=1000 single
endpoints priority=250 single -- --skipEndpointsIpv6 true
recovery priority=2000 buckets=8 parallelity=3 size=small single
# cluster
recovery_cluster priority=2000 cluster buckets=8 -- --extremeVerbosity true
recovery_server priority=2000 buckets=3 parallelity=3 size=small single
shell_api priority=500 single -- $EncryptionAtRest
shell_api_multi,shell_api_multi name=shell_api_multi priority=500 single -- --optionsJson [{"http":true},{"ssl":true,"suffix":"ssl"}] $EncryptionAtRest
# Shell client tests Single Server; distribute evenly
shell_client priority=250 parallelity=2 buckets=4 single size=medium -- $EncryptionAtRest
# shell_client_multi tests run quite long, so want to have a separate job for each protocol
shell_client_multi priority=1500 parallelity=2 single suffix=http2 -- --http2 true
shell_client_multi priority=1500 parallelity=2 single suffix=http -- --http true
shell_client_transaction,shell_client_transaction name=shell_client_transaction priority=750 single -- $EncryptionAtRest --optionsJson [{"http":true,"suffix":"http"},{"http2":true,"suffix":"http2"}]
shell_client_traffic,shell_client_traffic,shell_client_traffic,shell_client_traffic name=shell_client_traffic priority=250 parallelity=1 single -- --optionsJson [{"http":true,"suffix":"http"},{"http2":true,"suffix":"http2"},{"http":true,"ssl":true,"suffix":"http-ssl"},{"http2":true,"ssl":true,"suffix":"http2-ssl"}]
shell_client_aql buckets=19 priority=250 single
# C++ unit tests are executed in single env
gtest_iresearch !coverage priority=1000 parallelity=3 size=medium single
gtest_arangodb !coverage priority=1000 parallelity=3 size=large single
# Fuerte tests are executed in single env
fuerte !coverage priority=500 single suffix=http
fuerte !coverage priority=500 single suffix=ssl-http -- --protocol ssl
rta_makedata single -- --extraArgs:experimental-vector-index true
# Cluster Tests
arangobench priority=1000 size=small cluster -- --dumpAgencyOnError true
chaos cluster !coverage !full -- --dumpAgencyOnError true parallelity=8
chaos_subq !coverage full priority=1000 size=medium+ --javascript.v8-max-heap=32768 --javascript.gc-interval=25 -- --dumpAgencyOnError true
deadlock cluster !coverage -- --dumpAgencyOnError true
restart priority=1000 size=medium cluster -- --dumpAgencyOnError true --forceJson true
load_balancing priority=500 size=medium cluster -- --dumpAgencyOnError true
load_balancing_auth priority=500 size=small cluster -- --dumpAgencyOnError true
#replication2_client size=small cluster
#replication2_server size=medium cluster buckets=3 -- --dumpAgencyOnError true
# see also DISABLED restart tests when re-enabling!
resilience_analyzers priority=500 size=medium cluster -- --dumpAgencyOnError true
resilience_failover priority=1200 size=medium buckets=4 cluster -- --dumpAgencyOnError true
resilience_move priority=600 size=medium buckets=3 cluster -- --dumpAgencyOnError true
resilience_sharddist size=small cluster -- --dumpAgencyOnError true
shell_api priority=500 cluster -- $EncryptionAtRest
shell_api_multi priority=500 cluster suffix=http -- $EncryptionAtRest
shell_api_multi priority=500 cluster suffix=https -- $EncryptionAtRest --protocol ssl
# shell_client_multi tests run quite long, so want to have a separate job for each protocol
shell_client_multi priority=1500 cluster suffix=http -- --http true
shell_client_multi priority=1500 cluster suffix=http2 -- --http2 true
# different number of buckets in cluster
shell_client_aql priority=1000 size=medium+ cluster buckets=20 -- --dumpAgencyOnError true
shell_client priority=500 cluster size=medium+ buckets=20 -- --dumpAgencyOnError true
shell_client_transaction priority=500 cluster parallelity=5 size=small buckets=5 -- --dumpAgencyOnError true
#shell_client_replication2_recovery priority=500 size=medium cluster -- --dumpAgencyOnError true
rta_makedata cluster parallelity=6 -- --extraArgs:experimental-vector-index true
rta_makedata enterprise cluster name=rta_makedata_force_oneshard parallelity=5 -- --forceOneShard true --extraArgs:experimental-vector-index true
# shell aql vector index tests
shell_client_aql_vector priority=1000 size=medium+ cluster buckets=1 -- --dumpAgencyOnError true
# Common Tests
importing,export name=import_export parallelity=5 size=small cluster -- --dumpAgencyOnError true
importing,export name=import_export parallelity=3 size=small single -- --dumpAgencyOnError true
hot_backup enterprise single size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
hot_backup enterprise cluster size=large parallelity=5 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
# frequent restarts impose more threats to the SUT, increase parallelity.
server_parameters priority=1000 parallelity=3 buckets=3 single -- --dumpAgencyOnError true
server_parameters priority=1000 parallelity=5 buckets=10 size=large cluster -- --dumpAgencyOnError true
server_permissions priority=1000 parallelity=5 size=small buckets=2 -- --dumpAgencyOnError true
server_secrets priority=1000 parallelity=5 size=small -- --dumpAgencyOnError true
# Dump Tests
dump cluster size=medium parallelity=5 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump single size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump cluster suffix=_vpack size=medium parallelity=5 -- --dumpAgencyOnError true --dumpVPack true --extraArgs:experimental-vector-index true
dump single suffix=_vpack size=medium -- --dumpAgencyOnError true --dumpVPack true --extraArgs:experimental-vector-index true
# BTS-1852 enable after fix:
# dump size=medium enterprise cluster name=dump_singleshard_cluster -- --forceOneShard true --cluster true --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_authentication size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_jwt size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_multiple single size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_multiple cluster size=medium parallelity=5 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_non_parallel size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_maskings enterprise size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_encrypted enterprise size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_mixed_cluster_single size=medium -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_mixed_single_cluster size=large -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_mixed_cluster_single suffix=vpack size=medium -- --dumpAgencyOnError true --dumpVPack true --extraArgs:experimental-vector-index true
dump_mixed_single_cluster suffix=vpack size=large -- --dumpAgencyOnError true --dumpVPack true --extraArgs:experimental-vector-index true
# takes long, needs to go first. However, doesn't utilize the SUT hard.
dump_with_crashes single size=medium parallelity=1 priority=2000 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_with_crashes full cluster size=medium parallelity=1 priority=2000 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
dump_with_crashes_non_parallel size=medium parallelity=1 priority=2000 -- --dumpAgencyOnError true --extraArgs:experimental-vector-index true
# Audit Tests
audit_client,audit_server enterprise name=audit -- --dumpAgencyOnError true
# Full Tests Single Server
shell_fuzzer full cluster priority=500 parallelity=6 -- --dumpAgencyOnError true
shell_fuzzer full single priority=500 -- --dumpAgencyOnError true
authentication_parameters single full priority=1000
config single full priority=1000
foxx_manager single full priority=500
queryCacheAuthorization single full priority=500
readOnly single full priority=500
replication_aql priority=1000 single full
replication_random priority=1000 single full
# Full Cluster Tests
authentication full cluster priority=500 buckets=3 -- --dumpAgencyOnError true
chaos !coverage full cluster priority=9600 size=large -- --dumpAgencyOnError true --skipNightly false
chaos_subq !coverage full cluster priority=1000 size=medium+ --javascript.v8-max-heap=16384 -- --dumpAgencyOnError true
resilience_failover_failure full cluster priority=500 -- --dumpAgencyOnError true
resilience_failover_view full cluster priority=500 -- --dumpAgencyOnError true
resilience_transactions full cluster priority=500 -- --dumpAgencyOnError true
wal_cleanup full cluster priority=2500 -- --dumpAgencyOnError true
# Common Full Tests
communication full priority=1000 parallelity=2 single
communication_ssl full priority=1000 parallelity=2 single
communication full priority=1000 size=large parallelity=6 cluster
communication_ssl full priority=1000 size=large parallelity=6 cluster