Skip to content

Commit 7b7c20c

Browse files
DushistovDushistov
committed
ci: run tests that requires SG on CI
1 parent 1cc2779 commit 7b7c20c

19 files changed

+718
-20
lines changed

‎.github/workflows/main.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ jobs:
9696
cargo install cargo-valgrind
9797
python ci/build_and_run_tests.py --rust-only-with-valigrind
9898
shell: bash
99+
- name: Run tests that requires SG up and running
100+
if: matrix.os == 'ubuntu-20.04' && matrix.rust == 'stable'
101+
run: |
102+
set -e
103+
python ci/docker-up-down.py up
104+
python ci/build_and_run_tests.py --with-server-only
105+
python ci/docker-up-down.py down
99106
ios-tests:
100107
name: Check work with iOS
101108
runs-on: macos-13

‎ci/build_and_run_tests.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import os, time, sys
55
from pathlib import Path
6-
from subprocess import check_call
6+
from subprocess import check_call, Popen
77
from multiprocessing import cpu_count
88
from typing import List
99

@@ -77,9 +77,13 @@ def build_and_test_rust_part_for_ios(src_root: str) -> None:
7777

7878
@show_timing
7979
def run_tests_that_require_server(src_root: str) -> None:
80+
my_env = os.environ.copy()
81+
my_env["SG_PASS"] = "password"
82+
my_env["SG_USER"] = "sguser"
83+
my_env["SG_URL"] = 'ws://localhost:4884/scratch-30/'
8084
for test_name in ["test_double_replicator_restart", "test_wrong_sync_packets_order"]:
8185
check_call(["cargo", "test", "--release", "-p", "couchbase-lite",
82-
"-vv", test_name, "--", "--ignored"], cwd = src_root)
86+
"-vv", test_name, "--", "--ignored"], cwd = src_root, env = my_env)
8387

8488
@show_timing
8589
def main() -> None:
@@ -114,5 +118,6 @@ def main() -> None:
114118
if WITH_SERVER_TESTS in tests:
115119
run_tests_that_require_server(src_root)
116120

121+
117122
if __name__ == "__main__":
118123
main()

‎ci/docker-up-down.py

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env python3
2+
3+
import time, sys, os
4+
from pathlib import Path
5+
from subprocess import check_call, Popen
6+
from typing import List
7+
8+
def show_timing(function):
9+
def _wrapper(*args, **kwargs):
10+
start = time.time()
11+
ret = function(*args, **kwargs)
12+
elapsed = (time.time() - start)
13+
print("%s elapsed time: %f" % (function.__name__, elapsed))
14+
return ret
15+
return _wrapper
16+
17+
def get_src_root_path(my_path: str) -> str:
18+
my_path = os.path.dirname(os.path.realpath(my_path))
19+
return my_path
20+
21+
def path_to_docker_config(src_root: str) -> str:
22+
# TODO: use os.environ["CORE_SRC"]/Replicator/tests/data/docker,
23+
# when we switch to new version of couchbase-lite-core
24+
return os.path.join(src_root, "ci", "docker")
25+
26+
def wait_success_process(cmd: List[str]) -> bool:
27+
for i in range(0, 120):
28+
process = Popen(cmd)
29+
process.wait()
30+
if process.returncode == 0:
31+
return True
32+
print("%d wait command 5 seconds more" % i)
33+
time.sleep(5)
34+
return False
35+
36+
@show_timing
37+
def wait_couchbase_and_syncway_in_docker_up() -> None:
38+
if not wait_success_process(["curl", "-sIkL", "-H", "Authorization: Basic YWRtaW46cGFzc3dvcmQ=",
39+
"--fail", "http://localhost:4985/scratch"]):
40+
raise Exception("Wating of SG up and running FAILED")
41+
print("SG is up")
42+
if not wait_success_process(["curl", "-sIkL", "-H", "Authorization: Basic YWRtaW46cGFzc3dvcmQ=", "--fail", "http://localhost:4885/scratch-30"]):
43+
raise Exception("Wating of SG Legacy up and running FAILED")
44+
print("SG Legacy is up")
45+
46+
47+
48+
@show_timing
49+
def run_couchbase_and_syncway_in_docker(src_root: str) -> None:
50+
docker_path = path_to_docker_config(src_root)
51+
docker_exe = "docker"
52+
if "DOCKER" in os.environ:
53+
docker_exe = os.environ["DOCKER"]
54+
my_env = os.environ.copy()
55+
my_env["SSL"] = "false"
56+
check_call(["sudo", "/bin/sh", "-c", "SSL=false " + docker_exe + " compose up --build -d"], cwd = docker_path)
57+
wait_couchbase_and_syncway_in_docker_up()
58+
59+
@show_timing
60+
def setup_users_in_sg() -> None:
61+
check_call(["curl", "--fail", "-k", "--location", "--request", "POST", "http://localhost:4985/scratch/_user/",
62+
"--header", "Content-Type: application/json",
63+
"--header", "Authorization: Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==",
64+
"--data-raw", '{"name": "sguser", "password": "password", "collection_access": {"flowers": {"roses": {"admin_channels": ["*"]}, "tulips": {"admin_channels": ["*"]}, "lavenders": {"admin_channels": ["*"]}}}}'])
65+
check_call(["curl", "--fail", "-k", "--location", "--request", "POST", "http://localhost:4885/scratch-30/_user/",
66+
"--header", "Content-Type: application/json",
67+
"--header", "Authorization: Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==",
68+
"--data-raw", '{"name": "sguser", "password": "password", "collection_access": {"flowers": {"roses": {"admin_channels": ["*"]}, "tulips": {"admin_channels": ["*"]}, "lavenders": {"admin_channels": ["*"]}}}}'])
69+
70+
@show_timing
71+
def stop_couchbase_and_syncway_in_docker(src_root: str) -> None:
72+
docker_path = path_to_docker_config(src_root)
73+
docker_exe = "docker"
74+
if "DOCKER" in os.environ:
75+
docker_exe = os.environ["DOCKER"]
76+
check_call(["sudo", docker_exe, "compose", "down"], cwd = docker_path)
77+
78+
79+
def main() -> None:
80+
ci_dir = Path(get_src_root_path(sys.argv[0]))
81+
src_root = ci_dir.parent
82+
if sys.argv[1] == "up":
83+
run_couchbase_and_syncway_in_docker(src_root)
84+
setup_users_in_sg()
85+
elif sys.argv[1] == "down":
86+
stop_couchbase_and_syncway_in_docker(src_root)
87+
88+
if __name__ == "__main__":
89+
main()

‎ci/docker/README.md

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
This docker folder contains the `docker-compose.yml` file for starting and configuring Couchbase Server and Sync Gateways for running non-walrus replicator tests with Sync Gateway.
2+
3+
When running `docker compose up`, there will be one Couchbase Server and two Sync Gateways started as follows:
4+
5+
### 1. Sync Gateway for .SyncServerCollection Tests
6+
7+
| Config | Value |
8+
| ----------- | ----------- |
9+
| Database | scratch |
10+
| Port | 4984 |
11+
| Admin Port | 4985 |
12+
| Collections | flowers.roses, flowers.tulips, and flowers.lavenders |
13+
14+
#### Sync Function
15+
```JS
16+
function (doc, oldDoc) {
17+
if (doc.isRejected == 'true')
18+
throw({'forbidden':'read_only'});
19+
channel(doc.channels);
20+
}
21+
```
22+
23+
### 2. Legacy Sync Gateway for .SyncServer30 Tests
24+
25+
| Config | Value |
26+
| ----------- | ----------- |
27+
| Database | scratch-30 |
28+
| Port | 4884 |
29+
| Admin Port | 4885 |
30+
31+
** SG 3.0 doesn't support collection.
32+
33+
#### Sync Function
34+
35+
Use default sync function which is as follows
36+
```JS
37+
function (doc, oldDoc) {
38+
channel(doc.channels);
39+
}
40+
```
41+
42+
The Admin Credentials of both Sync Gateways are `admin/password` or `Administrator/password`.
43+
44+
### Docker Compose Environment Variables
45+
46+
The `docker-compose.yml` has 3 environment variables for configuration.
47+
48+
| Variable | Description |
49+
| ------------- | -------------- |
50+
| SG_DEB | Sync Gateway 3.1+ deb file URL. Default is SG 3.1.0 Ubuntu ARM64 URL. |
51+
| SG_LEGACY_DEB | Sync Gateway 3.0 deb file URL. Default is SG 3.0.7 Ubuntu ARM64 URL. |
52+
| SSL | Boolean flag to configure Sync Gateway for SSL endpoints. Default is true. |
53+
54+
Note: If you are using Mac x86-64, you must configure SG_DEB and SG_LEGACY_DEB to use x86_64 versions.
55+
56+
To configure environment variables, create `.env` file with the variables in key=value format.
57+
58+
**Sample .env file for Mac x86-64**
59+
```
60+
SG_DEB=https://packages.couchbase.com/releases/couchbase-sync-gateway/3.1.0/couchbase-sync-gateway-enterprise_3.1.0_x86_64.deb
61+
SG_LEGACY_DEB=https://packages.couchbase.com/releases/couchbase-sync-gateway/3.0.7/couchbase-sync-gateway-enterprise_3.0.7_x86_64.deb
62+
```
63+
64+
### Some Commands
65+
66+
| Commands | Description |
67+
| ------------------- | -------------- |
68+
| docker compose up | Start all docker containers as specified in the `docker-compose.yml`. If the docker images and containers do not exists, they will be built first |
69+
| docker compose down | Stop running docker containers without deleting the containers |
70+
| docker compose down --rmi all | Stop running docker containers and deleting the images and containers |

‎ci/docker/cbs/Dockerfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
ARG COUCHBASE_VERSION=7.1.4
2+
3+
FROM couchbase/server:enterprise-$COUCHBASE_VERSION
4+
5+
COPY configure-node.sh /etc/service/config-couchbase/run
6+
RUN chmod +x /etc/service/config-couchbase/run
7+
RUN chown -R couchbase:couchbase /etc/service

‎ci/docker/cbs/configure-node.sh

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/bin/sh
2+
3+
# Log all subsequent commands to logfile. FD 3 is now the console
4+
# for things we want to show up in "docker logs".
5+
LOGFILE=/opt/couchbase/var/lib/couchbase/logs/container-startup.log
6+
exec 3>&1 1>>${LOGFILE} 2>&1
7+
8+
CONFIG_DONE_FILE=/opt/couchbase/var/lib/couchbase/container-configured
9+
config_done() {
10+
touch ${CONFIG_DONE_FILE}
11+
echo "Couchbase Admin UI: http://localhost:8091" \
12+
"\nLogin credentials: Administrator / password" | tee /dev/fd/3
13+
echo "Stopping config-couchbase service"
14+
sv stop /etc/service/config-couchbase
15+
}
16+
17+
if [ -e ${CONFIG_DONE_FILE} ]; then
18+
echo "Container previously configured." | tee /dev/fd/3
19+
config_done
20+
else
21+
echo "Configuring Couchbase Server. Please wait (~60 sec)..." | tee /dev/fd/3
22+
fi
23+
24+
export PATH=/opt/couchbase/bin:${PATH}
25+
26+
wait_for_uri() {
27+
expected=$1
28+
shift
29+
uri=$1
30+
echo "Waiting for $uri to be available..."
31+
while true; do
32+
status=$(curl -s -w "%{http_code}" -o /dev/null $*)
33+
if [ "x$status" = "x$expected" ]; then
34+
break
35+
fi
36+
echo "$uri not up yet, waiting 2 seconds..."
37+
sleep 2
38+
done
39+
echo "$uri ready, continuing"
40+
}
41+
42+
panic() {
43+
cat <<EOF 1>&3
44+
45+
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
46+
Error during initial configuration - aborting container
47+
Here's the log of the configuration attempt:
48+
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
49+
EOF
50+
cat $LOGFILE 1>&3
51+
echo 1>&3
52+
kill -HUP 1
53+
exit
54+
}
55+
56+
couchbase_cli_check() {
57+
couchbase-cli $* || {
58+
echo Previous couchbase-cli command returned error code $?
59+
panic
60+
}
61+
}
62+
63+
curl_check() {
64+
status=$(curl -sS -w "%{http_code}" -o /tmp/curl.txt $*)
65+
cat /tmp/curl.txt
66+
rm /tmp/curl.txt
67+
if [ "$status" -lt 200 -o "$status" -ge 300 ]; then
68+
echo
69+
echo Previous curl command returned HTTP status $status
70+
panic
71+
fi
72+
}
73+
74+
wait_for_uri 200 http://127.0.0.1:8091/ui/index.html
75+
76+
echo "Setting up cluster:"
77+
couchbase_cli_check cluster-init -c 127.0.0.1 --cluster-name lite-core-test --cluster-username Administrator \
78+
--cluster-password password --services data,index,query,fts --cluster-ramsize 512 --cluster-index-ramsize 512 \
79+
--cluster-fts-ramsize 256 --index-storage-setting default
80+
echo
81+
82+
echo "Checking credentials with curl:"
83+
curl_check http://127.0.0.1:8091/settings/web -d port=8091 -d username=Administrator -d password=password -u Administrator:password
84+
echo
85+
86+
echo "Creating lite-core bucket :"
87+
couchbase_cli_check bucket-create -c 127.0.0.1 \
88+
-u Administrator -p password --bucket lite-core \
89+
--bucket-type couchbase --bucket-ramsize 512 \
90+
--bucket-replica 0 --enable-flush 1
91+
echo
92+
93+
echo "Creating scope"
94+
couchbase_cli_check collection-manage -c 127.0.0.1 \
95+
-u Administrator -p password --bucket lite-core \
96+
--create-scope flowers
97+
98+
echo "Creating collections:"
99+
couchbase_cli_check collection-manage -c 127.0.0.1 \
100+
-u Administrator -p password --bucket lite-core \
101+
--create-collection flowers.roses
102+
103+
couchbase_cli_check collection-manage -c 127.0.0.1 \
104+
-u Administrator -p password --bucket lite-core \
105+
--create-collection flowers.tulips
106+
107+
couchbase_cli_check collection-manage -c 127.0.0.1 \
108+
-u Administrator -p password --bucket lite-core \
109+
--create-collection flowers.lavenders
110+
111+
echo "Creating RBAC 'admin' user on lite-core bucket"
112+
couchbase_cli_check user-manage --set \
113+
--rbac-username admin --rbac-password password \
114+
--roles 'bucket_full_access[lite-core],bucket_admin[lite-core]' --auth-domain local \
115+
-c 127.0.0.1 -u Administrator -p password
116+
echo
117+
118+
echo "Configuration completed!" | tee /dev/fd/3
119+
120+
config_done

‎ci/docker/docker-compose.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
version: "3.9"
2+
services:
3+
lite-core-cbs:
4+
build:
5+
context: ./cbs
6+
args:
7+
- COUCHBASE_VERSION=${COUCHBASE_VERSION:-7.6.1}
8+
ports:
9+
- "8091:8091"
10+
- "8092:8092"
11+
- "8093:8093"
12+
- "8094:8094"
13+
- "11207:11207"
14+
- "11210:11210"
15+
- "11211:11211"
16+
- "18091:18091"
17+
- "18092:18092"
18+
- "18093:18093"
19+
- "18094:18094"
20+
21+
lite-core-sg:
22+
build:
23+
context: ./sg
24+
args:
25+
- SG_DEB_ARM64=${SG_DEB_ARM64:-https://packages.couchbase.com/releases/couchbase-sync-gateway/3.1.5/couchbase-sync-gateway-enterprise_3.1.5_aarch64.deb}
26+
- SG_DEB_AMD64=${SG_DEB_AMD64:-https://packages.couchbase.com/releases/couchbase-sync-gateway/3.1.5/couchbase-sync-gateway-enterprise_3.1.5_x86_64.deb}
27+
- SSL=${SSL:-true}
28+
ports:
29+
- "4984:4984"
30+
- "4985:4985"
31+
depends_on:
32+
- "lite-core-cbs"
33+
34+
lite-core-sg-legacy:
35+
build:
36+
context: ./sg
37+
args:
38+
- SG_DEB_ARM64=${SG_LEGACY_DEB_ARM64:-https://packages.couchbase.com/releases/couchbase-sync-gateway/3.0.9/couchbase-sync-gateway-enterprise_3.0.9_aarch64.deb}
39+
- SG_DEB_AMD64=${SG_LEGACY_DEB_AMD64:-https://packages.couchbase.com/releases/couchbase-sync-gateway/3.0.9/couchbase-sync-gateway-enterprise_3.0.9_x86_64.deb}
40+
- SSL=${SSL:-true}
41+
- LEGACY_MODE=true
42+
ports:
43+
- "4884:4884"
44+
- "4885:4885"
45+
depends_on:
46+
- "lite-core-cbs"

0 commit comments

Comments
 (0)