Pre release

.github/workflows/main.yml

@@ -11,7 +11,7 @@ on:
     - cron: '00 02 */4 * *'
 env:
   RUST_BACKTRACE: 1
-  CLC_STABLE_HASH: "aecd4740aafb180c24f25de6c4188f39eb530c46"
+  CLC_STABLE_HASH: "b963be478a9b97fd149326dc69581f6733b23c23"
 jobs:
   rust-tests:
     name: Rust tests
@@ -96,6 +96,13 @@ jobs:
           cargo install cargo-valgrind
           python ci/build_and_run_tests.py --rust-only-with-valigrind
         shell: bash
+      - name: Run tests that requires SG up and running
+        if: matrix.os == 'ubuntu-20.04' && matrix.rust == 'stable'
+        run: |
+          set -e
+          python ci/docker-up-down.py up
+          python ci/build_and_run_tests.py --with-server-only
+          python ci/docker-up-down.py down
   ios-tests:
     name: Check work with iOS
     runs-on: macos-13
@@ -219,9 +226,4 @@ jobs:
           RUSTFLAGS: -Zsanitizer=address -Cdebuginfo=0
           RUSTDOCFLAGS: -Zsanitizer=address
           ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=0"
-          # Work around https://github.com/rust-lang/rust/issues/59125 by
-          # disabling backtraces. In an ideal world we'd probably suppress the
-          # leak sanitization, but we don't care about backtraces here, so long
-          # as the other tests have them.
-          RUST_BACKTRACE: "0"
         run: cargo -Z build-std test --no-default-features --features="build-cpp,use-couchbase-lite-sqlite,use-tokio-websocket,use-native-tls,with-asan" --target x86_64-unknown-linux-gnu

ci/build_and_run_tests.py

@@ -3,7 +3,7 @@
 
 import os, time, sys
 from pathlib import Path
-from subprocess import check_call
+from subprocess import check_call, Popen
 from multiprocessing import cpu_count
 from typing import List
 
@@ -67,7 +67,7 @@ def build_and_test_rust_part_for_ios(src_root: str) -> None:
     cpp_src = os.environ["CORE_SRC"]
     cpp_build_dir = os.path.join(cpp_src, "build-ios")
     mkdir_if_not_exists(cpp_build_dir)
-    cmake_conf_cmd = ["cmake", cpp_src, "-DCMAKE_OSX_ARCHITECTURES=arm64", "-DCMAKE_OSX_SYSROOT=iphoneos",  "-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0", "-DCMAKE_SYSTEM_NAME=iOS", "-DDISABLE_LTO_BUILD=True", "-DMAINTAINER_MODE=False", "-DENABLE_TESTING=False", "-DLITECORE_BUILD_TESTS=False", "-DSQLITE_ENABLE_RTREE=True", "-DCMAKE_C_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_C_COMPILER=/usr/bin/clang", "-DCMAKE_CXX_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++", "-DCMAKE_ASM_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_ASM_COMPILER=/usr/bin/clang", "-DCMAKE_BUILD_TYPE=Debug"]
+    cmake_conf_cmd = ["cmake", cpp_src, "-DCMAKE_OSX_ARCHITECTURES=arm64", "-DCMAKE_OSX_SYSROOT=iphoneos",  "-DCMAKE_OSX_DEPLOYMENT_TARGET=14.2", "-DCMAKE_SYSTEM_NAME=iOS", "-DDISABLE_LTO_BUILD=True", "-DMAINTAINER_MODE=False", "-DENABLE_TESTING=False", "-DLITECORE_BUILD_TESTS=False", "-DSQLITE_ENABLE_RTREE=True", "-DCMAKE_C_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_C_COMPILER=/usr/bin/clang", "-DCMAKE_CXX_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++", "-DCMAKE_ASM_FLAGS=-fPIC --target=aarch64-apple-ios", "-DCMAKE_ASM_COMPILER=/usr/bin/clang", "-DCMAKE_BUILD_TYPE=Debug"]
     print("configure build with cmake " + " ".join(cmake_conf_cmd))
     check_call(cmake_conf_cmd, cwd = cpp_build_dir)
     check_call(["cmake", "--build", ".", "--", "LiteCoreStatic", "FleeceStatic", "BLIPStatic"], cwd = cpp_build_dir)
@@ -77,9 +77,13 @@ def build_and_test_rust_part_for_ios(src_root: str) -> None:
 
 @show_timing
 def run_tests_that_require_server(src_root: str) -> None:
+    my_env = os.environ.copy()
+    my_env["SG_PASS"] = "password"
+    my_env["SG_USER"] = "sguser"
+    my_env["SG_URL"]  = 'ws://localhost:4884/scratch-30/'
     for test_name in ["test_double_replicator_restart", "test_wrong_sync_packets_order"]:
         check_call(["cargo", "test", "--release", "-p", "couchbase-lite",
-                    "-vv", test_name, "--", "--ignored"], cwd = src_root)
+                    "-vv", test_name, "--", "--ignored"], cwd = src_root, env = my_env)
 
 @show_timing
 def main() -> None:
@@ -114,5 +118,6 @@ def main() -> None:
     if WITH_SERVER_TESTS in tests:
         run_tests_that_require_server(src_root)
 
+
 if __name__ == "__main__":
     main()

ci/docker-up-down.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+import time, sys, os
+from pathlib import Path
+from subprocess import check_call, Popen
+from typing import List
+
+def show_timing(function):
+    def _wrapper(*args, **kwargs):
+        start = time.time()
+        ret = function(*args, **kwargs)
+        elapsed = (time.time() - start)
+        print("%s elapsed time: %f" % (function.__name__, elapsed))
+        return ret
+    return _wrapper
+
+def get_src_root_path(my_path: str) -> str:
+    my_path = os.path.dirname(os.path.realpath(my_path))
+    return my_path
+
+def path_to_docker_config(src_root: str) -> str:
+    # TODO: use os.environ["CORE_SRC"]/Replicator/tests/data/docker,
+    # when we switch to new version of couchbase-lite-core
+    return os.path.join(src_root, "ci", "docker")
+
+def wait_success_process(cmd: List[str]) -> bool:
+    for i in range(0, 120):
+        process = Popen(cmd)
+        process.wait()
+        if process.returncode == 0:
+            return True
+        print("%d wait command 5 seconds more" % i)
+        time.sleep(5)
+    return False
+
+@show_timing
+def wait_couchbase_and_syncway_in_docker_up() -> None:
+    if not wait_success_process(["curl", "-sIkL", "-H", "Authorization: Basic YWRtaW46cGFzc3dvcmQ=",
+                         "--fail", "http://localhost:4985/scratch"]):
+        raise Exception("Wating of SG up and running FAILED")
+    print("SG is up")
+    if not wait_success_process(["curl", "-sIkL", "-H", "Authorization: Basic YWRtaW46cGFzc3dvcmQ=", "--fail", "http://localhost:4885/scratch-30"]):
+        raise Exception("Wating of SG Legacy up and running FAILED")
+    print("SG Legacy is up")
+
+
+
+@show_timing
+def run_couchbase_and_syncway_in_docker(src_root: str) -> None:
+    docker_path = path_to_docker_config(src_root)
+    docker_exe = "docker"
+    if "DOCKER" in os.environ:
+        docker_exe = os.environ["DOCKER"]
+    my_env = os.environ.copy()
+    my_env["SSL"] = "false"
+    check_call(["sudo", "/bin/sh", "-c", "SSL=false " + docker_exe + " compose up --build -d"], cwd = docker_path)
+    wait_couchbase_and_syncway_in_docker_up()
+
+@show_timing
+def setup_users_in_sg() -> None:
+    check_call(["curl", "--fail",  "-k", "--location", "--request", "POST", "http://localhost:4985/scratch/_user/",
+                "--header", "Content-Type: application/json",
+                "--header", "Authorization: Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==",
+                "--data-raw", '{"name": "sguser", "password": "password", "collection_access": {"flowers": {"roses": {"admin_channels": ["*"]}, "tulips": {"admin_channels": ["*"]}, "lavenders": {"admin_channels": ["*"]}}}}'])
+    check_call(["curl", "--fail",  "-k", "--location", "--request", "POST", "http://localhost:4885/scratch-30/_user/",
+                "--header", "Content-Type: application/json",
+                "--header", "Authorization: Basic QWRtaW5pc3RyYXRvcjpwYXNzd29yZA==",
+                "--data-raw", '{"name": "sguser", "password": "password", "collection_access": {"flowers": {"roses": {"admin_channels": ["*"]}, "tulips": {"admin_channels": ["*"]}, "lavenders": {"admin_channels": ["*"]}}}}'])
+
+@show_timing
+def stop_couchbase_and_syncway_in_docker(src_root: str) -> None:
+    docker_path = path_to_docker_config(src_root)
+    docker_exe = "docker"
+    if "DOCKER" in os.environ:
+        docker_exe = os.environ["DOCKER"]
+    check_call(["sudo", docker_exe, "compose", "down"], cwd = docker_path)
+
+
+def main() -> None:
+    ci_dir = Path(get_src_root_path(sys.argv[0]))
+    src_root = ci_dir.parent
+    if sys.argv[1] == "up":
+        run_couchbase_and_syncway_in_docker(src_root)
+        setup_users_in_sg()        
+    elif sys.argv[1] == "down":
+        stop_couchbase_and_syncway_in_docker(src_root)
+
+if __name__ == "__main__":
+    main()

ci/docker/README.md

@@ -0,0 +1,70 @@
+This docker folder contains the `docker-compose.yml` file for starting and configuring Couchbase Server and Sync Gateways for running non-walrus replicator tests with Sync Gateway.
+
+When running `docker compose up`, there will be one Couchbase Server and two Sync Gateways started as follows:
+
+### 1. Sync Gateway for .SyncServerCollection Tests
+
+ | Config      | Value       |
+ | ----------- | ----------- |
+ | Database    | scratch     |
+ | Port        | 4984        |
+ | Admin Port  | 4985        |
+ | Collections | flowers.roses, flowers.tulips, and flowers.lavenders |
+
+ #### Sync Function
+ ```JS
+    function (doc, oldDoc) { 
+        if (doc.isRejected == 'true') 
+            throw({'forbidden':'read_only'}); 
+        channel(doc.channels); 
+    }
+ ``` 
+
+### 2. Legacy Sync Gateway for .SyncServer30 Tests
+
+ | Config      | Value       |
+ | ----------- | ----------- |
+ | Database    | scratch-30  |
+ | Port        | 4884        |
+ | Admin Port  | 4885        |
+
+ ** SG 3.0 doesn't support collection.
+
+ #### Sync Function
+
+ Use default sync function which is as follows
+ ```JS
+    function (doc, oldDoc) {
+        channel(doc.channels); 
+    }
+ ``` 
+
+The Admin Credentials of both Sync Gateways are `admin/password` or `Administrator/password`.
+
+### Docker Compose Environment Variables
+
+The `docker-compose.yml` has 3 environment variables for configuration.
+
+ | Variable      |   Description  |
+ | ------------- | -------------- |
+ | SG_DEB        | Sync Gateway 3.1+ deb file URL. Default is SG 3.1.0 Ubuntu ARM64 URL.      |
+ | SG_LEGACY_DEB | Sync Gateway 3.0 deb file URL. Default is SG 3.0.7 Ubuntu ARM64 URL.       |
+ | SSL           | Boolean flag to configure Sync Gateway for SSL endpoints. Default is true. |
+
+ Note: If you are using Mac x86-64, you must configure SG_DEB and SG_LEGACY_DEB to use x86_64 versions. 
+
+ To configure environment variables, create `.env` file with the variables in key=value format.
+
+ **Sample .env file for Mac x86-64**
+```
+SG_DEB=https://packages.couchbase.com/releases/couchbase-sync-gateway/3.1.0/couchbase-sync-gateway-enterprise_3.1.0_x86_64.deb
+SG_LEGACY_DEB=https://packages.couchbase.com/releases/couchbase-sync-gateway/3.0.7/couchbase-sync-gateway-enterprise_3.0.7_x86_64.deb
+```
+
+### Some Commands
+
+ |      Commands       |   Description  |
+ | ------------------- | -------------- |
+ | docker compose up   | Start all docker containers as specified in the `docker-compose.yml`. If the docker images and containers do not exists, they will be built first |
+ | docker compose down | Stop running docker containers without deleting the containers |
+ | docker compose down --rmi all | Stop running docker containers and deleting the images and containers |

ci/docker/cbs/Dockerfile

@@ -0,0 +1,7 @@
+ARG COUCHBASE_VERSION=7.1.4
+
+FROM couchbase/server:enterprise-$COUCHBASE_VERSION
+
+COPY configure-node.sh /etc/service/config-couchbase/run
+RUN chmod +x /etc/service/config-couchbase/run
+RUN chown -R couchbase:couchbase /etc/service

ci/docker/cbs/configure-node.sh

@@ -0,0 +1,120 @@
+#!/bin/sh
+
+# Log all subsequent commands to logfile. FD 3 is now the console
+# for things we want to show up in "docker logs".
+LOGFILE=/opt/couchbase/var/lib/couchbase/logs/container-startup.log
+exec 3>&1 1>>${LOGFILE} 2>&1
+
+CONFIG_DONE_FILE=/opt/couchbase/var/lib/couchbase/container-configured
+config_done() {
+  touch ${CONFIG_DONE_FILE}
+  echo "Couchbase Admin UI: http://localhost:8091" \
+     "\nLogin credentials: Administrator / password" | tee /dev/fd/3
+  echo "Stopping config-couchbase service"
+  sv stop /etc/service/config-couchbase
+}
+
+if [ -e ${CONFIG_DONE_FILE} ]; then
+  echo "Container previously configured." | tee /dev/fd/3
+  config_done
+else
+  echo "Configuring Couchbase Server.  Please wait (~60 sec)..." | tee /dev/fd/3
+fi
+
+export PATH=/opt/couchbase/bin:${PATH}
+
+wait_for_uri() {
+  expected=$1
+  shift
+  uri=$1
+  echo "Waiting for $uri to be available..."
+  while true; do
+    status=$(curl -s -w "%{http_code}" -o /dev/null $*)
+    if [ "x$status" = "x$expected" ]; then
+      break
+    fi
+    echo "$uri not up yet, waiting 2 seconds..."
+    sleep 2
+  done
+  echo "$uri ready, continuing"
+}
+
+panic() {
+  cat <<EOF 1>&3
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+Error during initial configuration - aborting container
+Here's the log of the configuration attempt:
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+EOF
+  cat $LOGFILE 1>&3
+  echo 1>&3
+  kill -HUP 1
+  exit
+}
+
+couchbase_cli_check() {
+  couchbase-cli $* || {
+    echo Previous couchbase-cli command returned error code $?
+    panic
+  }
+}
+
+curl_check() {
+  status=$(curl -sS -w "%{http_code}" -o /tmp/curl.txt $*)
+  cat /tmp/curl.txt
+  rm /tmp/curl.txt
+  if [ "$status" -lt 200 -o "$status" -ge 300 ]; then
+    echo
+    echo Previous curl command returned HTTP status $status
+    panic
+  fi
+}
+
+wait_for_uri 200 http://127.0.0.1:8091/ui/index.html
+
+echo "Setting up cluster:"
+couchbase_cli_check cluster-init -c 127.0.0.1 --cluster-name lite-core-test --cluster-username Administrator \
+  --cluster-password password --services data,index,query,fts --cluster-ramsize 512 --cluster-index-ramsize 512 \
+  --cluster-fts-ramsize 256 --index-storage-setting default
+echo
+
+echo "Checking credentials with curl:"
+curl_check http://127.0.0.1:8091/settings/web -d port=8091 -d username=Administrator -d password=password -u Administrator:password
+echo
+
+echo "Creating lite-core bucket  :"
+couchbase_cli_check bucket-create -c 127.0.0.1 \
+  -u Administrator -p password --bucket lite-core \
+  --bucket-type couchbase --bucket-ramsize 512 \
+  --bucket-replica 0 --enable-flush 1
+echo
+
+echo "Creating scope"
+couchbase_cli_check collection-manage -c 127.0.0.1 \
+ -u Administrator -p password --bucket lite-core \
+ --create-scope flowers
+
+echo "Creating collections:"
+couchbase_cli_check collection-manage -c 127.0.0.1 \
+ -u Administrator -p password --bucket lite-core \
+ --create-collection flowers.roses
+
+couchbase_cli_check collection-manage -c 127.0.0.1 \
+  -u Administrator -p password --bucket lite-core \
+  --create-collection flowers.tulips
+
+couchbase_cli_check collection-manage -c 127.0.0.1 \
+  -u Administrator -p password --bucket lite-core \
+  --create-collection flowers.lavenders
+
+echo "Creating RBAC 'admin' user on lite-core bucket"
+couchbase_cli_check user-manage --set \
+  --rbac-username admin --rbac-password password \
+  --roles 'bucket_full_access[lite-core],bucket_admin[lite-core]' --auth-domain local \
+  -c 127.0.0.1 -u Administrator -p password
+echo
+
+echo "Configuration completed!" | tee /dev/fd/3
+
+config_done