CI | Warp run action on Containerized and Non Containerized deployments

.github/workflows/warp-nc-tests.yaml

@@ -0,0 +1,31 @@
+
+name: Warp NC Tests
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  warp-nc-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout noobaa-core
+        uses: actions/checkout@v4
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Create Warp logs directory
+        run: |
+          set -x
+          cd ./noobaa-core
+          mkdir -p logs/warp-test-logs
+          chmod 777 logs/warp-test-logs
+
+      - name: Run NC Warp tests
+        run: |
+          set -x
+          cd ./noobaa-core
+          make test-nc-warp
+

.github/workflows/warp-tests.yaml

@@ -0,0 +1,30 @@
+name: Warp Tests
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  warp-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout noobaa-core
+        uses: actions/checkout@v4
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Create Warp logs directory
+        run: |
+          set -x
+          cd ./noobaa-core
+          mkdir -p logs/warp-test-logs
+          chmod 777 logs/warp-test-logs
+
+      - name: Run Warp tests
+        run: |
+          set -x
+          cd ./noobaa-core
+          make test-warp
+

Makefile

@@ -337,6 +337,22 @@ test-cephs3: tester
 	@$(call remove_docker_network)
 .PHONY: test-cephs3
 
+test-warp: tester
+	@echo "\033[1;34mRunning warp tests with Postgres.\033[0m"
+	@$(call create_docker_network)
+	@$(call run_postgres)
+	@echo "\033[1;34mRunning warp tests\033[0m"
+	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/warp/run_warp_on_test_container.sh"
+	@$(call stop_noobaa)
+	@$(call stop_postgres)
+	@$(call remove_docker_network)
+.PHONY: test-warp
+
+test-nc-warp: tester
+	@echo "\033[1;34mRunning warp tests on NC environment\033[0m"
+	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/warp/run_nc_warp_on_test_container.sh"
+.PHONY: test-nc-warp
+
 test-nsfs-cephs3: tester
 	@echo "\033[1;34mRunning Ceph S3 tests on NSFS Standalone platform\033[0m"
 	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh"

docs/CI & Tests/warp.md

@@ -0,0 +1,95 @@
+# Warp Github Action, Tests and Tool
+
+1. [Introduction](#introduction)
+2. [Warp GitHub actions](#warp-github-actions)
+3. [Warp Makefile Targets](#warp-makefile-targets)
+4. [run_warp.js Tool](#run_warpjs-tool)
+5. [Manual Warp Installation](#manual-warp-installation)
+
+
+
+## Introduction
+
+[Warp](https://github.com/minio/warp) is a benchmarking tool for S3-compatible object storage systems, NooBaa CI runs Warp as performance/burst/scale tests for the NooBaa system on both containerized and Non Containerized flavors.
+Currently, Warp runs as part of our PR tests and in the future we will add long Warp runs as part of our nightly CI process. 
+
+## Warp GitHub actions
+
+NooBaa CI contains 2 Github actions that build, configure and run Warp. These Github actions run automatically on every PR and on every push, and can run by workflow dispatch manually.
+* [Warp Tests](../../.github/workflows/warp-tests.yaml) - Based on NooBaa Tester image, runs Warp on standard NooBaa (db configuration).
+* [Warp NC Tests](../../.github/workflows/warp-nc-tests.yaml) - Based on NooBaa Tester image, runs Warp on non-containerized NooBaa (ConfigFS configuration).
+
+Our next goal is to add longer Warp runs as part of NooBaa's nightly CI process.
+
+## Warp Makefile Targets
+
+One can run Warp tests on NooBaa using Warp Makefile targets - 
+* `make run-warp` - Based on NooBaa Tester image, runs Warp on standard NooBaa (db configuration).
+* `make run-nc-warp` - Based on NooBaa Tester image, runs Warp on non-containerized NooBaa (ConfigFS configuration).
+
+The above makefile targets, build NooBaa tester image, and later deploy NooBaa (DB/ConfigFS deployments), install warp, create default account and bucket and runs the run_warp.js tool.
+
+## `run_warp.js` Tool
+
+The `run_warp.js` script is designed to execute Warp performance tests for the NooBaa system. This script provides a streamlined way to configure and run Warp tests with various parameters.
+
+
+#### Usage
+
+The script can be executed using Node.js. Below is the usage syntax:
+
+```bash
+node run_warp.js [options]
+```
+
+#### Command-Line Arguments
+
+The following arguments are supported by the script:
+
+| Argument            | Description                                                                 | Default Value            |
+|---------------------|-----------------------------------------------------------------------------|--------------------------|
+| `--op`              | Warp operation to run (`mix`, `put`, `get`, etc.).                         | `mixed`                 |
+| `--concurrency`     | Number of workers to run the tests.                                         | `5` (default workers)    |
+| `--bucket`          | Bucket name to run the tests on.                                            | `warp-benchmark-bucket` |
+| `--duration`        | Duration of the tests (e.g., `30s`, `1m`, `1h`).                           | `10m`                   |
+| `--disable-multipart` | Disable multipart upload (`true` or `false`).                              | `true`                  |
+| `--access-key`      | Access key for the tests.                                                   | Derived from account.   |
+| `--secret-key`      | Secret key for the tests.                                                   | Derived from account.   |
+| `--obj-size`        | Object size for the tests (e.g., `1k`, `1m`, `1g`).                        | `1k`                    |
+| `--account-name`    | Account name to use for the tests.                                          | `warp_account`          |
+| `--help`            | Display usage information.                                                 | N/A                     |
+
+
+#### Warp Command Construction
+
+The Warp command is constructed dynamically based on the provided arguments. Below is an example of the command:
+
+```bash
+warp mixed --host=localhost:443 --access-key=<access_key> --secret-key=<secret_key> --bucket=<bucket> --obj.size=1k --duration=10m --disable-multipart=true --tls --insecure --concurrent 5
+```
+
+#### Future Improvements
+
+1. **Support for Additional Warp Features**:
+   - Add support for more Warp operations and configurations.
+   - Implement logging of test results in CSV format.
+
+2. **Nightly CI Automation**:
+   - Integrate the Warp tests into nightly CI/CD pipelines for automated performance testing.
+
+---
+
+
+## Manual Warp Installation
+
+Linux latest warp release - https://github.com/minio/warp/releases/download/v1.1.4/warp_Linux_x86_64.tar.gz
+
+Darwin latest warp release - https://github.com/minio/warp/releases/download/v1.1.4/warp_Darwin_x86_64.tar.gz
+
+```
+curl -L <warp_release_link> -o warp.tar.gz
+tar -xzf warp.tar.gz
+chmod +x warp
+mv warp /usr/local/bin/warp
+warp --version
+```

src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js

@@ -12,9 +12,8 @@ dbg.set_process_name('test_ceph_s3');
 
 const fs = require('fs');
 const os_utils = require('../../../util/os_utils');
-const test_utils = require('../../system_tests/test_utils');
-const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants');
 const { CEPH_TEST } = require('./test_ceph_s3_constants.js');
+const { get_access_keys, create_account } = require('../nc_test_utils');
 
 async function main() {
     try {
@@ -73,26 +72,7 @@ async function ceph_test_setup() {
     console.info('CEPH TEST CONFIGURATION: DONE');
 }
 
-/**
- * get_access_keys returns account access keys using noobaa-cli
- * @param {string} account_name 
- */
-async function get_access_keys(account_name) {
-    const options = { name: account_name, show_secrets: true };
-    const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.STATUS, options);
-    const json_account = JSON.parse(res);
-    const account_data = json_account.response.reply;
-    return account_data.access_keys[0];
-}
 
-/**
- * create_account creates accounts using noobaa-cli
- * @param {{ name?: string, uid?: number, gid?: number, anonymous?: boolean }} [options] 
- */
-async function create_account(options = {}) {
-    const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, options);
-    console.log('Account Created', res);
-}
 
 if (require.main === module) {
     main();

src/test/system_tests/nc_test_utils.js

@@ -0,0 +1,50 @@
+/* Copyright (C) 2016 NooBaa */
+'use strict';
+
+const test_utils = require('./test_utils');
+const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants');
+
+/**
+ * get_access_keys returns account access keys using noobaa-cli
+ * @param {string} account_name 
+ */
+async function get_access_keys(account_name) {
+    const account_data = await get_account(account_name);
+    return account_data.access_keys[0];
+}
+
+/**
+ * get_account returns account data using noobaa-cli
+ * @param {string} account_name 
+ */
+async function get_account(account_name) {
+    const options = { name: account_name, show_secrets: true };
+    const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.STATUS, options);
+    const json_account = JSON.parse(res);
+    const account_data = json_account.response.reply;
+    return account_data;
+}
+
+/**
+ * create_account creates account using noobaa-cli
+ * @param {{ name?: string, uid?: number, gid?: number, anonymous?: boolean }} [options] 
+ */
+async function create_account(options = {}) {
+    const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, options);
+    console.log('NC Account Created', res);
+}
+
+/**
+ * create_bucket creates bucket using noobaa-cli
+ * @param {{ name?: string, owner?: string, path?: string}} [options] 
+ */
+async function create_bucket(options = {}) {
+    const res = await test_utils.exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, options);
+    console.log('NC Bucket Created', res);
+}
+
+// EXPORTS
+exports.get_account = get_account;
+exports.get_access_keys = get_access_keys;
+exports.create_account = create_account;
+exports.create_bucket = create_bucket;

src/test/system_tests/warp/configure_warp.js

@@ -0,0 +1,31 @@
+/* Copyright (C) 2016 NooBaa */
+"use strict";
+
+const dbg = require('../../../util/debug_module')(__filename);
+dbg.set_process_name('test_warp_s3');
+
+const { WARP_TEST } = require('./warp_constants.js');
+const { create_warp_account, create_warp_bucket } = require('./warp_utils.js');
+
+async function main() {
+    try {
+        await warp_test_setup();
+    } catch (err) {
+        console.error(`Warp Setup Failed: ${err}`);
+        process.exit(1);
+    }
+    process.exit(0);
+}
+
+async function warp_test_setup() {
+    console.info('WARP TEST CONFIGURATION:', JSON.stringify(WARP_TEST));
+    await create_warp_account();
+    await create_warp_bucket();
+    console.info('WARP TEST SETUP DONE');
+}
+
+if (require.main === module) {
+    main();
+}
+
+

src/test/system_tests/warp/run_nc_warp_on_test_container.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+export PS4='\e[36m+ ${FUNCNAME:-main}\e[0m@\e[32m${BASH_SOURCE}:\e[35m${LINENO} \e[0m'
+
+set -e
+# It will add to the logs every line that we run
+set -x
+
+# ====================================================================================
+# Set the environment variables
+export email='[email protected]'
+export password=123456789
+
+export ENDPOINT_PORT=6001
+export ENDPOINT_SSL_PORT=6443
+export S3_SERVICE_HOST=localhost
+
+export CEPH_TEST_LOGS_DIR=/logs/warp-nc-test-logs
+export CONFIG_DIR=/etc/noobaa.conf.d/
+export FS_ROOT_1=/tmp/nsfs_root1/
+export FS_ROOT_2=/tmp/nsfs_root2/
+export WARP_BUCKET_PATH=${FS_ROOT_1}/warp-benchmark-bucket/
+export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write'
+
+# ====================================================================================
+
+# 1. Create configuration directory
+# 2. Create config.json file
+mkdir -p ${CONFIG_DIR}
+config='{"ALLOW_HTTP":true, "ENDPOINT_FORKS":2}'
+echo "$config" > ${CONFIG_DIR}/config.json
+
+# 1. Create root directory for bucket creation
+# 2. Add permission to all users
+# this will allow the new accounts to create directories (buckets),
+# else we would see [Error: Permission denied] { code: 'EACCES' }
+mkdir -p ${FS_ROOT_1}
+mkdir -p ${FS_ROOT_2}
+mkdir -p ${WARP_BUCKET_PATH}
+chmod 777 ${FS_ROOT_1}
+chmod 777 ${FS_ROOT_2}
+chmod 777 ${WARP_BUCKET_PATH}
+
+# Create the logs directory
+mkdir -p ${CEPH_TEST_LOGS_DIR}
+
+
+# ====================================================================================
+
+# Install warp
+curl -L https://github.com/minio/warp/releases/download/v1.1.4/warp_Linux_x86_64.tar.gz -o warp_Linux_x86_64.tar.gz
+tar -xzf warp_Linux_x86_64.tar.gz
+chmod +x warp
+mv warp /usr/local/bin/warp
+warp --version
+
+# ====================================================================================
+
+# Deploy standalone NooBaa on the test container
+# And create the accounts needed for the Ceph tests
+./src/deploy/NVA_build/standalone_deploy_nsfs.sh
+
+# ====================================================================================
+
+cd /root/node_modules/noobaa-core/
+
+# Configure the warp test 
+node ./src/test/system_tests/warp/configure_warp.js
+
+# ====================================================================================
+
+# Run the warp tests
+node ./src/test/system_tests/warp/run_warp.js
+
+# ====================================================================================