Introduce run-async subtool (#391)

* feature/callback: add callback in to run_remote logic

* feature/callback: test fix for gnupg

* feature/callback: remove cat gpg.conf command from action

* feature/callback: try another way of fix gnupg problem

* feature/callback: remove actions-setup-docker

* feature/callback: different poetry install

* feature/callback: shell way to install poetry

* feature/callback: try another actions lib to install poetry

* feature/callback: fix file

* feature/callback: poetry version

* feature/callback: test ci without python3.6

* feature/callback: fix formatting

* feature/callback: fix what flake8 found

* feature/callback: add callback url

* feature/callback: change github env variable name to avoid problem with github actions env

* feature/callback: remove branch, workflow id from request, make github token not required

* feature/callback: fix callback url formatting

* feature/callback: callback_headers statement in function

* feature/async-run: add async run

* feature/async-run: private key fix

* feature/async-run: return actual poetry lock

* feature/async-run: delete poetry.lock file

* feature/async-run: poetry.lock from master

* feature/async-run: remove callback action

* feature/async-run: fix path

* feature/async-run: fix path

* feature/async-run: fix path for python

* feature/async-run: command line, terraform install

* feature/async-run: add sudo

* feature/async-run: add /dev/null redirect for useless stdout from gpg

* feature/async-run: add copy of private key in right directory

* feature/async-run: rename pem file naming, file writing change

* feature/async-run: fixing pem formatting

* feature/async-run: module path

* feature/async-run: module copy fix and module path fix

* feature/async-run: docker redis, add info in output of copy files

* feature/async-run: docker redis install fix

* feature/async-run: fix for branches with slash

* feature/async-run: fix for branches with slash, add sudo to docker run

* feature/async-run: remove fix for branch with slashes

* feature/async-run: add run for service, add github token pass to run variables

* feature/async-run: token env variable pass, code format

* feature/async-run: remove duplicate code

* feature/async-run: testing infrastructure repo url

* feature/async-run: cleanup code

* feature/async-run: cleanup and fix diff between master

* feature/async-run: remove unused files

* feature/async-run: fix formatting

* feature/async-run: flake8 formatting

* feature/async-run: fix commands

* feature/async-run: fix commands

* feature/async-run: return commands

* feature/async-run: add job name env pass

* feature/async-run: add job name parameter

* Updated run-async not to run stack

* Bumping version from 0.9.55 to 0.10.0

---------

Co-authored-by: filipe oliveira <[email protected]>

Author: anton-tokarev-softeq

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.9.55"
+version = "0.10.0"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/cli.py

@@ -21,6 +21,7 @@
 from redisbench_admin.extract.extract import extract_command_logic
 from redisbench_admin.grafana_api.args import create_grafana_api_arguments
 from redisbench_admin.grafana_api.grafana_api import grafana_api_command_logic
+from redisbench_admin.run_async.run_async import run_async_command_logic
 from redisbench_admin.run_local.args import create_run_local_arguments
 from redisbench_admin.run_local.run_local import run_local_command_logic
 from redisbench_admin.run_remote.args import create_run_remote_arguments
@@ -76,6 +77,8 @@ def main():
 
     if requested_tool == "run-remote":
         parser = create_run_remote_arguments(parser)
+    elif requested_tool == "run-async":
+        parser = create_run_remote_arguments(parser)
     elif requested_tool == "run-local":
         parser = create_run_local_arguments(parser)
     elif requested_tool == "extract":
@@ -100,6 +103,7 @@ def main():
             "compare",
             "run-local",
             "run-remote",
+            "run-async",
             "deploy",
             "export",
             "extract",
@@ -143,6 +147,8 @@ def main():
         run_local_command_logic(args, project_name, project_version)
     if requested_tool == "run-remote":
         run_remote_command_logic(args, project_name, project_version)
+    if requested_tool == "run-async":
+        run_async_command_logic(argv, args, project_name, project_version)
     if requested_tool == "export":
         export_command_logic(args, project_name, project_version)
     if requested_tool == "extract":

redisbench_admin/run_async/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/run_async/async_env.py

@@ -0,0 +1,91 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import logging
+import tarfile
+
+from redisbench_admin.run_async.terraform import (
+    retrieve_inventory_info,
+    terraform_spin_or_reuse_env,
+)
+
+
+def tar_files(filename="archive.tar", path="./../../"):
+    logging.info("Creating tar archive: {} for path {}".format(filename, path))
+    with tarfile.open(filename, "w") as tar:
+        tar.add(path, arcname=".")
+    return filename
+
+
+def async_env_setup(
+    args,
+    benchmark_config,
+    remote_envs,
+    repetition,
+    test_name,
+    tf_bin_path,
+    tf_github_actor,
+    tf_github_org,
+    tf_github_repo,
+    tf_github_sha,
+    tf_setup_name_sufix,
+    tf_triggering_env,
+    tf_timeout_secs=7200,
+    tf_override_name=None,
+    tf_folder_path=None,
+):
+    server_plaintext_port = args.db_port
+    db_ssh_port = args.db_ssh_port
+    client_ssh_port = args.client_ssh_port
+    username = args.user
+    if args.inventory is not None:
+        (
+            status,
+            client_public_ip,
+            server_private_ip,
+            server_public_ip,
+        ) = retrieve_inventory_info(args.inventory)
+        if status is False:
+            logging.error(
+                "Missing one of the required keys for inventory usage. Exiting..."
+            )
+            exit(1)
+        logging.info("Using the following connection addresses.")
+        logging.info("client_public_ip={}".format(client_public_ip))
+        logging.info("server_public_ip={}".format(server_public_ip))
+        logging.info("server_private_ip={}".format(server_private_ip))
+    else:
+        (
+            client_public_ip,
+            _,
+            _,
+            server_private_ip,
+            server_public_ip,
+            username,
+        ) = terraform_spin_or_reuse_env(
+            benchmark_config,
+            remote_envs,
+            repetition,
+            test_name,
+            tf_bin_path,
+            tf_github_actor,
+            tf_github_org,
+            tf_github_repo,
+            tf_github_sha,
+            tf_setup_name_sufix,
+            tf_triggering_env,
+            tf_timeout_secs,
+            tf_override_name,
+            tf_folder_path,
+        )
+    return (
+        client_public_ip,
+        server_plaintext_port,
+        server_private_ip,
+        server_public_ip,
+        db_ssh_port,
+        client_ssh_port,
+        username,
+    )

redisbench_admin/run_async/benchmark.py

@@ -0,0 +1,49 @@
+from redisbench_admin.utils.benchmark_config import (
+    prepare_benchmark_definitions,
+    process_benchmark_definitions_remote_timeouts,
+)
+
+
+class BenchmarkClass:
+    def __init__(
+        self,
+        benchmark_defs_result=None,
+        benchmark_definitions=None,
+        default_metrics=None,
+        exporter_timemetric_path=None,
+        default_specs=None,
+        clusterconfig=None,
+        remote_envs_timeout=None,
+        benchmark_runs_plan=None,
+    ):
+        self.benchmark_defs_result = benchmark_defs_result
+        self.benchmark_definitions = benchmark_definitions
+        self.default_metrics = default_metrics
+        self.exporter_timemetric_path = exporter_timemetric_path
+        self.default_specs = default_specs
+        self.clusterconfig = clusterconfig
+        self.benchmark_artifacts_table_headers = [
+            "Setup",
+            "Test-case",
+            "Artifact",
+            "link",
+        ]
+        self.benchmark_artifacts_table_name = "Benchmark client artifacts"
+        self.benchmark_runs_plan = benchmark_runs_plan
+
+        self.remote_envs_timeout = remote_envs_timeout
+
+    def prepare_benchmark_definitions(self, args):
+        (
+            self.benchmark_defs_result,
+            self.benchmark_definitions,
+            self.default_metrics,
+            self.exporter_timemetric_path,
+            self.default_specs,
+            self.clusterconfig,
+        ) = prepare_benchmark_definitions(args)
+
+    def populate_remote_envs_timeout(self):
+        self.remote_envs_timeout = process_benchmark_definitions_remote_timeouts(
+            self.benchmark_definitions
+        )

redisbench_admin/run_async/log.py

@@ -0,0 +1,5 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/run_async/render_files.py

@@ -0,0 +1,79 @@
+import jinja2
+
+
+def renderServiceFile(access_key, region, secret_key, gh_token, job_name, args, argv):
+    environment = jinja2.Environment()
+    template = environment.from_string(
+        """[Unit]
+Description=Redisbench-admin run service
+
+[Service]
+Type=oneshot
+Environment="PATH=/home/ubuntu/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+Environment="AWS_ACCESS_KEY_ID={{access_key}}"
+Environment="AWS_DEFAULT_REGION={{region}}"
+Environment="AWS_SECRET_ACCESS_KEY={{secret_key}}"
+{% if gh_token != None %}Environment="GH_TOKEN={{gh_token}}"{% endif %}
+{% if job_name != None %}Environment="CIRCLE_JOB={{job_name}}"{% endif %}
+WorkingDirectory=/home/ubuntu/work_dir/tests/benchmarks
+Type=simple
+User=ubuntu
+ExecStart=/home/ubuntu/work_dir/redisbench-admin/.venv/bin/python /home/ubuntu/work_dir/redisbench-admin/run.py run-remote {{args}}
+
+[Install]
+WantedBy=multi-user.target
+    """
+    )
+    if "--private_key" not in argv:
+        argv.append("--private_key")
+        argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem")
+    else:
+        argv[
+            argv.index(args.private_key)
+        ] = "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
+    if len(args.module_path) != 0:
+        argv[argv.index(args.module_path[0])] = (
+            "/home/ubuntu/work_dir/tests/benchmarks/"
+            + args.module_path[0].split("/")[-1]
+        )
+    argv_str = " ".join(argv)
+    with open("redisbench-admin.service", mode="w", encoding="utf-8") as results:
+        results.write(
+            template.render(
+                access_key=access_key,
+                region=region,
+                secret_key=secret_key,
+                gh_token=gh_token,
+                job_name=job_name,
+                args=argv_str,
+            )
+        )
+
+
+def renderRunFile():
+    with open("run.py", mode="w", encoding="utf-8") as run_file:
+        run_file.write(
+            """#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from redisbench_admin.cli import main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\\.pyw|\\.exe)?$", "", sys.argv[0])
+    sys.exit(main())
+"""
+        )
+
+
+def savePemFile(pem_data):
+    with open("benchmarks.redislabs.pem", mode="w", encoding="utf-8") as pem_file:
+        pem_data = pem_data.replace("-----BEGIN RSA PRIVATE KEY-----", "")
+        pem_data = pem_data.replace("-----END RSA PRIVATE KEY-----", "")
+        pem_data = pem_data.replace(" ", "\n")
+        pem_data = (
+            "-----BEGIN RSA PRIVATE KEY-----"
+            + pem_data
+            + "-----END RSA PRIVATE KEY-----"
+        )
+        pem_file.write(pem_data)