Merge pull request #11 from sourceplusplus/update

test: probe config tests

Author: BFergerson

.github/workflows/e2e.yml

@@ -41,7 +41,7 @@ jobs:
         run: |
           until $(curl --header "Authorization: Bearer ${{ env.SPP_JWT_TOKEN }}" --insecure --output /dev/null --silent --fail https://localhost:12800/health); do printf '.'; sleep 5; done
       - run: curl -sS https://webinstall.dev/jq | bash
-      - run: sleep 30s #wait for services to connect; todo: use health endpoint
+      - run: sleep 45s #wait for services to connect; todo: use health endpoint
 
       - name: Verify probe connected
         run: |

build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 python {
-    pip("apache-skywalking:0.7.0")
+    pip("apache-skywalking:0.8.0")
     pip("vertx-eventbus-client:1.0.0")    
 }
 

e2e/docker-compose.yml

@@ -12,7 +12,6 @@ services:
       - "5000:5000"
     environment:
       - SPP_PLATFORM_HOST=spp-platform
-      - SPP_OAP_HOST=skywalking-oap
       - GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/usr/local/bin/config/spp-platform.crt
   spp-platform:
     image: sourceplusplus/spp-platform:latest

e2e/spp-probe.yml

@@ -36,8 +36,6 @@ skywalking:
   logging:
     level: "INFO"
   agent:
-    is_cache_enhanced_class: true
-    class_cache_mode: "FILE"
     service_name: "spp"
   collector:
     backend_service: "spp-platform:11801"

requirements.txt

@@ -1,5 +1,5 @@
 vertx-eventbus-client~=1.0.0
-apache-skywalking~=0.7.0
-nopdb~=0.1.0
-pyhumps~=3.0.2
+apache-skywalking~=0.8.0
+nopdb~=0.2.0
+pyhumps~=3.7.2
 PyYAML~=6.0

settings.gradle

@@ -1,5 +1,5 @@
 pluginManagement {
     plugins {
-        id 'com.avast.gradle.docker-compose' version "0.14.11" apply false
+        id 'com.avast.gradle.docker-compose' version "0.16.8" apply false
     }
 }

setup.py

@@ -10,7 +10,7 @@
       license='Apache License, Version 2.0',
       packages=setuptools.find_packages(),
       install_requires=['vertx-eventbus-client>=1.0.0',
-                        'apache-skywalking>=0.7.0',
-                        'nopdb>=0.1.0',
-                        'pyhumps>=3.0.2',
+                        'apache-skywalking>=0.8.0',
+                        'nopdb>=0.2.0',
+                        'pyhumps>=3.7.2',
                         'PyYAML>=6.0'])

sourceplusplus/SourcePlusPlus.py

@@ -16,14 +16,22 @@
 
 class SourcePlusPlus(object):
 
-    def get_config_value(self, env, default, true_default):
-        env_value = os.getenv(env)
-        if env_value is not None:
-            return env_value
-        elif default is not None:
-            return default
-        else:
-            return true_default
+    @staticmethod
+    def __set_config_default(config_dict, config_name, env, default):
+        config_path = config_name.split(".")
+        tmp_config = config_dict
+        for i in range(len(config_path)):
+            if i == len(config_path) - 1 and tmp_config.get(config_path[i]) is None:
+                if default is bool:
+                    tmp_config[config_path[i]] = str(os.getenv(env, default)).lower() == "true"
+                else:
+                    tmp_config[config_path[i]] = os.getenv(env, default)
+
+            if tmp_config.get(config_path[i]) is None:
+                tmp_config[config_path[i]] = {}
+                tmp_config = tmp_config[config_path[i]]
+            else:
+                tmp_config = tmp_config[config_path[i]]
 
     def __init__(self, args: dict = None):
         if args is None:
@@ -33,41 +41,40 @@ def __init__(self, args: dict = None):
         if os.path.exists(probe_config_file):
             probe_config = yaml.full_load(open(probe_config_file, "r"))
 
-        # ensure probe_config has required keys
-        if probe_config.get("spp") is None:
-            probe_config["spp"] = {}
-        if probe_config.get("skywalking") is None:
-            probe_config["skywalking"] = {}
-        if probe_config["skywalking"].get("collector") is None:
-            probe_config["skywalking"]["collector"] = {}
-        if probe_config["skywalking"].get("agent") is None:
-            probe_config["skywalking"]["agent"] = {}
-
-        # set default values
-        probe_config["spp"]["probe_id"] = self.get_config_value(
-            "SPP_PROBE_ID", probe_config["spp"].get("probe_id"), str(uuid.uuid4())
+        # set spp default values
+        self.__set_config_default(
+            probe_config, "spp.probe_id", "SPP_PROBE_ID",
+            str(uuid.uuid4())
         )
-        probe_config["spp"]["platform_host"] = self.get_config_value(
-            "SPP_PLATFORM_HOST", probe_config["spp"].get("platform_host"), "localhost"
+        self.__set_config_default(
+            probe_config, "spp.platform_host", "SPP_PLATFORM_HOST",
+            "localhost"
         )
-        probe_config["spp"]["platform_port"] = self.get_config_value(
-            "SPP_PLATFORM_PORT", probe_config["spp"].get("platform_port"), 12800
+        self.__set_config_default(
+            probe_config, "spp.platform_port", "SPP_PLATFORM_PORT",
+            12800
         )
-        probe_config["spp"]["verify_host"] = str(self.get_config_value(
-            "SPP_TLS_VERIFY_HOST", probe_config["spp"].get("verify_host"), True
-        )).lower() == "true"
-        probe_config["spp"]["ssl_enabled"] = str(self.get_config_value(
-            "SPP_HTTP_SSL_ENABLED", probe_config["spp"].get("ssl_enabled"), True
-        )).lower() == "true"
-        probe_config["skywalking"]["agent"]["service_name"] = self.get_config_value(
-            "SPP_SERVICE_NAME", probe_config["skywalking"]["agent"].get("service_name"), "spp"
+        self.__set_config_default(
+            probe_config, "spp.verify_host", "SPP_TLS_VERIFY_HOST",
+            True
+        )
+        self.__set_config_default(
+            probe_config, "spp.ssl_enabled", "SPP_HTTP_SSL_ENABLED",
+            True
+        )
+        self.__set_config_default(
+            probe_config, "skywalking.agent.service_name", "SPP_SERVICE_NAME",
+            "spp"
         )
 
-        skywalking_port = self.get_config_value("SPP_OAP_PORT", 11800, 11800)
-        probe_config["skywalking"]["collector"]["backend_service"] = self.get_config_value(
-            "SPP_SKYWALKING_BACKEND_SERVICE",
-            probe_config["skywalking"]["collector"].get("backend_service"),
-            probe_config["spp"]["platform_host"] + ":" + str(skywalking_port)
+        # set sw default values
+        self.__set_config_default(
+            probe_config, "skywalking.collector.backend_service", "SPP_SKYWALKING_BACKEND_SERVICE",
+            probe_config["spp"]["platform_host"] + ":11800"
+        )
+        self.__set_config_default(
+            probe_config, "skywalking.plugin.toolkit.log.transmit_formatted", "SPP_SKYWALKING_LOG_TRANSMIT_FORMATTED",
+            True
         )
 
         for key, val in args.items():
@@ -89,8 +96,8 @@ def attach(self):
             collector_address=self.probe_config["skywalking"]["collector"]["backend_service"],
             service_name=self.probe_config["skywalking"]["agent"]["service_name"],
             log_reporter_active=True,
-            force_tls=self.probe_config["spp"]["ssl_enabled"] is True,
-            log_reporter_formatted=False
+            force_tls=self.probe_config["spp"]["ssl_enabled"],
+            log_reporter_formatted=self.probe_config["skywalking"]["plugin"]["toolkit"]["log"]["transmit_formatted"]
         )
         agent.start()
 

tests/probe_config/__init__.py

@@ -0,0 +1,9 @@
+spp:
+  platform_host: "spp-platform"
+  platform_port: 12800
+  ssl_enabled: false
+skywalking:
+  logging:
+    level: "WARN"
+  agent:
+    service_name: "tutorial-jvm"

tests/probe_config/resources/base-config.yml

@@ -0,0 +1,45 @@
+spp:
+  platform_host: "spp-platform"
+  platform_port: 12800
+  quiet_mode: false
+  verify_host: false
+  platform_certificate: |
+    MIIFLzCCAxegAwIBAgIGAYG9v8lfMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNVBAMM
+    DHNwcC1wbGF0Zm9ybTAeFw0yMjA3MDIwNzExMTFaFw0yMzA3MDIwNzExMTFaMBcx
+    FTATBgNVBAMMDHNwcC1wbGF0Zm9ybTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
+    AgoCggIBAJglr80YnYGgFu3ZmTqU1ZuYX9Na9W78kYfeEtE/v/LPzXOPCU3duHvI
+    q5W5Ev6fvww6iRK32tQmZ84CTr3L5HMIbgKHFHe7TFp+0Eb+2MdqFp1K8dRobUKd
+    vehIQsRi5I+I7lq0bQwT7cgysT/+sYUBWr6ZZSIe3Emqn+xHuwX9MXHSNzTRwZzQ
+    EYIECbih8Zxa7stxp3wowKB3Z9cLppfJ04JQ5eEXaNrVNUfPZb+eGCEcnPthN5vZ
+    WSz0G0OLkEaU5UH8FsvSBrpJmmHRIjEiYUpEITusk/QEPTBGS7nPPyhEvVj75/Q0
+    euMUZU7xSQfBblHO3nj/YoskCPCGI7wG7ah1FiXGDq6sfUAeG285CdBYKZYI2gWr
+    pPya21L7aOy9qLZw2SJ4HQ92qow9fHV52meDPeuyLJjhzHZoCKuOus0IdjztYJ8p
+    bAoZJtsjW9nVh0VCBIYpwkjZzWBsJvCObWJS/UBHKZwNH2sM7DQVqsJQTG2RGxts
+    d7GEn9AEzzehjBRQXhW22OH5PkujFw2vu23J6kUw33P5lDUOybAK4Bp10dP+tTzS
+    60MjspN+N0ZKcgiYegB0/W7Pkc2pGrmVpmv1F4znCuXkn8nUSqzGZ0eX5ehZZ1Wa
+    j4Nc2/WlIvuAdPM/SEn4gzZoZ+FdOQEoZmsDdjOxBwihgeNF3WefAgMBAAGjgYAw
+    fjArBgNVHREBAf8EITAfggxzcHAtcGxhdGZvcm2HBH8AAAGCCWxvY2FsaG9zdDAd
+    BgNVHQ4EFgQUQYUL1pWw3KzgUXBO+2Gh8mwsSxQwHwYDVR0jBBgwFoAUQYUL1pWw
+    3KzgUXBO+2Gh8mwsSxQwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC
+    AgEAQyBpimXMYDuIEL8PXCab8EzoRfsu+Ot5ZdcwkAE/93H0YnTHr7+79WzTnIcj
+    VW/MQ4keDQSljjA6x9Te2+qH2eMFuPPa0X5iXkLQaJro3kaVzi/NWbHkJ2RSemsw
+    6lqx0G0f1ojoIXcgmurwlTRyoU/5wikX2WTS8VSA9Tylv5K6nYXaEU5eMCCdt9rI
+    5RQGMatQDNPmwJCgwkW2+zpDJTBSPl7i7qym5+ljCKD+Vuawq9SRTU1I3BZXB1bN
+    LdlZLrCbnM9nSW64oXG7X5h2t2N8PifCg/kwiE5UHO9snv1a6bjHyVyBXOhS4zJT
+    JhM8i8911uCaxc6e+fBknT4t2V7DET3MCxUQ2fvJ75JgBD3OPo6qcpdOwWwrQiyC
+    nITRe+LLhZQfyIsfbXoNE930uFNQvoYoHIovrOm6nUPCzBRx9YOZpdOkbYi3V6tA
+    ooUtSLrQVO++tk4pZsvLRwZ5Yku+/J/nzjYDk+dtCxd5bM3vHAfhEjzvijouVCA3
+    +7VjhiWpBSNeUt7ENX9s2GGs0OjUCAoOFoOENDcZdu7u9hrzUIhHsFGTyL0DQ1lT
+    Ip40RbmWHOunjHYEZRbJhB2Mo+hcIMGpARIq3WSYQFZnxBbchqnUnVvvo/bE5yB8
+    g6TYnkFuMFHg/LvA7RX1v2Xha5YSEHBBV7V2emBmZc1Tpkk=
+skywalking:
+  logging:
+    level: "INFO"
+  agent:
+    service_name: "spp"
+  collector:
+    backend_service: "spp-platform:11801"
+  plugin:
+    toolkit:
+      log:
+        transmit_formatted: false

tests/probe_config/resources/extended-config.yml

@@ -0,0 +1,48 @@
+import os
+import unittest
+from unittest import mock
+
+from sourceplusplus.SourcePlusPlus import SourcePlusPlus
+
+
+class TestSum(unittest.TestCase):
+
+    def test_default_config(self):
+        spp = SourcePlusPlus()
+        self.assertEqual("localhost", spp.probe_config["spp"]["platform_host"])
+        self.assertEqual(12800, spp.probe_config["spp"]["platform_port"])
+        self.assertEqual(True, spp.probe_config["spp"]["verify_host"])
+        self.assertEqual(True, spp.probe_config["spp"]["ssl_enabled"])
+        self.assertIsNotNone(spp.probe_config["spp"]["probe_id"])
+        self.assertEqual("localhost:11800", spp.probe_config["skywalking"]["collector"]["backend_service"])
+        self.assertEqual("spp", spp.probe_config["skywalking"]["agent"]["service_name"])
+
+    @mock.patch.dict('os.environ', {
+        'SPP_PROBE_CONFIG_FILE': '%s/resources/base-config.yml' % os.path.dirname(__file__)
+    })
+    def test_config_from_env(self):
+        spp = SourcePlusPlus()
+        self.assertEqual("spp-platform", spp.probe_config["spp"]["platform_host"])
+        self.assertEqual(12800, spp.probe_config["spp"]["platform_port"])
+        self.assertEqual(True, spp.probe_config["spp"]["verify_host"])
+        self.assertEqual(False, spp.probe_config["spp"]["ssl_enabled"])
+        self.assertEqual("spp-platform:11800", spp.probe_config["skywalking"]["collector"]["backend_service"])
+        self.assertEqual("tutorial-jvm", spp.probe_config["skywalking"]["agent"]["service_name"])
+        self.assertEqual("WARN", spp.probe_config["skywalking"]["logging"]["level"])
+
+    @mock.patch.dict('os.environ', {
+        'SPP_PROBE_CONFIG_FILE': '%s/resources/extended-config.yml' % os.path.dirname(__file__)
+    })
+    def test_config_from_env_extended(self):
+        spp = SourcePlusPlus()
+        self.assertEqual("spp-platform", spp.probe_config["spp"]["platform_host"])
+        self.assertEqual(12800, spp.probe_config["spp"]["platform_port"])
+        self.assertEqual(False, spp.probe_config["spp"]["quiet_mode"])
+        self.assertEqual(False, spp.probe_config["spp"]["verify_host"])
+        self.assertIsNotNone(spp.probe_config["spp"]["platform_certificate"])
+        self.assertIsNotNone(spp.probe_config["spp"]["probe_id"])
+        self.assertEqual(True, spp.probe_config["spp"]["ssl_enabled"])
+        self.assertEqual("INFO", spp.probe_config["skywalking"]["logging"]["level"])
+        self.assertEqual("spp", spp.probe_config["skywalking"]["agent"]["service_name"])
+        self.assertEqual("spp-platform:11801", spp.probe_config["skywalking"]["collector"]["backend_service"])
+        self.assertEqual(False, spp.probe_config["skywalking"]["plugin"]["toolkit"]["log"]["transmit_formatted"])