Enh: CLI add cli to generate/import encryption key. If need, reload the key in the daemon.

Enh: TEST now UDP key is set with CLI call.

Author: naparuba

bin/opsbro

@@ -57,17 +57,16 @@ if __name__ == '__main__':
     parser.error = old_error
 
     # We need to import our libs, but can be an install or directly in the tarball (if executed directly from the install tar ball)
-    install_less = False
     # In windows exe, __file__ is not set
     if not hasattr(sys, 'frozen'):
         my_file = os.path.abspath(os.path.dirname(__file__))
     else:
         my_file = os.path.dirname(os.path.abspath(sys.argv[0]))
     my_root_dir = os.path.dirname(my_file)
+    
     # Both files must be present
     tar_ball_detection_files = (os.path.join(my_root_dir, 'README.md'), os.path.join(my_root_dir, 'setup.py'))
     if os.path.exists(tar_ball_detection_files[0]) and os.path.exists(tar_ball_detection_files[1]):
-        install_less = True
         # Be sure that the opsbro lib is the install root one
         sys.path.insert(0, my_root_dir)
         from opsbro.defaultpaths import remap_from_install_dir

data/core-configuration/packs/core-cli-gossip/cli/cli.py

@@ -5,9 +5,12 @@
 #    Gabes Jean, [email protected]
 
 from __future__ import print_function
+import os
 import sys
 import time
 import itertools
+import uuid
+import base64
 
 from opsbro.characters import CHARACTERS
 from opsbro.log import cprint, logger, sprintf
@@ -234,6 +237,65 @@ def do_zone_list():
             cprint(sub_zname, color='cyan')
 
 
+def _save_key(key_string, zone_name, key_path):
+    with open(key_path, 'wb') as f:
+        f.write(key_string)
+    
+    cprint('%s OK the key is saved as file %s' % (CHARACTERS.check, key_path))
+    
+    # Try to send the information to the agent, so it can reload the key
+    try:
+        get_opsbro_json('/agent/zones-keys/reload/%s' % zone_name)
+    except get_request_errors():
+        cprint('  | The agent seems to not be started. Skipping hot key reload.', color='grey')
+        return
+
+
+def do_zone_key_generate(zone, erase=False):
+    from opsbro.defaultpaths import DEFAULT_CFG_DIR
+    from opsbro.configurationmanager import ZONE_KEYS_DIRECTORY_NAME
+    print_h1('Generate a new key for the zone %s' % zone)
+    
+    key_path = os.path.join(DEFAULT_CFG_DIR, ZONE_KEYS_DIRECTORY_NAME, '%s.key' % zone)
+    
+    if os.path.exists(key_path) and not erase:
+        cprint('ERROR: the key %s is already existing', color='red')
+        cprint('  %s Note: You can use the --erase parameter to erase over an existing key' % (CHARACTERS.corner_bottom_left), color='grey')
+        sys.exit(2)
+    
+    k = uuid.uuid1().hex[:16]
+    b64_k = base64.b64encode(k)
+    cprint('Encryption key for the zone ', end='')
+    cprint(zone, color='magenta', end='')
+    cprint(' :', end='')
+    cprint(b64_k, color='green')
+    _save_key(b64_k, zone, key_path)
+
+
+def do_zone_key_import(zone, key, erase=False):
+    from opsbro.defaultpaths import DEFAULT_CFG_DIR
+    from opsbro.configurationmanager import ZONE_KEYS_DIRECTORY_NAME
+    
+    key_path = os.path.join(DEFAULT_CFG_DIR, ZONE_KEYS_DIRECTORY_NAME, '%s.key' % zone)
+    
+    if os.path.exists(key_path) and not erase:
+        cprint('ERROR: the key %s is already existing', color='red')
+        cprint('  %s Note: You can use the --erase parameter to erase over an existing key' % (CHARACTERS.corner_bottom_left), color='grey')
+        sys.exit(2)
+    # check key is base64(len16)
+    try:
+        raw_key = base64.b64decode(key)
+    except TypeError:  # bad key
+        cprint('ERROR: the key is not valid. (not base4 encoded)', color='red')
+        sys.exit(2)
+    if len(raw_key) != 16:
+        cprint('ERROR: the key is not valid. (not 128bits)', color='red')
+        sys.exit(2)
+    
+    # Note: key is the original b64 encoded one, we did check it
+    _save_key(key, zone, key_path)
+
+
 def __print_detection_spinner(timeout):
     spinners = itertools.cycle(CHARACTERS.spinners)
     start = time.time()
@@ -430,30 +492,30 @@ def do_wait_members(name='', display_name='', group='', count=1, timeout=30):
 
 
 exports = {
-    do_members         : {
+    do_members          : {
         'keywords'   : ['gossip', 'members'],
         'args'       : [
             {'name': '--detail', 'type': 'bool', 'default': False, 'description': 'Show detail mode for the cluster members'},
         ],
         'description': 'List the cluster members'
     },
 
-    do_members_history : {
+    do_members_history  : {
         'keywords'   : ['gossip', 'history'],
         'args'       : [
         ],
         'description': 'Show the history of the gossip nodes'
     },
 
-    do_join            : {
+    do_join             : {
         'keywords'   : ['gossip', 'join'],
         'description': 'Join another node cluster',
         'args'       : [
             {'name': 'seed', 'default': '', 'description': 'Other node to join. For example 192.168.0.1:6768'},
         ],
     },
 
-    do_leave           : {
+    do_leave            : {
         'keywords'   : ['gossip', 'leave'],
         'description': 'Put in leave a cluster node',
         'args'       : [
@@ -462,7 +524,7 @@ def do_wait_members(name='', display_name='', group='', count=1, timeout=30):
         ],
     },
 
-    do_zone_change     : {
+    do_zone_change      : {
         'keywords'             : ['gossip', 'zone', 'change'],
         'args'                 : [
             {'name': 'name', 'default': '', 'description': 'Change to the zone'},
@@ -471,15 +533,34 @@ def do_wait_members(name='', display_name='', group='', count=1, timeout=30):
         'description'          : 'Change the zone of the node'
     },
 
-    do_zone_list       : {
+    do_zone_list        : {
         'keywords'             : ['gossip', 'zone', 'list'],
         'args'                 : [
         ],
         'allow_temporary_agent': {'enabled': True, },
         'description'          : 'List all known zones of the node'
     },
 
-    do_detect_nodes    : {
+    do_zone_key_generate: {
+        'keywords'   : ['gossip', 'zone', 'key', 'generate'],
+        'args'       : [
+            {'name': '--zone', 'description': 'Name of zone to generate a key for'},
+            {'name': '--erase', 'type': 'bool', 'default': False, 'description': 'Erase the key if already exiting.'},
+        ],
+        'description': 'Generate a gossip encryption key for the zone'
+    },
+    
+    do_zone_key_import  : {
+        'keywords'   : ['gossip', 'zone', 'key', 'import'],
+        'args'       : [
+            {'name': '--zone', 'description': 'Name of zone to import the key for'},
+            {'name': '--key', 'description': 'Key to import.'},
+            {'name': '--erase', 'type': 'bool', 'default': False, 'description': 'Erase the key if already exiting.'},
+        ],
+        'description': 'Import a gossip encryption key for the zone'
+    },
+    
+    do_detect_nodes     : {
         'keywords'   : ['gossip', 'detect'],
         'args'       : [
             {'name': '--auto-join', 'default': False, 'description': 'Try to join the first detected proxy node. If no proxy is founded, join the first one.', 'type': 'bool'},
@@ -488,7 +569,7 @@ def do_wait_members(name='', display_name='', group='', count=1, timeout=30):
         'description': 'Try to detect (broadcast) others nodes in the network'
     },
 
-    do_wait_event      : {
+    do_wait_event       : {
         'keywords'   : ['gossip', 'events', 'wait'],
         'args'       : [
             {'name': 'event-type', 'description': 'Name of the event to wait for'},
@@ -497,15 +578,15 @@ def do_wait_members(name='', display_name='', group='', count=1, timeout=30):
         'description': 'Wait until the event is detected'
     },
 
-    do_gossip_add_event: {
+    do_gossip_add_event : {
         'keywords'   : ['gossip', 'events', 'add'],
         'args'       : [
             {'name': 'event-type', 'description': 'Name of the event to add'},
         ],
         'description': 'Add a event to the gossip members'
     },
 
-    do_wait_members    : {
+    do_wait_members     : {
         'keywords'   : ['gossip', 'wait-members'],
         'args'       : [
             {'name': '--name', 'description': 'Name of the members to wait for be alive'},

opsbro/configurationmanager.py

@@ -16,6 +16,8 @@
 logger = LoggerFactory.create_logger('configuration')
 
 
+ZONE_KEYS_DIRECTORY_NAME = 'zone_keys'
+
 class ConfigurationManager(object):
     # The parameter for the main cluster class is list here, and we will give back to it what we did read in the
     # local.yaml file (one set ones)
@@ -120,6 +122,8 @@ def get_parameters_for_cluster_from_configuration(self):
 
     def load_main_cfg_dir(self, cfg_dir):
         self.main_cfg_directory = cfg_dir
+        # also compute other directories from this
+        self.zone_keys_directory = os.path.join(self.main_cfg_directory, ZONE_KEYS_DIRECTORY_NAME)
         self.load_cfg_dir(self.main_cfg_directory, load_focus='agent')
 
 
@@ -184,8 +188,7 @@ def load_agent_parameters(self, o):
         if 'zone' in o:
             zone = o['zone']
             zonemgr = self.get_zonemgr()
-            zone_keys_directory = os.path.join(self.main_cfg_directory, 'zone_keys')
-            zonemgr.add_zone(zone, zone_keys_directory)
+            zonemgr.add_zone(zone)
 
         # grok all others data so we can use them in our checks
         cluster_parameters = self.__class__.cluster_parameters

opsbro/encrypter.py

@@ -7,6 +7,7 @@
 
 from .log import LoggerFactory
 from .util import bytes_to_unicode, unicode_to_bytes
+from .library import libstore
 
 # Global logger for this part
 logger = LoggerFactory.create_logger('gossip')
@@ -87,6 +88,22 @@ def load_zone_encryption_key(self, zone_encryption_key, zone_name):
         logger.debug('Loading the encryption key %s for the zone %s' % (key_fingerprint, zone_name))
 
 
+    # We did load a zone, or maybe we need to check that the zone did not change it's key
+    # if so, reload it
+    def load_or_reload_key_for_zone_if_need(self, zone_name):
+        from .configurationmanager import configmgr
+        # If the zone have a key, load it into the encrypter so we will be
+        # able to use it to exchange with this zone
+        # The key can be a file in the zone key directory, with the name of the zone.key
+        zone_keys_directory = configmgr.zone_keys_directory
+        key_file = os.path.join(zone_keys_directory, '%s.key' % zone_name)
+        if os.path.exists(key_file):
+            logger.debug('The zone %s have a key file (%s)' % (zone_name, key_file))
+            with open(key_file, 'rb') as f:
+                encryption_key = f.read().strip()
+                self.load_zone_encryption_key(encryption_key, zone_name)
+    
+    
     def _get_key_from_zone(self, zone_name):
         if zone_name is None or zone_name not in self.fingerprints_from_zone:
             from .gossip import gossiper

opsbro/gossip.py

@@ -1773,6 +1773,14 @@ def get_zones():
             return jsoner.dumps(zonemgr.get_zones())
 
 
+        # Reload the key for the zone: zone_name
+        @http_export('/agent/zones-keys/reload/:zone_name', method='GET', protected=True)
+        def get_reload_zone_key(zone_name):
+            response.content_type = 'application/json'
+            encrypter = libstore.get_encrypter()
+            return jsoner.dumps(encrypter.load_or_reload_key_for_zone_if_need(zone_name))
+        
+        
         @http_export('/agent/event/:event_type', method='GET', protected=True)
         def get_event(event_type):
             response.content_type = 'application/json'

opsbro/zonemanager.py

@@ -7,6 +7,7 @@
 # Global logger for this part
 logger = LoggerFactory.create_logger('configuration')
 
+
 class Zone(object):
     def __init__(self):
         pass
@@ -69,26 +70,19 @@ def __get_sub_zones_rec(self, zname, sub_zones_set, level):
             self.__get_sub_zones_rec(sub_zone_name, sub_zones_set, level + 1)
 
 
-    def add_zone(self, zone, zone_keys_directory):
+    def add_zone(self, zone):
         name = zone.get('name', '')
         if not name:
             return
         self.zones[name] = zone
         # We did change, dirty the trees
         self._dirty_tree = True
-
+        
         # If the zone have a key, load it into the encrypter so we will be
         # able to use it to exchange with this zone
         # The key can be a file in the zone key directory, with the name of the zone.key
-        key_file = os.path.join(zone_keys_directory, '%s.key' % name)
-        if os.path.exists(key_file):
-            logger.debug('The zone %s have a key file (%s)' % (name, key_file))
-            with open(key_file, 'rb') as f:
-                encryption_key = f.read().strip()
-                encrypter = libstore.get_encrypter()
-                encrypter.load_zone_encryption_key(encryption_key, name)
-        else:
-            logger.debug('The zone %s do not have a key file (%s)' % (name, key_file))
+        encrypter = libstore.get_encrypter()
+        encrypter.load_or_reload_key_for_zone_if_need(name)
 
 
     def get_top_zones_from(self, zname):

test/docker-files/docker-file-DEMO-HTTP-PYTHON3-http-node-client.txt

@@ -15,7 +15,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    test/test_demo1_http.sh "NODE-CLIENT"

test/docker-files/docker-file-DEMO-HTTP-PYTHON3-http-node-haproxy.txt

@@ -20,7 +20,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 
 RUN        opsbro  packs overload global.haproxy

test/docker-files/docker-file-DEMO-HTTP-PYTHON3-http-node-http-1.txt

@@ -17,7 +17,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    /etc/init.d/apache2 start; test/test_demo1_http.sh "NODE-HTTP-1"

test/docker-files/docker-file-DEMO-HTTP-PYTHON3-http-node-http-2.txt

@@ -16,7 +16,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    /etc/init.d/apache2 start; test/test_demo1_http.sh "NODE-HTTP-2"

test/docker-files/docker-file-DEMO-HTTP-http-node-client.txt

@@ -15,7 +15,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    test/test_demo1_http.sh "NODE-CLIENT"

test/docker-files/docker-file-DEMO-HTTP-http-node-haproxy.txt

@@ -20,7 +20,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 
 RUN        opsbro  packs overload global.haproxy

test/docker-files/docker-file-DEMO-HTTP-http-node-http-1.txt

@@ -17,7 +17,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    /etc/init.d/apache2 start; test/test_demo1_http.sh "NODE-HTTP-1"

test/docker-files/docker-file-DEMO-HTTP-http-node-http-2.txt

@@ -16,7 +16,7 @@ WORKDIR       /root/opsbro-oss
 RUN       python setup.py install
 
 # Ask for an encrypted test
-RUN       echo -n "NGNjZWI2ZmEyMzEyMTFlOA==" > /etc/opsbro/zone_keys/internet.key
+RUN       opsbro gossip zone key import --zone internet --key "NGNjZWI2ZmEyMzEyMTFlOA=="
 
 # The node1 will try to connect to node2 and auto join it
 ENTRYPOINT    /etc/init.d/apache2 start; test/test_demo1_http.sh "NODE-HTTP-2"