Fix: more python3 stuff

Fix: Compliance get url now works with python3 & https ^^
Add: first compliance url with python3

Author: naparuba

data/global-configuration/packs/core-cli-compliance/cli/cli.py

@@ -5,7 +5,6 @@
 #    Gabes Jean, [email protected]
 
 from __future__ import print_function
-import json
 import time
 import sys
 
@@ -15,6 +14,7 @@
 from opsbro.cli import get_opsbro_local, put_opsbro_json
 from opsbro.cli_display import print_h1, print_h2, get_terminal_size
 from opsbro.compliancemgr import COMPLIANCE_LOG_COLORS, COMPLIANCE_STATES, COMPLIANCE_STATE_COLORS
+from opsbro.jsonmgr import jsoner
 
 
 def __print_rule_entry(rule):
@@ -74,7 +74,7 @@ def do_compliance_state(compliance_name=''):
         return
 
     try:
-        compliances = json.loads(r)
+        compliances = jsoner.loads(r)
     except ValueError as exp:  # bad json
         logger.error('Bad return from the server %s' % exp)
         return
@@ -87,12 +87,12 @@ def do_compliance_state(compliance_name=''):
         if pack_name not in packs:
             packs[pack_name] = {}
         packs[pack_name][cname] = compliance
-    pnames = packs.keys()
+    pnames = list(packs.keys())  # python3
     pnames.sort()
     for pname in pnames:
         pack_entries = packs[pname]
         cprint('* Pack %s' % pname, color='blue')
-        cnames = pack_entries.keys()
+        cnames = list(pack_entries.keys())  # python3
         cnames.sort()
         for cname in cnames:
             cprint('  - %s' % pname, color='blue', end='')
@@ -109,7 +109,7 @@ def do_compliance_history():
         return
 
     try:
-        histories = json.loads(r)
+        histories = jsoner.loads(r)
     except ValueError as exp:  # bad json
         logger.error('Bad return from the server %s' % exp)
         return
@@ -147,7 +147,7 @@ def do_compliance_wait_compliant(compliance_name, timeout=30, exit_if_ok=True):
     spinners = itertools.cycle(CHARACTERS.spinners)
 
     current_state = COMPLIANCE_STATES.UNKNOWN
-    for i in xrange(timeout):
+    for i in range(timeout):  # no xrange in python3
         uri = '/compliance/state'
         try:
             (code, r) = get_opsbro_local(uri)
@@ -156,7 +156,7 @@ def do_compliance_wait_compliant(compliance_name, timeout=30, exit_if_ok=True):
             return
 
         try:
-            compliances = json.loads(r)
+            compliances = jsoner.loads(r)
         except ValueError as exp:  # bad json
             logger.error('Bad return from the server %s' % exp)
             return
@@ -179,7 +179,7 @@ def do_compliance_wait_compliant(compliance_name, timeout=30, exit_if_ok=True):
         except:  # beware, maybe we don't have a tty
             terminal_width = 100
         cprint('\r' + ' ' * terminal_width, end='')
-        cprint('\r %s ' % spinners.next(), color='blue', end='')
+        cprint('\r %s ' % next(spinners), color='blue', end='')  # no spinnner.next in python3
         cprint('%s' % compliance_name, color='magenta', end='')
         cprint(' is ', end='')
         cprint('%15s ' % current_state, color=COMPLIANCE_STATE_COLORS.get(current_state, 'cyan'), end='')

data/global-configuration/packs/core-functions/compliancebackends/compliancebackend_geturl.py

@@ -1,13 +1,16 @@
 from __future__ import print_function
-import urllib2
 import shutil
 import os
-import urlparse
-import ssl
+
+try:  # Python 2
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
 import hashlib
 
 from opsbro.compliancemgr import InterfaceComplianceDriver
 from opsbro.util import make_dir
+from opsbro.httpclient import get_http_exceptions, httper
 
 
 class GetURLDriver(InterfaceComplianceDriver):
@@ -16,17 +19,6 @@ class GetURLDriver(InterfaceComplianceDriver):
 
     def __init__(self):
         super(GetURLDriver, self).__init__()
-        
-        # NOTE: ssl.SSLContext is only availabe on last python 2.7 versions
-        if hasattr(ssl, 'SSLContext'):
-            # NOTE: was before, but seems to be not as large as default context
-            ## self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
-            self.ssl_context = ssl.create_default_context()
-            self.ssl_context.options &= ~ssl.OP_NO_SSLv3  # reenable SSLv3 if need
-            self.ssl_context.check_hostname = False
-            self.ssl_context.verify_mode = ssl.CERT_NONE
-        else:
-            self.ssl_context = None
 
 
     # environments:   <- take first to win
@@ -67,7 +59,7 @@ def launch(self, rule):
             rule.set_error()
             return
 
-        parsed_uri = urlparse.urlparse(url)
+        parsed_uri = urlparse(url)
         file_name = os.path.basename(parsed_uri.path)
         self.logger.debug("TRY DOWNLOADING %s => %s " % (url, file_name))
 
@@ -95,12 +87,8 @@ def launch(self, rule):
 
         self.logger.debug('START DOWNLOAd', url)
         try:
-            args = {}
-            if self.ssl_context:
-                args['context'] = self.ssl_context
-            filedata = urllib2.urlopen(url, **args)
-            data = filedata.read()
-        except Exception as exp:
+            data = httper.get(url)
+        except get_http_exceptions() as exp:
             err = 'ERROR: downloading the uri: %s did fail withthe error: %s' % (url, exp)
             rule.add_error(err)
             rule.set_error()

opsbro/cli.py

@@ -213,7 +213,7 @@ def wait_for_agent_started(timeout=30, visual_wait=False, exit_if_stopped=False,
         if agent_state == 'ok':
             break
         if visual_wait:
-            cprint('\r %s ' % spinners.next(), color='blue', end='')
+            cprint('\r %s ' % next(spinners), color='blue', end='')  # note: spinners.next() do not exists in python3
             cprint(' agent is still ', end='')
             cprint('initializing', color='yellow', end='')
             cprint(' (collector, detector,... are not finish)', end='')

opsbro/httpclient.py

@@ -1,5 +1,6 @@
 import json
 import urllib
+import ssl
 
 try:
     from httplib import HTTPException
@@ -8,9 +9,9 @@
 from socket import error as SocketError
 
 try:
-    from urllib2 import Request, build_opener, URLError, HTTPError, HTTPHandler, HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler
+    from urllib2 import Request, build_opener, URLError, HTTPError, HTTPHandler, HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, HTTPSHandler
 except ImportError:
-    from urllib.request import Request, build_opener, HTTPHandler, HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler
+    from urllib.request import Request, build_opener, HTTPHandler, HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, HTTPSHandler
     from urllib.error import URLError, HTTPError
 
 _HTTP_EXCEPTIONS = None
@@ -27,23 +28,34 @@ def get_http_exceptions():
 
 class Httper(object):
     def __init__(self):
-        pass
+        # NOTE: ssl.SSLContext is only availabe on last python 2.7 versions
+        if hasattr(ssl, 'SSLContext'):
+            # NOTE: was before, but seems to be not as large as default context
+            ## self.ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+            self.ssl_context = ssl.create_default_context()
+            self.ssl_context.options &= ~ssl.OP_NO_SSLv3  # reenable SSLv3 if need
+            self.ssl_context.check_hostname = False
+            self.ssl_context.verify_mode = ssl.CERT_NONE
+        else:
+            self.ssl_context = None
 
 
-    @staticmethod
-    def get(uri, params={}, headers={}, with_status_code=False, timeout=10, user=None, password=None):
+    def get(self, uri, params={}, headers={}, with_status_code=False, timeout=10, user=None, password=None):
         data = None  # always none in GET
 
         if params:
             uri = "%s?%s" % (uri, urllib.urlencode(params))
 
-        # We need both user & password to use them
-        if not user or not  password:
-            handler = HTTPHandler
-        else:
+        # SSL, user/password and basic
+        # NOTE: currently don't manage ssl & user/password
+        if uri.startswith('https://'):
+            handler = HTTPSHandler(context=self.ssl_context)
+        elif user and password:
             passwordMgr = HTTPPasswordMgrWithDefaultRealm()
             passwordMgr.add_password(None, uri, user, password)
             handler = HTTPBasicAuthHandler(passwordMgr)
+        else:
+            handler = HTTPHandler
 
         url_opener = build_opener(handler)
 

opsbro/jsonmgr.py

@@ -1,5 +1,5 @@
 import json
-from .util import byteify
+from .util import byteify, string_decode
 
 
 # Class to wrap several things to json, like manage some utf8 things and such things
@@ -13,6 +13,7 @@ def dumps(self, o):
 
 
     def loads(self, s):
+        s = string_decode(s)
         return json.loads(s)
 
 

opsbro/log.py

@@ -16,6 +16,9 @@
 
 from .misc.colorama import init as init_colorama
 
+# Lasy load to avoid recursive import
+string_decode = None
+
 
 def is_tty():
     # TODO: what about windows? how to have beautiful & Windows?
@@ -59,17 +62,33 @@ def sprintf(s, color='', end=''):
     stdout_utf8 = codecs.getwriter("utf-8")(sys.stdout)
 
 
+    # stdout_utf8.errors = 'ignore'
+    
     def cprint(s, color='', on_color='', end='\n'):
+        global string_decode
+        if string_decode is None:
+            from .util import string_decode
+            string_decode = string_decode
         if not isinstance(s, basestring):
             s = str(s)
-        if (PY3 and isinstance(s, bytes)):
-            print('\n\nTYPE OF s: %s\n\n' % type(s))
-            s = s.decode('utf8', 'ignore')
-        if end == '':
-            stdout_utf8.write(s)
-        else:
-            stdout_utf8.write(s)
-            stdout_utf8.write('\n')
+        # Python 2 and 3: good luck for unicode in a terminal.
+        # It's a nightmare to manage all of this, if you have a common code
+        # that allow to run WITHOUT a terminal, I take it :)
+        if PY3:
+            s = string_decode(s)
+            raw_bytes, consumed = stdout_utf8.encode(s, 'strict')
+            if end == '':
+                sys.stdout.buffer.write(raw_bytes)
+                # stdout_utf8.write(s)
+            else:
+                sys.stdout.buffer.write(raw_bytes)
+                sys.stdout.buffer.write(b'\n')
+        else:  # PY2
+            if end == '':
+                stdout_utf8.write(s)
+            else:
+                stdout_utf8.write(s)
+                stdout_utf8.write('\n')
 
 
     def sprintf(s, color='', end=''):

opsbro/unixclient.py

@@ -6,7 +6,6 @@
 
 
 import os
-import json
 import socket
 
 try:  # Python 2
@@ -30,6 +29,7 @@
     from urllib.parse import urlsplit
 
 from .log import logger
+from .jsonmgr import jsoner
 
 #### For local socket handling
 DEFAULT_SOCKET_TIMEOUT = 5
@@ -151,7 +151,7 @@ def get_json(uri, local_socket='', params={}, multi=False, method='GET', timeout
         r = "[{0}]".format(r.replace("}{", "},{"))
 
     try:
-        d = json.loads(r)
+        d = jsoner.loads(r)
     except ValueError as exp:  # bad json
         logger.debug("ERROR local unix get json raw return did raise an exception  in bad json (%s) %s" % (r, exp))
         logger.error('Bad return from the server %s: "%s"' % (exp, r))

spike/test_https.py

@@ -0,0 +1,11 @@
+
+# -*- coding: utf-8 -*-
+import sys
+
+
+sys.path.insert(0, '.')
+
+uri = 'https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz'
+from opsbro.httpclient import get_http_exceptions, httper
+r = httper.get(uri)
+print('Result: r')

test/docker-files/docker-file-COMPLIANCE-GET-URL-debian9.txt renamed to test/docker-files/docker-file-COMPLIANCE-GET-URL-PYTHON2-debian9.txt

@@ -13,6 +13,7 @@ RUN        /apt_get_install   python-apt
 
 RUN        /apt_get_install   procps
 
+
 ADD        . /root/opsbro-oss
 
 

test/docker-files/docker-file-COMPLIANCE-GET-URL-PYTHON3-debian9.txt

@@ -0,0 +1,31 @@
+FROM debian:9
+MAINTAINER Jean Gabes <[email protected]>
+
+ADD        test/docker-helper/  /
+
+RUN        /apt_get_install  python3
+# All need for debian, so faster test (we are testging feature here, not install)
+RUN        /apt_get_install   python3-jinja2
+RUN        /apt_get_install   python3-crypto
+RUN        /apt_get_install   python3-setuptools
+RUN        /apt_get_install   python3-apt
+
+# Set python3 as default python
+RUN        update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
+
+RUN        /apt_get_install   procps
+
+ADD        . /root/opsbro-oss
+
+
+WORKDIR    /root/opsbro-oss
+
+RUN        python setup.py install
+
+# Copy the mail pack into a local one to enable modification
+RUN        opsbro  packs overload global.linux
+# Copy the new compliance value
+ADD        test/test-files/test-compliance-get-uri/compliance/install-cockroachdb.yml   /var/lib/opsbro/local-configuration/packs/linux/compliance/
+
+ENTRYPOINT     test/test_feature_compliance-get-url.sh