From cfebf54c3930b9fa144251e2a6439cd825ac2a1a Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Wed, 21 Aug 2019 22:41:48 -0300 Subject: [PATCH 1/9] Refactor fucntion get client It was realized refactor in __get_es_client function, for better maintenance. --- cmreslogging/handlers.py | 75 ++++++---------------------------------- 1 file changed, 11 insertions(+), 64 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 52e250a..615e45c 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -1,28 +1,19 @@ """ Elasticsearch logging handler """ -import logging import datetime +import logging import socket -from threading import Timer, Lock from enum import Enum -from elasticsearch import helpers as eshelpers -from elasticsearch import Elasticsearch, RequestsHttpConnection - -try: - from requests_kerberos import HTTPKerberosAuth, DISABLED - CMR_KERBEROS_SUPPORTED = True -except ImportError: - CMR_KERBEROS_SUPPORTED = False +from threading import Lock, Timer -try: - from requests_aws4auth import AWS4Auth - AWS4AUTH_SUPPORTED = True -except ImportError: - AWS4AUTH_SUPPORTED = False +from elasticsearch import helpers as eshelpers from cmreslogging.serializers import CMRESSerializer +from .client_es import (AWS_SIGNED_AUTH, BASIC_AUTH, KERBEROS_AUTH, NO_AUTH, + FactoryClientES) + class CMRESHandler(logging.Handler): """ Elasticsearch log handler @@ -39,10 +30,10 @@ class AuthType(Enum): - Basic authentication - Kerberos or SSO authentication (on windows and linux) """ - NO_AUTH = 0 - BASIC_AUTH = 1 - KERBEROS_AUTH = 2 - AWS_SIGNED_AUTH = 3 + NO_AUTH = NO_AUTH + BASIC_AUTH = BASIC_AUTH + KERBEROS_AUTH = KERBEROS_AUTH + AWS_SIGNED_AUTH = AWS_SIGNED_AUTH class IndexNameFrequency(Enum): """ Index type supported @@ -209,52 +200,8 @@ def __schedule_flush(self): self._timer.start() def __get_es_client(self): - if self.auth_type == CMRESHandler.AuthType.NO_AUTH: - if self._client is None: - self._client = Elasticsearch(hosts=self.hosts, - use_ssl=self.use_ssl, - verify_certs=self.verify_certs, - connection_class=RequestsHttpConnection, - serializer=self.serializer) - return self._client - - if self.auth_type == CMRESHandler.AuthType.BASIC_AUTH: - if self._client is None: - return Elasticsearch(hosts=self.hosts, - http_auth=self.auth_details, - use_ssl=self.use_ssl, - verify_certs=self.verify_certs, - connection_class=RequestsHttpConnection, - serializer=self.serializer) - return self._client - - if self.auth_type == CMRESHandler.AuthType.KERBEROS_AUTH: - if not CMR_KERBEROS_SUPPORTED: - raise EnvironmentError("Kerberos module not available. Please install \"requests-kerberos\"") - # For kerberos we return a new client each time to make sure the tokens are up to date - return Elasticsearch(hosts=self.hosts, - use_ssl=self.use_ssl, - verify_certs=self.verify_certs, - connection_class=RequestsHttpConnection, - http_auth=HTTPKerberosAuth(mutual_authentication=DISABLED), - serializer=self.serializer) - - if self.auth_type == CMRESHandler.AuthType.AWS_SIGNED_AUTH: - if not AWS4AUTH_SUPPORTED: - raise EnvironmentError("AWS4Auth not available. Please install \"requests-aws4auth\"") - if self._client is None: - awsauth = AWS4Auth(self.aws_access_key, self.aws_secret_key, self.aws_region, 'es') - self._client = Elasticsearch( - hosts=self.hosts, - http_auth=awsauth, - use_ssl=self.use_ssl, - verify_certs=True, - connection_class=RequestsHttpConnection, - serializer=self.serializer - ) - return self._client - raise ValueError("Authentication method not supported") + return FactoryClientES.get_client(cmrs_handler=self) def test_es_source(self): """ Returns True if the handler can ping the Elasticsearch servers From 27f41099f43ab66807bd276ce45314459a02e49f Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Wed, 21 Aug 2019 22:45:41 -0300 Subject: [PATCH 2/9] Added new file It was added new file for decouple non-class action. --- cmreslogging/client_es.py | 119 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 cmreslogging/client_es.py diff --git a/cmreslogging/client_es.py b/cmreslogging/client_es.py new file mode 100644 index 0000000..35db43d --- /dev/null +++ b/cmreslogging/client_es.py @@ -0,0 +1,119 @@ +from elasticsearch import Elasticsearch, RequestsHttpConnection + +MSG_KERBEROS = "Kerberos module not available. Please install \"requests-kerberos\"" +MSG_AWS = "AWS4Auth not available. Please install \"requests-aws4auth\"" +try: + from requests_kerberos import HTTPKerberosAuth, DISABLED + CMR_KERBEROS_SUPPORTED = True +except ImportError: + CMR_KERBEROS_SUPPORTED = False + +try: + from requests_aws4auth import AWS4Auth + AWS4AUTH_SUPPORTED = True +except ImportError: + AWS4AUTH_SUPPORTED = False + +NO_AUTH = 0 +BASIC_AUTH = 1 +KERBEROS_AUTH = 2 +AWS_SIGNED_AUTH = 3 + + +class FactoryClientES: + @staticmethod + def get_client(cmrs_handler): + for klass in ClientElasticSearch.__subclasses__(): + if klass.TYPE_CLIENT == cmrs_handler.auth_type.value: + return klass(cmrs_handler).get() + raise ValueError("Authentication method not supported") + + +class ClientElasticSearch: + + def __init__(self, cmrs_handler): + self._cmrs_handler = cmrs_handler + + @staticmethod + def _validation_environment_error(value, msg_error): + if not value: + raise EnvironmentError(msg_error) + + +class ClientNotAuth(ClientElasticSearch): + TYPE_CLIENT = NO_AUTH + + def __init__(self, cmrs_handler): + super(ClientNotAuth, self).__init__(cmrs_handler) + + def get(self): + if self._cmrs_handler._client is None: + self._cmrs_handler._client = Elasticsearch(hosts=self._cmrs_handler.hosts, + use_ssl=self._cmrs_handler.use_ssl, + verify_certs=self._cmrs_handler.verify_certs, + connection_class=RequestsHttpConnection, + serializer=self._cmrs_handler.serializer) + + return self._cmrs_handler._client + + +class ClientBasicAuth(ClientElasticSearch): + TYPE_CLIENT = BASIC_AUTH + + def __init__(self, cmrs_handler): + super(ClientBasicAuth, self).__init__(cmrs_handler) + + def get(self): + + if self._cmrs_handler._client is None: + self._cmrs_handler._client = Elasticsearch(hosts=self._cmrs_handler.hosts, + http_auth=self._cmrs_handler.auth_details, + use_ssl=self._cmrs_handler.use_ssl, + verify_certs=self._cmrs_handler.verify_certs, + connection_class=RequestsHttpConnection, + serializer=self._cmrs_handler.serializer) + + return self._cmrs_handler._client + + +class ClientKerberos(ClientElasticSearch): + TYPE_CLIENT = KERBEROS_AUTH + + def __init__(self, cmrs_handler): + super(ClientKerberos, self).__init__(cmrs_handler) + self._cmrs_handler._validation_environment_error(CMR_KERBEROS_SUPPORTED, MSG_KERBEROS) + + def get(self): + + # For kerberos we return a new client each time to make sure the tokens are up to date + return Elasticsearch(hosts=self._cmrs_handler.hosts, + use_ssl=self._cmrs_handler.use_ssl, + verify_certs=self._cmrs_handler.verify_certs, + connection_class=RequestsHttpConnection, + http_auth=HTTPKerberosAuth(mutual_authentication=DISABLED), + serializer=self._cmrs_handler.serializer) + + +class ClientAmazon(ClientElasticSearch): + TYPE_CLIENT = AWS_SIGNED_AUTH + + def __init__(self, cmrs_handler): + super(ClientAmazon, self).__init__(cmrs_handler) + + self._validation_environment_error(AWS4AUTH_SUPPORTED, MSG_AWS) + + def get(self): + + if self._cmrs_handler._client is None: + awsauth = AWS4Auth(self._cmrs_handler.aws_access_key, + self._cmrs_handler.aws_secret_key, + self._cmrs_handler.aws_region, 'es') + self._cmrs_handler._client = Elasticsearch( + hosts=self._cmrs_handler.hosts, + http_auth=awsauth, + use_ssl=self._cmrs_handler.use_ssl, + verify_certs=True, + connection_class=RequestsHttpConnection, + serializer=self._cmrs_handler.serializer) + + return self._cmrs_handler._client From bcbb1af700e8d05827043c6d13a88a1b6ddd0b99 Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:23:40 -0300 Subject: [PATCH 3/9] Added test to file test_client_es It was added test client to file client_es.py --- tests/test_client_es.py | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_client_es.py diff --git a/tests/test_client_es.py b/tests/test_client_es.py new file mode 100644 index 0000000..ade99dd --- /dev/null +++ b/tests/test_client_es.py @@ -0,0 +1,53 @@ +import os +import sys +import unittest +from enum import Enum + +from cmreslogging.client_es import (ClientAmazon, ClientBasicAuth, + ClientKerberos, ClientNotAuth, + FactoryClientES) +from cmreslogging.handlers import CMRESHandler + +sys.path.insert(0, os.path.abspath('.')) + + +class AuthTypeFake(Enum): + NO_AUTH = 10 + + +class ClientESTestCase(unittest.TestCase): + + def crms_handler(self, type_client_es): + return CMRESHandler(hosts=[{'host': 'localhost', 'port': 9200}], + auth_type=type_client_es, + es_index_name="my_python_index", + es_additional_fields={'App': 'MyAppName', 'Environment': 'Dev'}) + + def test_get_client_no_auth(self): + cmr_handler = self.crms_handler(CMRESHandler.AuthType.NO_AUTH) + client = FactoryClientES.get_client(cmr_handler) + self.assertIsInstance(client, ClientNotAuth) + + def test_get_client_auth_basic(self): + cmr_handler = self.crms_handler(CMRESHandler.AuthType.BASIC_AUTH) + client = FactoryClientES.get_client(cmr_handler) + self.assertIsInstance(client, ClientBasicAuth) + + def test_get_client_amazon(self): + cmr_handler = self.crms_handler(CMRESHandler.AuthType.AWS_SIGNED_AUTH) + client = FactoryClientES.get_client(cmr_handler) + self.assertIsInstance(client, ClientAmazon) + + def test_get_client_kerberos(self): + cmr_handler = self.crms_handler(CMRESHandler.AuthType.KERBEROS_AUTH) + client = FactoryClientES.get_client(cmr_handler) + self.assertIsInstance(client, ClientKerberos) + + def test_execption_value_error(self): + cmr_handler = self.crms_handler(AuthTypeFake.NO_AUTH) + with self.assertRaises(ValueError): + FactoryClientES.get_client(cmr_handler) + + +if __name__ == '__main__': + unittest.main() From 05262985e3084df55d244a1a60db62f9eb34ba15 Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:24:42 -0300 Subject: [PATCH 4/9] Adapted to python 2.7 It was adapted to python 2.7 --- cmreslogging/client_es.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/cmreslogging/client_es.py b/cmreslogging/client_es.py index 35db43d..6b98cc4 100644 --- a/cmreslogging/client_es.py +++ b/cmreslogging/client_es.py @@ -20,15 +20,6 @@ AWS_SIGNED_AUTH = 3 -class FactoryClientES: - @staticmethod - def get_client(cmrs_handler): - for klass in ClientElasticSearch.__subclasses__(): - if klass.TYPE_CLIENT == cmrs_handler.auth_type.value: - return klass(cmrs_handler).get() - raise ValueError("Authentication method not supported") - - class ClientElasticSearch: def __init__(self, cmrs_handler): @@ -44,7 +35,7 @@ class ClientNotAuth(ClientElasticSearch): TYPE_CLIENT = NO_AUTH def __init__(self, cmrs_handler): - super(ClientNotAuth, self).__init__(cmrs_handler) + ClientElasticSearch.__init__(self, cmrs_handler) def get(self): if self._cmrs_handler._client is None: @@ -61,7 +52,7 @@ class ClientBasicAuth(ClientElasticSearch): TYPE_CLIENT = BASIC_AUTH def __init__(self, cmrs_handler): - super(ClientBasicAuth, self).__init__(cmrs_handler) + ClientElasticSearch.__init__(self, cmrs_handler) def get(self): @@ -80,10 +71,10 @@ class ClientKerberos(ClientElasticSearch): TYPE_CLIENT = KERBEROS_AUTH def __init__(self, cmrs_handler): - super(ClientKerberos, self).__init__(cmrs_handler) - self._cmrs_handler._validation_environment_error(CMR_KERBEROS_SUPPORTED, MSG_KERBEROS) + ClientElasticSearch.__init__(self, cmrs_handler) def get(self): + self._validation_environment_error(CMR_KERBEROS_SUPPORTED, MSG_KERBEROS) # For kerberos we return a new client each time to make sure the tokens are up to date return Elasticsearch(hosts=self._cmrs_handler.hosts, @@ -98,12 +89,11 @@ class ClientAmazon(ClientElasticSearch): TYPE_CLIENT = AWS_SIGNED_AUTH def __init__(self, cmrs_handler): - super(ClientAmazon, self).__init__(cmrs_handler) - - self._validation_environment_error(AWS4AUTH_SUPPORTED, MSG_AWS) + ClientElasticSearch.__init__(self, cmrs_handler) def get(self): + self._validation_environment_error(AWS4AUTH_SUPPORTED, MSG_AWS) if self._cmrs_handler._client is None: awsauth = AWS4Auth(self._cmrs_handler.aws_access_key, self._cmrs_handler.aws_secret_key, @@ -117,3 +107,19 @@ def get(self): serializer=self._cmrs_handler.serializer) return self._cmrs_handler._client + + +class FactoryClientES: + + CLIENTS = {NO_AUTH: ClientNotAuth, + AWS_SIGNED_AUTH: ClientAmazon, + BASIC_AUTH: ClientBasicAuth, + KERBEROS_AUTH: ClientKerberos} + + @staticmethod + def get_client(cmrs_handler): + klass = FactoryClientES.CLIENTS.get(cmrs_handler.auth_type.value) + if not klass: + raise ValueError("Authentication method not supported") + + return klass(cmrs_handler) From 6b19f4c964fdc9c41f27167cda3e2286de223dda Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:25:34 -0300 Subject: [PATCH 5/9] Called function correctly It was called funciont correctly --- cmreslogging/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 615e45c..4600ead 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -201,7 +201,7 @@ def __schedule_flush(self): def __get_es_client(self): - return FactoryClientES.get_client(cmrs_handler=self) + return FactoryClientES.get_client(cmrs_handler=self).get() def test_es_source(self): """ Returns True if the handler can ping the Elasticsearch servers From 3972f6b052f73051ffb7b65d3cb21a8fcbd3e89a Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:26:34 -0300 Subject: [PATCH 6/9] Added coverage to file client_es. It was coverage to file client_es. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a0d2466..5188acd 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ commands = # pylint ./cmreslogging -r n --files-output=y '--msg-template="\{path\}:\{line\}: [\{msg_id\}(\{symbol\}), \{obj\}] \{msg\}"' coverage erase coverage run -a --source=./cmreslogging --branch tests/test_cmreshandler.py + coverage run -a --source=./cmreslogging --branch tests/test_client_es.py coverage run -a --source=./cmreslogging --branch tests/test_cmresserializer.py coverage xml -i coverage html From ebe0c566c3a6003c82ac99badb980dd05111afd7 Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:39:11 -0300 Subject: [PATCH 7/9] upadate travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 57772f5..04fa28e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ addons: - oracle-java8-installer before_install: + - sudo apt-get install python-software-properties + - sudo add-apt-repository ppa:webupd8team/java + - sudo apt-get update - sudo update-java-alternatives -s java-8-oracle - export JAVA_HOME=/usr/lib/jvm/java-8-oracle/jre - java -version From 61adb680e4c2848949570d7e7c4cc63c16a6e51e Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 00:42:30 -0300 Subject: [PATCH 8/9] update travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 04fa28e..42dc5df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: addons: apt: packages: - - oracle-java8-installer + - oracle-java8-set-default before_install: - sudo apt-get install python-software-properties From b9f2d8d10d1a925f6bd5b9f5445bce0a0f34f7b3 Mon Sep 17 00:00:00 2001 From: "laerte.allan" Date: Thu, 22 Aug 2019 10:28:52 -0300 Subject: [PATCH 9/9] Return format default --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42dc5df..57772f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,9 @@ python: addons: apt: packages: - - oracle-java8-set-default + - oracle-java8-installer before_install: - - sudo apt-get install python-software-properties - - sudo add-apt-repository ppa:webupd8team/java - - sudo apt-get update - sudo update-java-alternatives -s java-8-oracle - export JAVA_HOME=/usr/lib/jvm/java-8-oracle/jre - java -version