diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..4f1cc30 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,11 @@ +name: Run pre-commit checks + +on: + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 52e250a..5b98825 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -11,12 +11,14 @@ try: from requests_kerberos import HTTPKerberosAuth, DISABLED + CMR_KERBEROS_SUPPORTED = True except ImportError: CMR_KERBEROS_SUPPORTED = False try: from requests_aws4auth import AWS4Auth + AWS4AUTH_SUPPORTED = True except ImportError: AWS4AUTH_SUPPORTED = False @@ -79,6 +81,7 @@ class IndexNameFrequency(Enum): __LOGGING_FILTER_FIELDS = ['msecs', 'relativeCreated', 'levelno', + 'msg', 'created'] @staticmethod @@ -133,6 +136,7 @@ def __init__(self, verify_ssl=__DEFAULT_VERIFY_SSL, buffer_size=__DEFAULT_BUFFER_SIZE, flush_frequency_in_sec=__DEFAULT_FLUSH_FREQ_INSEC, + max_retries=10, es_index_name=__DEFAULT_ES_INDEX_NAME, index_name_frequency=__DEFAULT_INDEX_FREQUENCY, es_doc_type=__DEFAULT_ES_DOC_TYPE, @@ -186,6 +190,7 @@ def __init__(self, self.verify_certs = verify_ssl self.buffer_size = buffer_size self.flush_frequency_in_sec = flush_frequency_in_sec + self.max_retries = max_retries self.es_index_name = es_index_name self.index_name_frequency = index_name_frequency self.es_doc_type = es_doc_type @@ -276,6 +281,13 @@ def __get_es_datetime_str(timestamp): current_date = datetime.datetime.utcfromtimestamp(timestamp) return "{0!s}.{1:03d}Z".format(current_date.strftime('%Y-%m-%dT%H:%M:%S'), int(current_date.microsecond / 1000)) + def error_handler(self, exception): + """ + This function can be implemented by the client to handle errors that happen when trying to write logs to ES. + :param exception python exception that raised while writing to ES + """ + pass + def flush(self): """ Flushes the buffer into ES :return: None @@ -297,12 +309,17 @@ def flush(self): } for log_record in logs_buffer ) + # say we send 100 records, and get errors from 3. + # Then 97 will be uploaded successfully and an exception will be raised with a summary of the 3 errors. + # that's why we don't need the return value from the bulk. eshelpers.bulk( client=self.__get_es_client(), actions=actions, - stats_only=True + stats_only=True, + max_retries=self.max_retries, ) except Exception as exception: + self.error_handler(exception) if self.raise_on_indexing_exceptions: raise exception diff --git a/requirements/requirements_py27.txt b/requirements/requirements_py27.txt deleted file mode 100644 index 588b2f5..0000000 --- a/requirements/requirements_py27.txt +++ /dev/null @@ -1,3 +0,0 @@ -elasticsearch==5.4.0 -requests==2.18.1 -enum==0.4.6 diff --git a/setup.py b/setup.py index 08baacc..6a3fbf1 100644 --- a/setup.py +++ b/setup.py @@ -22,25 +22,21 @@ 'requests' ] -# If python version is above 3.4 (built in enums supported enums) -if sys.version_info <= (3,4): - dependencies.append('enum') - print("List of dependencies : {0}".format(str(dependencies))) setup( - name='CMRESHandler', + name='vycmreshandler', # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='1.0.0', + version='1.0.1', - description='Elasticsearch Log handler for the logging library', + description='Voyantis fork - Elasticsearch Log handler for the logging library', long_description=long_description, # The project's main homepage. - url='https://github.com/cmanaha/python-elasticsearch-logger', + url='https://github.com/Voyantis/python-elasticsearch-logger', # Author details author='Carlos Manzanedo Rueda', @@ -67,7 +63,6 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', ],