Prep for 8.0.22 release (#1786)

* Fix singleton show_indices

Addresses #1775

Fixed some Pylance errors with re-doing how CLIAction.get_list_object() works (return the value now instead of setting the attribute), and instead of setting self.list_object = None in the early lines of `__init__`, move it to the end so all of the dependencies needed by get_list_object() are addressed.

Added a bit of typing to object_class.py as well.

In show.py, the missing step was

```python
action.list_object.get_index_stats()
```

This was missed when the IndexList class was updated to change how and when stats were collected.

* Added a few more index patterns for de-facto omission

```diff
diff --git a/curator/defaults/settings.py b/curator/defaults/settings.py
index 800febf..bc0f298 100644
--- a/curator/defaults/settings.py
+++ b/curator/defaults/settings.py
@@ -12,8 +12,9 @@ CLICK_DRYRUN = {
 }
 DATA_NODE_ROLES = ['data', 'data_content', 'data_hot', 'data_warm']
 EXCLUDE_SYSTEM = (
-    '-.kibana*,-.security*,-.watch*,-.triggered_watch*,'
-    '-.ml*,-.geoip_databases*,-.logstash*,-.tasks*'
+    '-.apm-agent-configuration,-.apm-custom-link,-.async-search,-.fleet*,'
+    '-.geoip_databases*,-.inference,-.kibana*,-.logstash*,-.ml*,-.secrets-inference,'
+    '-.security*,-.slo-*,-.tasks*,-.transform-telemetry,-.watch*'
 )
 VERSION_MIN = (7, 14, 0)
 VERSION_MAX = (8, 99, 99)
```

* Correct inverted ILM allow behavior

* Add tiered-debug module, address index inclusion issues

Fixes #1770

Author: untergeek

curator/_version.py

@@ -1,3 +1,3 @@
 """Curator Version"""
 
-__version__ = '8.0.21'
+__version__ = '8.0.22'

curator/actions/alias.py

@@ -3,11 +3,14 @@
 import logging
 
 # pylint: disable=import-error
+from curator.debug import debug, begin_end
 from curator.exceptions import ActionError, MissingArgument, NoIndices
 from curator.helpers.date_ops import parse_date_pattern, parse_datemath
 from curator.helpers.testers import verify_index_list
 from curator.helpers.utils import report_failure
 
+logger = logging.getLogger(__name__)
+
 
 class Alias:
     """Alias Action Class"""
@@ -34,17 +37,22 @@ def __init__(self, name=None, extra_settings=None, **kwargs):
         #: :py:meth:`~.curator.actions.Alias.add` and
         #: :py:meth:`~.curator.actions.Alias.remove`
         self.actions = []
-        #: The :py:class:`~.elasticsearch.Elasticsearch` client object which will
-        #: later be set by :py:meth:`~.curator.actions.Alias.add` or
-        #: :py:meth:`~.curator.actions.Alias.remove`
-        self.client = None
         #: Any extra things to add to the alias, like filters, or routing. Gets
         #: the value from param ``extra_settings``.
         self.extra_settings = extra_settings
-        self.loggit = logging.getLogger('curator.actions.alias')
         #: Preset default value to ``False``.
         self.warn_if_no_indices = False
 
+    @property
+    def client(self):
+        """The :py:class:`~.elasticsearch.Elasticsearch` client object"""
+        return self._client
+
+    @client.setter
+    def client(self, value):
+        self._client = value
+
+    @begin_end()
     def add(self, ilo, warn_if_no_indices=False):
         """
         Create ``add`` statements for each index in ``ilo`` for :py:attr:`name`, then
@@ -55,25 +63,24 @@ def add(self, ilo, warn_if_no_indices=False):
         :type ilo: :py:class:`~.curator.indexlist.IndexList`
         """
         verify_index_list(ilo)
-        self.loggit.debug('ADD -> ILO = %s', ilo.indices)
-        if not self.client:
-            self.client = ilo.client
+        debug.lv5('ADD -> ILO = %s', ilo.indices)
+        self.client = ilo.client
         self.name = parse_datemath(self.client, self.name)
         try:
             ilo.empty_list_check()
         except NoIndices as exc:
             # Add a warning if there are no indices to add, if so set in options
             if warn_if_no_indices:
                 self.warn_if_no_indices = True
-                self.loggit.warning(
+                logger.warning(
                     'No indices found after processing filters. Nothing to add to %s',
                     self.name,
                 )
                 return
             # Re-raise the exceptions.NoIndices so it will behave as before
             raise NoIndices('No indices to add to alias') from exc
         for index in ilo.working_list():
-            self.loggit.debug(
+            debug.lv1(
                 'Adding index %s to alias %s with extra settings %s',
                 index,
                 self.name,
@@ -83,6 +90,7 @@ def add(self, ilo, warn_if_no_indices=False):
             add_dict['add'].update(self.extra_settings)
             self.actions.append(add_dict)
 
+    @begin_end()
     def remove(self, ilo, warn_if_no_indices=False):
         """
         Create ``remove`` statements for each index in ``ilo`` for :py:attr:`name`,
@@ -92,17 +100,16 @@ def remove(self, ilo, warn_if_no_indices=False):
         :type ilo: :py:class:`~.curator.indexlist.IndexList`
         """
         verify_index_list(ilo)
-        self.loggit.debug('REMOVE -> ILO = %s', ilo.indices)
-        if not self.client:
-            self.client = ilo.client
+        debug.lv5('REMOVE -> ILO = %s', ilo.indices)
+        self.client = ilo.client
         self.name = parse_datemath(self.client, self.name)
         try:
             ilo.empty_list_check()
         except NoIndices as exc:
             # Add a warning if there are no indices to add, if so set in options
             if warn_if_no_indices:
                 self.warn_if_no_indices = True
-                self.loggit.warning(
+                logger.warning(
                     'No indices found after processing filters. '
                     'Nothing to remove from %s',
                     self.name,
@@ -114,22 +121,21 @@ def remove(self, ilo, warn_if_no_indices=False):
         aliases = self.client.indices.get_alias(expand_wildcards=['open', 'closed'])
         for index in ilo.working_list():
             if index in aliases:
-                self.loggit.debug('Index %s in get_aliases output', index)
+                debug.lv3('Index %s in get_aliases output', index)
                 # Only remove if the index is associated with the alias
                 if self.name in aliases[index]['aliases']:
-                    self.loggit.debug(
-                        'Removing index %s from alias %s', index, self.name
-                    )
+                    debug.lv1('Removing index %s from alias %s', index, self.name)
                     self.actions.append(
                         {'remove': {'index': index, 'alias': self.name}}
                     )
                 else:
-                    self.loggit.debug(
+                    debug.lv2(
                         'Can not remove: Index %s is not associated with alias %s',
                         index,
                         self.name,
                     )
 
+    @begin_end()
     def check_actions(self):
         """
         :returns: :py:attr:`actions` for use with the
@@ -140,13 +146,13 @@ def check_actions(self):
             if not self.warn_if_no_indices:
                 raise ActionError('No "add" or "remove" operations')
             raise NoIndices('No "adds" or "removes" found.  Taking no action')
-        self.loggit.debug('Alias actions: %s', self.actions)
+        debug.lv5('Alias actions: %s', self.actions)
 
         return self.actions
 
     def do_dry_run(self):
         """Log what the output would be, but take no action."""
-        self.loggit.info('DRY-RUN MODE.  No changes will be made.')
+        logger.info('DRY-RUN MODE.  No changes will be made.')
         for item in self.check_actions():
             job = list(item.keys())[0]
             index = item[job]['index']
@@ -157,15 +163,16 @@ def do_dry_run(self):
                 f"DRY-RUN: alias: {job.rstrip('e')}ing index \"{index}\" "
                 f"{'to' if job == 'add' else 'from'} alias \"{alias}\""
             )
-            self.loggit.info(msg)
+            logger.info(msg)
 
+    @begin_end()
     def do_action(self):
         """
         :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` for
         :py:attr:`name` with :py:attr:`actions`
         """
-        self.loggit.info('Updating aliases...')
-        self.loggit.info('Alias actions: %s', self.actions)
+        logger.info('Updating aliases...')
+        debug.lv3('Alias actions: %s', self.actions)
         try:
             self.client.indices.update_aliases(actions=self.actions)
         # pylint: disable=broad-except

curator/actions/allocation.py

@@ -3,11 +3,14 @@
 import logging
 
 # pylint: disable=import-error
+from curator.debug import debug, begin_end
 from curator.exceptions import MissingArgument
 from curator.helpers.testers import verify_index_list
 from curator.helpers.waiters import wait_for_it
 from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv
 
+logger = logging.getLogger(__name__)
+
 
 class Allocation:
     """Allocation Action Class"""
@@ -58,7 +61,6 @@ def __init__(
         #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from
         #: :py:attr:`index_list`
         self.client = ilo.client
-        self.loggit = logging.getLogger('curator.actions.allocation')
         bkey = f'index.routing.allocation.{allocation_type}.{key}'
         #: Populated at instance creation time. Value is built from the passed params
         #: ``allocation_type``, ``key``, and ``value``, e.g.
@@ -75,31 +77,32 @@ def do_dry_run(self):
         """Log what the output would be, but take no action."""
         show_dry_run(self.index_list, 'allocation', settings=self.settings)
 
+    @begin_end()
     def do_action(self):
         """
         :py:meth:`~.elasticsearch.client.IndicesClient.put_settings` to indices in
         :py:attr:`index_list` with :py:attr:`settings`.
         """
-        self.loggit.debug(
+        debug.lv3(
             'Cannot get change shard routing allocation of closed indices.  '
             'Omitting any closed indices.'
         )
         self.index_list.filter_closed()
         self.index_list.empty_list_check()
-        self.loggit.info(
+        logger.info(
             'Updating %s selected indices: %s',
             len(self.index_list.indices),
             self.index_list.indices,
         )
-        self.loggit.info('Updating index setting %s', self.settings)
+        debug.lv1('Updating index setting %s', self.settings)
         try:
             index_lists = chunk_index_list(self.index_list.indices)
             for lst in index_lists:
                 self.client.indices.put_settings(
                     index=to_csv(lst), settings=self.settings
                 )
                 if self.wfc:
-                    self.loggit.debug(
+                    debug.lv3(
                         'Waiting for shards to complete relocation for indices: %s',
                         to_csv(lst),
                     )

curator/actions/close.py

@@ -3,9 +3,12 @@
 import logging
 import warnings
 from elasticsearch8.exceptions import ElasticsearchWarning
+from curator.debug import begin_end, debug
 from curator.helpers.testers import verify_index_list
 from curator.helpers.utils import chunk_index_list, report_failure, show_dry_run, to_csv
 
+logger = logging.getLogger(__name__)
+
 
 class Close:
     """Close Action Class"""
@@ -31,22 +34,22 @@ def __init__(self, ilo, delete_aliases=False, skip_flush=False):
         #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from
         #: :py:attr:`index_list`
         self.client = ilo.client
-        self.loggit = logging.getLogger('curator.actions.close')
 
     def do_dry_run(self):
         """Log what the output would be, but take no action."""
         show_dry_run(
             self.index_list, 'close', **{'delete_aliases': self.delete_aliases}
         )
 
+    @begin_end()
     def do_action(self):
         """
         :py:meth:`~.elasticsearch.client.IndicesClient.close` open indices in
         :py:attr:`index_list`
         """
         self.index_list.filter_closed()
         self.index_list.empty_list_check()
-        self.loggit.info(
+        logger.info(
             'Closing %s selected indices: %s',
             len(self.index_list.indices),
             self.index_list.indices,
@@ -55,16 +58,16 @@ def do_action(self):
             index_lists = chunk_index_list(self.index_list.indices)
             for lst in index_lists:
                 lst_as_csv = to_csv(lst)
-                self.loggit.debug('CSV list of indices to close:  %s', lst_as_csv)
+                debug.lv3('CSV list of indices to close:  %s', lst_as_csv)
                 if self.delete_aliases:
-                    self.loggit.info('Deleting aliases from indices before closing.')
-                    self.loggit.debug('Deleting aliases from:  %s', lst)
+                    debug.lv1('Deleting aliases from indices before closing.')
+                    debug.lv3('Deleting aliases from:  %s', lst)
                     try:
                         self.client.indices.delete_alias(index=lst_as_csv, name='*')
-                        self.loggit.debug('Deleted aliases from: %s', lst)
+                        debug.lv3('Deleted aliases from: %s', lst)
                     # pylint: disable=broad-except
                     except Exception as err:
-                        self.loggit.warning(
+                        logger.warning(
                             'Some indices may not have had aliases.  Exception: %s', err
                         )
                 if not self.skip_flush:

curator/actions/cluster_routing.py

@@ -3,10 +3,13 @@
 import logging
 
 # pylint: disable=import-error
+from curator.debug import debug, begin_end
 from curator.helpers.testers import verify_client_object
 from curator.helpers.utils import report_failure
 from curator.helpers.waiters import wait_for_it
 
+logger = logging.getLogger(__name__)
+
 
 class ClusterRouting:
     """ClusterRouting Action Class"""
@@ -49,7 +52,6 @@ def __init__(
         verify_client_object(client)
         #: An :py:class:`~.elasticsearch.Elasticsearch` client object
         self.client = client
-        self.loggit = logging.getLogger('curator.actions.cluster_routing')
         #: Object attribute that gets the value of param ``wait_for_completion``
         self.wfc = wait_for_completion
         #: Object attribute that gets the value of param ``wait_interval``
@@ -81,22 +83,21 @@ def __init__(
 
     def do_dry_run(self):
         """Log what the output would be, but take no action."""
-        self.loggit.info('DRY-RUN MODE.  No changes will be made.')
+        logger.info('DRY-RUN MODE.  No changes will be made.')
         msg = f'DRY-RUN: Update cluster routing transient settings: {self.settings}'
-        self.loggit.info(msg)
+        logger.info(msg)
 
+    @begin_end()
     def do_action(self):
         """
         :py:meth:`~.elasticsearch.client.ClusterClient.put_settings` to the cluster with
         :py:attr:`settings`.
         """
-        self.loggit.info('Updating cluster settings: %s', self.settings)
+        logger.info('Updating cluster settings: %s', self.settings)
         try:
             self.client.cluster.put_settings(transient=self.settings)
             if self.wfc:
-                self.loggit.debug(
-                    'Waiting for shards to complete routing and/or rebalancing'
-                )
+                debug.lv3('Waiting for shards to complete routing and/or rebalancing')
                 wait_for_it(
                     self.client,
                     'cluster_routing',