diff --git a/consul/assets/configuration/spec.yaml b/consul/assets/configuration/spec.yaml index 39448f373a7fe..da68e820076fc 100644 --- a/consul/assets/configuration/spec.yaml +++ b/consul/assets/configuration/spec.yaml @@ -109,6 +109,20 @@ files: - - + - name: services_tags_keys_include + description: | + If set, only tags with keys matching this list will be sent to Datadog. + This is helpful if you have a lot of tags on services that are not + relevant to Datadog (ingress routing tags, etc). Tags should be specified + here in lowercase. Otherwise, the check will downcase tags from Consul before comparing. + value: + type: array + items: + type: string + example: + - + - + - name: max_services description: | Increase the maximum number of queried services. diff --git a/consul/changelog.d/20306.added b/consul/changelog.d/20306.added new file mode 100644 index 0000000000000..8fdc03cf2211c --- /dev/null +++ b/consul/changelog.d/20306.added @@ -0,0 +1 @@ +Add a new feature to filter Consul service tags being sent to Datadog using an allow list. It can be configured using the `services_tags_keys_include` option. diff --git a/consul/datadog_checks/consul/config_models/instance.py b/consul/datadog_checks/consul/config_models/instance.py index 11cb2e2afb164..c0ca477df4dbf 100644 --- a/consul/datadog_checks/consul/config_models/instance.py +++ b/consul/datadog_checks/consul/config_models/instance.py @@ -91,6 +91,7 @@ class InstanceConfig(BaseModel): service: Optional[str] = None services_exclude: Optional[tuple[str, ...]] = None services_include: Optional[tuple[str, ...]] = None + services_tags_keys_include: Optional[tuple[str, ...]] = None single_node_install: Optional[bool] = None skip_proxy: Optional[bool] = None tags: Optional[tuple[str, ...]] = None diff --git a/consul/datadog_checks/consul/consul.py b/consul/datadog_checks/consul/consul.py index 7ea41109f30df..e44df6ce9198b 100644 --- a/consul/datadog_checks/consul/consul.py +++ b/consul/datadog_checks/consul/consul.py @@ -106,6 +106,9 @@ def __init__(self, name, init_config, instances): 'service_whitelist', self.instance.get('services_include', default_services_include) ) self.services_exclude = set(self.instance.get('services_exclude', self.init_config.get('services_exclude', []))) + self.services_tags_keys_include = set( + self.instance.get("services_tags_keys_include", self.init_config.get("services_tags_keys_include", [])) + ) self.max_services = self.instance.get('max_services', self.init_config.get('max_services', MAX_SERVICES)) self.threads_count = self.instance.get('threads_count', self.init_config.get('threads_count', THREADS_COUNT)) if self.threads_count > 1: @@ -312,6 +315,18 @@ def _cull_services_list(self, services): return services + def _cull_services_tags_list(self, services): + if self.services_tags_keys_include: + # services is a dict of {service_name: [tags]} where tags is a list + # of string having the form of "tagkey=tagvalue" + for service in services: + tags = services[service] + # get the tagkey (the part before the "=") and check it against the include list + tags = [t for t in tags if t.split("=")[0].lower() in self.services_tags_keys_include] + services[service] = tags + + return services + @staticmethod def _get_service_tags(service, tags): service_tags = ['consul_service_id:{}'.format(service)] @@ -397,6 +412,7 @@ def check(self, _): self.count_all_nodes(main_tags) services = self._cull_services_list(services) + tags = self._cull_services_tags_list(services) # {node_id: {"up: 0, "passing": 0, "warning": 0, "critical": 0} nodes_to_service_status = defaultdict(lambda: defaultdict(int)) diff --git a/consul/datadog_checks/consul/data/conf.yaml.example b/consul/datadog_checks/consul/data/conf.yaml.example index 92601618a8339..cb6a60bf30a18 100644 --- a/consul/datadog_checks/consul/data/conf.yaml.example +++ b/consul/datadog_checks/consul/data/conf.yaml.example @@ -126,6 +126,16 @@ instances: # - # - + ## @param services_tags_keys_include - list of strings - optional + ## If set, only tags with keys matching this list will be sent to Datadog. + ## This is helpful if you have a lot of tags on services that are not + ## relevant to Datadog (ingress routing tags, etc). Tags should be specified + ## here in lowercase. Otherwise, the check will downcase tags from Consul before comparing. + # + # services_tags_keys_include: + # - + # - + ## @param max_services - number - optional - default: 50 ## Increase the maximum number of queried services. # diff --git a/consul/tests/consul_mocks.py b/consul/tests/consul_mocks.py index be6cd803f221b..d97016561b5f0 100644 --- a/consul/tests/consul_mocks.py +++ b/consul/tests/consul_mocks.py @@ -93,6 +93,14 @@ def mock_get_services_in_cluster(): } +def mock_get_n_custom_tagged_services_in_cluster(n, tags): + svcs = {} + for i in range(n): + k = "service-{}".format(i) + svcs[k] = tags + return svcs + + def mock_get_n_services_in_cluster(n): dct = {} for i in range(n): diff --git a/consul/tests/test_unit.py b/consul/tests/test_unit.py index 6e93105a5734c..1f06466fd4604 100644 --- a/consul/tests/test_unit.py +++ b/consul/tests/test_unit.py @@ -54,6 +54,42 @@ def test_get_nodes_with_service(aggregator): aggregator.assert_metric('consul.catalog.services_count', value=1, tags=expected_tags) +def test_cull_services_tags_keys(aggregator): + consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) + consul_mocks.mock_check(consul_check, consul_mocks._get_consul_mocks()) + + all_tags = { + "active", + "standby", + "unwanted.tag=unwantedvalue", + "unwanted.tag.but.actually.wanted=wantedvalue", + "wanted.tag", + "unwanted.tag.noequals", + } + + include_tags = {'active', 'standby', 'unwanted.tag.but.actually.wanted', 'wanted.tag'} + + expected_tags = { + "active", + "standby", + "unwanted.tag.but.actually.wanted=wantedvalue", + "wanted.tag", + } + + unwanted_tags = { + "unwanted.tag=unwantedvalue", + "unwanted.tag.noequals", + } + + consul_check.services_tags_keys_include = include_tags + services = consul_mocks.mock_get_n_custom_tagged_services_in_cluster(6, all_tags) + + services = consul_check._cull_services_tags_list(services) + for service in services: + assert unwanted_tags.isdisjoint(set(services[service])) + assert expected_tags == set(services[service]) + + def test_get_peers_in_cluster(aggregator): my_mocks = consul_mocks._get_consul_mocks() consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])