Skip to content

Commit 5aeac27

Browse files
lnatapovlnatapov
committed
[WIP] Adding audit logging role and functional tests
1 parent 1936df5 commit 5aeac27

10 files changed

Lines changed: 336 additions & 0 deletions

File tree

‎.zuul.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,28 @@
200200
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
201201
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-cloudkitty-fvt.yml"
202202

203+
- job:
204+
name: functional-audit-logging-tests-osp18
205+
dependencies: ["telemetry-openstack-meta-content-provider-master"]
206+
parent: telemetry-operator-multinode-logging
207+
description: |
208+
Run audit logging functional tests on osp18
209+
extra-vars: *functional_autoscaling_extra_vars
210+
irrelevant-files: []
211+
roles:
212+
- zuul: github.com/openstack-k8s-operators/ci-framework
213+
- zuul: github.com/infrawatch/feature-verification-tests
214+
required-projects: *required_projects
215+
vars:
216+
cifmw_update_containers: false
217+
cifmw_extras:
218+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml"
219+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml"
220+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
221+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
222+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-audit-logging.yml"
223+
- "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-audit-logging-fvt.yml"
224+
203225
- project:
204226
name: infrawatch/feature-verification-tests
205227
periodic:
@@ -220,6 +242,16 @@
220242
- ^.zuul.yaml$
221243
dependencies:
222244
- telemetry-openstack-meta-content-provider-master
245+
- functional-audit-logging-tests-osp18:
246+
files:
247+
- ^roles/telemetry_audit_logging/.*$
248+
- ^roles/common/.*$
249+
- ^ci/vars-audit-logging-fvt.yml$
250+
- ^ci/run_audit_logging_tests.yml$
251+
- ^ci/report_result.yml$
252+
- ^.zuul.yaml$
253+
dependencies:
254+
- telemetry-openstack-meta-content-provider-master
223255
- feature-verification-tests-noop:
224256
files: *irrelevant_files
225257
- functional-tests-osp18

‎ci/run_audit_logging_tests.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
---
2+
- name: "Verify all the applicable projects, pods & services for audit logging"
3+
hosts: "{{ cifmw_target_hook_host | default('localhost') }}"
4+
gather_facts: true
5+
ignore_errors: true
6+
environment:
7+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
8+
PATH: "{{ cifmw_path }}"
9+
vars_files:
10+
- vars/common.yml
11+
- vars/osp18_env.yml
12+
vars:
13+
common_pod_status_str: "Running"
14+
common_pod_nspace: openshift-logging
15+
common_pod_list:
16+
- logging-loki-audit-compactor-0
17+
- logging-loki-audit-distributor
18+
- logging-loki-audit-gateway
19+
- logging-loki-audit-index-gateway
20+
- logging-loki-audit-ingester
21+
- logging-loki-audit-querier
22+
- logging-loki-audit-frontend
23+
- collectd-with-cadf--to-loki-audit
24+
25+
common_project_list:
26+
- openshift-logging
27+
- openshift
28+
29+
common_service_nspace: openshift-logging
30+
common_service_list:
31+
- logging-loki-audit-compactor-grpc
32+
- logging-loki-audit-compactor-http
33+
- logging-loki-audit-distributor-grpc
34+
- logging-loki-audit-distributor-http
35+
- logging-loki-audit-gateway-http
36+
- logging-loki-audit-gossip-ring
37+
- logging-loki-audit-index-gateway-grpc
38+
- logging-loki-audit-index-gateway-http
39+
- logging-loki-audit-ingester-grpc
40+
- logging-loki-audit-ingester-http
41+
- logging-loki-audit-querier-grpc
42+
- logging-loki-audit-querier-http
43+
- logging-loki-audit-frontend-grpc
44+
- logging-loki-audit-frontend-http
45+
46+
tasks:
47+
- name: "Verify audit logging infrastructure components"
48+
ansible.builtin.import_role:
49+
name: common
50+
51+
- name: "Verify audit logging for OpenStack services"
52+
ansible.builtin.import_role:
53+
name: telemetry_audit_logging

‎ci/vars-audit-logging-fvt.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
pre_tests_01_run_audit_logging_tests:
3+
source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_audit_logging_tests.yml"
4+
type: playbook
5+
config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
6+
7+
cifmw_run_tests: true
8+
cifmw_test_operator_tempest_include_list: |
9+
^tempest.*\[.*\bsmoke\b.*\]
10+
11+
post_tests_99_collect_results:
12+
source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
13+
type: playbook

‎roles/telemetry_audit_logging/README.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
telemetry_audit_logging
2+
=======================
3+
4+
Verify that OpenStack API audit events are emitted in CADF format and forwarded
5+
to the dedicated audit Loki stack. The scenarios follow the manual verification
6+
steps in
7+
[Manually enable audit logs](https://redhat.atlassian.net/wiki/spaces/CLOUDOPS/pages/389779771/Manually+enable+audit+logs).
8+
9+
For each OpenStack service the role:
10+
11+
1. Generates API activity with an ``openstack`` command
12+
2. Checks the service API pod logs for CADF audit notifications using ``oc logs``
13+
14+
After all service checks, the role verifies Loki log separation using ``oc`` and
15+
``curl`` with an OpenShift bearer token, as documented in the forwarding section:
16+
17+
* Default Loki (``logging-loki``) must **not** contain CADF audit events
18+
* Audit Loki (``logging-loki-audit``) **must** contain CADF audit events
19+
20+
Requirements
21+
------------
22+
23+
* OpenStack deployed with audit logging enabled
24+
* Audit Loki stack and ClusterLogForwarder configured in ``openshift-logging``
25+
* ``oc``, ``curl``, and ``openstack`` CLI access via ``openstackclient``
26+
27+
Service verification steps
28+
--------------------------
29+
30+
+----------+---------------------------+-----------------------------------------------+
31+
| Service | API activity | Pod log check |
32+
+==========+===========================+===============================================+
33+
| Barbican | ``openstack secret list`` | ``grep barbican-api`` |
34+
| Cinder | ``openstack volume list`` | ``grep cinder-api`` |
35+
| Glance | ``openstack image list`` | ``grep glance-default`` |
36+
| Keystone | ``openstack project list``| ``oc get pod -l service=keystone ...`` |
37+
| Neutron | ``openstack network list``| ``grep neutron`` |
38+
| Nova | ``openstack server list`` | ``grep nova-api`` |
39+
+----------+---------------------------+-----------------------------------------------+
40+
41+
Role Variables
42+
--------------
43+
44+
| Variable | Default | Description |
45+
|----------|---------|-------------|
46+
| ``openstack_cmd`` | ``oc rsh openstackclient openstack`` | OpenStack CLI command |
47+
| ``audit_logging_namespace`` | ``openshift-logging`` | Namespace for Loki routes |
48+
| ``audit_logging_default_loki_route`` | ``logging-loki`` | Main Loki route name |
49+
| ``audit_logging_loki_route`` | ``logging-loki-audit`` | Audit Loki route name |
50+
| ``audit_logging_loki_tenant`` | ``application`` | Loki tenant in query URL |
51+
| ``audit_logging_loki_logql_query`` | ``{log_type="application"} \|= "..."`` | LogQL query for Loki checks |
52+
| ``audit_logging_services`` | see ``defaults/main.yml`` | List of service scenarios |
53+
54+
Each service entry supports:
55+
56+
* ``name`` - service identifier
57+
* ``trigger_args`` - arguments passed to ``openstack_cmd``
58+
* ``pod_pattern`` - substring used to locate the API pod
59+
* ``pod_label_selector`` - optional ``oc get pod -l`` selector (used for Keystone)
60+
61+
Example Playbook
62+
----------------
63+
64+
.. code-block:: yaml
65+
66+
- hosts: localhost
67+
gather_facts: true
68+
environment:
69+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
70+
PATH: "{{ cifmw_path }}"
71+
vars_files:
72+
- vars/osp18_env.yml
73+
tasks:
74+
- name: "Run audit logging service tests"
75+
ansible.builtin.import_role:
76+
name: telemetry_audit_logging
77+
78+
License
79+
-------
80+
81+
Apache 2

‎roles/telemetry_audit_logging/defaults/main.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
openstack_cmd: "oc rsh openstackclient openstack"
3+
audit_logging_openstack_namespace: openstack
4+
audit_logging_namespace: openshift-logging
5+
audit_logging_default_loki_route: logging-loki
6+
audit_logging_loki_route: logging-loki-audit
7+
audit_logging_loki_tenant: application
8+
audit_logging_cadf_event_pattern: "http://schemas.dmtf.org/cloud/audit/1.0/event"
9+
audit_logging_loki_logql_query: >-
10+
{log_type="application"} |= "http://schemas.dmtf.org/cloud/audit/1.0/event"
11+
audit_logging_loki_query_retries: 6
12+
audit_logging_loki_query_delay: 10
13+
audit_logging_propagation_delay: 30
14+
15+
# Verify steps from the manual audit logging guide for each service.
16+
audit_logging_services:
17+
- name: barbican
18+
trigger_args: "secret list"
19+
pod_pattern: barbican-api
20+
21+
- name: cinder
22+
trigger_args: "volume list"
23+
pod_pattern: cinder-api
24+
25+
- name: glance
26+
trigger_args: "image list"
27+
pod_pattern: glance-default
28+
29+
- name: keystone
30+
trigger_args: "project list"
31+
pod_label_selector: "service=keystone"
32+
pod_pattern: keystone
33+
34+
- name: neutron
35+
trigger_args: "network list"
36+
pod_pattern: neutron
37+
38+
- name: nova
39+
trigger_args: "server list"
40+
pod_pattern: nova-api

‎roles/telemetry_audit_logging/meta/main.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
galaxy_info:
3+
author: Red Hat CloudOps team
4+
description: Verify OpenStack API audit events are emitted and stored in Loki
5+
company: Red Hat
6+
7+
license: Apache-2.0
8+
9+
min_ansible_version: "2.9"
10+
11+
galaxy_tags: []
12+
13+
dependencies: []

‎roles/telemetry_audit_logging/tasks/main.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
- name: "Setup Loki route URLs for audit log queries"
3+
ansible.builtin.include_tasks: setup_loki_env.yml
4+
5+
- name: "Verify audit logging for each OpenStack service"
6+
ansible.builtin.include_tasks: verify_service_audit_logs.yml
7+
loop: "{{ audit_logging_services }}"
8+
loop_control:
9+
loop_var: audit_service
10+
11+
- name: "Verify audit events are separated into the audit Loki stack"
12+
ansible.builtin.include_tasks: verify_loki_separation.yml

‎roles/telemetry_audit_logging/tasks/setup_loki_env.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
---
2+
- name: "Get default Loki route host"
3+
ansible.builtin.command:
4+
cmd: >
5+
oc get route {{ audit_logging_default_loki_route }}
6+
-n {{ audit_logging_namespace }}
7+
-o jsonpath={.spec.host}
8+
register: audit_default_loki_route
9+
changed_when: false
10+
11+
- name: "Get audit Loki route host"
12+
ansible.builtin.command:
13+
cmd: >
14+
oc get route {{ audit_logging_loki_route }}
15+
-n {{ audit_logging_namespace }}
16+
-o jsonpath={.spec.host}
17+
register: audit_loki_route
18+
changed_when: false
19+
20+
- name: "Set Loki query URLs"
21+
ansible.builtin.set_fact:
22+
audit_default_loki_query_url: >-
23+
https://{{ audit_default_loki_route.stdout }}/api/logs/v1/{{ audit_logging_loki_tenant }}/loki/api/v1/query_range
24+
audit_loki_query_url: >-
25+
https://{{ audit_loki_route.stdout }}/api/logs/v1/{{ audit_logging_loki_tenant }}/loki/api/v1/query_range

‎roles/telemetry_audit_logging/tasks/verify_loki_separation.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
---
2+
- name: "Wait for audit events to reach Loki"
3+
ansible.builtin.pause:
4+
seconds: "{{ audit_logging_propagation_delay }}"
5+
6+
- name: "Query default Loki for CADF audit events"
7+
ansible.builtin.shell:
8+
cmd: |
9+
curl -sk \
10+
-H "Authorization: Bearer $(oc whoami --show-token)" \
11+
"{{ audit_default_loki_query_url }}" \
12+
--data-urlencode 'query={{ audit_logging_loki_logql_query }}'
13+
register: audit_default_loki_query
14+
changed_when: false
15+
failed_when: audit_default_loki_query.rc != 0
16+
17+
- name: "TEST Verify default Loki does not contain CADF audit events"
18+
ansible.builtin.assert:
19+
that:
20+
- (audit_default_loki_query.stdout | from_json).data.result | length == 0
21+
fail_msg: >-
22+
FAILED: Default Loki ({{ audit_logging_default_loki_route }}) returned CADF audit events.
23+
Audit events should only be forwarded to {{ audit_logging_loki_route }}.
24+
success_msg: >-
25+
SUCCESS: Default Loki does not contain CADF audit events.
26+
27+
- name: "Query audit Loki for CADF audit events"
28+
ansible.builtin.shell:
29+
cmd: |
30+
curl -sk \
31+
-H "Authorization: Bearer $(oc whoami --show-token)" \
32+
"{{ audit_loki_query_url }}" \
33+
--data-urlencode 'query={{ audit_logging_loki_logql_query }}'
34+
register: audit_loki_query
35+
changed_when: false
36+
until: >
37+
audit_loki_query.rc == 0 and
38+
(audit_loki_query.stdout | from_json).data.result | length > 0
39+
retries: "{{ audit_logging_loki_query_retries }}"
40+
delay: "{{ audit_logging_loki_query_delay }}"
41+
failed_when: false
42+
43+
- name: "TEST Verify audit Loki contains CADF audit events"
44+
ansible.builtin.assert:
45+
that:
46+
- audit_loki_query.rc == 0
47+
- (audit_loki_query.stdout | from_json).data.result | length > 0
48+
fail_msg: >-
49+
FAILED: Audit Loki ({{ audit_logging_loki_route }}) did not return CADF audit events.
50+
LogQL: {{ audit_logging_loki_logql_query }}
51+
success_msg: >-
52+
SUCCESS: Audit Loki contains CADF audit events.

‎roles/telemetry_audit_logging/tasks/verify_service_audit_logs.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
- name: "Generate API activity for {{ audit_service.name }}"
3+
ansible.builtin.command:
4+
cmd: "{{ openstack_cmd }} {{ audit_service.trigger_args }}"
5+
register: audit_trigger
6+
changed_when: false
7+
failed_when: audit_trigger.rc != 0
8+
9+
- name: "TEST Check {{ audit_service.name }} API pod logs for CADF audit events"
10+
ansible.builtin.shell:
11+
cmd: |
12+
oc logs $(oc get pod{% if audit_service.pod_label_selector is defined %} -l {{ audit_service.pod_label_selector }}{% endif %} -n {{ audit_logging_openstack_namespace }} -o name | grep {{ audit_service.pod_pattern }} | head -1) | grep "{{ audit_logging_cadf_event_pattern }}"
13+
register: audit_pod_logs
14+
changed_when: false
15+
failed_when: audit_pod_logs.rc != 0

0 commit comments

Comments
 (0)