feat: Integrate listener operator

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]).
+- Added Listener support for NiFi ([#784]).
 - Adds new telemetry CLI arguments and environment variables ([#782]).
   - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept.
   - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation.
@@ -52,6 +53,7 @@ All notable changes to this project will be documented in this file.
 [#782]: https://github.com/stackabletech/nifi-operator/pull/782
 [#785]: https://github.com/stackabletech/nifi-operator/pull/785
 [#787]: https://github.com/stackabletech/nifi-operator/pull/787
+[#784]: https://github.com/stackabletech/nifi-operator/pull/784
 [#789]: https://github.com/stackabletech/nifi-operator/pull/789
 [#793]: https://github.com/stackabletech/nifi-operator/pull/793
 [#794]: https://github.com/stackabletech/nifi-operator/pull/794

deploy/helm/nifi-operator/crds/crds.yaml

@@ -194,20 +194,6 @@ spec:
                           description: Allow all proxy hosts by turning off host header validation. See <https://github.com/stackabletech/docker-images/pull/694>
                           type: boolean
                       type: object
-                    listenerClass:
-                      default: cluster-internal
-                      description: |-
-                        This field controls which type of Service the Operator creates for this NifiCluster:
-
-                        * cluster-internal: Use a ClusterIP service
-
-                        * external-unstable: Use a NodePort service
-
-                        This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change.
-                      enum:
-                        - cluster-internal
-                        - external-unstable
-                      type: string
                     sensitiveProperties:
                       description: These settings configure the encryption of sensitive properties in NiFi processors. NiFi supports encrypting sensitive properties in processors as they are written to disk. You can configure the encryption algorithm and the key to use. You can also let the operator generate an encryption key for you.
                       properties:
@@ -790,11 +776,15 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                     roleConfig:
                       default:
+                        listenerClass: cluster-internal
                         podDisruptionBudget:
                           enabled: true
                           maxUnavailable: null
                       description: This is a product-agnostic RoleConfig, which is sufficient for most of the products.
                       properties:
+                        listenerClass:
+                          default: cluster-internal
+                          type: string
                         podDisruptionBudget:
                           default:
                             enabled: true

deploy/helm/nifi-operator/templates/roles.yaml

@@ -90,6 +90,17 @@ rules:
     verbs:
       - create
       - patch
+  - apiGroups:
+      - listeners.stackable.tech
+    resources:
+      - listeners
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+      - create
+      - delete
   - apiGroups:
       - {{ include "operator.name" . }}.stackable.tech
     resources:

docs/modules/nifi/examples/getting_started/getting_started.sh

@@ -143,15 +143,17 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-users
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1
+
 EOF
 # end::install-nifi[]
 

docs/modules/nifi/examples/getting_started/getting_started.sh.j2

@@ -143,12 +143,13 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-users
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

docs/modules/nifi/pages/usage_guide/custom-components.adoc

@@ -162,12 +162,13 @@ spec:
     - name: nifi-processors
       configMap:
         name: nifi-processors
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     configOverrides:
       nifi.properties:
         nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/  # <2>
@@ -281,12 +282,13 @@ spec:
     - name: nifi-processors
       persistentVolumeClaim:
         claimName: nifi-processors
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     configOverrides:
       nifi.properties:
         nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/  # <2>

docs/modules/nifi/pages/usage_guide/index.adoc

@@ -26,11 +26,12 @@ spec:
       - name: nifi-client-certs
         secret:
           secretName: nifi-client-certs
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
   roleGroups:
     default:
       config:

docs/modules/nifi/pages/usage_guide/listenerclass.adoc

@@ -1,19 +1,14 @@
 = Service exposition with ListenerClasses
 :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes.
 
-Apache NiFi offers a web UI and an API.
-The Operator deploys a service called `<name>` (where `<name>` is the name of the NifiCluster) through which NiFi can be reached.
-
-This service can have either the `cluster-internal` or `external-unstable` type.
-`external-stable` is not supported for NiFi at the moment.
-Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
-
-This is how the listener class is configured:
+The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod.
+The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`:
 
 [source,yaml]
 ----
 spec:
-  clusterConfig:
-    listenerClass: cluster-internal  # <1>
+  nodes:
+    roleConfig:
+      listenerClass: external-unstable  # <1>
 ----
-<1> The default `cluster-internal` setting.
+<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` or a custom ListenerClass (the default setting is `cluster-internal`).

docs/modules/nifi/pages/usage_guide/monitoring.adoc

@@ -127,7 +127,7 @@ spec:
           - __meta_kubernetes_pod_container_port_number
         targetLabel: __address__
         replacement: ${1}.${2}.${3}.svc.cluster.local:${4}
-        regex: (.+);(.+?)(?:-metrics)?;(.+);(.+)
+        regex: (.+);(.+?)(?:-headless)?;(.+);(.+)
   selector:
     matchLabels:
       prometheus.io/scrape: "true"
@@ -138,4 +138,4 @@ spec:
 <1> Authorization via Bearer Token stored in a secret
 <2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod
 
-NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix.
+NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-headless` thus we need to regex this suffix.

examples/simple-nifi-cluster.yaml

@@ -51,13 +51,13 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-admin-user
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
-    config:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

rust/operator-binary/src/config/jvm.rs

@@ -1,12 +1,12 @@
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
     memory::{BinaryMultiple, MemoryQuantity},
-    role_utils::{self, GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role},
+    role_utils::{self, JavaCommonConfig, JvmArgumentOverrides, Role},
 };
 
 use crate::{
     config::{JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY},
-    crd::{NifiConfig, NifiConfigFragment},
+    crd::{NifiConfig, NifiConfigFragment, NifiNodeRoleConfig},
 };
 
 // Part of memory resources allocated for Java heap
@@ -29,7 +29,7 @@ pub enum Error {
 /// Create the NiFi bootstrap.conf
 pub fn build_merged_jvm_config(
     merged_config: &NifiConfig,
-    role: &Role<NifiConfigFragment, GenericRoleConfig, JavaCommonConfig>,
+    role: &Role<NifiConfigFragment, NifiNodeRoleConfig, JavaCommonConfig>,
     role_group: &str,
 ) -> Result<JvmArgumentOverrides, Error> {
     let heap_size = MemoryQuantity::try_from(

rust/operator-binary/src/config/mod.rs

@@ -14,14 +14,15 @@ use stackable_operator::{
         ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config,
         validate_all_roles_and_groups_config,
     },
-    role_utils::{GenericRoleConfig, JavaCommonConfig, Role},
+    role_utils::{JavaCommonConfig, Role},
 };
 use strum::{Display, EnumIter};
 
 use crate::{
     crd::{
-        HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiRole, NifiStorageConfig, PROTOCOL_PORT,
-        sensitive_properties, v1alpha1, v1alpha1::NifiClusteringBackend,
+        HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiNodeRoleConfig, NifiRole,
+        NifiStorageConfig, PROTOCOL_PORT, sensitive_properties,
+        v1alpha1::{self, NifiClusteringBackend},
     },
     operations::graceful_shutdown::graceful_shutdown_config_properties,
     security::{
@@ -112,7 +113,7 @@ pub enum Error {
 pub fn build_bootstrap_conf(
     merged_config: &NifiConfig,
     overrides: BTreeMap<String, String>,
-    role: &Role<NifiConfigFragment, GenericRoleConfig, JavaCommonConfig>,
+    role: &Role<NifiConfigFragment, NifiNodeRoleConfig, JavaCommonConfig>,
     role_group: &str,
 ) -> Result<String, Error> {
     let mut bootstrap = BTreeMap::new();
@@ -736,7 +737,7 @@ pub fn build_state_management_xml(clustering_backend: &NifiClusteringBackend) ->
 pub fn validated_product_config(
     resource: &v1alpha1::NifiCluster,
     version: &str,
-    role: &Role<NifiConfigFragment, GenericRoleConfig, JavaCommonConfig>,
+    role: &Role<NifiConfigFragment, NifiNodeRoleConfig, JavaCommonConfig>,
     product_config: &ProductConfigManager,
 ) -> Result<ValidatedRoleConfigByPropertyKind, Error> {
     let mut roles = HashMap::new();