diff --git a/CHANGELOG.md b/CHANGELOG.md index f907dcb4..84f390bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]). +- Added Listener support for NiFi ([#784]). - Adds new telemetry CLI arguments and environment variables ([#782]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. @@ -52,6 +53,7 @@ All notable changes to this project will be documented in this file. [#782]: https://github.com/stackabletech/nifi-operator/pull/782 [#785]: https://github.com/stackabletech/nifi-operator/pull/785 [#787]: https://github.com/stackabletech/nifi-operator/pull/787 +[#784]: https://github.com/stackabletech/nifi-operator/pull/784 [#789]: https://github.com/stackabletech/nifi-operator/pull/789 [#793]: https://github.com/stackabletech/nifi-operator/pull/793 [#794]: https://github.com/stackabletech/nifi-operator/pull/794 diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index b3884ece..1fa6e8d0 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -194,20 +194,6 @@ spec: description: Allow all proxy hosts by turning off host header validation. See type: boolean type: object - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this NifiCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - type: string sensitiveProperties: description: These settings configure the encryption of sensitive properties in NiFi processors. NiFi supports encrypting sensitive properties in processors as they are written to disk. You can configure the encryption algorithm and the key to use. You can also let the operator generate an encryption key for you. properties: @@ -790,11 +776,15 @@ spec: x-kubernetes-preserve-unknown-fields: true roleConfig: default: + listenerClass: cluster-internal podDisruptionBudget: enabled: true maxUnavailable: null description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. properties: + listenerClass: + default: cluster-internal + type: string podDisruptionBudget: default: enabled: true diff --git a/deploy/helm/nifi-operator/templates/roles.yaml b/deploy/helm/nifi-operator/templates/roles.yaml index 17127ac4..0e0e61fa 100644 --- a/deploy/helm/nifi-operator/templates/roles.yaml +++ b/deploy/helm/nifi-operator/templates/roles.yaml @@ -90,6 +90,17 @@ rules: verbs: - create - patch + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get + - list + - watch + - patch + - create + - delete - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh b/docs/modules/nifi/examples/getting_started/getting_started.sh index 3aed9fe5..fb25f97b 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh @@ -143,15 +143,17 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + roleConfig: + listenerClass: external-unstable roleGroups: default: replicas: 1 + EOF # end::install-nifi[] diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 index 668eb772..b9248c9e 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 @@ -143,12 +143,13 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + roleConfig: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/docs/modules/nifi/pages/usage_guide/custom-components.adoc b/docs/modules/nifi/pages/usage_guide/custom-components.adoc index 210aa925..f9f1e305 100644 --- a/docs/modules/nifi/pages/usage_guide/custom-components.adoc +++ b/docs/modules/nifi/pages/usage_guide/custom-components.adoc @@ -162,12 +162,13 @@ spec: - name: nifi-processors configMap: name: nifi-processors - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + roleConfig: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> @@ -281,12 +282,13 @@ spec: - name: nifi-processors persistentVolumeClaim: claimName: nifi-processors - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + roleConfig: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> diff --git a/docs/modules/nifi/pages/usage_guide/index.adoc b/docs/modules/nifi/pages/usage_guide/index.adoc index 13dea386..adaf1f60 100644 --- a/docs/modules/nifi/pages/usage_guide/index.adoc +++ b/docs/modules/nifi/pages/usage_guide/index.adoc @@ -26,11 +26,12 @@ spec: - name: nifi-client-certs secret: secretName: nifi-client-certs - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true nodes: + roleConfig: + listenerClass: external-unstable roleGroups: default: config: diff --git a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc index 8ff77c87..03d0d47f 100644 --- a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc +++ b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc @@ -1,19 +1,14 @@ = Service exposition with ListenerClasses :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes. -Apache NiFi offers a web UI and an API. -The Operator deploys a service called `` (where `` is the name of the NifiCluster) through which NiFi can be reached. - -This service can have either the `cluster-internal` or `external-unstable` type. -`external-stable` is not supported for NiFi at the moment. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. - -This is how the listener class is configured: +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod. +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`: [source,yaml] ---- spec: - clusterConfig: - listenerClass: cluster-internal # <1> + nodes: + roleConfig: + listenerClass: external-unstable # <1> ---- -<1> The default `cluster-internal` setting. +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` or a custom ListenerClass (the default setting is `cluster-internal`). diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 355f96b6..e3e102ea 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -127,7 +127,7 @@ spec: - __meta_kubernetes_pod_container_port_number targetLabel: __address__ replacement: ${1}.${2}.${3}.svc.cluster.local:${4} - regex: (.+);(.+?)(?:-metrics)?;(.+);(.+) + regex: (.+);(.+?)(?:-headless)?;(.+);(.+) selector: matchLabels: prometheus.io/scrape: "true" @@ -138,4 +138,4 @@ spec: <1> Authorization via Bearer Token stored in a secret <2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod -NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix. +NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-headless` thus we need to regex this suffix. diff --git a/examples/simple-nifi-cluster.yaml b/examples/simple-nifi-cluster.yaml index b084efc5..a8b2d033 100644 --- a/examples/simple-nifi-cluster.yaml +++ b/examples/simple-nifi-cluster.yaml @@ -51,13 +51,13 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-admin-user - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index f9d3c25c..7960a2e9 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,12 +1,12 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - role_utils::{self, GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, + role_utils::{self, JavaCommonConfig, JvmArgumentOverrides, Role}, }; use crate::{ config::{JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY}, - crd::{NifiConfig, NifiConfigFragment}, + crd::{NifiConfig, NifiConfigFragment, NifiNodeRoleConfig}, }; // Part of memory resources allocated for Java heap @@ -29,7 +29,7 @@ pub enum Error { /// Create the NiFi bootstrap.conf pub fn build_merged_jvm_config( merged_config: &NifiConfig, - role: &Role, + role: &Role, role_group: &str, ) -> Result { let heap_size = MemoryQuantity::try_from( diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index d1df2426..3ddfa6da 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -14,14 +14,15 @@ use stackable_operator::{ ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, validate_all_roles_and_groups_config, }, - role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, + role_utils::{JavaCommonConfig, Role}, }; use strum::{Display, EnumIter}; use crate::{ crd::{ - HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiRole, NifiStorageConfig, PROTOCOL_PORT, - sensitive_properties, v1alpha1, v1alpha1::NifiClusteringBackend, + HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiNodeRoleConfig, NifiRole, + NifiStorageConfig, PROTOCOL_PORT, sensitive_properties, + v1alpha1::{self, NifiClusteringBackend}, }, operations::graceful_shutdown::graceful_shutdown_config_properties, security::{ @@ -112,7 +113,7 @@ pub enum Error { pub fn build_bootstrap_conf( merged_config: &NifiConfig, overrides: BTreeMap, - role: &Role, + role: &Role, role_group: &str, ) -> Result { let mut bootstrap = BTreeMap::new(); @@ -736,7 +737,7 @@ pub fn build_state_management_xml(clustering_backend: &NifiClusteringBackend) -> pub fn validated_product_config( resource: &v1alpha1::NifiCluster, version: &str, - role: &Role, + role: &Role, product_config: &ProductConfigManager, ) -> Result { let mut roles = HashMap::new(); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 819935c3..c16a591a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -20,34 +20,35 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::SecretFormat, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ListenerOperatorVolumeSourceBuilderError, SecretFormat}, }, }, client::Client, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, - config::fragment, - crd::{authentication::oidc, git_sync}, + crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, k8s_openapi::{ DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, - EnvVar, EnvVarSource, Node, ObjectFieldSelector, Probe, SecretVolumeSource, - Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, + EnvVar, EnvVarSource, ObjectFieldSelector, Probe, SecretVolumeSource, + TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kube::{ Resource, ResourceExt, - api::ListParams, core::{DeserializeGuard, error_boundary}, - runtime::{controller::Action, reflector::ObjectRef}, + runtime::controller::Action, }, - kvp::{Label, Labels, ObjectLabels}, + kvp::{Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::env_vars_from_rolegroup_config, @@ -81,11 +82,14 @@ use crate::{ validated_product_config, }, crd::{ - APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, CurrentlySupportedListenerClasses, - HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, - NifiConfigFragment, NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, - v1alpha1, + APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, + METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiConfigFragment, NifiNodeRoleConfig, + NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, v1alpha1, + }, + listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, + group_listener_name, }, operations::{ graceful_shutdown::add_graceful_shutdown_config, @@ -103,6 +107,10 @@ use crate::{ build_tls_volume, check_or_generate_oidc_admin_password, check_or_generate_sensitive_key, tls::{KEYSTORE_NIFI_CONTAINER_MOUNT, KEYSTORE_VOLUME_NAME, TRUSTSTORE_VOLUME_NAME}, }, + service::{ + build_rolegroup_headless_service, build_rolegroup_metrics_service, + rolegroup_headless_service_name, + }, }; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; @@ -131,9 +139,6 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("object defines no spec"))] - ObjectHasNoSpec, - #[snafu(display("object defines no namespace"))] ObjectHasNoNamespace, @@ -147,11 +152,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply global Service"))] - ApplyRoleService { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to fetch deployed StatefulSets"))] FetchStatefulsets { source: stackable_operator::client::Error, @@ -213,24 +213,6 @@ pub enum Error { #[snafu(display("Failed to find information about file [{}] in product config", kind))] ProductConfigKindNotSpecified { kind: String }, - #[snafu(display("Failed to find any nodes in cluster {obj_ref}",))] - MissingNodes { - source: stackable_operator::client::Error, - obj_ref: ObjectRef, - }, - - #[snafu(display("Failed to find service {obj_ref}"))] - MissingService { - source: stackable_operator::client::Error, - obj_ref: ObjectRef, - }, - - #[snafu(display("Failed to find an external port to use for proxy hosts"))] - ExternalPort, - - #[snafu(display("Could not build role service fqdn"))] - NoRoleServiceFqdn, - #[snafu(display("Bootstrap configuration error"))] BootstrapConfig { #[snafu(source(from(config::Error, Box::new)))] @@ -250,12 +232,6 @@ pub enum Error { container_name: String, }, - #[snafu(display("failed to validate resources for {rolegroup}"))] - ResourceValidation { - source: fragment::ValidationError, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, @@ -362,6 +338,21 @@ pub enum Error { #[snafu(display("Failed to determine the state of the version upgrade procedure"))] ClusterVersionUpdateState { source: upgrade::Error }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, + #[snafu(display("failed to apply group listener"))] + ApplyGroupListener { + source: stackable_operator::cluster_resources::Error, + }, + + #[snafu(display("failed to configure listener"))] + ListenerConfiguration { source: crate::listener::Error }, + + #[snafu(display("failed to configure service"))] + ServiceConfiguration { source: crate::service::Error }, } type Result = std::result::Result; @@ -384,11 +375,6 @@ pub async fn reconcile_nifi( .context(InvalidNifiClusterSnafu)?; let client = &ctx.client; - let namespace = &nifi - .metadata - .namespace - .clone() - .with_context(|| ObjectHasNoNamespaceSnafu {})?; let resolved_product_image: ResolvedProductImage = nifi .spec @@ -450,20 +436,6 @@ pub async fn reconcile_nifi( .map(Cow::Borrowed) .unwrap_or_default(); - let node_role_service = build_node_role_service(nifi, &resolved_product_image)?; - cluster_resources - .add(client, node_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - - // This is read back to obtain the hosts that we later need to fill in the proxy_hosts variable - let updated_role_service = client - .get::(&nifi.name_any(), namespace) - .await - .with_context(|_| MissingServiceSnafu { - obj_ref: ObjectRef::new(&nifi.name_any()).within(namespace), - })?; - let authentication_config = NifiAuthenticationConfig::try_from( AuthenticationClassResolved::from(nifi, client) .await @@ -523,8 +495,24 @@ pub async fn reconcile_nifi( ) .context(InvalidGitSyncSpecSnafu)?; - let rg_service = - build_node_rolegroup_service(nifi, &resolved_product_image, &rolegroup)?; + let role_group_service_recommended_labels = build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + ); + + let role_group_service_selector = + Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) + .context(LabelBuildSnafu)?; + + let rg_headless_service = build_rolegroup_headless_service( + nifi, + &rolegroup, + role_group_service_recommended_labels.clone(), + role_group_service_selector.clone().into(), + ) + .context(ServiceConfigurationSnafu)?; let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; @@ -533,7 +521,7 @@ pub async fn reconcile_nifi( // Since we cannot predict which of the addresses a user might decide to use we will simply // add all of them to the setting for now. // For more information see - let proxy_hosts = get_proxy_hosts(client, nifi, &updated_role_service).await?; + let proxy_hosts = get_proxy_hosts(client, nifi, &resolved_product_image).await?; let rg_configmap = build_node_rolegroup_config_map( nifi, @@ -574,12 +562,30 @@ pub async fn reconcile_nifi( ) .await?; + if resolved_product_image.product_version.starts_with("1.") { + let rg_metrics_service = build_rolegroup_metrics_service( + nifi, + &rolegroup, + role_group_service_recommended_labels, + role_group_service_selector.into(), + ) + .context(ServiceConfigurationSnafu)?; + + cluster_resources + .add(client, rg_metrics_service) + .await + .with_context(|_| ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; + } + cluster_resources - .add(client, rg_service) + .add(client, rg_headless_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), })?; + cluster_resources .add(client, rg_configmap) .await @@ -602,13 +608,34 @@ pub async fn reconcile_nifi( } let role_config = nifi.role_config(&nifi_role); - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, + if let Some(NifiNodeRoleConfig { + common: GenericRoleConfig { + pod_disruption_budget: pdb, + }, + listener_class, }) = role_config { add_pdbs(pdb, nifi, &nifi_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; + + let role_group_listener = build_group_listener( + nifi, + build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &nifi_role.to_string(), + "none", + ), + listener_class.to_owned(), + group_listener_name(nifi, &nifi_role.to_string()), + ) + .context(ListenerConfigurationSnafu)?; + + cluster_resources + .add(client, role_group_listener) + .await + .context(ApplyGroupListenerSnafu)?; } // Only add the reporting task in case it is enabled. @@ -674,52 +701,6 @@ pub async fn reconcile_nifi( Ok(Action::await_change()) } -/// The node-role service is the primary endpoint that should be used by clients that do not -/// perform internal load balancing including targets outside of the cluster. -pub fn build_node_role_service( - nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, -) -> Result { - let role_name = NifiRole::Node.to_string(); - - let role_svc_name = nifi.node_role_service_name(); - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(&role_svc_name) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - type_: Some(nifi.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(vec![ServicePort { - name: Some(HTTPS_PORT_NAME.to_string()), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), - selector: Some( - Labels::role_selector(nifi, APP_NAME, &role_name) - .context(LabelBuildSnafu)? - .into(), - ), - external_traffic_policy: match nifi.spec.cluster_config.listener_class { - CurrentlySupportedListenerClasses::ClusterInternal => None, - CurrentlySupportedListenerClasses::ExternalUnstable => Some("Local".to_string()), - }, - ..ServiceSpec::default() - }), - status: None, - }) -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_config_map( @@ -727,7 +708,7 @@ async fn build_node_rolegroup_config_map( resolved_product_image: &ResolvedProductImage, authentication_config: &NifiAuthenticationConfig, authorization_config: &NifiAuthorizationConfig, - role: &Role, + role: &Role, rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, merged_config: &NifiConfig, @@ -838,76 +819,19 @@ async fn build_node_rolegroup_config_map( }) } -/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup -/// -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -fn build_node_rolegroup_service( - nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, -) -> Result { - let mut enabled_ports = vec![ServicePort { - name: Some(HTTPS_PORT_NAME.to_string()), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]; - - // NiFi 2.x.x offers nifi-api/flow/metrics/prometheus at the HTTPS_PORT, therefore METRICS_PORT is only required for NiFi 1.x.x... - if resolved_product_image.product_version.starts_with("1.") { - enabled_ports.push(ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }) - } - - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(rolegroup.object_name()) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(enabled_ports), - selector: Some( - Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(LabelBuildSnafu)? - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }) -} - const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`] (from [`build_node_rolegroup_service`]). +/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_statefulset( nifi: &v1alpha1::NifiCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, rolegroup_ref: &RoleGroupRef, - role: &Role, + role: &Role, rolegroup_config: &HashMap>, merged_config: &NifiConfig, authentication_config: &NifiAuthenticationConfig, @@ -983,25 +907,22 @@ async fn build_node_rolegroup_statefulset( } if let NifiAuthenticationConfig::Oidc { oidc, .. } = authentication_config { - env_vars.extend( - oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts( - oidc.client_credentials_secret_ref.clone(), - ), - ); + env_vars.extend(AuthenticationProvider::client_credentials_env_var_mounts( + oidc.client_credentials_secret_ref.clone(), + )); } env_vars.extend(authorization_config.get_env_vars()); let node_address = format!( - "$POD_NAME.{}-node-{}.{}.svc.{}", - rolegroup_ref.cluster.name, - rolegroup_ref.role_group, - &nifi + "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", + service_name = rolegroup_headless_service_name(&rolegroup_ref.object_name()), + namespace = &nifi .metadata .namespace .as_ref() .context(ObjectHasNoNamespaceSnafu)?, - cluster_info.cluster_domain, + cluster_domain = cluster_info.cluster_domain, ); let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; @@ -1049,6 +970,15 @@ async fn build_node_rolegroup_statefulset( .as_slice(), ); + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" + .to_string(), + ]); + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" + .to_string(), + ]); + prepare_args.extend(vec![ "echo Templating config files".to_string(), "config-utils template /stackable/nifi/conf/nifi.properties".to_string(), @@ -1112,6 +1042,8 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .resources( ResourceRequirementsBuilder::new() .with_cpu_request("500m") @@ -1188,6 +1120,8 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .add_container_port(HTTPS_PORT_NAME, HTTPS_PORT.into()) .add_container_port(PROTOCOL_PORT_NAME, PROTOCOL_PORT.into()) .add_container_port(BALANCE_PORT_NAME, BALANCE_PORT.into()) @@ -1218,6 +1152,33 @@ async fn build_node_rolegroup_statefulset( } let mut pod_builder = PodBuilder::new(); + + let recommended_object_labels = build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + nifi, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + let listener_pvc = build_group_listener_pvc( + &group_listener_name(nifi, &rolegroup_ref.role), + &unversioned_recommended_labels, + ) + .context(ListenerConfigurationSnafu)?; + add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; // Add user configured extra volumes if any are specified @@ -1385,12 +1346,10 @@ async fn build_node_rolegroup_statefulset( build_tls_volume( nifi, KEYSTORE_VOLUME_NAME, - vec![ - &nifi_cluster_name, - &build_reporting_task_service_name(&nifi_cluster_name), - ], + vec![&build_reporting_task_service_name(&nifi_cluster_name)], SecretFormat::TlsPkcs12, &requested_secret_lifetime, + LISTENER_VOLUME_NAME, ) .context(SecuritySnafu)?, ) @@ -1440,12 +1399,7 @@ async fn build_node_rolegroup_statefulset( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(MetadataBuildSnafu)? .build(), spec: Some(StatefulSetSpec { @@ -1464,7 +1418,9 @@ async fn build_node_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.object_name()), + service_name: Some(rolegroup_headless_service_name( + &rolegroup_ref.object_name(), + )), template: pod_template, update_strategy: Some(StatefulSetUpdateStrategy { type_: if rolling_update_supported { @@ -1495,6 +1451,7 @@ async fn build_node_rolegroup_statefulset( &NifiRepository::State.repository(), Some(vec!["ReadWriteOnce"]), ), + listener_pvc, ]), ..StatefulSetSpec::default() }), @@ -1502,29 +1459,10 @@ async fn build_node_rolegroup_statefulset( }) } -fn external_node_port(nifi_service: &Service) -> Result { - let external_ports = nifi_service - .spec - .as_ref() - .with_context(|| ObjectHasNoSpecSnafu {})? - .ports - .as_ref() - .with_context(|| ExternalPortSnafu {})? - .iter() - .filter(|p| p.name == Some(HTTPS_PORT_NAME.to_string())) - .collect::>(); - - let port = external_ports - .first() - .with_context(|| ExternalPortSnafu {})?; - - port.node_port.with_context(|| ExternalPortSnafu {}) -} - async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, - nifi_service: &Service, + resolved_product_image: &ResolvedProductImage, ) -> Result { let host_header_check = nifi.spec.cluster_config.host_header_check.clone(); @@ -1540,52 +1478,24 @@ async fn get_proxy_hosts( return Ok("*".to_string()); } - let node_role_service_fqdn = nifi - .node_role_service_fqdn(&client.kubernetes_cluster_info) - .context(NoRoleServiceFqdnSnafu)?; - let reporting_task_service_name = reporting_task::build_reporting_task_fqdn_service_name( - nifi, - &client.kubernetes_cluster_info, - ) - .context(ReportingTaskSnafu)?; - let mut proxy_hosts_set = HashSet::from([ - node_role_service_fqdn.clone(), - format!("{node_role_service_fqdn}:{HTTPS_PORT}"), - format!("{reporting_task_service_name}:{HTTPS_PORT}"), + // Address and port are injected from the listener volume during the prepare container + let mut proxy_hosts = HashSet::from([ + "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), ]); + proxy_hosts.extend(host_header_check.additional_allowed_hosts); - proxy_hosts_set.extend(host_header_check.additional_allowed_hosts); - - // In case NodePort is used add them as well - if nifi.spec.cluster_config.listener_class - == CurrentlySupportedListenerClasses::ExternalUnstable - { - let external_port = external_node_port(nifi_service)?; + // Reporting task only exists for NiFi 1.x + if resolved_product_image.product_version.starts_with("1.") { + let reporting_task_service_name = reporting_task::build_reporting_task_fqdn_service_name( + nifi, + &client.kubernetes_cluster_info, + ) + .context(ReportingTaskSnafu)?; - let cluster_nodes = client - .list::(&(), &ListParams::default()) - .await - .with_context(|_| MissingNodesSnafu { - obj_ref: ObjectRef::from_obj(nifi), - })?; - - // We need the addresses of all nodes to add these to the NiFi proxy setting - // Since there is no real convention about how to label these addresses we will simply - // take all published addresses for now to be on the safe side. - proxy_hosts_set.extend( - cluster_nodes - .into_iter() - .flat_map(|node| { - node.status - .unwrap_or_default() - .addresses - .unwrap_or_default() - }) - .map(|node_address| format!("{}:{external_port}", node_address.address)), - ); + proxy_hosts.insert(format!("{reporting_task_service_name}:{HTTPS_PORT}")); } - let mut proxy_hosts = Vec::from_iter(proxy_hosts_set); + let mut proxy_hosts = Vec::from_iter(proxy_hosts); proxy_hosts.sort(); Ok(proxy_hosts.join(",")) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 430fd8d7..122b4ea7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -38,13 +38,9 @@ use stackable_operator::{ schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, time::Duration, - utils::{ - cluster_info::KubernetesClusterInfo, - crds::{raw_object_list_schema, raw_object_schema}, - }, + utils::crds::{raw_object_list_schema, raw_object_schema}, versioned::versioned, }; -use strum::Display; use tls::NifiTls; pub const APP_NAME: &str = "nifi"; @@ -67,15 +63,9 @@ const DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object has no namespace associated"))] - NoNamespace, - #[snafu(display("the NiFi role [{role}] is missing from spec"))] MissingNifiRole { role: String }, - #[snafu(display("the NiFi node role group [{role_group}] is missing from spec"))] - MissingNifiRoleGroup { role_group: String }, - #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, } @@ -105,7 +95,7 @@ pub mod versioned { // no doc - docs in Role struct. #[serde(default, skip_serializing_if = "Option::is_none")] - pub nodes: Option>, + pub nodes: Option>, // no doc - docs in ProductImage struct. pub image: ProductImage, @@ -164,18 +154,6 @@ pub mod versioned { #[schemars(schema_with = "raw_object_list_schema")] pub extra_volumes: Vec, - /// This field controls which type of Service the Operator creates for this NifiCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, - // Docs are on the struct #[serde(default)] pub create_reporting_task_job: CreateReportingTaskJob, @@ -211,21 +189,6 @@ impl HasStatusCondition for v1alpha1::NifiCluster { } impl v1alpha1::NifiCluster { - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_name(&self) -> String { - self.name_any() - } - - /// The fully-qualified domain name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Option { - Some(format!( - "{}.{}.svc.{}", - self.node_role_service_name(), - self.metadata.namespace.as_ref()?, - cluster_info.cluster_domain, - )) - } - /// Metadata about a metastore rolegroup pub fn node_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { RoleGroupRef { @@ -235,7 +198,7 @@ impl v1alpha1::NifiCluster { } } - pub fn role_config(&self, role: &NifiRole) -> Option<&GenericRoleConfig> { + pub fn role_config(&self, role: &NifiRole) -> Option<&NifiNodeRoleConfig> { match role { NifiRole::Node => self.spec.nodes.as_ref().map(|n| &n.role_config), } @@ -246,31 +209,6 @@ impl v1alpha1::NifiCluster { &self.spec.cluster_config.tls.server_secret_class } - /// List all pods expected to form the cluster - /// - /// We try to predict the pods here rather than looking at the current cluster state in order to - /// avoid instance churn. - pub fn pods(&self) -> Result + '_, Error> { - let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; - Ok(self - .spec - .nodes - .iter() - .flat_map(|role| &role.role_groups) - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .flat_map(move |(rolegroup_name, rolegroup)| { - let rolegroup_ref = self.node_rolegroup_ref(rolegroup_name); - let ns = ns.clone(); - (0..rolegroup.replicas.unwrap_or(0)).map(move |i| PodRef { - namespace: ns.clone(), - role_group_service_name: rolegroup_ref.object_name(), - pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - }) - })) - } - /// Retrieve and merge resource configs for role and role groups pub fn merged_config(&self, role: &NifiRole, role_group: &str) -> Result { // Initialize the result with all default values as baseline @@ -344,26 +282,6 @@ pub fn default_allow_all() -> bool { true } -// TODO: Temporary solution until listener-operator is finished -#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum CurrentlySupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - ClusterInternal, - #[serde(rename = "external-unstable")] - ExternalUnstable, -} - -impl CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { - match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - } - } -} - #[derive(strum::Display, Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub enum StoreType { @@ -616,23 +534,25 @@ pub struct NifiStorageConfig { pub state_repo: PvcConfig, } -/// Reference to a single `Pod` that is a component of a [`NifiCluster`] -/// Used for service discovery. -// TODO: this should move to operator-rs -pub struct PodRef { - pub namespace: String, - pub role_group_service_name: String, - pub pod_name: String, +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NifiNodeRoleConfig { + #[serde(flatten)] + pub common: GenericRoleConfig, + + #[serde(default = "node_default_listener_class")] + pub listener_class: String, } -impl PodRef { - pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> String { - format!( - "{pod_name}.{service_name}.{namespace}.svc.{cluster_domain}", - pod_name = self.pod_name, - service_name = self.role_group_service_name, - namespace = self.namespace, - cluster_domain = cluster_info.cluster_domain - ) +impl Default for NifiNodeRoleConfig { + fn default() -> Self { + NifiNodeRoleConfig { + listener_class: node_default_listener_class(), + common: Default::default(), + } } } + +fn node_default_listener_class() -> String { + "cluster-internal".to_string() +} diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs new file mode 100644 index 00000000..fb0e4ab8 --- /dev/null +++ b/rust/operator-binary/src/listener.rs @@ -0,0 +1,79 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, + }, + crd::listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, + k8s_openapi::api::core::v1::PersistentVolumeClaim, + kube::ResourceExt, + kvp::{Labels, ObjectLabels}, +}; + +use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, v1alpha1}; + +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("listener object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build listener object meta data"))] + BuildObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerPersistentVolume { + source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, + }, +} + +pub fn build_group_listener( + nifi: &v1alpha1::NifiCluster, + object_labels: ObjectLabels, + listener_class: String, + listener_group_name: String, +) -> Result { + Ok(Listener { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(listener_group_name) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(BuildObjectMetaSnafu)? + .build(), + spec: ListenerSpec { + class_name: Some(listener_class), + ports: Some(vec![ListenerPort { + name: HTTPS_PORT_NAME.into(), + port: HTTPS_PORT.into(), + protocol: Some("TCP".into()), + }]), + ..Default::default() + }, + status: None, + }) +} + +pub fn build_group_listener_pvc( + group_listener_name: &String, + unversioned_recommended_labels: &Labels, +) -> Result { + ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(group_listener_name.to_string()), + unversioned_recommended_labels, + ) + .context(BuildListenerPersistentVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerPersistentVolumeSnafu) +} + +pub fn group_listener_name(nifi: &v1alpha1::NifiCluster, role_name: &String) -> String { + format!("{cluster_name}-{role_name}", cluster_name = nifi.name_any(),) +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 3beb2105..65dfebec 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -34,10 +34,12 @@ use crate::{ mod config; mod controller; mod crd; +mod listener; mod operations; mod product_logging; mod reporting_task; mod security; +mod service; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index b36841b8..1efd71b3 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -8,7 +8,7 @@ //! Due to changes in the JWT validation in 1.25.0, the issuer refers to the FQDN of the Pod that was created, e.g.: //! { //! "sub": "admin", -//! "iss": "test-nifi-node-default-0.test-nifi-node-default.default.svc.cluster.local:8443", +//! "iss": "test-nifi-node-default-0.test-nifi-node-default-headless.default.svc.cluster.local:8443", //! } //! which was different in e.g. 1.23.2 //! { @@ -51,6 +51,7 @@ use stackable_operator::{ use crate::{ controller::build_recommended_labels, crd::{APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, + listener::LISTENER_VOLUME_NAME, security::{ authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, build_tls_volume, @@ -295,7 +296,6 @@ fn build_reporting_task_job( format!("-n {nifi_connect_url}"), user_name_command, format!("-p \"$(cat {admin_password_file})\""), - format!("-v {product_version}"), format!("-m {METRICS_PORT}"), format!("-c {REPORTING_TASK_CERT_VOLUME_MOUNT}/ca.crt"), ]; @@ -357,6 +357,7 @@ fn build_reporting_task_job( // There is no correct way to configure this job since it's an implementation detail. // Also it will be dropped when support for 1.x is removed. &Duration::from_days_unchecked(1), + LISTENER_VOLUME_NAME, ) .context(SecretVolumeBuildFailureSnafu)?, ) diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs index eebc0a30..bc304b30 100644 --- a/rust/operator-binary/src/security/mod.rs +++ b/rust/operator-binary/src/security/mod.rs @@ -50,6 +50,7 @@ pub fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, + listener_scope: &str, ) -> Result { tls::build_tls_volume( nifi, @@ -57,6 +58,7 @@ pub fn build_tls_volume( service_scopes, secret_format, requested_secret_lifetime, + listener_scope, ) .context(TlsSnafu) } diff --git a/rust/operator-binary/src/security/tls.rs b/rust/operator-binary/src/security/tls.rs index 8559a4ef..932ab381 100644 --- a/rust/operator-binary/src/security/tls.rs +++ b/rust/operator-binary/src/security/tls.rs @@ -27,6 +27,7 @@ pub(crate) fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, + listener_scope: &str, ) -> Result { let mut secret_volume_source_builder = SecretOperatorVolumeSourceBuilder::new(nifi.server_tls_secret_class()); @@ -42,8 +43,8 @@ pub(crate) fn build_tls_volume( Ok(VolumeBuilder::new(volume_name) .ephemeral( secret_volume_source_builder - .with_node_scope() .with_pod_scope() + .with_listener_volume_scope(listener_scope) .with_format(secret_format) .with_auto_tls_cert_lifetime(*requested_secret_lifetime) .build() diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs new file mode 100644 index 00000000..2e0e9a79 --- /dev/null +++ b/rust/operator-binary/src/service.rs @@ -0,0 +1,124 @@ +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kvp::{Label, ObjectLabels}, + role_utils::RoleGroupRef, +}; + +use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}; + +const METRICS_SERVICE_SUFFIX: &str = "metrics"; +const HEADLESS_SERVICE_SUFFIX: &str = "headless"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, +} + +/// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup +/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. +pub fn build_rolegroup_headless_service( + nifi: &v1alpha1::NifiCluster, + role_group_ref: &RoleGroupRef, + object_labels: ObjectLabels, + selector: BTreeMap, +) -> Result { + Ok(Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(rolegroup_headless_service_name( + &role_group_ref.object_name(), + )) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(MetadataBuildSnafu)? + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(headless_service_ports()), + selector: Some(selector), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }) +} + +/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. +pub fn build_rolegroup_metrics_service( + nifi: &v1alpha1::NifiCluster, + role_group_ref: &RoleGroupRef, + object_labels: ObjectLabels, + selector: BTreeMap, +) -> Result { + Ok(Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(rolegroup_metrics_service_name( + &role_group_ref.object_name(), + )) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(MetadataBuildSnafu)? + .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(metrics_service_ports()), + selector: Some(selector), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }) +} + +fn headless_service_ports() -> Vec { + vec![ServicePort { + name: Some(HTTPS_PORT_NAME.into()), + port: HTTPS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +fn metrics_service_ports() -> Vec { + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +/// Returns the metrics rolegroup service name `---`. +fn rolegroup_metrics_service_name(role_group_ref_object_name: &str) -> String { + format!("{role_group_ref_object_name}-{METRICS_SERVICE_SUFFIX}") +} + +/// Returns the headless rolegroup service name `---`. +pub fn rolegroup_headless_service_name(role_group_ref_object_name: &str) -> String { + format!("{role_group_ref_object_name}-{HEADLESS_SERVICE_SUFFIX}") +} diff --git a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 index a6153bd6..9319e116 100644 --- a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 @@ -30,7 +30,6 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - listenerClass: external-unstable zookeeperConfigMapName: test-nifi-znode authentication: - authenticationClass: nifi-users @@ -51,6 +50,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: external-unstable podOverrides: spec: initContainers: diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-range-limit.yaml b/tests/templates/kuttl/external-access/00-range-limit.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-range-limit.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/external-access/10-assert.yaml.j2 b/tests/templates/kuttl/external-access/10-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/15-create-listener-class.yaml b/tests/templates/kuttl/external-access/15-create-listener-class.yaml new file mode 100644 index 00000000..7d5c2ee4 --- /dev/null +++ b/tests/templates/kuttl/external-access/15-create-listener-class.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst < 15_listener-class.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/15_listener-class.yaml b/tests/templates/kuttl/external-access/15_listener-class.yaml new file mode 100644 index 00000000..46f0c64a --- /dev/null +++ b/tests/templates/kuttl/external-access/15_listener-class.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml new file mode 100644 index 00000000..e0766c49 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 b/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 new file mode 100644 index 00000000..ab2a9536 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-nifi-znode +spec: + clusterRef: + name: test-zk diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml new file mode 100644 index 00000000..25907e6a --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-nifi +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true nificlusters.nifi.stackable.tech/test-nifi --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-nifi-node-default +status: + readyReplicas: 2 + replicas: 2 +--- +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-nifi-node +status: + expectedPods: 2 + currentHealthy: 2 + disruptionsAllowed: 1 +--- +--- +apiVersion: v1 +kind: Service +metadata: + name: test-nifi-node +spec: + type: NodePort # external-unstable +--- diff --git a/tests/templates/kuttl/external-access/30-install-nifi.yaml b/tests/templates/kuttl/external-access/30-install-nifi.yaml new file mode 100644 index 00000000..e7126be2 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-install-nifi.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst < 30_nifi.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/30_nifi.yaml.j2 b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 new file mode 100644 index 00000000..dd621497 --- /dev/null +++ b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 @@ -0,0 +1,60 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: simple-nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: simple-nifi-admin-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: simple-nifi-admin-credentials +stringData: + admin: > + passwordWithSpecialCharacter\@<&>"' +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: test-nifi +spec: + image: +{% if test_scenario['values']['nifi'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + zookeeperConfigMapName: test-nifi-znode + authentication: + - authenticationClass: simple-nifi-users + hostHeaderCheck: + allowAll: false + sensitiveProperties: + keySecret: nifi-sensitive-property-key +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: test-external-unstable-$NAMESPACE + roleGroups: + default: + replicas: 2 diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 index 58440246..bfca700f 100644 --- a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 @@ -7,9 +7,8 @@ spec: image: productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" pullPolicy: IfNotPresent - clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} servers: diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 index e5c1e80c..9d8b5415 100644 --- a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -16,7 +16,6 @@ spec: catalogLabelSelector: matchLabels: trino: trino - listenerClass: external-unstable {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 index 16705a35..11097e5c 100644 --- a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -14,7 +14,6 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - listenerClass: external-unstable authentication: - authenticationClass: nifi-users sensitiveProperties: @@ -50,6 +49,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: external-unstable configOverrides: nifi.properties: {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 7ab671ce..3666b063 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -54,7 +54,7 @@ data: import urllib3 # As of 2022-08-29 we cant use "https://nifi:8443" here because

The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.

- ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" + ENDPOINT = f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443" USERNAME = "admin" PASSWORD = open("/nifi-users/admin").read() diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml index d21fff01..81e42630 100644 --- a/tests/templates/kuttl/iceberg/interactive-nifi.yaml +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -8,7 +8,6 @@ spec: clusterConfig: authentication: - authenticationClass: nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-interactive-sensitive-property-key autoGenerate: true @@ -39,6 +38,8 @@ spec: storage: "1" storageClassName: secrets.stackable.tech nodes: + roleConfig: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ diff --git a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 index 8df853c6..7bc54924 100644 --- a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 @@ -44,12 +44,13 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} zookeeperConfigMapName: nifi-with-ldap-znode - listenerClass: external-unstable nodes: config: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: external-unstable roleGroups: default: config: {} diff --git a/tests/templates/kuttl/ldap/test_nifi.py b/tests/templates/kuttl/ldap/test_nifi.py index b885c09f..3b5cee3e 100755 --- a/tests/templates/kuttl/ldap/test_nifi.py +++ b/tests/templates/kuttl/ldap/test_nifi.py @@ -8,61 +8,71 @@ def get_token(nifi_host, username, password): nifi_headers = { - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', + "content-type": "application/x-www-form-urlencoded; charset=UTF-8", } - data = {'username': username, 'password': password} + data = {"username": username, "password": password} # TODO: handle actual errors when connecting properly - nifi_url = nifi_host + '/nifi-api/access/token' - response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem') + nifi_url = nifi_host + "/nifi-api/access/token" + response = requests.post( + nifi_url, headers=nifi_headers, data=data, verify=False + ) # , cert='./tmp/cacert.pem') if response.ok: - nifi_token = response.content.decode('utf-8') + nifi_token = response.content.decode("utf-8") return "Bearer " + nifi_token else: print(f"Failed to get token: {response.status_code}: {response.content}") exit(-1) -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-u", "--user", required=True, - help="Username to connect as") - all_args.add_argument("-p", "--password", required=True, - help="Password for the user") - all_args.add_argument("-n", "--namespace", required=True, - help="Namespace the test is running in") - all_args.add_argument("-c", "--count", required=True, - help="The expected number of Nodes") + all_args.add_argument("-u", "--user", required=True, help="Username to connect as") + all_args.add_argument( + "-p", "--password", required=True, help="Password for the user" + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="Namespace the test is running in" + ) + all_args.add_argument( + "-c", "--count", required=True, help="The expected number of Nodes" + ) args = vars(all_args.parse_args()) # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" - token = get_token(host, args['user'], args['password']) - headers = {'Authorization': token} - node_count = int(args['count']) + host = f"https://test-nifi-node-default-1.test-nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" + token = get_token(host, args["user"], args["password"]) + headers = {"Authorization": token} + node_count = int(args["count"]) x = 0 while x < 15: - url = host + '/nifi-api/controller/cluster' - cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem') + url = host + "/nifi-api/controller/cluster" + cluster = requests.get( + url, headers=headers, verify=False + ) # , cert='/tmp/cacert.pem') if cluster.status_code != 200: print("Waiting for cluster...") else: - cluster_data = json.loads(cluster.content.decode('utf-8')) - nodes = cluster_data['cluster']['nodes'] + cluster_data = json.loads(cluster.content.decode("utf-8")) + nodes = cluster_data["cluster"]["nodes"] if len(nodes) != node_count: - print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}") + print( + f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}" + ) else: connected = True for node in nodes: - if node['status'] != "CONNECTED": - print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED") + if node["status"] != "CONNECTED": + print( + f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED" + ) connected = False if connected: print("Test succeeded!") diff --git a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 index bb446c0b..659476a1 100644 --- a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 +++ b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 @@ -37,12 +37,13 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} zookeeperConfigMapName: nifi-znode - listenerClass: external-unstable nodes: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} gracefulShutdownTimeout: 1s # let the tests run faster + roleConfig: + listenerClass: external-unstable configOverrides: nifi.properties: # speed up tests diff --git a/tests/templates/kuttl/oidc-opa/test.py b/tests/templates/kuttl/oidc-opa/test.py index a7e96d55..608561d5 100644 --- a/tests/templates/kuttl/oidc-opa/test.py +++ b/tests/templates/kuttl/oidc-opa/test.py @@ -17,7 +17,7 @@ namespace = os.environ["NAMESPACE"] tls = os.environ["OIDC_USE_TLS"] nifi_version = os.environ["NIFI_VERSION"] -nifi = f"test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local" +nifi = f"test-nifi-node-default-0.test-nifi-node-default-headless.{namespace}.svc.cluster.local" keycloak_service = f"keycloak.{namespace}.svc.cluster.local" keycloak_base_url = ( diff --git a/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 index 103845c4..2080faf6 100644 --- a/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 @@ -8,7 +8,6 @@ spec: productVersion: "{{ test_scenario['values']['zookeeper'] }}" pullPolicy: IfNotPresent clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -16,6 +15,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 index 2b9adf37..9e6ddfe1 100644 --- a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 @@ -15,7 +15,6 @@ spec: pullPolicy: IfNotPresent clusterConfig: zookeeperConfigMapName: nifi-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - authenticationClass: nifi-users hostHeaderCheck: @@ -38,6 +37,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/smoke_v1/test_nifi.py b/tests/templates/kuttl/smoke_v1/test_nifi.py index 60740afe..5c72c1a6 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py index fd383ab0..336660c4 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://nifi-node-default-0.nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-0.nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout diff --git a/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 index 201b102f..cda0e02f 100644 --- a/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 @@ -9,7 +9,6 @@ spec: productVersion: "{{ test_scenario['values']['zookeeper'] }}" pullPolicy: IfNotPresent clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -17,6 +16,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 index 455a47f4..06cfae19 100644 --- a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 @@ -17,7 +17,6 @@ spec: {% if test_scenario['values']['use-zookeeper-manager'] == 'true' %} zookeeperConfigMapName: nifi-znode {% endif %} - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - authenticationClass: nifi-users hostHeaderCheck: @@ -40,6 +39,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/smoke_v2/test_nifi.py b/tests/templates/kuttl/smoke_v2/test_nifi.py index 60740afe..5c72c1a6 100755 --- a/tests/templates/kuttl/smoke_v2/test_nifi.py +++ b/tests/templates/kuttl/smoke_v2/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 index 8bcf1c8e..1f66464c 100644 --- a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 @@ -8,9 +8,9 @@ commands: - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} {% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" {% else %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" {% endif %} # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/old_input | grep -Eov 'Error|\b0\b'" diff --git a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 index a71c5a37..6f10d137 100644 --- a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 @@ -9,7 +9,7 @@ commands: {% if test_scenario['values']['nifi_new'].split(',')[0].startswith('1.') %} - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/new_input | grep -Eov 'Error|\b0\b'" # This tests that the number of input records stays the same after the upgrade. diff --git a/tests/templates/kuttl/upgrade/test_nifi.py b/tests/templates/kuttl/upgrade/test_nifi.py index b885c09f..3b5cee3e 100755 --- a/tests/templates/kuttl/upgrade/test_nifi.py +++ b/tests/templates/kuttl/upgrade/test_nifi.py @@ -8,61 +8,71 @@ def get_token(nifi_host, username, password): nifi_headers = { - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', + "content-type": "application/x-www-form-urlencoded; charset=UTF-8", } - data = {'username': username, 'password': password} + data = {"username": username, "password": password} # TODO: handle actual errors when connecting properly - nifi_url = nifi_host + '/nifi-api/access/token' - response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem') + nifi_url = nifi_host + "/nifi-api/access/token" + response = requests.post( + nifi_url, headers=nifi_headers, data=data, verify=False + ) # , cert='./tmp/cacert.pem') if response.ok: - nifi_token = response.content.decode('utf-8') + nifi_token = response.content.decode("utf-8") return "Bearer " + nifi_token else: print(f"Failed to get token: {response.status_code}: {response.content}") exit(-1) -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-u", "--user", required=True, - help="Username to connect as") - all_args.add_argument("-p", "--password", required=True, - help="Password for the user") - all_args.add_argument("-n", "--namespace", required=True, - help="Namespace the test is running in") - all_args.add_argument("-c", "--count", required=True, - help="The expected number of Nodes") + all_args.add_argument("-u", "--user", required=True, help="Username to connect as") + all_args.add_argument( + "-p", "--password", required=True, help="Password for the user" + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="Namespace the test is running in" + ) + all_args.add_argument( + "-c", "--count", required=True, help="The expected number of Nodes" + ) args = vars(all_args.parse_args()) # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" - token = get_token(host, args['user'], args['password']) - headers = {'Authorization': token} - node_count = int(args['count']) + host = f"https://test-nifi-node-default-1.test-nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" + token = get_token(host, args["user"], args["password"]) + headers = {"Authorization": token} + node_count = int(args["count"]) x = 0 while x < 15: - url = host + '/nifi-api/controller/cluster' - cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem') + url = host + "/nifi-api/controller/cluster" + cluster = requests.get( + url, headers=headers, verify=False + ) # , cert='/tmp/cacert.pem') if cluster.status_code != 200: print("Waiting for cluster...") else: - cluster_data = json.loads(cluster.content.decode('utf-8')) - nodes = cluster_data['cluster']['nodes'] + cluster_data = json.loads(cluster.content.decode("utf-8")) + nodes = cluster_data["cluster"]["nodes"] if len(nodes) != node_count: - print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}") + print( + f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}" + ) else: connected = True for node in nodes: - if node['status'] != "CONNECTED": - print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED") + if node["status"] != "CONNECTED": + print( + f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED" + ) connected = False if connected: print("Test succeeded!") diff --git a/tests/templates/kuttl/upgrade/test_nifi_metrics.py b/tests/templates/kuttl/upgrade/test_nifi_metrics.py index 86076038..f0f00d0c 100755 --- a/tests/templates/kuttl/upgrade/test_nifi_metrics.py +++ b/tests/templates/kuttl/upgrade/test_nifi_metrics.py @@ -4,18 +4,34 @@ import time from requests.exceptions import ConnectionError -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-m", "--metric", required=False, default="nifi_amount_bytes_read", - help="The name of a certain metric to check") - all_args.add_argument("-n", "--namespace", required=True, - help="The namespace the test is running in") - all_args.add_argument("-p", "--port", required=False, default="8081", - help="The port where metrics are exposed") - all_args.add_argument("-t", "--timeout", required=False, default="120", - help="The timeout in seconds to wait for the metrics port to be opened") + all_args.add_argument( + "-m", + "--metric", + required=False, + default="nifi_amount_bytes_read", + help="The name of a certain metric to check", + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="The namespace the test is running in" + ) + all_args.add_argument( + "-p", + "--port", + required=False, + default="8081", + help="The port where metrics are exposed", + ) + all_args.add_argument( + "-t", + "--timeout", + required=False, + default="120", + help="The timeout in seconds to wait for the metrics port to be opened", + ) args = vars(all_args.parse_args()) metric_name = args["metric"] @@ -23,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://test-nifi-node-default-0.test-nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout @@ -35,7 +51,9 @@ print("Test metrics succeeded!") exit(0) else: - print(f"Could not find metric [{metric_name}] in response:\n {response.text}") + print( + f"Could not find metric [{metric_name}] in response:\n {response.text}" + ) time.sleep(timeout) except ConnectionError: # NewConnectionError is expected until metrics are available diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 110f22d0..c35b2f99 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -152,6 +152,11 @@ tests: - nifi - zookeeper-latest - openshift + - name: external-access + dimensions: + - nifi + - zookeeper-latest + - openshift suites: - name: nightly patch: