airflow.yaml

blocks:
  - name: git_sync_dedicated_env_block
    content:
      $cndi.comment(dag-repo-credentials-embedded): Airflow DAG Repo Credentials Embedded
      GIT_SYNC_USERNAME: "{{ $cndi.get_prompt_response(airflow_git_sync_username) }}"
      GIT_SYNC_PASSWORD: "{{ $cndi.get_prompt_response(airflow_git_sync_password) }}"

  - name: git_sync_dedicated
    content:
      apiVersion: v1
      kind: Secret
      metadata:
        name: airflow-git-credentials
        namespace: airflow
      stringData:
        GIT_SYNC_USERNAME: $cndi_on_ow.seal_secret_from_env_var(GIT_SYNC_USERNAME)
        GIT_SYNC_PASSWORD: $cndi_on_ow.seal_secret_from_env_var(GIT_SYNC_PASSWORD)

  - name: git_sync_shared
    content:
      apiVersion: v1
      kind: Secret
      metadata:
        name: airflow-git-credentials
        namespace: airflow
      stringData:
        GIT_SYNC_USERNAME: $cndi_on_ow.seal_secret_from_env_var(GIT_USERNAME)
        GIT_SYNC_PASSWORD: $cndi_on_ow.seal_secret_from_env_var(GIT_TOKEN)

  - name: airflow_public_config
    content:
      webserver:
        expose_config: "True"
        instance_name: CNDI
        enable_proxy_fix: "True"
        base_url: "https://{{ $cndi.get_arg(hostname) }}"
      operators:
        default_owner: CNDI

  - name: airflow_private_config
    content:
      webserver:
        expose_config: "True"
        instance_name: CNDI
        enable_proxy_fix: "True"
      operators:
        default_owner: CNDI

  - name: deploy_airflow_ingress_blk
    content:
      apiVersion: networking.k8s.io/v1
      kind: Ingress
      metadata:
        name: airflow-ingress
        namespace: airflow
        annotations:
          cert-manager.io/cluster-issuer: cluster-issuer
          kubernetes.io/tls-acme: "true"
          external-dns.alpha.kubernetes.io/hostname: "{{ $cndi.get_prompt_response(airflow_hostname) }}"
      spec:
        ingressClassName: public
        tls:
          - hosts:
              - "{{ $cndi.get_prompt_response(airflow_hostname) }}"
            secretName: cluster-issuer-private-key-airflow
        rules:
          - host: "{{ $cndi.get_prompt_response(airflow_hostname) }}"
            http:
              paths:
                - path: /
                  pathType: Prefix
                  backend:
                    service:
                      name: airflow-webserver
                      port:
                        number: 8080
prompts:
  - $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/cluster/core-prompts.yaml):
      {}
  - $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/{{ $cndi.get_prompt_response(deployment_target_provider) }}/core-prompts.yaml):
      {}

  - name: deploy_airflow_ingress
    default: true
    message: >-
      Do you want to expose the Airflow UI to the web?
    type: Confirm

  - name: airflow_hostname
    default: airflow.example.com
    message: >-
      What hostname should Airflow be accessible at?
    type: Input
    validators:
      - hostname # FQDN
      - min_length: 3
    condition:
      - "{{ $cndi.get_prompt_response(deploy_airflow_ingress) }}"
      - ==
      - true

  - name: airflow_dag_repo_url
    default: https://github.com/polyseam/demo-dag-bag
    message: >-
      What is the URL of the Git repository containing your Airflow DAGs?
    type: Input

  - name: airflow_share_credentials
    default: true
    message: >-
      Do you want to use your cluster credentials for Airflow's Git Sync?
    type: Confirm
    condition:
      - "{{ $cndi.get_prompt_response(git_credentials_mode) }}"
      - ==
      - token

  - name: airflow_git_sync_username
    message: >-
      What is the username for your DAG Git repository?
    type: Input
    condition:
      - "{{ $cndi.get_prompt_response(airflow_share_credentials) }}"
      - ==
      - false

  - name: airflow_git_sync_password
    message: >-
      What is the password for your DAG Git repository?
    type: Input
    condition:
      - "{{ $cndi.get_prompt_response(airflow_share_credentials) }}"
      - ==
      - false
  - name: postgresql_user_password
    message: "Please enter a password for the PostgreSQL database used as Airflow's storage backend:"
    type: Secret
    validators:
      - min_length: 3
    default: "{{ $cndi.get_random_string(12) }}"

outputs:
  cndi_config:
    cndi_version: v2
    project_name: "{{ $cndi.get_prompt_response(project_name) }}"
    provider: "{{ $cndi.get_prompt_response(deployment_target_provider) }}"
    distribution: "{{ $cndi.get_prompt_response(deployment_target_distribution) }}"
    infrastructure:
      cndi:
        $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/cluster/observability-config.yaml):
          condition:
            - "{{ $cndi.get_prompt_response(deploy_grafana_ingress) }}"
            - ==
            - true

        $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/cluster/argocd-config.yaml):
          condition:
            - "{{ $cndi.get_prompt_response(deploy_argocd_ingress) }}"
            - ==
            - true

        external_dns:
          $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/external-dns/config/{{ $cndi.get_prompt_response(dns_provider) }}.yaml):
            condition:
              - "{{ $cndi.get_prompt_response(enable_external_dns) }}"
              - ==
              - true

        cert_manager:
          email: "{{ $cndi.get_prompt_response(cert_manager_email) }}"

        nodes:
          $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/{{ $cndi.get_prompt_response(deployment_target_provider) }}/basic-node-pool.yaml):
            {}

    cluster_manifests:
      external-dns-secret:
        $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/external-dns/secret/{{ $cndi.get_prompt_response(dns_provider) }}.yaml):
          condition:
            - "{{ $cndi.get_prompt_response(enable_external_dns) }}"
            - ==
            - true

      $cndi.comment(airflow-git-sync-secret): Airflow Credentials
      git-sync-credentials-secret:
        $cndi.get_block(git_sync_dedicated):
          condition:
            - "{{ $cndi.get_prompt_response(airflow_share_credentials) }}"
            - ==
            - false
        $cndi.get_block(git_sync_shared):
          condition:
            - "{{ $cndi.get_prompt_response(airflow_share_credentials) }}"
            - ==
            - true

      postgresql-connection-string-secret:
        apiVersion: v1
        kind: Secret
        metadata:
          name: postgresql-connection-string-secret
          namespace: airflow
        type: Opaque
        stringData:
          connection: $cndi_on_ow.seal_secret_from_env_var(FQDN_POSTGRESQL_CONNECTION_STRING)

      airflow-fernet-key-secret:
        apiVersion: v1
        kind: Secret
        metadata:
          name: airflow-fernet-key
          namespace: airflow
        type: Opaque
        stringData:
          fernet-key: $cndi_on_ow.seal_secret_from_env_var(AIRFLOW_FERNET_KEY)

      airflow-webserver-key-secret:
        apiVersion: v1
        kind: Secret
        metadata:
          name: airflow-webserver-key
          namespace: airflow
        type: Opaque
        stringData:
          webserver-key: $cndi_on_ow.seal_secret_from_env_var(AIRFLOW_WEBSERVER_KEY)

      cnpg-cluster:
        apiVersion: postgresql.cnpg.io/v1
        kind: Cluster
        metadata:
          name: cnpg-airflow-cluster
          namespace: airflow
          annotations:
            $cndi.comment(why_propagation_policy): PrunePropagationPolicy=background should prevent PDBs from blocking cluster destruction
            argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true,PrunePropagationPolicy=background
        spec:
          imageName: "ghcr.io/cloudnative-pg/postgresql:16"
          instances: 3
          storage:
            size: 1Gi
          replicationSlots:
            highAvailability:
              enabled: true
          postgresql:
            pg_hba:
              - >-
                  host airflow-db airflow all password
          bootstrap:
            initdb:
              database: airflow-db
              owner: airflow
              secret:
                name: cnpg-database-user-auth-secret
          superuserSecret:
            name: cnpg-cluster-superuser-auth-secret

      cnpg-cluster-superuser-auth-secret:
        apiVersion: v1
        kind: Secret
        metadata:
          name: cnpg-cluster-superuser-auth-secret
          namespace: airflow
        type: kubernetes.io/basic-auth
        stringData:
          username: $cndi_on_ow.seal_secret_from_env_var(POSTGRESQL_CLUSTER_SUPERUSER)
          password: $cndi_on_ow.seal_secret_from_env_var(POSTGRESQL_CLUSTER_SUPERUSER_PASSWORD)

      cnpg-database-user-auth-secret:
        apiVersion: v1
        kind: Secret
        metadata:
          name: cnpg-database-user-auth-secret
          namespace: airflow
        type: kubernetes.io/basic-auth
        stringData:
          username: $cndi_on_ow.seal_secret_from_env_var(POSTGRESQL_USER)
          password: $cndi_on_ow.seal_secret_from_env_var(POSTGRESQL_USER_PASSWORD)

      airflow-ingress:
        $cndi.get_block(deploy_airflow_ingress_blk):
          condition:
            - "{{ $cndi.get_prompt_response(deploy_airflow_ingress) }}"
            - ==
            - true

      airflow-ns:
        apiVersion: v1
        kind: Namespace
        metadata:
          name: airflow

      cnpg-system-ns:
        apiVersion: v1
        kind: Namespace
        metadata:
          name: cnpg-system

    applications:
      cnpg:
        targetRevision: 0.23.0
        destinationNamespace: cnpg-system
        repoURL: "https://cloudnative-pg.github.io/charts"
        chart: cloudnative-pg
        syncPolicy:
          syncOptions:
            - ServerSideApply=true

      airflow:
        targetRevision: 1.11.0
        destinationNamespace: airflow
        repoURL: "https://airflow.apache.org"
        chart: airflow
        values:
          executor: LocalKubernetesExecutor
          data:
            metadataSecretName: postgresql-connection-string-secret
          postgresql:
            enabled: false
          dags:
            gitSync:
              enabled: true
              $cndi.comment(airflow-dag-repo-url): Airflow DAG Repo URL
              repo: "{{ $cndi.get_prompt_response(airflow_dag_repo_url) }}"
              credentialsSecret: airflow-git-credentials
              branch: main
              wait: 40
              $cndi.comment(subPath): Folder path in DAG repo to sync
              subPath: dags
          enableBuiltInSecretEnvVars:
            AIRFLOW__CORE__FERNET_KEY: false
            AIRFLOW__WEBSERVER__SECRET_KEY: false
          fernetKeySecretName: airflow-fernet-key
          webserverSecretKeySecretName: airflow-webserver-key
          config:
            $cndi.get_block(airflow_public_config):
              args:
                hostname: "{{ $cndi.get_prompt_response(airflow_hostname) }}"
              condition:
                - "{{ $cndi.get_prompt_response(deploy_airflow_ingress) }}"
                - ==
                - true
            $cndi.get_block(airflow_private_config):
              condition:
                - "{{ $cndi.get_prompt_response(deploy_airflow_ingress) }}"
                - ==
                - false
          logs:
            persistence:
              enabled: true
              size: 15Gi
              storageClassName: rwm
          createUserJob:
            useHelmHooks: false
            applyCustomEnv: false
          migrateDatabaseJob:
            jobAnnotations:
              "argocd.argoproj.io/hook": Sync
            useHelmHooks: false
            applyCustomEnv: false

  env:
    $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/git-credentials-{{ $cndi.get_prompt_response(git_credentials_mode) }}-env.yaml):
      {}
    $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/{{ $cndi.get_prompt_response(deployment_target_provider) }}/env.yaml):
      {}
    $cndi.get_block(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/cluster/env.yaml):
      {}

    $cndi.comment(airflow-heading): Airflow Key Parameters
    AIRFLOW_FERNET_KEY: "{{ $cndi.get_random_string(32) }}"
    AIRFLOW_WEBSERVER_KEY: "{{ $cndi.get_random_string(16) }}"
    $cndi.comment(postgres-heading): PostgreSQL Connection Parameters
    POSTGRESQL_DB: airflow-db
    POSTGRESQL_USER: airflow
    POSTGRESQL_USER_PASSWORD: "{{ $cndi.get_prompt_response(postgresql_user_password) }}"
    FQDN_POSTGRESQL_CONNECTION_STRING: "postgresql://airflow:{{ $cndi.get_prompt_response(postgresql_user_password) }}@cnpg-airflow-cluster-rw:5432/airflow-db"
    POSTGRESQL_CLUSTER_SUPERUSER: postgres
    POSTGRESQL_CLUSTER_SUPERUSER_PASSWORD: "{{ $cndi.get_random_string(24) }}"
    $cndi.get_block(git_sync_dedicated_env_block):
      condition:
        - "{{ $cndi.get_prompt_response(airflow_share_credentials) }}"
        - ==
        - false
  readme:
    project_name: "# {{ $cndi.get_prompt_response(project_name) }}"
    $cndi.get_string(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/common/cluster/core-readme.md):
      {}
    $cndi.get_string(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/{{ $cndi.get_prompt_response(deployment_target_provider) }}/core.md):
      {}
    $cndi.get_string(https://raw.githubusercontent.com/polyseam/cndi/main/blocks/{{ $cndi.get_prompt_response(deployment_target_provider) }}/{{ $cndi.get_prompt_response(deployment_target_distribution) }}.md):
      {}
    airflow-section: |
      # Apache Airflow Deployment Guide

      This Template deploys a standalone production-ready [Apache Airflow](https://airflow.apache.org) application using Helm on a Kubernetes cluster.
      Apache Airflow is an open-source platform designed to programmatically author, schedule, and monitor workflows, offering a robust foundation for managing complex data pipelines.

      ## Walkthroughs

      For detailed guidance on deploying Airflow across different environments, we offer step-by-step walkthroughs for each supported deployment target.
      These walkthroughs illustrate how to deploy a production-grade Airflow cluster using CNDI's `airflow` Template on various platforms:

      - **AWS EKS:** [eks/airflow](https://github.com/polyseam/cndi/blob/main/docs/walkthroughs/eks/airflow.md)
      - **GCP GKE:** [gke/airflow](https://github.com/polyseam/cndi/blob/main/docs/walkthroughs/gke/airflow.md)
      - **Azure AKS:** [aks/airflow](https://github.com/polyseam/cndi/blob/main/docs/walkthroughs/aks/airflow.md)
      - **Local Development:** [dev/airflow](https://github.com/polyseam/cndi/blob/main/docs/walkthroughs/dev/airflow.md)

      ## Configuration

      Customizing your Airflow deployment is straightforward with the Helm chart's extensive configuration options, adjustable in the `cndi_config.yaml` file under `applications.airflow.values`. Key configuration areas include:

      - **Executor:** Choose between Sequential, Local, Celery, and Kubernetes executors for task execution.
      - **Persistence:** Configure persistent storage solutions for DAGs and logs, ensuring data durability.
      - **Webserver:** Customize service type and ingress settings for external web access.
      - **Scheduler:** Fine-tune scheduler behavior and resource allocations to optimize performance.

      For comprehensive details on all configurable values, please refer to the [chart's values.yaml file](https://github.com/apache/airflow/blob/main/chart/values.yaml) or the [official chart documentation](https://airflow.apache.org/docs/helm-chart/stable/index.html).

      ## Verifying Airflow Accessibility

      ### Steps for Verification

      **Access the Web Interface**:
        - Open a web browser and navigate to the domain you've configured for Airflow, e.g., `https://airflow.yourdomain.com`.
        - You should be greeted with Airflow's login screen.

      **Default Credentials**:
        - By default, the login credentials for a fresh Airflow installation are:
          - **Username:** `admin`
          - **Password:** `admin`
        - It's strongly recommended to change these default credentials during the initial setup to ensure your Airflow instance is secure.

      **Verify that Airflow is connected to the private DAG repository**:
        - If correct, the private DAGs should be visible on the Airflow UI. If not,you should go back and make sure that the private DAG repository is properly connected to Airflow with the correct credentials that located in the `.env` file