tmgstevens · bhagya7893 · Dec 23, 2019 · Jan 29, 2020 · Jan 29, 2020 · Jan 29, 2020
diff --git a/.gitignore b/.gitignore
@@ -3,5 +3,5 @@
 *.iml
 .idea/
 .DS_Store
-site.retry
-enable_kerberos.retry
+*.retry
+*.pyc
diff --git a/README.md b/README.md
@@ -143,6 +143,7 @@ sudo ln -s /path/to/dynamic_inventory_cm hosts
 ```
 
 **Set up SSH public key authentication for remote host(s)**
+
 If you do not have ~/.ssh/id_rsa.pub and ~/.ssh/id_rsa files then you need to generate them with the ssh-keygen command before this:
 ```
 ANSIBLE_HOST_KEY_CHECKING=False ansible all -m authorized_key -a key="{{ lookup('file', '~/.ssh/id_rsa.pub') }} user=$USER" -k

diff --git a/action_plugins/scm_hosts.py b/action_plugins/scm_hosts.py
diff --git a/action_plugins/scm_hosts.pyc b/action_plugins/scm_hosts.pyc
diff --git a/cm_roles_test.yml b/cm_roles_test.yml
@@ -0,0 +1,8 @@
+---
+# Cloudera playbook
+
+- name: Install Cloudera Manager roles
+  hosts: scm_server
+  roles:
+    - cm_roles
+  tags: cm_roles_test
diff --git a/group_vars/all b/group_vars/all
@@ -2,9 +2,89 @@
 
 ansible_become: true
 tmp_dir: /tmp
-krb5_realm: AD.SEC.CLOUDERA.COM
+agent_tls: true
+krb5_realm: MIT.EXAMPLE.COM
 ad_domain: "{{ krb5_realm.lower() }}"
-computer_ou: ou=Hosts,ou=morhidi,ou=HadoopClusters,ou=morhidi,dc=ad,dc=sec,dc=cloudera,dc=com
-domain: vpc.cloudera.com
-kdc: w2k8-1.ad.sec.cloudera.com
-admin_server: w2k8-1.ad.sec.cloudera.com
+computer_ou: OU=computer_hosts,OU=hadoop_prd,DC=ad,DC=sec,DC=example,DC=com 
+domain: MIT.EXAMPLE.COM
+kdc: tristan3-1.vpc.cloudera.com
+admin_server: tristan3-1.vpc.cloudera.com
+enc_types: rc4-hmac DES-CBC-MD5 DES-CBC-CRC
+ad_account_prefix: prefix_
+kdc_account_creation_host_override: tristan3-1.vpc.cloudera.com
+
+## ------------------------------------------------------------------------------------------------------------
+## Cluster software installation options
+## ------------------------------------------------------------------------------------------------------------
+
+# Version of CDH to install
+cluster_version_cdh: 5.16.2
+#cluster_version_cdh: 7.x
+
+# Version of Cloudera Manager to install
+cluster_version_cm:  5.16.2
+#cluster_version_cm:  "{{ cluster_version_cdh }}"
+#cluster_version_cm:  7.x.0
+
+# Version of CDS Powered by Apache Spark (note: not installed if CDH6/7 is also selected)
+cluster_version_cds: 2.4.0.cloudera2
+
+# Helper variables for major and minor versions
+cluster_version_cdh_major: "{{ cluster_version_cdh.split('.')[0] }}"
+cluster_version_cdh_minor: "{{ cluster_version_cdh.split('.')[1] }}"
+cluster_version_cm_major: "{{ cluster_version_cm.split('.')[0] }}"
+cluster_version_cm_minor: "{{ cluster_version_cm.split('.')[1] }}"
+
+cloudera_archive_protocol: https://
+cloudera_archive: archive.cloudera.com
+cloudera_archive_authn: ""
+
+configs_by_version:
+  "5":
+    scm_repo_url:    "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm5/redhat/{{ ansible_distribution_major_version }}/x86_64/cm/{{ cluster_version_cm }}/"
+    scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm5/redhat/{{ ansible_distribution_major_version }}/x86_64/cm/RPM-GPG-KEY-cloudera"
+    scm_parcel_repositories: 
+      - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cdh5/parcels/{{ cluster_version_cdh }}/"
+      - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/spark2/parcels/{{ cluster_version_cds }}/"
+      - "http://cloudera-build-3-us-central-1.gce.cloudera.com/s3/build/723506/parcels"
+      - "http://cloudera-build-3-us-central-1.gce.cloudera.com/s3/build/338985/parcels"
+    scm_csds:
+      - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/spark2/csd/SPARK2_ON_YARN-{{cluster_version_cds}}.jar"
+    scm_prepare_database_script_path: "/usr/share/cmf/schema/scm_prepare_database.sh"
+  "6":
+    scm_repo_url:    "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm6/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum"
+    scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cm6/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum/RPM-GPG-KEY-cloudera"
+    scm_parcel_repositories: 
+      - "{{ cloudera_archive_protocol }}{{ cloudera_archive }}/cdh6/{{ cluster_version_cdh }}/parcels"
+      - http://tristan3-1.vpc.cloudera.com/parcels/keytrustee-kms-6.1.0-parcels/6.1.0/parcels/
+      - http://tristan3-1.vpc.cloudera.com/parcels/keytrustee-server-6.1.0-parcels/6.1.0/parcels/
+    scm_prepare_database_script_path: "/opt/cloudera/cm/schema/scm_prepare_database.sh"
+  "7":
+    scm_repo_url:    "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cm7/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum"
+    scm_repo_gpgkey: "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cm7/{{ cluster_version_cm }}/redhat{{ ansible_distribution_major_version }}/yum/RPM-GPG-KEY-cloudera"
+    scm_parcel_repositories:
+      - "{{ cloudera_archive_protocol }}{{ cloudera_archive_authn }}@{{ cloudera_archive }}/p/cdh7/{{ cluster_version_cdh }}/parcels"
+    scm_prepare_database_script_path: "/opt/cloudera/cm/schema/scm_prepare_database.sh"
+
+scm_default_user: admin
+scm_default_pass: admin
+scm_port: 7180
+scm_license_file: /path/to/cloudera_license.txt
+scm_parcel_repositories:            "{{ configs_by_version[cluster_version_cdh_major].scm_parcel_repositories }}"
+scm_prepare_database_script_path:   "{{ configs_by_version[cluster_version_cm_major].scm_prepare_database_script_path }}"
+scm_repo_url:                       "{{ configs_by_version[cluster_version_cm_major].scm_repo_url }}"
+scm_repo_gpgkey:                    "{{ configs_by_version[cluster_version_cm_major].scm_repo_gpgkey }}"
+scm_csds:                           "{{ configs_by_version[cluster_version_cm_major].scm_csds | default([]) }}"
+
+
+## ------------------------------------------------------------------------------------------------------------
+## Java installation options
+## ------------------------------------------------------------------------------------------------------------
+
+java_installation_strategy: package       # can be set to 'none', 'package' or 'rpm'
+
+java_package: java-1.8.0-openjdk-devel
+java_rpm_location: /tmp/jdk-8u181-linux-x64.rpm
+java_rpm_remote_src: no
+java_jce_location: /tmp/jce_policy-8.zip
+java_jce_remote_src: no
diff --git a/group_vars/ca.yml b/group_vars/ca.yml
@@ -0,0 +1,17 @@
+---
+openssl_path: "openssl"
+ca_root_location: "/ca"
+ca_root_key_password: password
+ca_countryname_default: GB
+ca_state_or_province: England
+ca_org_name: Cloudera Inc
+ca_ou: PS
+ca_root_cn: Root CA
+ca_intermediate_location: "/ca/intermediate"
+ca_intermediate_key_password: password
+ca_intermediate_cn: Intermediate CA
+root_ca_cert_name: ca.cert.pem
+intermediate_ca_cert_name: intermediate.cert.pem
+chain_cert_name: ca-chain.cert.pem
+signed_certificates_local_location: "/tmp/ca/signedcerts"
+csr_certificates_local_location: "/tmp/ca/csrs"
diff --git a/group_vars/cdh_servers.yml b/group_vars/cdh_servers.yml
@@ -3,31 +3,69 @@
 db_hostname: "{{ hostvars[groups['db_server'][0]]['inventory_hostname'] }}"
 scm_hostname: "{{ hostvars[groups['scm_server'][0]]['inventory_hostname'] }}"
 
-cdh_version: 5.8.3
-cluster_display_name: cluster_1
+cluster_display_name: Cluster1
+
+cdh_tls: true
+log_base: /var/log
+trusted_realm: EXAMPLE.COM
 
 cdh_services:
+
   - type: hdfs
-    dfs_data_dir_list: /dfs/dn
-    fs_checkpoint_dir_list: /dfs/snn
-    dfs_name_dir_list: /dfs/nn
-    dfs_journalnode_edits_dir: /dfs/jn
+    nameservice: nameservice1
+    dfs_data_dir_list: /data/1/dfs/dn
+    fs_checkpoint_dir_list: /data/1/dfs/snn
+    dfs_name_dir_list: /data/1/dfs/nn
+    dfs_journalnode_edits_dir: /data/2/dfs/jn
+    hdfs_supergroup: hdfs
+    sentry_sync_path_prefixes: /user/hive/warehouse
+
+#  - type: cdsw
 
   - type: hive
 
+  - type: hbase
+    hbase_superuser: "@hbase"
+
   - type: hue
+    hue_timezone: Europe/London
+    leaflet_tile_layer: "http://osm.org/osm_tiles/{z}/{x}/{y}.png"
+    mapping_attribution: "Custom mapping"
+    secure_content_security_policy: "script-src 'self' 'unsafe-inline' 'unsafe-eval' *.google-analytics.com *.doubleclick.net *.mathjax.org data:;img-src 'self' *.google-analytics.com *.doubleclick.net *.gstatic.com data:;style-src 'self' 'unsafe-inline';connect-src 'self';child-src 'self' data:;object-src 'none'"
 
   - type: impala
-    scratch_dirs: /tmp/impala
+    impala_scratch_dirs: /data/1/impala/impalad
+
+#  - type: kafka
+#    kafka_super_users: kafka
+
+#  - type: keytrustee
+
+  - type: ks_indexer
 
   - type: oozie
+    oozie_from_email_address: [email protected]
+    oozie_email_smtp_host: mail.exampe.com
+    oozie_https_port: 11444
 
   - type: sentry
+    sentry_admin_group: hive,impala,hue,solr,kafka,group_sentry_admin
+    sentry_allow_connect: hive,impala,hue,hdfs,solr
+    sentry_user: sentry
+
+  - type: solr
 
   - type: spark
 
+  - type: spark2
+
   - type: yarn
-    yarn_nodemanager_local_dirs: /tmp/nm
-    yarn_nodemanager_log_dirs: /var/log/nm
+    yarn_nodemanager_local_dirs: /data/1/yarn/nm
+    yarn_nodemanager_log_dirs: /data/1/yarn/nm/log
+    CMJOBUSER: user1
+    YARN_ADMIN_ACL: nobody  mapred,hue
 
   - type: zookeeper
+    zookeeper_data_log_dir: /data/1/zookeeper
+    zookeeper_edits_dir: /data/1/zookeeper
+
-Original file line number
+Diff line change
@@ Expand Up / @@ -143,6 +143,7 @@ sudo ln -s /path/to/dynamic_inventory_cm hosts @@
     ```
     **Set up SSH public key authentication for remote host(s)**
     If you do not have ~/.ssh/id_rsa.pub and ~/.ssh/id_rsa files then you need to generate them with the ssh-keygen command before this:
     ```
     ANSIBLE_HOST_KEY_CHECKING=False ansible all -m authorized_key -a key="{{ lookup('file', '~/.ssh/id_rsa.pub') }} user=$USER" -k
@@ Expand Down @@