diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d77bf2a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Wizcorp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..341f17c --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +ElasticSearch Role +================== + +This role will install ElasticSearch and ElasticDump on your servers. + +Configuration +------------- + +### Inventory + +```ini +# +# Here, you put the list of your Couchbase cluster +# +[elasticsearch:children] +elasticsearch-somedc-prod + +[elasticsearch-somedc-prod:children] +elasticsearch1-somedc-prod + +# +# And you create one children per cluster +# +[elasticsearch1-somedc-prod] +node1.elasticsearch.somedc.prod1 ansible_ssh_host=172.16.0.80 +node2.elasticsearch.somedc.prod1 ansible_ssh_host=172.16.0.81 +node3.elasticsearch.somedc.prod1 ansible_ssh_host=172.16.0.82 + +# +# Configure the RAM size and # of replicates accordingly +# +[elasticsearch1-somedc-prod:vars] +elasticsearch_cluster_name = elasticsearch1-somedc-prod +# +# Auto-discovery is enabled by default. However, +# in some environment such as AWS, this will not +# work as intended. By setting autodiscovery +# to false, we will use unicast discovery +# instead (as in, we will add all nodes) +# to configuration instead of relying on multicasting +# +elasticsearch_autodiscovery = true + +# ElasticSearch heap size (default: 256m) +# http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/heap-sizing.html +elasticsearch_heap_size = 1g + +# Version of ElasticDump to install +elasticdump_version = v0.12.0 +``` + +### Execution + +You will need to run this on the whole cluster every time you +add or remove nodes to it if you are running this with autodiscovery +set to false. However, you may provision only the machine you are +adding if you are using autodiscovery. + +See also +-------- + +* [ElasticSearch](https://www.elastic.co/products/elasticsearch) +* [ElasticDump](https://github.com/taskrabbit/elasticsearch-dump) diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..564e0fc --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,6 @@ +elasticsearch_tcp_port: 19350 +elasticsearch_http_port: 9200 +elasticsearch_autodiscovery: true +elasticsearch_heap_size: 256m + +elasticdump_version: v0.12.0 diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..9b621dc --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,6 @@ +- name: "Restart ElasticSearch" + service: > + name=elasticsearch + state=restarted + tags: + - elasticsearch diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..f2723d0 --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,27 @@ +galaxy_info: + author: Marc Trudel + description: Installs ElasticSearch + company: Wizcorp K.K. + license: MIT + min_ansible_version: 1.8.1 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Debian + versions: + - wheezy + categories: + - database + - database:nosql + - monitoring +dependencies: + - role: aeriscloud.nodejs + - role: aeriscloud.repos + repositories: + centos6: + - elasticsearch + centos7: + - elasticsearch + - role: aeriscloud.yum diff --git a/tasks/centos.yml b/tasks/centos.yml new file mode 100644 index 0000000..02fbb67 --- /dev/null +++ b/tasks/centos.yml @@ -0,0 +1,44 @@ +- name: "Set elasticsearch version" + set_fact: + es_ver: "{{ elasticsearch_version[:3] }}" + tags: + - pkgs + - elasticsearch + +- name: "Download and install the Public Signing Key" + rpm_key: > + key=https://packages.elastic.co/GPG-KEY-elasticsearch + state=present + tags: + - pkgs + - elasticsearch + +- name: "Install Elasticsearch and its dependencies" + yum: > + name={{ item }} + state=present + enablerepo=elasticsearch-{{ es_ver }} + with_items: + - java-1.7.0-openjdk + - elasticsearch-{{ elasticsearch_version }} + tags: + - pkgs + - elasticsearch + notify: + - Restart ElasticSearch + +- name: "Configure Elasticsearch to automatically start during bootup" + service: + name=elasticsearch + enabled=yes + tags: + - pkgs + - elasticsearch + +- name: "Upload logrotate config" + template: > + src=logrotate.j2 + dest=/etc/logrotate.d/elasticsearch + tags: + - files + - elasticsearch diff --git a/tasks/debian.yml b/tasks/debian.yml new file mode 100644 index 0000000..5b407cc --- /dev/null +++ b/tasks/debian.yml @@ -0,0 +1,77 @@ +- name: "Set elasticsearch version" + set_fact: + es_ver: "{{ elasticsearch_version[:3] }}" + tags: + - elasticsearch + +- name: "Download and install the Public Signing Key" + apt_key: > + url=https://packages.elasticsearch.org/GPG-KEY-elasticsearch + state=present + tags: + - elasticsearch + - pkgs + +- name: "Add Elasticsearch apt repository" + apt_repository: > + repo="deb http://packages.elasticsearch.org/elasticsearch/{{ es_ver }}/debian stable main" + state=present + tags: + - elasticsearch + - repos + +- name: "Install Elasticsearch's dependencies" + apt: > + name={{ item }} + state=present + update_cache=yes + with_items: + - openjdk-7-jre-headless + tags: + - elasticsearch + - pkgs + +- name: "Check if Java 7 as the default" + file: > + src=/usr/lib/jvm/java-1.7.0-openjdk-amd64/jre/bin/java + dest=/etc/alternatives/java + state=link + register: java_set + tags: + - elasticsearch + - files + +- name: "Ensure Java 7 is the default" + shell: | + executable=/bin/bash + update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java + when: java_set.changed + tags: + - elasticsearch + +- name: "Install Elasticsearch" + apt: > + update_cache=yes + name=elasticsearch + notify: + - Restart ElasticSearch + tags: + - elasticsearch + - pkgs + +- name: "Configure Elasticsearch to automatically start during bootup" + service: > + name=elasticsearch + runlevel="95 10" + enabled=yes + tags: + - elasticsearch + - service + +- name: "Upload logrotate config" + template: > + src=logrotate.j2 + dest=/etc/logrotate.d/elasticsearch + tags: + - elasticsearch + - files diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..f2b155b --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,77 @@ +- include: centos.yml + when: ansible_distribution == 'CentOS' + tags: + - elasticsearch + +- include: debian.yml + when: ansible_distribution == 'Debian' + tags: + - elasticsearch + +- include: tuning.yml + tags: + - elasticsearch + +- name: "Install ElasticSearch plugins" + shell: | + chdir=/usr/share/elasticsearch + creates=/usr/share/elasticsearch/plugins/{{ item.dest }} + + bin/plugin -install {{ item.src }} + with_items: + - src: royrusso/elasticsearch-HQ + dest: HQ + - src: mobz/elasticsearch-head + dest: head + tags: + - elasticsearch + - plugins + +- name: "Setup ElasticSearch configuration" + template: > + src=elasticsearch.yml + dest=/etc/elasticsearch/elasticsearch.yml + mode=0640 + owner=root + group=elasticsearch + notify: + - Restart ElasticSearch + tags: + - elasticsearch + - configs + - files + +- name: "Make sure ElasticSearch is running" + service: > + name=elasticsearch + state=started + enabled=yes + tags: + - elasticsearch + - services + +- name: "Make sure the required node version is installed" + shell: | + creates={{ nvm_dir }}/{{ elasticsearch_node_version }}/bin + executable=/bin/bash + + source {{ nvm_dir }}/nvm.sh && \ + nvm install {{ elasticsearch_node_version }} + tags: + - elasticsearch + - nodejs + +- name: "Install elasticdump" + shell: | + executable=/bin/bash + chdir=/opt + + [ ! -d elasticdump ] && git clone https://github.com/taskrabbit/elasticsearch-dump.git elasticdump + cd elasticdump + git checkout {{ elasticdump_version }} + + source {{ nvm_dir }}/nvm.sh + nvm use {{ elasticsearch_node_version }} + npm install . + tags: + - elasticsearch diff --git a/tasks/tuning.yml b/tasks/tuning.yml new file mode 100644 index 0000000..c2a348c --- /dev/null +++ b/tasks/tuning.yml @@ -0,0 +1,37 @@ +- name: "Reduce swappiness and increase max_map_count" + sysctl: > + name={{ item.name }} + value={{ item.value }} + ignoreerrors=true + state=present + with_items: + - { name: 'vm.swappiness', value: 1 } + - { name: 'vm.max_map_count', value: 262144 } + tags: + - elasticsearch + - sysctl + +- name: "Lock the process address space into RAM" + lineinfile: > + line='bootstrap.mlockall: true' + dest=/etc/elasticsearch/elasticsearch.yml + tags: + - elasticsearch + +- name: "Configure ES_HEAP_SIZE" + lineinfile: > + line='ES_HEAP_SIZE={{ elasticsearch_heap_size }}' + regexp='^#?ES_HEAP_SIZE=' + dest=/etc/sysconfig/elasticsearch + when: ansible_distribution == 'CentOS' + tags: + - elasticsearch + +- name: "Configure ES_HEAP_SIZE" + lineinfile: > + line='ES_HEAP_SIZE={{ elasticsearch_heap_size }}' + regexp='^#?ES_HEAP_SIZE=' + dest=/etc/default/elasticsearch + when: ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' + tags: + - elasticsearch \ No newline at end of file diff --git a/templates/elasticsearch.yml b/templates/elasticsearch.yml new file mode 100644 index 0000000..607d71e --- /dev/null +++ b/templates/elasticsearch.yml @@ -0,0 +1,69 @@ +# this must be the same as for your elasticsearch cluster +cluster.name: {{ elasticsearch_cluster_name }} + +# you could also leave this out, but makes it easier to identify the graylog2 client instance +node.name: {{ inventory_hostname }} + +# use a different port if you run multiple elasticsearch nodes on one machine +transport.tcp.port: {{ elasticsearch_tcp_port }} + +network.host: {{ ansible_ssh_host }} + +# we don't need to run the embedded HTTP server here +http.enabled: true +# http.port: {{ elasticsearch_http_port }} + +# adapt these for discovery to work in your network! multicast can be tricky +{% if elasticsearch_autodiscovery|string == "True" %} +discovery.zen.ping.multicast.address: {{ ansible_ssh_host }} +discovery.zen.ping.multicast.group: 224.2.2.4 +{% else %} +discovery.zen.ping.multicast.enabled: false +discovery.zen.ping.unicast.hosts: +{% for host in groups[elasticsearch_cluster_name] %} + - {{ hostvars[host]['ansible_ssh_host'] }}:{{ elasticsearch_tcp_port }} +{% endfor %} +{% endif %} + +################################## Discovery ################################## + +# Discovery infrastructure ensures nodes can be found within a cluster +# and master node is elected. Multicast discovery is the default. + +# Set to ensure a node sees N other master eligible nodes to be considered +# operational within the cluster. Set this option to a higher value (2-4) +# for large clusters (>3 nodes): +# +# discovery.zen.minimum_master_nodes: 1 + +# Set the time to wait for ping responses from other nodes when discovering. +# Set this option to a higher value on a slow or congested network +# to minimize discovery failures: +# +# discovery.zen.ping.timeout: 3s + +# See +# for more information. + +# Unicast discovery allows to explicitly control which nodes will be used +# to discover the cluster. It can be used when multicast is not present, +# or to restrict the cluster communication-wise. +# +# 1. Disable multicast discovery (enabled by default): +# +# discovery.zen.ping.multicast.enabled: false +# +# 2. Configure an initial list of master nodes in the cluster +# to perform discovery when new nodes (master or data) are started: +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] + +# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# +# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# +# See +# for more information. +# +# See +# for a step-by-step tutorial. diff --git a/templates/logrotate.j2 b/templates/logrotate.j2 new file mode 100644 index 0000000..9c433df --- /dev/null +++ b/templates/logrotate.j2 @@ -0,0 +1,11 @@ +/var/log/elasticsearch/*.log { + daily + rotate 20 + size 100M + copytruncate + compress + delaycompress + missingok + notifempty + create 644 elasticsearch elasticsearch +} \ No newline at end of file diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 0000000..9e7b804 --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,3 @@ +elasticsearch_version: "1.5.2" + +elasticsearch_node_version: "v0.10.38" \ No newline at end of file