Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reshard and Rebalance the Redis cluster on Redis Node Removal #12

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
+76 −18
Open

Reshard and Rebalance the Redis cluster on Redis Node Removal #12

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d9a10f2
Reshard the cluster using the json template
shubham-cmyk Dec 23, 2022
2b3b2cc
some updates
shubham-cmyk Dec 23, 2022
cf65d85
final changes
shubham-cmyk Dec 23, 2022
e0417ee
fix
shubham-cmyk Jan 12, 2023
9d22fd4
remove comment
shubham-cmyk Jan 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
final changes
@shubham-cmyk
shubham-cmyk committed Dec 23, 2022
commit cf65d85479ccd925996379614c37208a0eb24076
24 changes: 12 additions & 12 deletions inventory/cluster.ini
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
[redis]
redis1 ansible_ssh_host=172.31.57.91
redis2 ansible_ssh_host=172.31.49.187
redis3 ansible_ssh_host=172.31.62.128
#redis4 ansible_ssh_host=172.31.55.4
redis1 ansible_ssh_host=192.168.1.13
redis2 ansible_ssh_host=192.168.1.14
redis3 ansible_ssh_host=192.168.1.15
redis4 ansible_ssh_host=192.168.1.16

[redis_cluster:children]
leader
follower

[leader]
redis-leader1 ansible_ssh_host=172.31.57.91 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready
redis-leader2 ansible_ssh_host=172.31.49.187 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready
redis-leader3 ansible_ssh_host=172.31.62.128 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready
#redis-leader4 ansible_ssh_host=172.31.55.4 redis_port=6379 exporter_port=9121 leader_id=3 node_status=remove
redis-leader1 ansible_ssh_host=192.168.1.13 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready
redis-leader2 ansible_ssh_host=192.168.1.14 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready
redis-leader3 ansible_ssh_host=192.168.1.15 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready
redis-leader4 ansible_ssh_host=192.168.1.16 redis_port=6379 exporter_port=9121 leader_id=3 node_status=ready

[follower]
redis-follower1 ansible_ssh_host=172.31.57.91 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready
redis-follower2 ansible_ssh_host=172.31.49.187 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready
redis-follower3 ansible_ssh_host=172.31.62.128 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready
#redis-follower4 ansible_ssh_host=172.31.55.4 redis_port=6380 exporter_port=9122 leader_id=3 node_status=remove
redis-follower1 ansible_ssh_host=192.168.1.13 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready
redis-follower2 ansible_ssh_host=192.168.1.14 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready
redis-follower3 ansible_ssh_host=192.168.1.15 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready
redis-follower4 ansible_ssh_host=192.168.1.16 redis_port=6380 exporter_port=9122 leader_id=3 node_status=ready
4 changes: 2 additions & 2 deletions playbooks/node.json.j2
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"removenode" : "{{ remove_node_id.stdout }}",
"presentnode" : "{{ transfer_node_id.stdout }}"
"removenode" : {{ (lookup('file', '/tmp/node-removal')) }} ,
"presentnode" : {{ (lookup('file', '/tmp/node-ready')) }}
}

84 changes: 60 additions & 24 deletions playbooks/remove-node.yml
View file
Original file line number Diff line number Diff line change
@@ -5,8 +5,20 @@
leader_redis_port: 6379
follower_redis_port: 6380
redis_slots: 3446

tasks:

- name: Ensuring the temporary file is created before execution
local_action:
module: file
path: /tmp/node-removal
state: touch

- name: Ensuring the temporary file is created before execution
local_action:
module: file
path: /tmp/node-ready
state: touch

- name: Ensure redis-cli is generated without password
set_fact:
redis_command: redis-cli
@@ -20,29 +32,41 @@
- name: Ensure if redis cluster exists or not
command: |-
{{ redis_command }} cluster nodes
register: command_result
register: command_result

- name: Check the Node id of the remove node
shell: |-
redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid
register: remove_node_id
when: >
node_status == "remove" and inventory_hostname in groups['leader']
# - name: Check the Node id of the remove node
# shell: |-
# redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid
# register: register_remove_node_id
# vars:
# remove_node_id : "{{register_remove_node_id.stdout}}"
# when: >
# node_status == "remove" and inventory_hostname in groups['leader']

- name: Check the Node id of transfer node
shell : |-
redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid
register: transfer_node_id
when: >
node_status == "ready" and inventory_hostname in groups['leader']
# - name: Check the Node id of transfer node
# shell : |-
# redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid
# register: register_transfer_node_id
# vars:
# transfer_node_id : "{{register_transfer_node_id.stdout}}"
# when: >
# node_status == "ready" and inventory_hostname in groups['leader']

# - name: Ensure the nodes need to removed are present in temporary file
# local_action:
# module: copy
# content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}"
# dest: /tmp/node-removal
# when: >
# node_status == 'remove'
- name: Ensure the nodes need to removed are present in temporary file
local_action:
module: copy
content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}"
dest: /tmp/node-removal
when: >
node_status == 'remove'

- name: Ensure the nodes need to transfer are present in temporary file
local_action:
module: copy
content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}"
dest: /tmp/node-ready
when: >
node_status == 'ready'

- name: Create the Json file with the remove and ready redis node
ansible.builtin.template:
@@ -53,8 +77,8 @@
- name: Reshard the Node to be removed
command : |-
{{ redis_command }} --cluster reshard {{ansible_ssh_host}}:{{redis_port}}
--cluster-from {{item.removenode}}
--cluster-to {{ item.presentnode }}
--cluster-from {{ item.removenode[0].split(' ')[0] }}
--cluster-to {{ item.presentnode[0].split(' ')[0] }}
--cluster-slots {{ redis_slots }}
--cluster-yes
when: >
@@ -65,7 +89,7 @@
- name: Ensuring the removed marked nodes are not part of the cluster
command: |-
{{ redis_command }} --cluster del-node
{{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode}}
{{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode[0].split(' ')[0] }}
with_items:
- "{{ lookup('file', '/tmp/node.json') | from_json }}"
when: >
@@ -82,3 +106,15 @@
module: file
path: /tmp/node.json
state: absent

- name: Ensuring the temporary file is removed after execution
local_action:
module: file
path: /tmp/node-removal
state: absent

- name: Ensuring the temporary file is removed after execution
local_action:
module: file
path: /tmp/node-ready
state: absent