setup-playbook.yaml

- hosts: all
  gather_facts: False
  user: "{{ remote_user }}"
  any_errors_fatal: true
  roles:
    - coreos_bootstrap
  pre_tasks:
  - name: Copy python interpreter to server
    delegate_to: localhost
    command: "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {{ lookup('env', 'SSH_OPTS') }} files/pypy2.7-v7.3.1-linux64.tar.bz2 {{ remote_user }}@{{ inventory_hostname }}:/home/{{ remote_user }}/"
    changed_when: false

- hosts: localhost
  tasks:
  - name: Verifiy that facts have been gathered for all hosts in inventory
    assert:
      that:
        - ansible_play_hosts == ansible_play_hosts_all

- hosts: masters:workers:ceph_nodes:ceph_daemon_nodes
  user: "{{ remote_user }}"
  become: yes
  tasks:
  - name: Ensure locksmithd is stopped
    systemd:
      daemon_reload: yes
      name: locksmithd.service
      state: stopped
      masked: yes
    ignore_errors: yes

- hosts: etcd
  user: "{{ remote_user }}"
  become: yes
  tasks:
    - name: Copy update.conf to etcd nodes
      copy:
        src: files/etcd/etcd_update.conf
        dest: /etc/coreos/update.conf

    - name: Copy backup script to etcd nodes
      copy:
        src: files/etcd/etcd_backup.sh
        dest: "/home/{{ remote_user }}/etcd_backup.sh"
        mode: 0744

    - name: Copy etcd backup service and timer to etcd nodes
      copy:
        src: files/etcd/{{ item }}
        dest: /etc/systemd/system/
      loop:
        - etcd-backup.service
        - etcd-backup.timer
        - etcd-prometheus-push.service
        - etcd-prometheus-push.timer

    - name: Start and enable etcd-backend.timer
      systemd:
        daemon_reload: yes
        name: "{{ item }}"
        state: started
        enabled: yes
      loop:
        - etcd-backup.timer
        - etcd-prometheus-push.timer

- hosts: etcd:prometheus:githubrunner
  user: "{{ remote_user }}"
  become: yes
  tasks:
    - name: Ensure locksmithd is started
      systemd:
        daemon_reload: yes
        name: locksmithd.service
        state: started
        enabled: yes

- hosts: all
  user: "{{ remote_user }}"
  become: yes
  roles:
  -  { role: proxy, when: nais_http_proxy is defined }
  - os-users
  - logs
  - update-engine
  - node_exporter
  tasks:

  - name: Render and copy /etc/resolv.conf
    template:
      src: templates/resolv.conf.j2
      dest: /etc/resolv.conf

  - name: Add systemd unit to fix issue 32728 (symlinks in /sys)
    copy:
      src: files/kube-issue-32728-fix.service
      dest: /etc/systemd/system/kube-issue-32728-fix.service
      owner: root
      group: root
      mode: 0644

  - name: Enable the fix for 32728 during boot
    systemd:
      daemon_reload: yes
      name: kube-issue-32728-fix.service
      state: started
      enabled: yes

  - name: Add nsswitch.conf
    copy:
      src: "files/nsswitch.conf"
      dest: "/etc/nsswitch.conf"

- hosts: prometheus
  gather_facts: true
  user: "{{ remote_user }}"
  become: yes
  any_errors_fatal: true
  roles:
    - prometheus

- hosts: masters:workers
  user: "{{ remote_user }}"
  become: yes
  roles:
  - k8s_common
  tasks:
  - name: Remove /root/.bashrc
    file:
      path: /root/.bashrc
      state: absent

- hosts: masters
  user: "{{ remote_user }}"
  roles:
  - fetch_existing_certificates

- hosts: masters:workers
  user: "{{ remote_user }}"
  tasks:
  - name: Fetch Kubelet-specific certificates (if exists)
    include_role:
      name: fetch_existing_certificates
      tasks_from: kubelet

- hosts: etcd
  user: "{{ remote_user }}"
  tasks:
  - name: Fetch Etcd-specific certificates (if exists)
    include_role:
      name: fetch_existing_certificates
      tasks_from: etcd
    tags:
      - fetch_etcd_certs

- hosts: localhost
  roles:
  - create_certificates

- hosts: etcd
  user: "{{ remote_user }}"
  become: yes
  roles:
    - etcd

# Configure kubernetes Master node
- hosts: masters
  user: "{{ remote_user }}"
  become: yes
  tasks:
  - name: Copy certificates
    copy:
      src: "target/pki/{{ item }}"
      dest: /etc/kubernetes/pki/
    loop:
      - admin.pem
      - admin-key.pem
      - ca.pem
      - ca-key.pem
      - front-proxy-ca.pem
      - front-proxy-ca-key.pem
      - front-proxy-client.pem
      - front-proxy-client-key.pem
      - kube-apiserver-server-key.pem
      - kube-apiserver-server.pem
      - kube-proxy.pem
      - kube-proxy-key.pem
      - sa.key
      - sa.pub

  - name: Copy kubelet certificates
    copy:
      src: "target/pki/kubelet/{{ ansible_hostname.split('.')[0] }}{{ item }}"
      dest: "/etc/kubernetes/pki/kubelet{{ item }}"
    loop:
      - ".pem"
      - "-key.pem"

- hosts: masters:workers
  user: "{{ remote_user }}"
  become: yes
  tasks:

  - name: Copy ca certificate
    copy:
      src: "target/pki/{{ item }}"
      dest: /etc/kubernetes/pki/ # to ensure idempotency
    loop:
      - ca.pem
    notify:
      - restart_kubelet
      - restart_kubeproxy

  - name: Ensure /var/lib/docker exists
    file:
      path: /var/lib/docker
      state: directory

  - name: Check docker service status
    shell: systemctl status docker.service 2>&1 > /dev/null
    register: docker_status
    ignore_errors: true

  - name: Copy docker selinux drop-in file
    copy:
      src: files/docker.service.d/selinux.conf
      dest: /etc/systemd/system/docker.service.d/selinux.conf
    notify:
      - restart_docker_if_running

  - name: Setup docker daemon
    copy:
      src: files/daemon.json
      dest: /etc/docker/daemon.json
    notify:
      - restart_docker_if_running

  - name: Enable docker
    systemd:
      daemon_reload: yes
      name: docker.service
      enabled: yes

  - name: Ensure /root/.docker exists
    file:
      path: /root/.docker
      state: directory

  - name: Add docker credentials if defined
    template:
      src: templates/docker-config.json.j2
      dest: /root/.docker/config.json
    when: docker_repo_url is defined

  - name: Disable swap automount
    mount:
      path: swap
      state: absent

  - name: Disable swap
    command: swapoff -a
    changed_when: false

  - name: Download kubernetes node binaries # using curl as get_url gave protocol error (most likely caused by internal webproxy)
    environment: "{{ proxy_env }}"
    shell: curl -L --insecure https://dl.k8s.io/v{{ k8s_version }}/kubernetes-node-linux-amd64.tar.gz | \
           tar xzf - -C /tmp kubernetes/node/bin/kubectl kubernetes/node/bin/kube-proxy kubernetes/node/bin/kubelet && \
           mv /tmp/kubernetes/node/bin/kube-proxy {{ install_dir }}/bin/kube-proxy-{{ k8s_version }} && \
           mv /tmp/kubernetes/node/bin/kubectl {{ install_dir }}/bin/kubectl-{{ k8s_version }} && \
           mv /tmp/kubernetes/node/bin/kubelet {{ install_dir }}/bin/kubelet-{{ k8s_version }}
    args:
      creates: "{{ install_dir }}/bin/kubelet-{{ k8s_version }}"

  - name: Symlink kubernetes node binaries
    file:
      src: "{{ install_dir }}/bin/{{ item }}-{{ k8s_version }}"
      dest: "{{ install_dir }}/bin/{{ item }}"
      state: link
      force: yes
    loop:
      - kubectl
      - kube-proxy
      - kubelet
    notify:
      - restart_kubelet
      - restart_kubeproxy

  - name: Make kubernetes node binaries executable
    file:
      path: "{{ install_dir }}/bin/{{ item }}-{{ k8s_version }}"
      mode: 0755
    loop:
      - kubectl
      - kube-proxy
      - kubelet

  - name: Install conntrack binary
    copy:
      src: files/bin/conntrack
      dest: /opt/bin/conntrack
      mode: 0755

  - name: Copy kube-proxy service file
    template:
      src: templates/kube-proxy.service.j2
      dest: /etc/systemd/system/kube-proxy.service
    notify:
      - restart_kubeproxy

  - name: Enable kube-proxy
    systemd:
      daemon_reload: yes
      name: kube-proxy.service
      state: started
      enabled: yes

  - name: Set max_user_watches kernel parameter
    sysctl:
      name: fs.inotify.max_user_watches
      value: 16384
      state: present
      sysctl_file: /etc/sysctl.d/60-sysctl.conf
      sysctl_set: yes
      reload: yes

  - name: Set max_map_count kernel parameter
    sysctl:
      name: vm.max_map_count
      value: 262144
      state: present
      sysctl_file: /etc/sysctl.d/60-sysctl.conf
      sysctl_set: yes
      reload: yes

  - name: Start and enable systemd-timesyncd
    systemd:
      name: systemd-timesyncd.service
      state: started
      enabled: yes

  handlers:
    # Only restart docker if already running at this stage so it don't start with wrong docker0 subnet on new nodes.
    # Flannel will start docker on new nodes
    - name: restart_docker_if_running
      systemd:
        daemon_reload: yes
        name: docker.service
        state: restarted
      when: docker_status.rc == 0

    - name: restart_kubelet
      systemd:
        daemon_reload: yes
        name: kubelet.service
        state: restarted
      # on new nodes kublet will not yet be install, ignore errors
      ignore_errors: yes

    - name: restart_kubeproxy
      systemd:
        daemon_reload: yes
        name: kube-proxy.service
        state: restarted

# Start control plane
- hosts: masters
  user: "{{ remote_user }}"
  become: yes
  tasks:

  - name: Copy kubelet master config
    copy:
      src: files/kubelet-master-config.yaml
      dest: /etc/kubernetes/kubeconfigs/kubelet-config.yaml
    notify:
      - restart_kubelet

  - name: Copy kubelet service manifest
    template:
      src: templates/master-kubelet.service.j2
      dest: /etc/systemd/system/kubelet.service
    notify:
      - restart_kubelet

  - name: Copy kubelet kubeconfig
    template:
      src: templates/kubeconfigs/master-kubelet.conf.j2
      dest: /etc/kubernetes/kubeconfigs/kubelet.conf
    notify:
      - restart_kubelet

  - name: Enable kubelet
    systemd:
      daemon_reload: yes
      name: kubelet.service
      state: started
      enabled: yes

  - name: Ensure kubectl config directory exists on master
    file:
      state: directory
      path: /root/.kube

  - name: Copy API server pod description
    copy:
      src: files/kubectl-config
      dest: /root/.kube/config

  - name: Ensure manifests directory exists on master
    file:
      state: directory
      path: /etc/kubernetes/manifests

  - name: Copy manifests and kubeconfigs
    template:
      src: "templates/{{ item }}.j2"
      dest: "/etc/kubernetes/{{ item }}"
    tags:
    - devel
    loop:
      - kube-apiserver-audit-policy.yaml
      - manifests/kube-apiserver.yaml
      - manifests/kube-scheduler.yaml
      - manifests/kube-controller-manager.yaml
      - kubeconfigs/kube-scheduler.conf
      - kubeconfigs/kube-controller-manager.conf

  handlers:
    - name: restart_docker
      systemd:
        daemon_reload: yes
        name: docker.service
        state: restarted

    - name: restart_kubelet
      systemd:
        daemon_reload: yes
        name: kubelet.service
        state: restarted

# Configure workers
- hosts: workers
  user: "{{ remote_user }}"
  become: yes
  tasks:
  - name: Check kubelet service status
    shell: systemctl status kubelet.service 2>&1 > /dev/null
    register: kubelet_status
    ignore_errors: true
  - name: Copy kubelet certificates
    copy:
      src: "target/pki/kubelet/{{ ansible_hostname.split('.')[0] }}{{ item }}"
      dest: "/etc/kubernetes/pki/kubelet{{ item }}"
    loop:
      - ".pem"
      - "-key.pem"
    notify:
      - restart_kubelet_if_running

  - name: Copy kube-proxy certificates
    copy:
      src: "target/pki/{{ item }}"
      dest: /etc/kubernetes/pki/
    loop:
      - kube-proxy.pem
      - kube-proxy-key.pem
    notify:
      - restart_kubeproxy

  - name: Copy Issuing Internal certificate
    copy:
      src: files/NAVIssuingCAIntern.crt
      dest: "{{ cert_dir }}/NAV_Issuing_Intern_CA{{ cert_postfix }}"
      force: no

  - name: Ensure certs.d directory exists worker nodes
    file:
      state: directory
      path: /etc/docker/certs.d/docker.adeo.no:5000/

  - name: Symlink Internal CA to docker
    file:
      src: "/etc/ssl/certs/NAV_Issuing_Intern_CA{{ cert_postfix }}"
      dest: /etc/docker/certs.d/docker.adeo.no:5000/ca.crt
      state: link
      force: yes

  - name: Copy kubelet worker config
    template:
      src: templates/kubeconfigs/kubelet-worker-config.yaml.j2
      dest: /etc/kubernetes/kubeconfigs/kubelet-config.yaml
    notify:
      - restart_kubelet_if_running

  - name: Copy kubelet service manifest
    template:
      src: templates/worker-kubelet.service.j2
      dest: /etc/systemd/system/kubelet.service
    notify:
      - restart_kubelet_if_running

  - name: Copy kubelet kubeconfig
    template:
      src: templates/kubeconfigs/worker-kubelet.conf.j2
      dest: /etc/kubernetes/kubeconfigs/kubelet.conf
    notify:
      - restart_kubelet_if_running

  - name: Enable kubelet
    systemd:
      daemon_reload: yes
      name: kubelet.service
      enabled: yes

  handlers:
    - name: restart_kubeproxy
      systemd:
        daemon_reload: yes
        name: kube-proxy.service
        state: restarted

    - name: restart_kubelet_if_running
      systemd:
        daemon_reload: yes
        name: kubelet.service
        state: restarted
      when: kubelet_status.rc == 0

- hosts: masters
  user: "{{ remote_user }}"
  become: yes
  tasks:
    - name: Generate cluster-admin kubeconfig, used by nsync
      shell: "{{ install_dir }}/bin/kubectl config set-cluster {{ cluster_name }} --server=https://{{ groups.masters|first }}:6443 --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --kubeconfig=/etc/kubernetes/kubeconfigs/cluster-admin.conf &&
             {{ install_dir }}/bin/kubectl config set-credentials {{ cluster_name }}-cluster-admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs --kubeconfig=/etc/kubernetes/kubeconfigs/cluster-admin.conf &&
             {{ install_dir }}/bin/kubectl config set-context {{ cluster_name }} --cluster={{ cluster_name }} --user={{ cluster_name }}-cluster-admin --kubeconfig=/etc/kubernetes/kubeconfigs/cluster-admin.conf &&
             {{ install_dir }}/bin/kubectl config use-context {{ cluster_name }} --kubeconfig=/etc/kubernetes/kubeconfigs/cluster-admin.conf"
      changed_when: false

    - name: Wait for API server to become available
      wait_for:
        port: 6443
        host: "{{ groups.masters|first }}"
        delay: 15
        timeout: 240
      changed_when: false

- hosts: masters[0]
  user: "{{ remote_user }}"
  become: yes
  tasks:
    - name: Copy kubeconfig
      fetch:
        src: "/etc/kubernetes/kubeconfigs/cluster-admin.conf"
        dest: "target/kubeconfigs/cluster-admin.conf"
        flat: yes

- hosts: masters:workers
  user: "{{ remote_user }}"
  become: yes
  roles:
    - generate_kubeconfigs
  tasks:
    - name: Check if node is registered with api-server
      shell: "/opt/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfigs/kubelet.conf get nodes -o name | grep {{ inventory_hostname }}"
      ignore_errors: yes
      register: node_status

    - name: Register node if unregistered
      shell: |
        cat <<EOF | /opt/bin/kubectl --kubeconfig /etc/kubernetes/kubeconfigs/kubelet.conf apply -f -
        { "kind": "Node", "apiVersion": "v1", "metadata": { "name": "{{ inventory_hostname }}" } }
        EOF
      when: node_status.stdout is not match("^node/.*")

    - name: Setup node taints
      command: "{{ install_dir }}/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfigs/kubelet.conf  --overwrite=true taint nodes {{ inventory_hostname }} {{ item }}"
      loop: "{{ node_taints|default([]) }}"
      changed_when: false

    - name: Setup node labels
      command: "{{ install_dir }}/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfigs/kubelet.conf --overwrite=true label nodes {{ inventory_hostname }} {{ item }}"
      loop: "{{ extra_node_labels|default([]) }}"
      changed_when: false

    - name: Ensure flanneld.service.d exists
      file:
        path: /etc/systemd/system/flanneld.service.d/
        state: directory

    - name: Copy flanneld.service.d opt-file
      template:
        src: templates/flanneld.service.d/flannel_opts.conf
        dest: /etc/systemd/system/flanneld.service.d/flannel_opts.conf
      notify:
        - restart_flanneld

    - name: Ensure flannel config dir exists
      file:
        path: /etc/kubernetes/flannel
        state: directory

    - name: Copy flanneld net conf
      template:
        src: templates/flannel-net-conf.j2
        dest: /etc/kubernetes/flannel/net-conf.json
      notify:
        - restart_flanneld

    - name: Enable flanneld
      systemd:
        daemon_reload: yes
        name: flanneld.service
        enabled: yes

  handlers:
    - name: restart_flanneld
      systemd:
        daemon_reload: yes
        name: flanneld.service
        state: restarted
      ignore_errors: yes
    - name: ensure_kubelet_is_running
      listen: restart_flanneld
      systemd:
        name: kubelet.service
        state: started

- hosts: masters
  user: "{{ remote_user }}"
  become: yes

  tasks:
  - name: Ensure addons directory exists
    file:
      state: directory
      path: /etc/kubernetes/addons

  - name: Copy kubernetes-addons manifest templates
    template:
      src: "templates/addons/{{ item }}.j2"
      dest: "/etc/kubernetes/addons/{{ item }}"
    loop:
      - core-dns.yaml
      - namespaces.yaml
      - debug.yaml

  - name: Copy kubernetes-addons manifests
    copy:
      src: "files/addons/{{ item }}"
      dest: "/etc/kubernetes/addons/{{ item }}"
    loop:
      - clusterrolebindings.yaml
      - psp-kube-system.yaml
      - psp-rbac.yaml

  - name: Wait for API server to become available
    wait_for:
      port: 8080
      host: 127.0.0.1
      delay: 5
    changed_when: false

  - name: Apply addons
    command: "{{ install_dir }}/bin/kubectl apply -f /etc/kubernetes/addons/{{ item }}"
    loop:
      - namespaces.yaml
      - clusterrolebindings.yaml
      - psp-kube-system.yaml
      - psp-rbac.yaml
      - core-dns.yaml
      - debug.yaml
    changed_when: false

- hosts: githubrunner
  gather_facts: true
  user: "{{ remote_user }}"
  become: yes
  any_errors_fatal: true
  roles:
    - github-runner