Update our deployment and use node4x and npm3 (#66)

Lots of improvements to the deployment processes by updating the playbook from https://github.com/bocoup/deployment-workflow

No more out of memory errors

.env.example

@@ -1,6 +1,3 @@
-CELERY_ID="string"
-CELERY_TOKEN="string"
-CELERY_URL="url"
-TEST_CELERY_ID="string"
-TEST_CELERY_TOKEN="string"
-TEST_CELERY_URL="url"
+NPM_CONFIG_PRODUCTION="false"
+TUMBLR_APIKEY="xxxx"
+URL_HASH_KEY="xxxx"

README.md

@@ -12,24 +12,11 @@ This repo is built on gulp, sass, and livereload.
 ### Environment variables
 Environment variables examples can be found in `.env.example`
 
-You can also use the heroku plugin for config to pull down the environment variables from the server
-
-Install the plug in using:
-
-```
-heroku plugins:install git://github.com/ddollar/heroku-config.git
-```
-
-Pull down the environment variables with:
-
-```
-heroku config:pull
-```
-
 `npm run dev` will automatically use the .env file you have locally. You can also export the variables to your environment manually with `export $(cat .env | xargs)`.
 
-### Running the server locally
+Production env is stored in the password management. It should live in `/mnt/secrets/tessel-env.json` on the server the currently used keys are set in the .env.example file.
 
+### Running the server locally
 
 ```
 npm install

deploy/ansible/group_vars/all.yml

@@ -33,9 +33,9 @@ apt_keys:
 # version codename, if necessary):
 # Git latest:     ppa:git-core/ppa
 # Node.js 4.2.x (LTS): deb https://deb.nodesource.com/node_4.x trusty main
-# Node.js 5.x.x: deb https://deb.nodesource.com/node_5.x trusty main
+# Node.js 5.x.x: deb https://deb.nodesource.com/node_4.x trusty main
 apt_ppas:
-  - "deb https://deb.nodesource.com/node_0.12 trusty main"
+  - "deb https://deb.nodesource.com/node_4.x trusty main"
   - "ppa:git-core/ppa"
 
 # Any apt packages to install. Apt package versions may be specified like
@@ -125,6 +125,13 @@ keep_n_most_recent: 3
 # after the build has completed. This file is relative to your project root.
 build_info_path: "public/build.txt"
 
+# If these variables are uncommented, add swap space to the machine when the
+# configure playbook is run. The swap configuration controlled by this is
+# meant to address installation problems on machines with minimal ram (e.g.
+# npm bails during install because it runs out of memory)
+swap_file_path: /swap
+swap_file_size: 2GB
+
 ###################
 # DEPLOY EXTRA VARS
 ###################

deploy/ansible/roles/base/tasks/main.yml

@@ -2,14 +2,28 @@
 # or the project is cloned or built. All system dependencies should be
 # installed here.
 
+- name: ensure apt cache is updated
+  apt: update_cache=yes cache_valid_time=3600
+
+- name: ensure all packages are upgraded safely
+  apt: upgrade=safe
+  when: env != "development"
+
 - name: add keys to apt
   apt_key: url={{item}} state=present
   with_items: apt_keys
 
 - name: add ppas to apt
-  apt_repository: repo="{{item}}" state=present update_cache=yes
+  apt_repository:
+    repo: "{{item}}"
+    state: present
   with_items: apt_ppas
 
 - name: install apt packages
-  apt: name={{item}} state=latest update_cache=yes
+  apt:
+    name: "{{item}}"
+    state: latest
   with_items: apt_packages
+
+- name: update npm to latest
+  npm: name=npm state=latest global=yes

deploy/ansible/roles/configure/tasks/main.yml

@@ -4,49 +4,61 @@
 - name: set hostname
   hostname: name={{hostname}}
 
+- name: ensure unattended upgrades are running
+  copy:
+    content: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Download-Upgradeable-Packages "1";
+      APT::Periodic::AutocleanInterval "7";
+      APT::Periodic::Unattended-Upgrade "1";
+    dest: /etc/apt/apt.conf.d/10periodic
+
 - name: add loopback references to our domain in /etc/hosts
   lineinfile:
-    dest=/etc/hosts
-    state=present
-    line="127.0.0.1 {{hostname}} {{site_fqdn}}"
+    dest: /etc/hosts
+    state: present
+    line: "127.0.0.1 {{hostname}} {{site_fqdn}}"
 
 - name: disallow password authentication
   lineinfile:
-    dest=/etc/ssh/sshd_config
-    state=present
-    regexp="^PasswordAuthentication"
-    line="PasswordAuthentication no"
+    dest: /etc/ssh/sshd_config
+    state: present
+    regexp: "^PasswordAuthentication"
+    line: "PasswordAuthentication no"
   notify: restart sshd
 
 - name: disallow challenge response authentication
   lineinfile:
-    dest=/etc/ssh/sshd_config
-    state=present
-    regexp="^ChallengeResponseAuthentication"
-    line="ChallengeResponseAuthentication no"
+    dest: /etc/ssh/sshd_config
+    state: present
+    regexp: "^ChallengeResponseAuthentication"
+    line: "ChallengeResponseAuthentication no"
   notify: restart sshd
 
 - name: ensure github.com is a known host
   lineinfile:
-    dest=/etc/ssh/ssh_known_hosts
-    state=present
-    create=yes
-    regexp="^github\\.com"
-    line="{{ lookup('pipe', 'ssh-keyscan -t rsa github.com') }}"
+    dest: /etc/ssh/ssh_known_hosts
+    state: present
+    create: yes
+    regexp: "^github\\.com"
+    line: "{{ lookup('pipe', 'ssh-keyscan -t rsa github.com') }}"
 
 - name: ensure ssh agent socket environment variable persists when sudoing
   lineinfile:
-    dest=/etc/sudoers
-    state=present
-    insertafter="^Defaults"
-    line='Defaults\tenv_keep += "SSH_AUTH_SOCK"'
-    validate="visudo -cf %s"
+    dest: /etc/sudoers
+    state: present
+    insertafter: "^Defaults"
+    line: "Defaults\tenv_keep += \"SSH_AUTH_SOCK\""
+    validate: "visudo -cf %s"
 
 - name: allow passwordless sudo - development only!
   lineinfile:
-    dest=/etc/sudoers
-    state=present
-    regexp="^%sudo"
-    line="%sudo\tALL=(ALL:ALL) NOPASSWD:ALL"
-    validate="visudo -cf %s"
+    dest: /etc/sudoers
+    state: present
+    regexp: "^%sudo"
+    line: "%sudo\tALL=(ALL:ALL) NOPASSWD:ALL"
+    validate: "visudo -cf %s"
   when: env == "development"
+
+- include: swap.yml
+  when: swap_file_path is defined and swap_file_size is defined

deploy/ansible/roles/configure/tasks/swap.yml

@@ -0,0 +1,36 @@
+- name: check if swap file exists
+  stat: path={{swap_file_path}}
+  register: swap_file_check
+
+- name: ensure swapfile exists
+  command: fallocate -l {{swap_file_size}} /swap
+  when: not swap_file_check.stat.exists
+  args:
+    creates: "{{swap_file_path}}"
+
+- name: ensure swap file has correct permissions
+  file: path={{swap_file_path}} owner=root group=root mode=0600
+
+- name: ensure swapfile is formatted
+  command: mkswap {{swap_file_path}}
+  when: not swap_file_check.stat.exists
+
+# the quotes around integers here can be removed when this is resolved
+# https://github.com/ansible/ansible-modules-core/issues/1861
+- name: ensure swap file can be mounted
+  mount:
+    name: none
+    src: "{{swap_file_path}}"
+    fstype: swap
+    opts: sw
+    passno: "0"
+    dump: "0"
+    state: present
+
+- name: ensure swap is activited
+  command: swapon -a
+
+- name: ensure swap is used as a last resort
+  sysctl:
+    name: vm.swappiness
+    value: 0

deploy/ansible/roles/deploy/tasks/build.yml

@@ -8,8 +8,19 @@
   shell: service tessel-io restart
   sudo: true
 
+- name: compare package.json of current deploy with previous deploy
+  command: diff {{site_path}}/package.json {{clone_path}}/package.json
+  register: package_diff
+  ignore_errors: true
+  no_log: true
+
+- name: copy existing npm modules
+  command: cp -R {{site_path}}/node_modules {{clone_path}}
+  when: package_diff.rc == 0
+
 - name: install npm modules
   npm: path="{{clone_path}}"
+  when: package_diff.rc != 0
 
 - name: build production version
   shell: npm run build

deploy/ansible/roles/deploy/tasks/checkout.yml

@@ -7,9 +7,9 @@
 
 - name: clone git repo into temp directory
   git:
-    repo={{synced_folder if local else git_repo}}
-    dest={{clone_path}}
-    version={{commit}}
+    repo: "{{synced_folder if local else git_repo}}"
+    dest: "{{clone_path}}"
+    version: "{{commit}}"
 
 - name: get sha of cloned repo
   command: git rev-parse HEAD

deploy/ansible/roles/deploy/tasks/main.yml

@@ -12,9 +12,10 @@
 
 - name: link sha-named clone to make it live
   file:
-    path={{site_path}}
-    state=link src={{base_path}}/{{ sha.stdout | default(commit) }}
-    force=yes
+    path: "{{site_path}}"
+    state: link
+    src: "{{base_path}}/{{ sha.stdout | default(commit) }}"
+    force: yes
 
 - name: update last-modification time of sha-named clone
   file: path={{base_path}}/{{ sha.stdout | default(commit) }} state=touch

deploy/ansible/roles/nginx/tasks/main.yml

@@ -11,18 +11,24 @@
   shell: touch {{item.directory}}/{{item.file}}
   args:
     creates: "{{item.directory}}/{{item.file}}"
-  with_items: confs + shared
+  with_flattened:
+    - confs
+    - shared
 
 - name: backup existing nginx config files
   shell: cp {{item.file}} {{item.file}}.backup
   args:
     chdir: "{{item.directory}}"
-  with_items: confs + shared
+  with_flattened:
+    - confs
+    - shared
 
 - name: generate new nginx config files
   template: src={{item.file}} dest={{item.directory}}/
   register: nginx_conf
-  with_items: confs + shared
+  with_flattened:
+    - confs
+    - shared
   notify: restart nginx
 
 - name: ensure nginx config is valid
@@ -34,7 +40,9 @@
 
 - name: remove temporary backups if new nginx config files are valid
   file: path={{item.directory}}/{{item.file}}.backup state=absent
-  with_items: confs + shared
+  with_flattened:
+    - confs
+    - shared
   when: nginx_test_valid | success
 
 - name: restore temporary backups if new nginx config files are invalid

package.json

@@ -10,8 +10,8 @@
     "dev": "gulp dev"
   },
   "engines": {
-    "node": "0.12.0",
-    "npm": "2.5.1"
+    "node": "^4.4.3",
+    "npm": "^3.8.6"
   },
   "dependencies": {
     "async": "0.9.0",