Merge branch 'develop', prepare 3.0

Author: yamalight

.eslintrc

@@ -17,7 +17,11 @@
     "object-curly-spacing": ["warn", "never"],
     "max-len": ["error", 120, 4],
     "no-confusing-arrow": "warn",
+    "global-require": "off",
+    "no-underscore-dangle": "off",
+    "no-await-in-loop": "off",
     "no-plusplus": "off",
+    "prefer-promise-reject-errors": "off",
     "comma-dangle": [
       "error",
       {
@@ -28,7 +32,6 @@
         "functions": "ignore"
       }
     ],
-    "react/jsx-filename-extension": "off",
     "prettier/prettier": [
       "error",
       {

.gitignore

@@ -2,4 +2,5 @@ node_modules
 .nyc_output
 coverage
 test/fixtures/auth.db
-test/fixtures/extensions/
+test/fixtures/deploying/
+test/fixtures/deploying-swarm/

.travis.yml

@@ -12,6 +12,7 @@ cache: yarn
 before_install:
   - curl -L https://github.com/docker/compose/releases/download/1.13.0/docker-compose-`uname -s`-`uname -m` > docker-compose
   - chmod +x docker-compose
+  - docker swarm init
 
 after_success:
   - docker rmi $(docker images -q)

README.md

@@ -12,8 +12,8 @@ Exoframe is a self-hosted tool that allows simple one-command deployments using
 
 ## Installation and Usage
 
-1. Make sure you have Docker [installed and running](https://docs.docker.com/engine/installation/) on your host.
-2. Pull and run Exoframe server using docker:
+1.  Make sure you have Docker [installed and running](https://docs.docker.com/engine/installation/) on your host.
+2.  Pull and run Exoframe server using docker:
 
 ```sh
 docker run -d \
@@ -23,6 +23,7 @@ docker run -d \
   -e EXO_PRIVATE_KEY=your_private_key \
   --label traefik.backend=exoframe-server \
   --label traefik.frontend.rule=Host:exoframe.your-host.com \
+  --restart always \
   --name exoframe-server \
   exoframe/server
 
@@ -48,10 +49,35 @@ docker run -d \
 --label traefik.frontend.rule=Host:exoframe.your-host.com
 ```
 
-3. Edit config file to fit your needs (see section below)
+3.  Edit config file to fit your needs (see section below)
 
 Then install [Exoframe CLI](https://github.com/exoframejs/exoframe), point it to your new Exoframe server and use it.
 
+## Installation and usage in Swarm mode
+
+Exoframe also supports running in [Swarm mode](https://docs.docker.com/engine/swarm/).  
+To run Exoframe server in swarm, you need to do the following:
+
+1.  Make sure you have Docker on your host.
+2.  Make sure your Docker has [Swarm mode enabled](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/).
+3.  Pull and run Exoframe server using Docker on your manager node:
+
+```
+docker service create \
+  --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
+  --mount type=bind,source=/path/to/exoframe-folder,target=/root/.exoframe \
+  --mount type=bind,source=/home/user/.ssh/authorized_keys,target=/root/.ssh/authorized_keys,readonly \
+  -e EXO_PRIVATE_KEY=your_private_key \
+  --label traefik.backend=exoframe-server \
+  --label traefik.frontend.rule=Host:exoframe.your-host.com \
+  --label traefik.port=8080 \
+  --constraint=node.role==manager \
+  --name exoframe-server \
+  exoframe/server
+```
+
+Note that both Exoframe server and Traefik will be run on your manager node.
+
 ## Configuration
 
 Exoframe stores its config in `~/.exoframe/server.config.yml`.  

package.json

@@ -6,7 +6,7 @@
   "bin": "index.js",
   "scripts": {
     "start": "node index.js",
-    "test": "NODE_ENV=testing jest --coverage --silent --maxWorkers=3 --ci",
+    "test": "NODE_ENV=testing jest --coverage --silent --maxWorkers=2 --ci",
     "coveralls": "cat ./coverage/lcov.info | coveralls",
     "build": "pkg -t node8.9.0-alpine -o exoframe-server ."
   },
@@ -17,29 +17,29 @@
   "author": "Tim Ermilov <[email protected]>",
   "license": "MIT",
   "dependencies": {
-    "chokidar": "^2.0.0",
+    "chokidar": "^2.0.3",
     "cors": "^2.8.4",
-    "dockerode": "^2.4.3",
-    "fastify": "^0.40.0",
-    "fastify-auth": "^0.1.0",
-    "highland": "^2.11.1",
-    "js-yaml": "^3.9.1",
-    "jsonwebtoken": "^8.1.1",
-    "lodash": "^4.17.4",
-    "lokijs": "^1.5.0",
+    "dockerode": "^2.5.5",
+    "fastify": "^1.4.0",
+    "fastify-auth": "^0.2.0",
+    "highland": "^2.13.0",
+    "js-yaml": "^3.11.0",
+    "jsonwebtoken": "^8.2.1",
+    "lodash": "^4.17.10",
+    "lokijs": "^1.5.4",
     "mkdirp": "^0.5.1",
-    "node-fetch": "^1.7.3",
+    "node-fetch": "^2.1.2",
     "rimraf": "^2.6.1",
     "semver-compare": "^1.0.0",
-    "sshpk": "^1.13.1",
-    "tar-fs": "^1.15.2",
+    "sshpk": "^1.14.1",
+    "tar-fs": "^1.16.2",
     "uuid": "^3.2.1",
-    "winston": "^2.3.1"
+    "winston": "^2.4.2"
   },
   "devDependencies": {
-    "coveralls": "^3.0.0",
+    "coveralls": "^3.0.1",
     "get-port": "^3.2.0",
-    "jest": "^22.1.4",
-    "pkg": "^4.3.0"
+    "jest": "^22.4.3",
+    "pkg": "^4.3.1"
   }
 }

src/config/index.js

@@ -10,20 +10,19 @@ const {spawn} = require('child_process');
 const logger = require('../logger');
 
 // construct paths
-const baseFolder =
-  process.env.NODE_ENV !== 'testing'
-    ? path.join(os.homedir(), '.exoframe')
-    : path.join(__dirname, '..', '..', 'test', 'fixtures');
+const baseFolder = path.join(os.homedir(), '.exoframe');
 const configPath = path.join(baseFolder, 'server.config.yml');
-const publicKeysPath =
-  process.env.NODE_ENV !== 'testing'
-    ? path.join(os.homedir(), '.ssh')
-    : path.join(__dirname, '..', '..', 'test', 'fixtures');
+const publicKeysPath = path.join(os.homedir(), '.ssh');
 const extensionsFolder = path.join(baseFolder, 'extensions');
+const recipesFolder = path.join(baseFolder, 'recipes');
+// dir for temporary files used to build docker images
+const tempDir = path.join(baseFolder, 'deploying');
 
 // export paths for others
 exports.baseFolder = baseFolder;
 exports.extensionsFolder = extensionsFolder;
+exports.recipesFolder = recipesFolder;
+exports.tempDockerDir = tempDir;
 
 // create base folder if doesn't exist
 try {
@@ -45,6 +44,19 @@ try {
   spawn('yarn', ['init', '-y'], {cwd: extensionsFolder});
 }
 
+// create recipes folder if doesn't exist
+try {
+  fs.statSync(recipesFolder);
+} catch (e) {
+  fs.mkdirSync(recipesFolder);
+}
+// init package.json if it doesn't exist
+try {
+  fs.statSync(path.join(recipesFolder, 'package.json'));
+} catch (e) {
+  spawn('yarn', ['init', '-y'], {cwd: recipesFolder});
+}
+
 // default config
 const defaultConfig = {
   debug: false,
@@ -58,6 +70,8 @@ const defaultConfig = {
   traefikName: 'exoframe-traefik',
   traefikArgs: [],
   exoframeNetwork: 'exoframe',
+  exoframeNetworkSwarm: 'exoframe-swarm',
+  swarm: false,
   publicKeysPath,
 };
 

src/docker/build.js

@@ -4,21 +4,13 @@ const tar = require('tar-fs');
 // our modules
 const logger = require('../logger');
 const docker = require('./docker');
-const {tempDockerDir, getProjectConfig, tagFromConfig, writeStatus} = require('../util');
+const {tempDockerDir} = require('../config');
+const {getProjectConfig, tagFromConfig, writeStatus} = require('../util');
 
-module.exports = ({username, resultStream}) =>
-  new Promise(async (resolve, reject) => {
-    // get packed stream
-    const tarStream = tar.pack(tempDockerDir);
-
-    // get project info
-    const config = getProjectConfig();
-
-    // construct image tag
-    const tag = tagFromConfig({username, config});
-    logger.debug('building with tag:', tag);
-    writeStatus(resultStream, {message: `Building image with tag: ${tag}`, level: 'verbose'});
+const noop = () => {};
 
+exports.buildFromParams = ({tarStream, tag, logLine = noop}) =>
+  new Promise(async (resolve, reject) => {
     // deploy as docker
     const log = [];
     // track errors
@@ -34,21 +26,21 @@ module.exports = ({username, resultStream}) =>
           // process log data
           if (data.stream && data.stream.length) {
             log.push(data.stream);
-            writeStatus(resultStream, {message: data.stream, level: 'verbose'});
+            logLine({message: data.stream, level: 'verbose'});
           } else if (data.error && data.error.length) {
             // process error data
             log.push(data.error);
-            writeStatus(resultStream, {message: data.error, level: 'error'});
+            logLine({message: data.error, level: 'error'});
             hasErrors = true;
           } else {
             // push everything else as-is
             log.push(s);
-            writeStatus(resultStream, {message: s, level: 'verbose'});
+            logLine({message: s, level: 'verbose'});
           }
         } catch (e) {
           if (s && s.length) {
             log.push(s);
-            writeStatus(resultStream, {message: s, level: 'verbose'});
+            logLine({message: s, level: 'verbose'});
           }
         }
       });
@@ -61,3 +53,22 @@ module.exports = ({username, resultStream}) =>
       resolve({log, image: tag});
     });
   });
+
+exports.build = async ({username, resultStream}) => {
+  // get packed stream
+  const tarStream = tar.pack(tempDockerDir);
+
+  // get project info
+  const config = getProjectConfig();
+
+  // construct image tag
+  const tag = tagFromConfig({username, config});
+  logger.debug('building with tag:', tag);
+  writeStatus(resultStream, {message: `Building image with tag: ${tag}`, level: 'verbose'});
+
+  // create logger function
+  const logLine = data => writeStatus(resultStream, data);
+
+  // return build
+  return exports.buildFromParams({tarStream, tag, logLine});
+};