Merge branch 'develop', prepare v5.0

yamalight · yamalight · commit a8b888690298 · 2019-07-17T12:52:13.000+02:00
diff --git a/.eslintrc b/.eslintrc
@@ -6,6 +6,7 @@
     "max-len": ["error", 120, 4],
     "camelcase": "off",
     "promise/param-names": "off",
+    "no-async-promise-executor": "off",
     "prefer-promise-reject-errors": "off",
     "no-control-regex": "off",
     "prettier/prettier": [
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,4 @@ test/fixtures/deploying/
 test/fixtures/secrets.db
 dist/
 bin/server-core.js
-bin/home.html
+bin/assets/
diff --git a/.travis.yml b/.travis.yml
@@ -5,14 +5,18 @@ language: node_js
 services:
   - docker
 
-node_js: 10
+node_js: 12
 
 cache: yarn
 
 before_install:
   - curl -L https://github.com/docker/compose/releases/download/1.23.2/docker-compose-`uname -s`-`uname -m` > docker-compose
   - chmod +x docker-compose
 
+script:
+  - yarn test
+  - yarn lint
+
 after_success:
   - docker rmi $(docker images -q)
   - yarn coveralls
diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,16 @@
-FROM docker/compose:1.24.0
+FROM node:12.4-alpine
 
-# install required libs and yarn
-RUN apk update && apk add --no-cache libstdc++ libgcc yarn
+# install docker-compose
+RUN apk update \
+  && apk add --no-cache py-pip python-dev \
+  libffi-dev openssl-dev gcc libc-dev make \
+  && pip install docker-compose
 
-# copy binary
-COPY exoframe-server /
+# create new workdir
+WORKDIR /app
+
+# copy source
+COPY bin/ /app
 
 # set environment to production
 ENV NODE_ENV production
@@ -13,4 +19,4 @@ ENV NODE_ENV production
 EXPOSE 8080
 
 # set binary as entry point
-ENTRYPOINT ["/exoframe-server"]
+CMD ["node", "exoframe-server.js"]
diff --git a/package.json b/package.json
@@ -1,46 +1,55 @@
 {
   "name": "exoframe-server",
-  "version": "4.1.2",
+  "version": "4.1.3-dev",
   "description": "Exoframe is a self-hosted tool that allows simple one-command deployments using Docker",
   "main": "bin/server-core.js",
   "bin": "bin/exoframe-server.js",
   "scripts": {
     "start": "node index.js",
+    "lint": "eslint src/ test/ bin/exoframe-server.js",
     "test": "NODE_ENV=testing jest --coverage --silent --maxWorkers=2 --ci",
     "coveralls": "cat ./coverage/lcov.info | coveralls",
     "build-bundle": "ncc build src/index.js -o dist",
-    "prepare-build": "mv dist/index.js bin/server-core.js && mv dist/home.html bin/home.html",
-    "build": "yarn build-bundle && yarn prepare-build && pkg -t node10.4.1-alpine -o exoframe-server ."
-  },
-  "pkg": {
-    "assets": "bin/home.html"
+    "prepare-build": "cp dist/index.js bin/server-core.js && cp -r dist/assets bin/",
+    "build": "yarn build-bundle && yarn prepare-build"
   },
   "files": [
     "bin/exoframe-server.js",
     "bin/server-core.js",
-    "bin/home.html"
+    "bin/assets/home.html",
+    "bin/assets/runner.js"
   ],
   "repository": "git@github.com:exoframejs/exoframe-server.git",
   "author": "Tim Ermilov <yamalight@gmail.com>",
   "license": "MIT",
   "devDependencies": {
-    "@zeit/ncc": "^0.15.0",
+    "@zeit/ncc": "^0.20.4",
+    "babel-eslint": "^10.0.2",
     "cors": "^2.8.4",
-    "coveralls": "^3.0.2",
+    "coveralls": "^3.0.5",
     "dockerode": "^2.5.6",
-    "fastify": "^1.14.0",
-    "fastify-auth": "^0.3.0",
-    "get-port": "^4.0.0",
+    "eslint": "^6.0.1",
+    "eslint-config-prettier": "^6.0.0",
+    "eslint-config-standard": "^13.0.1",
+    "eslint-plugin-import": "^2.18.0",
+    "eslint-plugin-node": "^9.1.0",
+    "eslint-plugin-prettier": "^3.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.0",
+    "exoframe-faas": "^1.0.1",
+    "fastify": "^2.6.0",
+    "fastify-auth": "^0.5.0",
+    "get-port": "^5.0.0",
     "highland": "^2.13.0",
     "jest": "^24.0.0",
     "js-yaml": "^3.12.0",
     "jsonwebtoken": "^8.3.0",
-    "lodash": "^4.17.11",
+    "lodash": "^4.17.14",
     "lokijs": "^1.5.5",
     "mkdirp": "^0.5.1",
     "nock": "^10.0.4",
-    "node-fetch": "^2.2.0",
-    "pkg": "^4.3.4",
+    "node-fetch": "^2.6.0",
+    "prettier": "^1.18.2",
     "rimraf": "^2.6.2",
     "semver-compare": "^1.0.0",
     "signale": "^1.3.0",
diff --git a/src/config/index.js b/src/config/index.js
@@ -15,6 +15,7 @@ const publicKeysPath = path.join(os.homedir(), '.ssh');
 const extensionsFolder = path.join(baseFolder, 'extensions');
 const recipesFolder = path.join(baseFolder, 'recipes');
 const pluginsFolder = path.join(baseFolder, 'plugins');
+const faasFolder = path.join(baseFolder, 'faas');
 // dir for temporary files used to build docker images
 const tempDir = path.join(baseFolder, 'deploying');
 
@@ -23,6 +24,7 @@ exports.baseFolder = baseFolder;
 exports.extensionsFolder = extensionsFolder;
 exports.recipesFolder = recipesFolder;
 exports.pluginsFolder = pluginsFolder;
+exports.faasFolder = faasFolder;
 exports.tempDockerDir = tempDir;
 
 // create base folder if doesn't exist
@@ -32,6 +34,13 @@ try {
   fs.mkdirSync(baseFolder);
 }
 
+// create faas folder if doesn't exist
+try {
+  fs.statSync(faasFolder);
+} catch (e) {
+  fs.mkdirSync(faasFolder);
+}
+
 // create extensions folder if doesn't exist
 try {
   fs.statSync(extensionsFolder);
@@ -95,7 +104,7 @@ let userConfig = defaultConfig;
 
 // config loaded promise
 let loadedResolve = () => {};
-let isConfigLoaded = new Promise(resolve => {
+const isConfigLoaded = new Promise(resolve => {
   loadedResolve = resolve;
 });
 
@@ -107,7 +116,11 @@ const reloadUserConfig = () => {
     logger.debug('loaded new config:', userConfig);
     loadedResolve();
   } catch (e) {
-    logger.error('error parsing user config:', e);
+    if (e.code === 'ENOENT') {
+      logger.warn('no config found, using default values..');
+    } else {
+      logger.error('error parsing user config:', e);
+    }
   }
 };
 
diff --git a/src/docker/start.js b/src/docker/start.js
@@ -162,7 +162,7 @@ exports.start = async ({image, username, folder, resultStream, existing = []}) =
   // generate host
   const host = getHost({serverConfig, name, config});
 
-  const Env = getEnv({username, config, name, project, host}).map((pair) => pair.join('='));
+  const Env = getEnv({username, config, name, project, host}).map(pair => pair.join('='));
 
   // construct restart policy
   let RestartPolicy = {};
diff --git a/src/docker/templates/index.js b/src/docker/templates/index.js
@@ -11,7 +11,8 @@ const imageTemplate = require('./image');
 const composeTemplate = require('./compose');
 const dockerfileTemplate = require('./dockerfile');
 const nodeTemplate = require('./node');
-const nginxTemplate = require('./nginx');
+const staticTemplate = require('./static');
+const {template: faasTemplate} = require('exoframe-faas');
 
 // load 3rd party templates
 module.exports = () => {
@@ -24,5 +25,7 @@ module.exports = () => {
     return require(templatePath);
   });
 
-  return [imageTemplate, composeTemplate, dockerfileTemplate, nodeTemplate, nginxTemplate].concat(userTemplates);
+  return [faasTemplate, imageTemplate, composeTemplate, dockerfileTemplate, nodeTemplate, staticTemplate].concat(
+    userTemplates
+  );
 };
diff --git a/src/docker/templates/static.js b/src/docker/templates/static.js
diff --git a/src/index.js b/src/index.js
@@ -13,11 +13,12 @@ const {initDocker} = require('./docker/init');
 const {initPlugins} = require('./plugins');
 
 // config
-const {getConfig, waitForConfig} = require('./config');
+const {getConfig, waitForConfig, faasFolder} = require('./config');
 
 // paths
 const setupAuth = require('./auth');
 const routes = require('./routes');
+const {setup: faas} = require('exoframe-faas');
 
 exports.startServer = async (port = 8080) => {
   // create server
@@ -43,6 +44,7 @@ exports.startServer = async (port = 8080) => {
   // register plugins
   await setupAuth(fastify)
     .register(routes)
+    .register(faas({faasFolder}))
     .ready();
 
   // start server
diff --git a/src/logger/index.js b/src/logger/index.js
@@ -2,19 +2,15 @@ const {Signale} = require('signale');
 
 // prepare level
 const levelTesting = process.env.NODE_ENV === 'testing' ? 'error' : false;
-const level = levelTesting || process.env.NODE_ENV === 'production' ? 'info' : 'debug';
+const level = levelTesting || process.env.NODE_ENV === 'production' ? 'warn' : 'debug';
 
 const logger = new Signale({
   scope: 'exoframe-server',
+  logLevel: level,
   types: {
-    debug: {
-      stream: level === 'debug' ? [process.stdout] : [],
-    },
+    // override info type to show it in warn logging
     info: {
-      stream: ['debug', 'info'].includes(level) ? [process.stdout] : [],
-    },
-    warn: {
-      stream: ['debug', 'info'].includes(level) ? [process.stdout] : [],
+      logLevel: 'warn',
     },
   },
 });
diff --git a/src/routes/deploy.js b/src/routes/deploy.js
@@ -7,14 +7,15 @@ const uuidv1 = require('uuid/v1');
 // our modules
 const logger = require('../logger');
 const util = require('../util');
-const {getConfig, tempDockerDir} = require('../config');
+const {getConfig, tempDockerDir, faasFolder} = require('../config');
 const docker = require('../docker/docker');
 const {pullImage} = require('../docker/init');
 const {build} = require('../docker/build');
 const {start} = require('../docker/start');
 const getTemplates = require('../docker/templates');
 const {removeContainer} = require('../docker/util');
 const {getPlugins} = require('../plugins');
+const {registerFunction} = require('exoframe-faas');
 
 // destruct locally used functions
 const {sleep, cleanTemp, unpack, getProjectConfig, projectFromConfig} = util;
@@ -33,7 +34,7 @@ const deploy = async ({username, folder, existing, resultStream}) => {
   // generate template props
   const templateProps = {
     config,
-    serverConfig,
+    serverConfig: {...serverConfig, faasFolder},
     existing,
     username,
     resultStream,
@@ -45,6 +46,9 @@ const deploy = async ({username, folder, existing, resultStream}) => {
       start,
       pullImage,
     },
+    faas: {
+      registerFunction,
+    },
     util: Object.assign({}, util, {
       logger,
       getPlugins,
diff --git a/src/routes/list.js b/src/routes/list.js
@@ -2,6 +2,8 @@
 const docker = require('../docker/docker');
 const {getConfig} = require('../config');
 const {getPlugins} = require('../plugins');
+const {functionToContainerFormat} = require('../util');
+const {listFunctions} = require('exoframe-faas');
 const logger = require('../logger');
 
 module.exports = fastify => {
@@ -15,6 +17,10 @@ module.exports = fastify => {
       // get config
       const config = getConfig();
 
+      // get functions
+      const functions = listFunctions({functionToContainerFormat});
+
+      // get containers
       const allContainers = await docker.listContainers({all: true});
       const userContainers = await Promise.all(
         allContainers
@@ -36,13 +42,13 @@ module.exports = fastify => {
         logger.debug('Running list with plugin:', plugin.config.name, result);
         if (result && plugin.config.exclusive) {
           logger.debug('List finished via exclusive plugin:', plugin.config.name);
-          reply.send({containers: userContainers, ...result});
+          reply.send({containers: userContainers.concat(functions), ...result});
           return;
         }
       }
 
       // return results
-      reply.send({containers: userContainers, services: []});
+      reply.send({containers: userContainers.concat(functions), services: []});
     },
   });
 };
diff --git a/src/routes/logs.js b/src/routes/logs.js
@@ -6,6 +6,7 @@ const logger = require('../logger');
 
 // our modules
 const docker = require('../docker/docker');
+const {getLogsForFunction} = require('exoframe-faas');
 
 const generateLogsConfig = follow => ({
   follow: Boolean(follow),
@@ -90,6 +91,16 @@ module.exports = fastify => {
       const {id} = request.params;
       const {follow} = request.query;
 
+      // try to get function logs first
+      const fnLogs = getLogsForFunction(id);
+      if (fnLogs) {
+        // wrap logs into stream
+        const logsStream = _(fnLogs).flatten();
+        // send wrapped highland stream as response
+        reply.send(new Readable().wrap(logsStream));
+        return;
+      }
+
       // run logs via plugins if available
       const plugins = getPlugins();
       for (const plugin of plugins) {
diff --git a/src/routes/remove.js b/src/routes/remove.js
@@ -2,6 +2,7 @@
 const docker = require('../docker/docker');
 const {removeContainer} = require('../docker/util');
 const {getPlugins} = require('../plugins');
+const {removeFunction} = require('exoframe-faas');
 const logger = require('../logger');
 
 // removal of normal containers
@@ -24,7 +25,7 @@ const removeUserContainer = async ({username, id, reply}) => {
     c => c.Labels['exoframe.user'] === username && c.Labels['exoframe.project'] === id
   );
   if (!containers.length) {
-    reply.code(404).send({error: 'Container not found!'});
+    reply.code(404).send({error: 'Container or function not found!'});
     return;
   }
   // remove all
@@ -42,6 +43,13 @@ module.exports = fastify => {
       const {username} = request.user;
       const {id} = request.params;
 
+      // try and remove function
+      if (await removeFunction({id, username})) {
+        // reply
+        reply.code(204).send('removed');
+        return;
+      }
+
       // run remove via plugins if available
       const plugins = getPlugins();
       for (const plugin of plugins) {
diff --git a/src/util/index.js b/src/util/index.js
diff --git a/test/remove.test.js b/test/remove.test.js
diff --git a/yarn.lock b/yarn.lock
