From 56928e782045ea16665bc98036d00cae98495cac Mon Sep 17 00:00:00 2001 From: Fredrik Lindberg Date: Thu, 29 Apr 2021 22:34:41 +0200 Subject: [PATCH] storage: initial redis support --- deps/docker-compose.yaml | 6 ++ package.json | 1 + src/account/account-manager.js | 10 +-- src/config.js | 17 ++++ src/controller/api-controller.js | 2 +- src/index.js | 67 +++++++------- src/storage/index.js | 70 ++++++++++++++- src/storage/inmemory-storage.js | 69 ++++++--------- src/storage/redis-storage.js | 145 +++++++++++++++++++++++++++++++ src/tunnel/tunnel-manager.js | 2 +- yarn.lock | 32 +++++++ 11 files changed, 342 insertions(+), 79 deletions(-) create mode 100644 deps/docker-compose.yaml create mode 100644 src/storage/redis-storage.js diff --git a/deps/docker-compose.yaml b/deps/docker-compose.yaml new file mode 100644 index 0000000..c99ffaa --- /dev/null +++ b/deps/docker-compose.yaml @@ -0,0 +1,6 @@ +version: "3.8" +services: + redis: + image: "redis:alpine" + ports: + - "6379:6379" \ No newline at end of file diff --git a/package.json b/package.json index 120f985..3480ead 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "koa-joi-router": "^7.0.0", "koa-router": "^9.4.0", "log4js": "^6.3.0", + "redis": "^3.1.2", "ws": "^7.3.1", "yargs": "^15.4.1" } diff --git a/src/account/account-manager.js b/src/account/account-manager.js index e618b1c..5a194a6 100644 --- a/src/account/account-manager.js +++ b/src/account/account-manager.js @@ -25,15 +25,13 @@ class AccountManager { async create() { let accountId; let maxTries = 100; + let created; do { accountId = Account.generateId(); - const created = await this._db.set(accountId, {}, {NX: true}); - if (!created) { - accountId = undefined; - } - } while (accountId === undefined && maxTries-- > 0); + created = await this._db.set(accountId, {}, {NX: true}); + } while (!created && accountId === undefined && maxTries-- > 0); - if (!accountId) { + if (!created) { return undefined; } diff --git a/src/config.js b/src/config.js index 4b0e7c0..737c460 100644 --- a/src/config.js +++ b/src/config.js @@ -63,6 +63,23 @@ const args = yargs default: false, description: 'Allow public account registration - NB: this allows public tunnel creation!' }) + .option('storage', { + type: 'string', + default: 'memory', + choices: ['memory', 'redis'], + }) + .option('storage-redis-url', { + type: 'string', + description: 'Redis connection URL', + coerce: (url) => { + try { + return new URL(url); + } catch (err) { + console.log(err.message); + process.exit(-1); + } + }, + }) .option('log-level', { type: 'string', default: 'info', diff --git a/src/controller/api-controller.js b/src/controller/api-controller.js index 3d5f4c7..70f643b 100644 --- a/src/controller/api-controller.js +++ b/src/controller/api-controller.js @@ -259,7 +259,7 @@ class ApiController { } const account = await this.accountManager.create(); - if (account === undefined) { + if (!account) { ctx.status = 503; return; } diff --git a/src/index.js b/src/index.js index e4c4042..067b920 100644 --- a/src/index.js +++ b/src/index.js @@ -5,27 +5,29 @@ import AdminController from './controller/admin-controller.js'; import Listener from './listener/index.js'; import Ingress from './ingress/index.js'; import Endpoint from './endpoint/index.js'; +import Storage from './storage/index.js'; export default () => { Logger.info("exposr"); - // Setup listeners - const listener = new Listener({ - http: { - port: Config.get('port') - } - }); + let listener; + try { + // Setup listeners + listener = new Listener({ + http: { + port: Config.get('port') + } + }); - // Setup tunnel connection endpoints (for clients to establish tunnels) - const endpoint = new Endpoint({ - ws: { - enabled: true, - baseUrl: Config.get('base-url') - } - }); + // Setup tunnel connection endpoints (for clients to establish tunnels) + const endpoint = new Endpoint({ + ws: { + enabled: true, + baseUrl: Config.get('base-url') + } + }); // Setup tunnel data ingress (incoming tunnel data) - try { const ingress = new Ingress({ http: { enabled: Config.get('ingress').includes('http'), @@ -40,23 +42,30 @@ export default () => { const adminController = Config.get('admin-enable') ? new AdminController(Config.get('admin-port')) : undefined; const apiController = new ApiController(); + const readyCallback = () => { + adminController.setReady(); + Logger.info("Service fully ready"); + }; + listener.listen((err) => { - if (err === undefined) { - if (adminController) { - adminController.setReady(); - Logger.info({ - message: "Admin interface enabled", - port: Config.get('admin-port') - }); - } else { - Logger.info({message: "Admin interface disabled"}); - } - Logger.info({ - message: "Ready", - base_url: Config.get('base-url'), - port: Config.get('port') - }); + if (err) { + Logger.error(err); + process.exit(-1); + } + if (adminController) { + Logger.info({ + message: "Admin interface enabled", + port: Config.get('admin-port') + }); + new Storage("default", { callback: readyCallback }); + } else { + Logger.info({message: "Admin interface disabled"}); } + Logger.info({ + message: "API endpoint available", + base_url: Config.get('base-url'), + port: Config.get('port') + }); }); const sigHandler = (signal) => { diff --git a/src/storage/index.js b/src/storage/index.js index 35f295b..c916689 100644 --- a/src/storage/index.js +++ b/src/storage/index.js @@ -1,9 +1,77 @@ import InMemoryStorage from './inmemory-storage.js'; +import RedisStorage from './redis-storage.js'; +import Config from '../config.js' +import assert from 'assert/strict'; class Storage { constructor(namespace, opts = {}) { - return new InMemoryStorage(namespace, opts); + const storageType = Config.get('storage'); + + const ready = () => { + opts.callback && process.nextTick(opts.callback); + }; + + if (storageType == 'memory') { + this.storage = new InMemoryStorage(ready); + } else if (storageType == 'redis') { + this.storage = new RedisStorage(ready); + } else { + throw new Error(`Unknown storage ${storageType}`); + } + this.ns = namespace; + this.key = opts.key; + } + + _key(key) { + return `${this.ns}:${key}`; } + + // Returns + // Object on success + // undefined on not found + // false on storage error + async get(key = undefined) { + if (!key) { + key = this.key; + } + assert(key !== undefined); + return this.storage.get(this._key(key)); + }; + + // Returns + // Object on success + // false on storage error + async set(arg1, arg2, arg3) { + // set(key, obj, opts) + if (arg1 != undefined && arg2 != undefined && arg3 != undefined) { + return this.storage.set(this._key(arg1), arg2, arg3); + } else if (arg1 != undefined && arg2 != undefined && arg3 == undefined) { + // set(key, obj) + if (typeof arg1 === 'string') { + return this.storage.set(this._key(arg1), arg2, {}); + // set(obj, opts) + } else { + return this.storage.set(this._key(this.key), arg1, arg2); + } + // set(obj) + } else if (arg1 != undefined && arg2 == undefined && arg3 == undefined) { + return this.storage.set(this._key(this.key), arg1, {}); + } else { + assert.fail("invalid call to set"); + } + } + + // Returns + // True if deleted + // undefined if not found + // False on storage error + async delete(key = undefined) { + if (!key) { + key = this.key; + } + assert(key !== undefined); + this.storage.delete(this._key(key)); + }; } export default Storage; \ No newline at end of file diff --git a/src/storage/inmemory-storage.js b/src/storage/inmemory-storage.js index 20ac72d..b44e46f 100644 --- a/src/storage/inmemory-storage.js +++ b/src/storage/inmemory-storage.js @@ -1,51 +1,37 @@ import assert from 'assert/strict'; import { Logger } from '../logger.js'; -const _DB = {}; - class InMemoryStorage { - constructor(namespace, opts) { - this.logger = Logger("in-memory-storage"); - this.logger.addContext("ns", namespace); - this.ns = namespace; - this.key = opts.key; - if (_DB[namespace] === undefined) { - _DB[namespace] = {}; + constructor(callback) { + if (InMemoryStorage.instance instanceof InMemoryStorage) { + process.nextTick(callback); + return InMemoryStorage.instance; } - this.db = _DB[namespace]; + InMemoryStorage.instance = this; + this.logger = Logger("in-memory-storage"); + this.db = {}; + process.nextTick(callback); } - async get(key = undefined) { - if (!key) { - key = this.key; - } + async get(key) { assert(key !== undefined); + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'get', + key + }); return this.db[key]; }; - async set(arg1, arg2, arg3) { - // set(key, obj, opts) - if (arg1 != undefined && arg2 != undefined && arg3 != undefined) { - return this._set(arg1, arg2, arg3); - } else if (arg1 != undefined && arg2 != undefined && arg3 == undefined) { - // set(key, obj) - if (typeof arg1 === 'string') { - return this._set(arg1, arg2, {}); - // set(obj, opts) - } else { - return this._set(this.key, arg1, arg2); - } - // set(obj) - } else if (arg1 != undefined && arg2 == undefined && arg3 == undefined) { - return this._set(this.key, arg1, {}); - } else { - assert.fail("invalid call to set"); - } - } - - async _set(key, data, opts = {}) { + async set(key, data, opts = {}) { assert(key !== undefined); - this.logger.isTraceEnabled() && this.logger.trace(`set=${key}, opts=${JSON.stringify(opts)}, data=${JSON.stringify(data)}`); + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'set', + opts, + key, + data: JSON.stringify(data), + }); if (opts.NX === true && this.db[key] !== undefined) { return false; } @@ -53,12 +39,13 @@ class InMemoryStorage { return this.db[key]; }; - async delete(key = undefined) { - if (!key) { - key = this.key; - } + async delete(key) { assert(key !== undefined); - this.logger.isTraceEnabled() && this.logger.trace(`delete=${key}`); + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'delete', + key + }); delete this.db[key]; }; diff --git a/src/storage/redis-storage.js b/src/storage/redis-storage.js new file mode 100644 index 0000000..756e6ab --- /dev/null +++ b/src/storage/redis-storage.js @@ -0,0 +1,145 @@ +import Redis from 'redis'; +import assert from 'assert/strict'; +import Config from '../config.js'; +import { Logger } from '../logger.js'; + +class RedisStorage { + constructor(callback) { + if (RedisStorage.instance instanceof RedisStorage) { + const redis = RedisStorage.instance._client; + if (redis.server_info?.redis_version != undefined) { + process.nextTick(callback); + } else { + redis.once('ready', callback); + } + return RedisStorage.instance; + } + RedisStorage.instance = this; + this.logger = Logger("redis-storage"); + const redisUrl = Config.get('storage-redis-url'); + if (!redisUrl) { + throw new Error("No Redis connection string provided"); + } + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'redis_new', + url: redisUrl, + }); + const redis = this._client = Redis.createClient({ + url: redisUrl.href, + connect_timeout: 2147483647, + }); + redis.once('ready', () => { + process.nextTick(callback); + }); + redis.on('connect', () => { + this.logger.info({ + operation: 'redis_connect', + url: redisUrl, + version: redis.server_info?.redis_version, + }); + this.connected = true; + }); + redis.on('error', (err) => { + this.logger.error({ + operation: 'redis_error', + message: err.message, + }); + }); + redis.on('reconnecting', (obj) => { + this.logger.info({ + operation: 'redis_reconnecting', + delay: obj.delay, + attempt: obj.attempt, + }); + this.connected = false; + }); + } + + get(key) { + assert(key !== undefined); + if (!this.connected) { + return false; + } + return new Promise((resolve, reject) => { + this._client.get(key, (err, data) => { + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'get', + key, + err, + data, + }); + if (err) { + return resolve(false); + } + resolve(JSON.parse(data)); + }); + }); + } + + set(key, data, opts = {}) { + assert(key !== undefined); + assert(data !== undefined); + if (!this.connected) { + return false; + } + return new Promise((resolve, reject) => { + const serialized = JSON.stringify(data); + const cb = (err, res) => { + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'set', + key, + data: serialized, + res, + err + }); + if (err) { + resolve(false); + } else { + resolve(data); + } + } + if (opts.NX) { + this._client.set(key, serialized, 'NX', cb); + } else { + this._client.set(key, serialized, cb); + } + }); + }; + + delete(key) { + assert(key !== undefined); + if (!this.connected) { + return false; + } + return new Promise(resolve => { + this._client.del(key, (res) => { + this.logger.isTraceEnabled() && + this.logger.trace({ + operation: 'get', + key + }); + resolve(true); + }); + }); + }; + + destroy(cb) { + this.logger.trace({ + operation: 'destroy', + message: 'initiated' + }); + this._client.quit(() => { + delete RedisStorage.instance; + process.nextTick(cb); + logger.trace({ + operation: 'destroy', + message: 'complete' + }); + }); + } +} + +export default RedisStorage; \ No newline at end of file diff --git a/src/tunnel/tunnel-manager.js b/src/tunnel/tunnel-manager.js index f262caa..0a447c2 100644 --- a/src/tunnel/tunnel-manager.js +++ b/src/tunnel/tunnel-manager.js @@ -19,7 +19,7 @@ class TunnelManager { if (tunnel == undefined) { const tunnelProps = await this.db.get(tunnelId); - if (tunnelProps === undefined) { + if (!tunnelProps) { return undefined; } tunnel = new Tunnel(tunnelId, tunnelProps); diff --git a/yarn.lock b/yarn.lock index 105b0e4..250685b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -266,6 +266,11 @@ delegates@1.0.0, delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +denque@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de" + integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ== + depd@^2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -845,6 +850,33 @@ raw-body@^2.3.3: iconv-lite "0.4.24" unpipe "1.0.0" +redis-commands@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" + integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== + +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60= + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ= + dependencies: + redis-errors "^1.0.0" + +redis@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c" + integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw== + dependencies: + denque "^1.5.0" + redis-commands "^1.7.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"