From 7b8db3db7acd4d8babcfa7415d9bb748ce71aea7 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 17:54:15 +0100 Subject: [PATCH 01/10] Rename host variables to source In preparation of a larger merge request which adds more ways on source matching the variables indicating a host match are renamed to the more generic term source. --- api/ceryx/db.py | 52 ++++++++++++++--------------- ceryx/nginx/lualib/ceryx/routes.lua | 4 +-- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/ceryx/db.py b/api/ceryx/db.py index d26b2ae..7ace8c9 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -39,39 +39,39 @@ def _route_key(self, source): def _settings_key(self, source): return self._prefixed_key(f"settings:{source}") - def _delete_target(self, host): - key = self._route_key(host) + def _delete_target(self, source): + key = self._route_key(source) self.client.delete(key) - def _delete_settings(self, host): - key = self._settings_key(host) + def _delete_settings(self, source): + key = self._settings_key(source) self.client.delete(key) - def _lookup_target(self, host, raise_exception=False): - key = self._route_key(host) + def _lookup_target(self, source, raise_exception=False): + key = self._route_key(source) target = self.client.get(key) - + if target is None and raise_exception: raise exceptions.NotFound("Route not found.") return target - def _lookup_settings(self, host): - key = self._settings_key(host) + def _lookup_settings(self, source): + key = self._settings_key(source) return self.client.hgetall(key) - def lookup_hosts(self, pattern="*"): + def lookup_sources(self, pattern="*"): lookup_pattern = self._route_key(pattern) left_padding = len(lookup_pattern) - 1 keys = self.client.keys(lookup_pattern) return [_str(key)[left_padding:] for key in keys] - def _set_target(self, host, target): - key = self._route_key(host) - self.client.set(key, target) + def _set_target(self, source, target, ttl=None): + key = self._route_key(source) + self.client.set(key, target, ex=ttl) - def _set_settings(self, host, settings): - key = self._settings_key(host) + def _set_settings(self, source, settings): + key = self._settings_key(source) self.client.hmset(key, settings) def _set_route(self, route: schemas.Route): @@ -80,30 +80,30 @@ def _set_route(self, route: schemas.Route): self._set_settings(route.source, redis_data["settings"]) return route - def get_route(self, host): - target = self._lookup_target(host, raise_exception=True) - settings = self._lookup_settings(host) + def get_route(self, source): + target = self._lookup_target(source, raise_exception=True) + settings = self._lookup_settings(source) route = schemas.Route.from_redis({ - "source": host, + "source": source, "target": target, "settings": settings }) return route def list_routes(self): - hosts = self.lookup_hosts() - routes = [self.get_route(host) for host in hosts] + sources = self.lookup_sources() + routes = [self.get_route(source) for source in sources] return routes def create_route(self, data: dict): route = schemas.Route.validate(data) return self._set_route(route) - def update_route(self, host: str, data: dict): - data["source"] = host + def update_route(self, source: str, data: dict): + data["source"] = source route = schemas.Route.validate(data) return self._set_route(route) - def delete_route(self, host: str): - self._delete_target(host) - self._delete_settings(host) + def delete_route(self, source: str): + self._delete_target(source) + self._delete_settings(source) diff --git a/ceryx/nginx/lualib/ceryx/routes.lua b/ceryx/nginx/lualib/ceryx/routes.lua index 98ce442..ede8aeb 100644 --- a/ceryx/nginx/lualib/ceryx/routes.lua +++ b/ceryx/nginx/lualib/ceryx/routes.lua @@ -16,7 +16,7 @@ end function getTargetForSource(source, redisClient) -- Construct Redis key and then - -- try to get target for host + -- try to get target for source local key = getRouteKeyForSource(source) local target, _ = redisClient:get(key) @@ -69,7 +69,7 @@ function getRouteForSource(source) if targetIsInValid(route.target) then return nil end - cache:set(host, res, 5) + cache:set(source, res, 5) ngx.log(ngx.DEBUG, "Caching from " .. source .. " to " .. route.target .. " for 5 seconds.") end From eb20351c53872970d6acc48752ec9528304a3e35 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 18:02:35 +0100 Subject: [PATCH 02/10] Add ability to use headers on proxy and redirect In case of a proxy request the headers are hidden for the client calling the proxy. In case of the redirect the headers are added before sending the redirect response back to the client. --- api/ceryx/schemas.py | 17 ++++++++++++++++- ceryx/nginx/lualib/ceryx/routes.lua | 14 ++++++++++++++ ceryx/nginx/lualib/router.lua | 23 +++++++++++++++-------- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py index b1dfbc6..71f1d24 100644 --- a/api/ceryx/schemas.py +++ b/api/ceryx/schemas.py @@ -1,6 +1,6 @@ import re import typesystem - +import json def ensure_protocol(url): starts_with_protocol = r"^https?://" @@ -15,6 +15,14 @@ def redis_to_boolean(value): return True if value == "1" else False +def object_to_redis(value: object): + return json.dumps(value) + + +def redis_to_object(value): + return json.loads(value) + + def ensure_string(value): redis_value = ( None if value is None @@ -30,6 +38,9 @@ def value_to_redis(field, value): if isinstance(field, typesystem.Reference): return field.target.validate(value).to_redis() + if isinstance(field, typesystem.Object): + return object_to_redis(value) + return ensure_string(value) @@ -40,6 +51,9 @@ def redis_to_value(field, redis_value): if isinstance(field, typesystem.Reference): return field.target.from_redis(redis_value) + if isinstance(field, typesystem.Object): + return redis_to_object(redis_value) + return ensure_string(redis_value) @@ -69,6 +83,7 @@ class Settings(BaseSchema): ), default="proxy", ) + headers = typesystem.Object(default={}, properties=typesystem.String(max_length=100)) certificate_path = typesystem.String(allow_null=True) key_path = typesystem.String(allow_null=True) diff --git a/ceryx/nginx/lualib/ceryx/routes.lua b/ceryx/nginx/lualib/ceryx/routes.lua index ede8aeb..f6925fd 100644 --- a/ceryx/nginx/lualib/ceryx/routes.lua +++ b/ceryx/nginx/lualib/ceryx/routes.lua @@ -1,4 +1,5 @@ local redis = require "ceryx.redis" +local cjson = require "cjson" local exports = {} @@ -49,6 +50,18 @@ function getModeForSource(source, redisClient) return mode end +function getHeadersForSource(source, redisClient) + ngx.log(ngx.DEBUG, "Get routing headers for " .. source .. ".") + local settings_key = getSettingsKeyForSource(source) + local headers, _ = cjson.decode(redisClient:hget(settings_key, "headers")) + + if headers == ngx.null or not headers then + headers = {} + end + + return headers +end + function getRouteForSource(source) local _ local route = {} @@ -74,6 +87,7 @@ function getRouteForSource(source) end route.mode = getModeForSource(source, redisClient) + route.headers = getHeadersForSource(source, redisClient) return route end diff --git a/ceryx/nginx/lualib/router.lua b/ceryx/nginx/lualib/router.lua index 6922b25..57c9ac8 100644 --- a/ceryx/nginx/lualib/router.lua +++ b/ceryx/nginx/lualib/router.lua @@ -16,26 +16,33 @@ function formatTarget(target) return target .. ngx.var.request_uri end -function redirect(source, target) - ngx.log(ngx.INFO, "Redirecting request for " .. source .. " to " .. target .. ".") + +function redirect(source, target, headers) + ngx.log(ngx.INFO, "Redirecting request for " .. source) + for k,v in pairs(headers) do + ngx.headers[k] = v + end return ngx.redirect(target, ngx.HTTP_MOVED_PERMANENTLY) end -function proxy(source, target) +function proxy(source, target, headers) ngx.var.target = target - ngx.log(ngx.INFO, "Proxying request for " .. source .. " to " .. target .. ".") + for k,v in pairs(headers) do + ngx.req.set_header(k, v) +end + ngx.log(ngx.INFO, "Proxying request for " .. source) end -function routeRequest(source, target, mode) +function routeRequest(source, target, mode, headers) ngx.log(ngx.DEBUG, "Received " .. mode .. " routing request from " .. source .. " to " .. target) target = formatTarget(target) if mode == "redirect" then - return redirect(source, target) + return redirect(source, target, headers) end - return proxy(source, target) + return proxy(source, target, headers) end if is_not_https then @@ -62,4 +69,4 @@ if route == nil then end -- Save found key to local cache for 5 seconds -routeRequest(host, route.target, route.mode) +routeRequest(host, route.target, route.mode, route.headers) From cfb97fb580f8268a64d954e251a1df41ee13dd08 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 18:11:52 +0100 Subject: [PATCH 03/10] Add ttl to routes It is now possible to add a ttl setting in seconds to the settings of a route which will set a redis EXPIRE tag on the respective route. The default is persistent. --- api/ceryx/db.py | 2 +- api/ceryx/schemas.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/api/ceryx/db.py b/api/ceryx/db.py index 7ace8c9..102c9ea 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -76,7 +76,7 @@ def _set_settings(self, source, settings): def _set_route(self, route: schemas.Route): redis_data = route.to_redis() - self._set_target(route.source, redis_data["target"]) + self._set_target(route.source, redis_data["target"], route.settings.get("ttl")) self._set_settings(route.source, redis_data["settings"]) return route diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py index 71f1d24..3683d92 100644 --- a/api/ceryx/schemas.py +++ b/api/ceryx/schemas.py @@ -23,6 +23,14 @@ def redis_to_object(value): return json.loads(value) +def integer_to_redis(value: int): + return str(value) + + +def redis_to_integer(value): + return int(value) + + def ensure_string(value): redis_value = ( None if value is None @@ -41,6 +49,9 @@ def value_to_redis(field, value): if isinstance(field, typesystem.Object): return object_to_redis(value) + if isinstance(field, typesystem.Integer): + return integer_to_redis(value) + return ensure_string(value) @@ -54,6 +65,9 @@ def redis_to_value(field, redis_value): if isinstance(field, typesystem.Object): return redis_to_object(redis_value) + if isinstance(field, typesystem.Integer): + return redis_to_integer(redis_value) + return ensure_string(redis_value) @@ -84,6 +98,7 @@ class Settings(BaseSchema): default="proxy", ) headers = typesystem.Object(default={}, properties=typesystem.String(max_length=100)) + ttl = typesystem.Integer(allow_null=True) certificate_path = typesystem.String(allow_null=True) key_path = typesystem.String(allow_null=True) From 27ae85bc120aa6523b8fcdaec2fb84740422e997 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 18:20:34 +0100 Subject: [PATCH 04/10] Add request_uri to possible sources The proxy now first tries to find a matching route based on the host and if this does not yield a result it will next try to find a route based on the request_uri (without leading slash). This commit depends on the presence of feature/headers. --- ceryx/nginx/lualib/router.lua | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/ceryx/nginx/lualib/router.lua b/ceryx/nginx/lualib/router.lua index 57c9ac8..b26b94c 100644 --- a/ceryx/nginx/lualib/router.lua +++ b/ceryx/nginx/lualib/router.lua @@ -5,17 +5,21 @@ local utils = require "ceryx.utils" local redisClient = redis:client() local host = ngx.var.host +local request_uri = ngx.var.request_uri:sub(2) local cache = ngx.shared.ceryx local is_not_https = (ngx.var.scheme ~= "https") -function formatTarget(target) +function formatTargetHostSource(target) target = utils.ensure_protocol(target) target = utils.ensure_no_trailing_slash(target) - return target .. ngx.var.request_uri end +function formatTargetRequestUriSource(target) + target = utils.ensure_protocol(target) + return target +end function redirect(source, target, headers) ngx.log(ngx.INFO, "Redirecting request for " .. source) @@ -36,8 +40,6 @@ end function routeRequest(source, target, mode, headers) ngx.log(ngx.DEBUG, "Received " .. mode .. " routing request from " .. source .. " to " .. target) - target = formatTarget(target) - if mode == "redirect" then return redirect(source, target, headers) end @@ -60,13 +62,19 @@ if is_not_https then end end -ngx.log(ngx.INFO, "HOST " .. host) +ngx.log(ngx.INFO, "Try host route for " .. host) local route = routes.getRouteForSource(host) -if route == nil then - ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.") - return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE) +if route ~= nil then + return routeRequest(host, formatTargetHostSource(route.target), route.mode, route.headers) +end + +ngx.log(ngx.INFO, "Try request_uri route for " .. request_uri) +route = routes.getRouteForSource(request_uri) + +if route ~= nil then + return routeRequest(request_uri, formatTargetRequestUriSource(route.target), route.mode, route.headers) end --- Save found key to local cache for 5 seconds -routeRequest(host, route.target, route.mode, route.headers) +ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.") +ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE) From 1d159992b6d6e44641a7774a95f48da4d3778d9f Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 18:27:01 +0100 Subject: [PATCH 05/10] Fix responder error on missing default route path In newer versions or some environments it may occur that the responder API complains about a missing, explicit default route key. --- api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api.py b/api/api.py index f404a45..61e2a75 100644 --- a/api/api.py +++ b/api/api.py @@ -8,7 +8,7 @@ client = RedisClient.from_config() -@api.route(default=True) +@api.route("/", default=True) def default(req, resp): if not req.url.path.endswith("/"): api.redirect(resp, f"{req.url.path}/") From 0187aa349ff1f0ed9b9a7865cdee874478a3d8b6 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 5 Mar 2020 18:29:39 +0100 Subject: [PATCH 06/10] Add explicit remove of hash before set When using redis EXPIRE or any other automated cleaning utility it may happen that a hash is not fully removed. If a new hash is set at the same key and there are null values the old stored values may be reused. --- api/ceryx/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/ceryx/db.py b/api/ceryx/db.py index 102c9ea..8d2c56b 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -71,6 +71,7 @@ def _set_target(self, source, target, ttl=None): self.client.set(key, target, ex=ttl) def _set_settings(self, source, settings): + self._delete_settings(source) key = self._settings_key(source) self.client.hmset(key, settings) From 66246bd3bfc236f32a31667bde830824d3e73727 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 25 Feb 2021 11:40:54 +0100 Subject: [PATCH 07/10] Add instructions for new features to README --- README.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f7c57c..ea54785 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,25 @@ docker-compose exec api bin/populate-api ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000"}' \ + -d '{"source":"publicly.accessible.domain", "target":"http://service.internal:8000"}' \ + http://ceryx-api-host/api/routes +``` + +The route can also feature a random UUID as source. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source":"some-random-uuid", "target":"http://service.internal:8000"}' \ + http://ceryx-api-host/api/routes +``` + +A route may also have request params in the target. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source":"some-random-uuid", "target":"http://service.internal:8000?foo=bar&x=y"}' \ http://ceryx-api-host/api/routes ``` @@ -148,6 +166,28 @@ curl -H "Content-Type: application/json" \ http://ceryx-api-host/api/routes ``` +### Provide authorization (and other headers) for target connection + +If the route should be authorized behind ceryx you may deploy an authorization header with the route. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"headers": {"authorization": "Bearer ..."}}}' \ + http://ceryx-api-host/api/routes +``` + +### Give routes a TTL after which they become invalid + +If necessary you can provide a TTL in seconds for your routes after which they are removed from ceryx. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"ttl": 20}}' \ + http://ceryx-api-host/api/routes +``` + ## Ceryx web UI The [Ceryx Web community project](https://github.com/parisk/ceryx-web) provides a sweet web UI From 5f2d67d3bdd1dc653899a569746ba082f521f8b1 Mon Sep 17 00:00:00 2001 From: Alexander Wellbrock Date: Thu, 25 Feb 2021 11:43:00 +0100 Subject: [PATCH 08/10] Add othermo as real world use-case --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ea54785..b8b0c23 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ Ceryx has proven to be extremely reliable in production systems, handling tens o - [**SourceLair**](https://www.sourcelair.com/): In-browser IDE for web applications, made publicly accessible via development web servers powered by Ceryx. - [**Stolos**](http://stolos.io/): Managed Docker development environments for enterprises. +- [**othermo**](https://www.othermo.de): Industry 4.0 for heating plants and municipal utilities, using Ceryx to implement the [Data By-Pass Pattern](https://www.eclipse.org/ditto/advanced-data-by-pass.html) with [Eclipse Ditto](https://www.eclipse.org/ditto). Do you use Ceryx in production as well? Please [open a Pull Request](https://github.com/sourcelair/ceryx/pulls) to include it here. We would love to have it in our list. From 63ac665106f9050352552ba5ade2ec126030f5ec Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Sat, 6 Mar 2021 17:54:56 +0000 Subject: [PATCH 09/10] Improve copy --- README.md | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index b8b0c23..b9db08e 100644 --- a/README.md +++ b/README.md @@ -103,25 +103,16 @@ docker-compose exec api bin/populate-api ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"publicly.accessible.domain", "target":"http://service.internal:8000"}' \ + -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000"}' \ http://ceryx-api-host/api/routes ``` -The route can also feature a random UUID as source. +A route may also have request parameters in the target. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"some-random-uuid", "target":"http://service.internal:8000"}' \ - http://ceryx-api-host/api/routes -``` - -A route may also have request params in the target. - -``` -curl -H "Content-Type: application/json" \ - -X POST \ - -d '{"source":"some-random-uuid", "target":"http://service.internal:8000?foo=bar&x=y"}' \ + -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000?foo=bar&x=y"}' \ http://ceryx-api-host/api/routes ``` @@ -130,8 +121,8 @@ curl -H "Content-Type: application/json" \ ``` curl -H "Content-Type: application/json" \ -X PUT \ - -d '{"source":"publicly.accessible.domain","target":"http://another-service.internal:8000"}' \ - http://ceryx-api-host/api/routes/publicly.accessible.domain + -d '{"source": "any-valid-hostname", "target": "http://another-service.internal:8000"}' \ + http://ceryx-api-host/api/routes/any-valid-hostname ``` ### Delete a route from Ceryx @@ -139,7 +130,7 @@ curl -H "Content-Type: application/json" \ ``` curl -H "Content-Type: application/json" \ -X DELETE \ - http://ceryx-api-host/api/routes/publicly.accessible.domain + http://ceryx-api-host/api/routes/any-valid-hostname ``` ### Enforce HTTPS @@ -149,7 +140,7 @@ You can enforce redirection from HTTP to HTTPS for any host you would like. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \ + -d '{"source": "www.sourcelair.com", "target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \ http://ceryx-api-host/api/routes ``` @@ -157,34 +148,34 @@ The above functionality works in `PUT` update requests as well. ### Redirect to target, instead of proxying -Instead of proxying the request to the targetm you can prompt the client to redirect the request there itself. +Instead of proxying the request to the target, you can prompt the client to redirect the request there itself. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \ + -d '{"source": "sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \ http://ceryx-api-host/api/routes ``` -### Provide authorization (and other headers) for target connection +### Include additional headers (e.g. `Authorization`) for target connection -If the route should be authorized behind ceryx you may deploy an authorization header with the route. +If the route should be authorized behind Ceryx you may deploy an `Authorization` header with the route. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"headers": {"authorization": "Bearer ..."}}}' \ + -d '{"source":"sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"headers": {"authorization": "Bearer ..."}}}' \ http://ceryx-api-host/api/routes ``` -### Give routes a TTL after which they become invalid +### Give routes a TTL -If necessary you can provide a TTL in seconds for your routes after which they are removed from ceryx. +You can provide a TTL (in seconds) for your routes after which they are removed from Ceryx. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"ttl": 20}}' \ + -d '{"source":"sourcelair.com", "target": "https://www.sourcelair.com", "settings": {"ttl": 20}}' \ http://ceryx-api-host/api/routes ``` From 75ed15ae73df8d4f4f62434ce9803683c37da3b7 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Sat, 6 Mar 2021 18:09:41 +0000 Subject: [PATCH 10/10] Fix identation --- ceryx/nginx/lualib/router.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceryx/nginx/lualib/router.lua b/ceryx/nginx/lualib/router.lua index b26b94c..0c6badb 100644 --- a/ceryx/nginx/lualib/router.lua +++ b/ceryx/nginx/lualib/router.lua @@ -33,7 +33,7 @@ function proxy(source, target, headers) ngx.var.target = target for k,v in pairs(headers) do ngx.req.set_header(k, v) -end + end ngx.log(ngx.INFO, "Proxying request for " .. source) end