diff --git a/README.md b/README.md index 7f7c57c..b9db08e 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,16 @@ docker-compose exec api bin/populate-api ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000"}' \ + -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000"}' \ + http://ceryx-api-host/api/routes +``` + +A route may also have request parameters in the target. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source": "any-valid-hostname", "target": "http://service.internal:8000?foo=bar&x=y"}' \ http://ceryx-api-host/api/routes ``` @@ -112,8 +121,8 @@ curl -H "Content-Type: application/json" \ ``` curl -H "Content-Type: application/json" \ -X PUT \ - -d '{"source":"publicly.accessible.domain","target":"http://another-service.internal:8000"}' \ - http://ceryx-api-host/api/routes/publicly.accessible.domain + -d '{"source": "any-valid-hostname", "target": "http://another-service.internal:8000"}' \ + http://ceryx-api-host/api/routes/any-valid-hostname ``` ### Delete a route from Ceryx @@ -121,7 +130,7 @@ curl -H "Content-Type: application/json" \ ``` curl -H "Content-Type: application/json" \ -X DELETE \ - http://ceryx-api-host/api/routes/publicly.accessible.domain + http://ceryx-api-host/api/routes/any-valid-hostname ``` ### Enforce HTTPS @@ -131,7 +140,7 @@ You can enforce redirection from HTTP to HTTPS for any host you would like. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"publicly.accessible.domain","target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \ + -d '{"source": "www.sourcelair.com", "target":"http://service.internal:8000", "settings": {"enforce_https": true}}' \ http://ceryx-api-host/api/routes ``` @@ -139,12 +148,34 @@ The above functionality works in `PUT` update requests as well. ### Redirect to target, instead of proxying -Instead of proxying the request to the targetm you can prompt the client to redirect the request there itself. +Instead of proxying the request to the target, you can prompt the client to redirect the request there itself. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source": "sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \ + http://ceryx-api-host/api/routes +``` + +### Include additional headers (e.g. `Authorization`) for target connection + +If the route should be authorized behind Ceryx you may deploy an `Authorization` header with the route. + +``` +curl -H "Content-Type: application/json" \ + -X POST \ + -d '{"source":"sourcelair.com", "target":"https://www.sourcelair.com", "settings": {"headers": {"authorization": "Bearer ..."}}}' \ + http://ceryx-api-host/api/routes +``` + +### Give routes a TTL + +You can provide a TTL (in seconds) for your routes after which they are removed from Ceryx. ``` curl -H "Content-Type: application/json" \ -X POST \ - -d '{"source":"sourcelair.com","target":"https://www.sourcelair.com", "settings": {"mode": "redirect"}}' \ + -d '{"source":"sourcelair.com", "target": "https://www.sourcelair.com", "settings": {"ttl": 20}}' \ http://ceryx-api-host/api/routes ``` @@ -158,6 +189,7 @@ Ceryx has proven to be extremely reliable in production systems, handling tens o - [**SourceLair**](https://www.sourcelair.com/): In-browser IDE for web applications, made publicly accessible via development web servers powered by Ceryx. - [**Stolos**](http://stolos.io/): Managed Docker development environments for enterprises. +- [**othermo**](https://www.othermo.de): Industry 4.0 for heating plants and municipal utilities, using Ceryx to implement the [Data By-Pass Pattern](https://www.eclipse.org/ditto/advanced-data-by-pass.html) with [Eclipse Ditto](https://www.eclipse.org/ditto). Do you use Ceryx in production as well? Please [open a Pull Request](https://github.com/sourcelair/ceryx/pulls) to include it here. We would love to have it in our list. diff --git a/api/api.py b/api/api.py index f404a45..61e2a75 100644 --- a/api/api.py +++ b/api/api.py @@ -8,7 +8,7 @@ client = RedisClient.from_config() -@api.route(default=True) +@api.route("/", default=True) def default(req, resp): if not req.url.path.endswith("/"): api.redirect(resp, f"{req.url.path}/") diff --git a/api/ceryx/db.py b/api/ceryx/db.py index d26b2ae..8d2c56b 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -39,71 +39,72 @@ def _route_key(self, source): def _settings_key(self, source): return self._prefixed_key(f"settings:{source}") - def _delete_target(self, host): - key = self._route_key(host) + def _delete_target(self, source): + key = self._route_key(source) self.client.delete(key) - def _delete_settings(self, host): - key = self._settings_key(host) + def _delete_settings(self, source): + key = self._settings_key(source) self.client.delete(key) - def _lookup_target(self, host, raise_exception=False): - key = self._route_key(host) + def _lookup_target(self, source, raise_exception=False): + key = self._route_key(source) target = self.client.get(key) - + if target is None and raise_exception: raise exceptions.NotFound("Route not found.") return target - def _lookup_settings(self, host): - key = self._settings_key(host) + def _lookup_settings(self, source): + key = self._settings_key(source) return self.client.hgetall(key) - def lookup_hosts(self, pattern="*"): + def lookup_sources(self, pattern="*"): lookup_pattern = self._route_key(pattern) left_padding = len(lookup_pattern) - 1 keys = self.client.keys(lookup_pattern) return [_str(key)[left_padding:] for key in keys] - def _set_target(self, host, target): - key = self._route_key(host) - self.client.set(key, target) + def _set_target(self, source, target, ttl=None): + key = self._route_key(source) + self.client.set(key, target, ex=ttl) - def _set_settings(self, host, settings): - key = self._settings_key(host) + def _set_settings(self, source, settings): + self._delete_settings(source) + key = self._settings_key(source) self.client.hmset(key, settings) def _set_route(self, route: schemas.Route): redis_data = route.to_redis() - self._set_target(route.source, redis_data["target"]) + self._set_target(route.source, redis_data["target"], route.settings.get("ttl")) self._set_settings(route.source, redis_data["settings"]) return route - def get_route(self, host): - target = self._lookup_target(host, raise_exception=True) - settings = self._lookup_settings(host) + def get_route(self, source): + target = self._lookup_target(source, raise_exception=True) + settings = self._lookup_settings(source) route = schemas.Route.from_redis({ - "source": host, + "source": source, "target": target, "settings": settings }) return route def list_routes(self): - hosts = self.lookup_hosts() - routes = [self.get_route(host) for host in hosts] + sources = self.lookup_sources() + routes = [self.get_route(source) for source in sources] return routes def create_route(self, data: dict): route = schemas.Route.validate(data) return self._set_route(route) - def update_route(self, host: str, data: dict): - data["source"] = host + def update_route(self, source: str, data: dict): + data["source"] = source route = schemas.Route.validate(data) return self._set_route(route) - def delete_route(self, host: str): - self._delete_target(host) - self._delete_settings(host) + def delete_route(self, source: str): + self._delete_target(source) + self._delete_settings(source) diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py index b1dfbc6..3683d92 100644 --- a/api/ceryx/schemas.py +++ b/api/ceryx/schemas.py @@ -1,6 +1,6 @@ import re import typesystem - +import json def ensure_protocol(url): starts_with_protocol = r"^https?://" @@ -15,6 +15,22 @@ def redis_to_boolean(value): return True if value == "1" else False +def object_to_redis(value: object): + return json.dumps(value) + + +def redis_to_object(value): + return json.loads(value) + + +def integer_to_redis(value: int): + return str(value) + + +def redis_to_integer(value): + return int(value) + + def ensure_string(value): redis_value = ( None if value is None @@ -30,6 +46,12 @@ def value_to_redis(field, value): if isinstance(field, typesystem.Reference): return field.target.validate(value).to_redis() + if isinstance(field, typesystem.Object): + return object_to_redis(value) + + if isinstance(field, typesystem.Integer): + return integer_to_redis(value) + return ensure_string(value) @@ -40,6 +62,12 @@ def redis_to_value(field, redis_value): if isinstance(field, typesystem.Reference): return field.target.from_redis(redis_value) + if isinstance(field, typesystem.Object): + return redis_to_object(redis_value) + + if isinstance(field, typesystem.Integer): + return redis_to_integer(redis_value) + return ensure_string(redis_value) @@ -69,6 +97,8 @@ class Settings(BaseSchema): ), default="proxy", ) + headers = typesystem.Object(default={}, properties=typesystem.String(max_length=100)) + ttl = typesystem.Integer(allow_null=True) certificate_path = typesystem.String(allow_null=True) key_path = typesystem.String(allow_null=True) diff --git a/ceryx/nginx/lualib/ceryx/routes.lua b/ceryx/nginx/lualib/ceryx/routes.lua index 98ce442..f6925fd 100644 --- a/ceryx/nginx/lualib/ceryx/routes.lua +++ b/ceryx/nginx/lualib/ceryx/routes.lua @@ -1,4 +1,5 @@ local redis = require "ceryx.redis" +local cjson = require "cjson" local exports = {} @@ -16,7 +17,7 @@ end function getTargetForSource(source, redisClient) -- Construct Redis key and then - -- try to get target for host + -- try to get target for source local key = getRouteKeyForSource(source) local target, _ = redisClient:get(key) @@ -49,6 +50,18 @@ function getModeForSource(source, redisClient) return mode end +function getHeadersForSource(source, redisClient) + ngx.log(ngx.DEBUG, "Get routing headers for " .. source .. ".") + local settings_key = getSettingsKeyForSource(source) + local headers, _ = cjson.decode(redisClient:hget(settings_key, "headers")) + + if headers == ngx.null or not headers then + headers = {} + end + + return headers +end + function getRouteForSource(source) local _ local route = {} @@ -69,11 +82,12 @@ function getRouteForSource(source) if targetIsInValid(route.target) then return nil end - cache:set(host, res, 5) + cache:set(source, res, 5) ngx.log(ngx.DEBUG, "Caching from " .. source .. " to " .. route.target .. " for 5 seconds.") end route.mode = getModeForSource(source, redisClient) + route.headers = getHeadersForSource(source, redisClient) return route end diff --git a/ceryx/nginx/lualib/router.lua b/ceryx/nginx/lualib/router.lua index 6922b25..0c6badb 100644 --- a/ceryx/nginx/lualib/router.lua +++ b/ceryx/nginx/lualib/router.lua @@ -5,37 +5,46 @@ local utils = require "ceryx.utils" local redisClient = redis:client() local host = ngx.var.host +local request_uri = ngx.var.request_uri:sub(2) local cache = ngx.shared.ceryx local is_not_https = (ngx.var.scheme ~= "https") -function formatTarget(target) +function formatTargetHostSource(target) target = utils.ensure_protocol(target) target = utils.ensure_no_trailing_slash(target) - return target .. ngx.var.request_uri end -function redirect(source, target) - ngx.log(ngx.INFO, "Redirecting request for " .. source .. " to " .. target .. ".") +function formatTargetRequestUriSource(target) + target = utils.ensure_protocol(target) + return target +end + +function redirect(source, target, headers) + ngx.log(ngx.INFO, "Redirecting request for " .. source) + for k,v in pairs(headers) do + ngx.headers[k] = v + end return ngx.redirect(target, ngx.HTTP_MOVED_PERMANENTLY) end -function proxy(source, target) +function proxy(source, target, headers) ngx.var.target = target - ngx.log(ngx.INFO, "Proxying request for " .. source .. " to " .. target .. ".") + for k,v in pairs(headers) do + ngx.req.set_header(k, v) + end + ngx.log(ngx.INFO, "Proxying request for " .. source) end -function routeRequest(source, target, mode) +function routeRequest(source, target, mode, headers) ngx.log(ngx.DEBUG, "Received " .. mode .. " routing request from " .. source .. " to " .. target) - target = formatTarget(target) - if mode == "redirect" then - return redirect(source, target) + return redirect(source, target, headers) end - return proxy(source, target) + return proxy(source, target, headers) end if is_not_https then @@ -53,13 +62,19 @@ if is_not_https then end end -ngx.log(ngx.INFO, "HOST " .. host) +ngx.log(ngx.INFO, "Try host route for " .. host) local route = routes.getRouteForSource(host) -if route == nil then - ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.") - return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE) +if route ~= nil then + return routeRequest(host, formatTargetHostSource(route.target), route.mode, route.headers) +end + +ngx.log(ngx.INFO, "Try request_uri route for " .. request_uri) +route = routes.getRouteForSource(request_uri) + +if route ~= nil then + return routeRequest(request_uri, formatTargetRequestUriSource(route.target), route.mode, route.headers) end --- Save found key to local cache for 5 seconds -routeRequest(host, route.target, route.mode) +ngx.log(ngx.INFO, "No $wildcard target configured for fallback. Exiting with Bad Gateway.") +ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)