Merge pull request #5 from strongloop/nginx-honour-cache-expiration

NGINX: fix cache expiration

Author: rmg

ephemeral-npm.lua

@@ -2,21 +2,30 @@ local cjson = require "cjson.safe"
 
 local _M = {}
 
+local function parseDuration(str)
+    -- TODO: parse inputs like "1h", "2d", "10m" and return them as seconds
+    return 5 * 60
+end
+
 function _M.init()
     local npmConfig = ngx.shared.npmConfig
-    local registry = os.getenv("npm_config_registry"):gsub("/+$", "")
-    local pattern = registry:gsub("%.", "%%."):gsub("%-", "%%-")
+    _M.registry = os.getenv("npm_config_registry"):gsub("/+$", "")
+    _M.hostPattern = _M.registry:gsub("%.", "%%."):gsub("%-", "%%-")
+    _M.MAXAGE = parseDuration(os.getenv("MAXAGE") or "5m")
     -- escape . and - which have special meaning in Lua patterns
-    npmConfig:set('npm_config_registry', registry)
-    npmConfig:set('npm_upstream_pattern', pattern)
+    npmConfig:set('npm_config_registry', _M.registry)
+    npmConfig:set('npm_upstream_pattern', _M.hostPattern)
+    npmConfig:set('MAXAGE', _M.MAXAGE)
 end
 
 function _M.getPackage()
     local uri = ngx.var.uri
     local meta = ngx.shared.npmMeta
     local body = meta:get(uri)
+    local base = ngx.var.scheme .. '://' .. ngx.var.http_host
     -- yep, our own shared memory cache implementation :-/
     if body == nil then
+        ngx.var.ephemeralCacheStatus = 'MISS'
         local res = ngx.location.capture('/-@-' .. uri,
                                         { copy_all_vars = true })
         body = res.body
@@ -27,19 +36,13 @@ function _M.getPackage()
             -- next time
             return ngx.redirect(uri, ngx.HTTP_MOVED_TEMPORARILY)
         end
-        meta:set(uri, body)
+        meta:set(uri, body, _M.MAXAGE)
+    else
+        ngx.var.ephemeralCacheStatus = 'HIT'
     end
+    body = string.gsub(body, _M.hostPattern, base)
     ngx.header["Content-Length"] = #body
     ngx.print(body)
 end
 
-function _M.filterPackageBody()
-    local npmConfig = ngx.shared.npmConfig
-    local upstream = npmConfig:get('npm_upstream_pattern')
-    -- need to construct URL because we may be proxying http<->https
-    local base = ngx.var.scheme .. '://' .. ngx.var.http_host
-    -- ngx.log(ngx.ERR, "Modifying JSON of " .. ngx.var.uri .. " to replace '" .. upstream .. "' with '" .. base .. "'")
-    ngx.arg[1] = string.gsub(ngx.arg[1], upstream, base)
-end
-
 return _M

nginx.conf

@@ -11,18 +11,17 @@ events {
 http {
     # We literally only care about 2 types of files/payloads.
     types {
-        application/json                      json;
         application/gzip                      tgz;
     }
-    default_type  application/octet-stream;
+    default_type  application/json;
     sendfile        on;
     keepalive_timeout  65;
     proxy_cache_path  /tmp/npm/cache levels=1:2 keys_zone=NPM:10m inactive=90d max_size=5g;
     proxy_temp_path   /tmp/npm/temp;
     proxy_cache_lock  on;
     log_format upstreamlog '$remote_addr - $remote_user [$time_local] '
                            '"$request" $status $body_bytes_sent '
-                           '"$http_referer" "$http_user_agent" "$upstream_cache_status"';
+                           '"$http_referer" "$http_user_agent" "$ephemeralCacheStatus"';
     access_log logs/access.log upstreamlog;
 
     lua_shared_dict npmConfig 32k;
@@ -32,7 +31,6 @@ http {
         local ephemeralNPM = require "ephemeral-npm"
         ephemeralNPM.init()
     }
-
     server {
         listen       4873;
         set $npm_config_registry '';
@@ -44,34 +42,27 @@ http {
             try_files          $uri @fetch-tgz;
         }
         location / {
+            set $ephemeralCacheStatus '-';
+            # We can't do caching at the proxy level because we can't easily
+            # parameterize cache lifetime for proxy_cache. Instead, we do
+            # the caching ourselves in Lua using a shared dict.
             content_by_lua_block {
                 local ephemeralNPM = require "ephemeral-npm"
                 ephemeralNPM.getPackage()
             }
         }
+        # This is just a straight though, uncaching proxy to the upstream
+        # registry.
+        # We have to use a path prefix instead of an internal @name because
+        # we need to be able to make sub-requests using ngx.location.capture()
         location /-@- {
             internal;
             rewrite                /-@-/(.+) /$1 break;
             resolver               127.0.0.1 ipv6=off;
             proxy_pass             $npm_config_registry;
             proxy_buffers          32 1m;
-            proxy_cache            NPM;
-            # need nginx-1.11.10 for this, which isn't in openresty yet:
-            # proxy_cache_background_update on;
-            proxy_cache_revalidate on;
-            # let our metadata be 20 minutes old, at most
-            proxy_cache_valid      200  20m;
-            proxy_cache_use_stale  error timeout invalid_header updating http_500 http_502 http_503 http_504;
-            # need to disable encoding in order to be able to filter the response body
+            # need to disable encoding in order to be able to process it locally
             proxy_set_header       Accept-Encoding "";
-            # modifying the response body will change the length
-            header_filter_by_lua_block {
-                ngx.header.content_length = nil
-            }
-            body_filter_by_lua_block {
-                local ephemeralNPM = require "ephemeral-npm"
-                ephemeralNPM.filterPackageBody()
-            }
         }
         location @fetch-tgz {
             internal;