Rate limiting

CHANGES.md

@@ -34,6 +34,7 @@ General:
 -   Set pwgen to `0.1.6` (was `^0.1.6`)
 -   Remove ESRI-portal related code from deployment charts
 -   Allow admin users to download history report of a dataset
+-   Add rate-limiting by IP
 
 UI:
 

docs/docs/how-to-create-local-users.md

@@ -10,7 +10,7 @@ Before start to use the `set-user-password script, you need to:
     -   `kubectl port-forward combined-db-0 5432:5432`
     -   If you didn't install magda to the default namespace, you can use: `kubectl port-forward -n [namespace] combined-db-0 5432:5432`
 
-After the installation is done, run `yset-user-password` will list help information as below:
+After the installation is done, run `yarn set-user-password` will list help information as below:
 
 ```
 Usage: set-user-password [options]

magda-gateway/package.json

@@ -18,9 +18,8 @@
   "dependencies": {
     "@magda/tenant-api": "^0.0.57-0",
     "@magda/typescript-common": "^0.0.57-0",
+    "@types/express-rate-limit": "^5.0.0",
     "bcrypt": "^5.0.0",
-    "is-uuid": "^1.0.2",
-    "uuid": "^8.2.0",
     "body-parser": "^1.13.2",
     "cheerio": "^0.22.0",
     "compression": "^1.7.2",
@@ -32,10 +31,12 @@
     "escape-string-regexp": "^1.0.5",
     "express": "^4.17.1",
     "express-basic-auth": "^1.1.5",
+    "express-rate-limit": "^5.1.3",
     "express-session": "^1.17.1",
     "gravatar": "^1.8.0",
     "helmet": "^3.12.0",
     "http-proxy": "^1.16.2",
+    "is-uuid": "^1.0.2",
     "isomorphic-fetch": "^2.2.1",
     "isomorphic-form-data": "^1.0.0",
     "jsonwebtoken": "^8.4.0",
@@ -54,13 +55,12 @@
     "request": "^2.88.0",
     "tsmonad": "^0.7.2",
     "urijs": "^1.18.12",
+    "uuid": "^8.2.0",
     "yargs": "^12.0.5"
   },
   "devDependencies": {
     "@magda/scripts": "^0.0.57-0",
     "@types/bcrypt": "^3.0.0",
-    "@types/is-uuid": "^1.0.0",
-    "@types/uuid": "^8.0.0",
     "@types/chai": "^4.0.1",
     "@types/compression": "^0.0.36",
     "@types/cookie": "^0.3.3",
@@ -71,6 +71,7 @@
     "@types/express": "^4.17.6",
     "@types/express-session": "^1.15.15",
     "@types/http-proxy": "^1.16.0",
+    "@types/is-uuid": "^1.0.0",
     "@types/lodash": "^4.14.74",
     "@types/mocha": "^2.2.41",
     "@types/nock": "^8.2.1",
@@ -85,6 +86,7 @@
     "@types/sinon": "^2.3.3",
     "@types/supertest": "^2.0.2",
     "@types/urijs": "1.19.1",
+    "@types/uuid": "^8.0.0",
     "@types/yargs": "^12.0.8",
     "chai": "^4.1.0",
     "mocha": "^3.4.2",

magda-gateway/src/buildApp.ts

@@ -25,6 +25,7 @@ import defaultConfig from "./defaultConfig";
 import { ProxyTarget } from "./createApiRouter";
 import setupTenantMode from "./setupTenantMode";
 import createPool from "./createPool";
+import rateLimit from "express-rate-limit";
 
 // Tell typescript about the semi-private __express field of ejs.
 declare module "ejs" {
@@ -260,6 +261,22 @@ export default function buildApp(app: express.Application, config: Config) {
         }
     }
 
+    const limiter = rateLimit({
+        windowMs: 5 * 60 * 1000, // 5 minutes
+        max: 200, // limit each IP/API Key to 200 requests per windowMs
+        keyGenerator: function (req /*, res*/) {
+            // let key = req.header("X-Magda-API-Key");
+            let key = req.connection.remoteAddress;
+            if (!key) {
+                key = req.ip;
+            }
+            return key;
+        }
+    });
+
+    // apply to all requests
+    app.use("/api/v0", limiter);
+
     // Proxy any other URL to magda-web
     app.use("/", createGenericProxy(config.web, apiRouterOptions));
 

yarn.lock

@@ -3153,6 +3153,13 @@
   resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
   integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
 
+"@types/express-rate-limit@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@types/express-rate-limit/-/express-rate-limit-5.0.0.tgz#46b0dbae748a53347a5e1c3bdbb5a54e3f5c34f9"
+  integrity sha512-tVRtvshGIYTNiUET3XJaONn1Li0V9mhi9Tb23Kf3rJ19hsgbCywkZBOf5Xhd9OXBO0Ssw7B+qAHL77xe/TiqUw==
+  dependencies:
+    "@types/express" "*"
+
 "@types/express-serve-static-core@*":
   version "4.17.7"
   resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.7.tgz#dfe61f870eb549dc6d7e12050901847c7d7e915b"
@@ -8724,6 +8731,11 @@ express-multipart-file-parser@^0.1.2:
     content-type "^1.0.4"
     raw-body "^2.3.2"
 
+express-rate-limit@^5.1.3:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-5.1.3.tgz#656bacce3f093034976346958a0f0199902c9174"
+  integrity sha512-TINcxve5510pXj4n9/1AMupkj3iWxl3JuZaWhCdYDlZeoCPqweGZrxbrlqTCFb1CT5wli7s8e2SH/Qz2c9GorA==
+
 express-session@^1.17.1:
   version "1.17.1"
   resolved "https://registry.yarnpkg.com/express-session/-/express-session-1.17.1.tgz#36ecbc7034566d38c8509885c044d461c11bf357"