Extended Request Data Collection (#5709)

* Headers collection config

* Revamp headers collection + extended collection

* Fix condition for extended collection

* Body collection config

* Check for RASP events

* Testing extend header collection

* RASP body collection tests

* Minor reformats

* Lint

* Clearer condition

* Truncate body on body collection

* Complete config test

* Fix lint

* Reporter init w/ appsec config

* Remove TODO

* Clarify collection comments

* Update test.ts with new configuration

* Fix linting

* Integration test for RASP request body collection

* Set tag when reported request body is truncated

* Integration test for data collection

* Fix linting

* Check for header names in integration test

* Set the correct value for request body exceeded size tag

* Add support for toJSON in request body truncation

* Invert condition for early return

* Avoid slicing on request body truncation and implement it in a for loop

* Rename reporter config

* Simplify condition logic

* Create block scope in switch case to limit the scope of declared vars

* Initialize Array with a fixed size

Co-authored-by: simon-id <simon.id@datadoghq.com>

* Switch to for loop

Co-authored-by: simon-id <simon.id@datadoghq.com>

* Set env var as strings

Co-authored-by: simon-id <simon.id@datadoghq.com>

* Set env var as string

Co-authored-by: simon-id <simon.id@datadoghq.com>

* Refactor headers group declaration + using set instead of arrays

* Cache res.getHeaders()

* Fix config and its test

* Fix config test for defaults values

* Check reported request body truncation on integration test

* Add a comment to clarify the test case

* Add a test case to check no request body is collected when feat is disabled

* Manage custom toJSON function in arrays

---------

Co-authored-by: simon-id <simon.id@datadoghq.com>

Author: 2 people

docs/test.ts

@@ -118,12 +118,18 @@ tracer.init({
       enabled: true,
     },
     rasp: {
-      enabled: true
+      enabled: true,
+      bodyCollection: true
     },
     stackTrace: {
       enabled: true,
       maxStackTraces: 5,
       maxDepth: 42
+    },
+    extendedHeadersCollection: {
+      enabled: true,
+      redaction: false,
+      maxHeaders: 42
     }
   },
   iast: {

index.d.ts

@@ -713,7 +713,12 @@ declare namespace tracer {
         /** Whether to enable RASP.
          * @default false
          */
-        enabled?: boolean
+        enabled?: boolean,
+
+        /** Whether to enable request body collection on RASP event
+         * @default false
+         */
+        bodyCollection?: boolean
       },
       /**
        * Configuration for stack trace reporting
@@ -733,6 +738,25 @@ declare namespace tracer {
          * @default 32
          */
         maxDepth?: number,
+      },
+      /**
+       * Configuration for extended headers collection tied to security events
+       */
+      extendedHeadersCollection?: {
+        /** Whether to enable extended headers collection
+         * @default false
+         */
+        enabled: boolean,
+
+        /** Whether to redact collected headers
+         * @default true
+         */
+        redaction: boolean,
+
+        /** Specifies the maximum number of headers collected.
+         * @default 50
+         */
+        maxHeaders: number,
       }
     }
 

integration-tests/appsec/data-collection.spec.js

@@ -0,0 +1,127 @@
+'use strict'
+
+const { assert } = require('chai')
+const getPort = require('get-port')
+const path = require('path')
+const Axios = require('axios')
+
+const {
+  createSandbox,
+  FakeAgent,
+  spawnProc
+} = require('../helpers')
+
+describe('ASM Data collection', () => {
+  let axios, sandbox, cwd, appPort, appFile, agent, proc
+
+  before(async () => {
+    sandbox = await createSandbox(['express'])
+    appPort = await getPort()
+    cwd = sandbox.folder
+    appFile = path.join(cwd, 'appsec/data-collection/index.js')
+    axios = Axios.create({
+      baseURL: `http://localhost:${appPort}`
+    })
+  })
+
+  after(async () => {
+    await sandbox.remove()
+  })
+
+  function startServer (extendedDataCollection) {
+    beforeEach(async () => {
+      agent = await new FakeAgent().start()
+
+      const env = {
+        APP_PORT: appPort,
+        DD_TRACE_AGENT_PORT: agent.port,
+        DD_APPSEC_ENABLED: true,
+        DD_APPSEC_RULES: path.join(cwd, 'appsec', 'data-collection', 'data-collection-rules.json')
+      }
+
+      if (extendedDataCollection) {
+        env.DD_APPSEC_COLLECT_ALL_HEADERS = true
+        env.DD_APPSEC_HEADER_COLLECTION_REDACTION_ENABLED = false
+        env.DD_APPSEC_MAX_COLLECTED_HEADERS = 25
+      }
+
+      proc = await spawnProc(appFile, { cwd, env, execArgv: [] })
+    })
+
+    afterEach(async () => {
+      proc.kill()
+      await agent.stop()
+    })
+  }
+
+  async function assertHeadersReported (requestHeaders, responseHeaders) {
+    await agent.assertMessageReceived(({ headers, payload }) => {
+      // Request headers
+      assert.equal(
+        Object.keys(payload[0][0].meta).filter(tagName => tagName.startsWith('http.request.headers.')).length,
+        requestHeaders.length
+      )
+      requestHeaders.forEach((headerName) => {
+        assert.property(payload[0][0].meta, `http.request.headers.${headerName}`)
+      })
+
+      // Response headers
+      assert.equal(
+        Object.keys(payload[0][0].meta).filter(tagName => tagName.startsWith('http.response.headers.')).length,
+        responseHeaders.length
+      )
+      responseHeaders.forEach((headerName) => {
+        assert.property(payload[0][0].meta, `http.response.headers.${headerName}`)
+      })
+    })
+  }
+
+  describe('Basic data collection', () => {
+    startServer(false)
+
+    it('should collect event headers', async () => {
+      const expectedRequestHeaders = [
+        'user-agent',
+        'accept',
+        'host',
+        'accept-encoding'
+      ]
+
+      const expectedResponseHeaders = [
+        'content-type',
+        'content-language'
+      ]
+
+      await axios.get('/', { headers: { 'User-Agent': 'Arachni/v1' } })
+      await assertHeadersReported(expectedRequestHeaders, expectedResponseHeaders)
+    })
+  })
+
+  describe('Extended data collection', () => {
+    startServer(true)
+
+    it('should collect extended headers', async () => {
+      const expectedRequestHeaders = [
+        'user-agent',
+        'accept',
+        'host',
+        'accept-encoding',
+        'connection'
+      ]
+
+      // DD_APPSEC_MAX_COLLECTED_HEADERS is set to 25, so it is expected to collect
+      // 22 x-datadog-res-XX headers + x-powered-by, content-type and content-language, for a total of 25.
+      const expectedResponseHeaders = [
+        ...Array.from({ length: 22 }, (_, i) =>
+          `x-datadog-res-${i}`
+        ),
+        'x-powered-by',
+        'content-type',
+        'content-language'
+      ]
+
+      await axios.get('/', { headers: { 'User-Agent': 'Arachni/v1' } })
+      await assertHeadersReported(expectedRequestHeaders, expectedResponseHeaders)
+    })
+  })
+})

integration-tests/appsec/data-collection/data-collection-rules.json

@@ -0,0 +1,32 @@
+{
+  "version": "2.1",
+  "metadata": {
+    "rules_version": "1.2.6"
+  },
+  "rules": [
+    {
+      "id": "arachni_rule",
+      "name": "Arachni",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              }
+            ],
+            "regex": "^Arachni\\/v"
+          },
+          "operator": "match_regex"
+        }
+      ]
+    }
+  ]
+}

integration-tests/appsec/data-collection/index.js

@@ -0,0 +1,27 @@
+'use strict'
+const tracer = require('dd-trace')
+tracer.init({
+  flushInterval: 0
+})
+
+const express = require('express')
+
+const app = express()
+const port = process.env.APP_PORT || 3000
+
+app.get('/', (req, res) => {
+  // Content headers
+  res.set('content-type', 'text/plain')
+  res.set('content-language', 'en')
+
+  // Custom headers
+  for (let i = 0; i < 25; i++) {
+    res.set(`x-datadog-res-${i}`, `ext-res-${i}`)
+  }
+
+  res.end('end')
+})
+
+app.listen(port, () => {
+  process.send({ port })
+})

integration-tests/appsec/index.spec.js

@@ -4,6 +4,7 @@ const getPort = require('get-port')
 const path = require('path')
 const Axios = require('axios')
 const { assert } = require('chai')
+const msgpack = require('@msgpack/msgpack')
 const { createSandbox, FakeAgent, spawnProc } = require('../helpers')
 
 describe('RASP', () => {
@@ -27,7 +28,7 @@ describe('RASP', () => {
     await sandbox.remove()
   })
 
-  function startServer (abortOnUncaughtException) {
+  function startServer (abortOnUncaughtException, collectRequestBody = false) {
     beforeEach(async () => {
       let execArgv = process.execArgv
       if (abortOnUncaughtException) {
@@ -42,7 +43,8 @@ describe('RASP', () => {
           APP_PORT: appPort,
           DD_APPSEC_ENABLED: true,
           DD_APPSEC_RASP_ENABLED: true,
-          DD_APPSEC_RULES: path.join(cwd, 'appsec/rasp/rasp_rules.json')
+          DD_APPSEC_RULES: path.join(cwd, 'appsec/rasp/rasp_rules.json'),
+          DD_APPSEC_RASP_COLLECT_REQUEST_BODY: collectRequestBody
         }
       }, stdOutputHandler, stdOutputHandler)
     })
@@ -60,6 +62,17 @@ describe('RASP', () => {
     })
   }
 
+  async function assertBodyReported (expectedBody, truncated) {
+    await agent.assertMessageReceived(({ headers, payload }) => {
+      assert.property(payload[0][0].meta_struct, 'http.request.body')
+      assert.deepStrictEqual(msgpack.decode(payload[0][0].meta_struct['http.request.body']), expectedBody)
+
+      if (truncated) {
+        assert.property(payload[0][0].meta, '_dd.appsec.rasp.request_body_size.exceeded')
+      }
+    })
+  }
+
   describe('--abort-on-uncaught-exception is not configured', () => {
     startServer(false)
 
@@ -339,4 +352,65 @@ describe('RASP', () => {
       })
     })
   })
+
+  describe('extended data collection', () => {
+    describe('with feature enabled', () => {
+      startServer(false, true)
+
+      it('should report body request', async () => {
+        const requestBody = { host: 'localhost/ifconfig.pro' }
+        try {
+          await axios.post('/ssrf', requestBody)
+        } catch (e) {
+          if (!e.response) {
+            throw e
+          }
+
+          await assertBodyReported(requestBody)
+        }
+      })
+
+      it('should report truncated body request', async () => {
+        const requestBody = {
+          host: 'localhost/ifconfig.pro',
+          objectWithLotsOfNodes: Object.fromEntries([...Array(300).keys()].map(i => [i, i])),
+          arr: Array(300).fill('foo')
+        }
+        try {
+          await axios.post('/ssrf', requestBody)
+        } catch (e) {
+          if (!e.response) {
+            throw e
+          }
+
+          const expectedReportedBody = {
+            host: 'localhost/ifconfig.pro',
+            objectWithLotsOfNodes: Object.fromEntries([...Array(256).keys()].map(i => [i, i])),
+            arr: Array(256).fill('foo')
+          }
+
+          await assertBodyReported(expectedReportedBody, true)
+        }
+      })
+    })
+
+    describe('with feature disabled', () => {
+      startServer(false, false)
+
+      it('should not report body request', async () => {
+        const requestBody = { host: 'localhost/ifconfig.pro' }
+        try {
+          await axios.post('/ssrf', requestBody)
+        } catch (e) {
+          if (!e.response) {
+            throw e
+          }
+
+          await agent.assertMessageReceived(({ headers, payload }) => {
+            assert.notProperty(payload[0][0].meta_struct, 'http.request.body')
+          })
+        }
+      })
+    })
+  })
 })

integration-tests/appsec/rasp/index.js

@@ -42,6 +42,8 @@ function httpGetPromise (host) {
   })
 }
 
+app.use(express.json())
+
 app.get('/crash', () => {
   process.nextTick(() => {
     throw new Error('Crash')
@@ -205,6 +207,11 @@ app.get('/ssrf/http/unhandled-promise', (req, res) => {
     .then(() => res.end('end'))
 })
 
+app.post('/ssrf', (req, res) => {
+  axios.get(`https://${req.body.host}`)
+    .then(() => res.end('end'))
+})
+
 app.listen(port, () => {
   process.send({ port })
 })

packages/dd-trace/src/appsec/index.js

@@ -58,7 +58,7 @@ function enable (_config) {
 
     remoteConfig.enableWafUpdate(_config.appsec)
 
-    Reporter.setRateLimit(_config.appsec.rateLimit)
+    Reporter.init(_config.appsec)
 
     apiSecuritySampler.configure(_config)