diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 999646eac74..89758e6be51 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -377,6 +377,7 @@ /packages/prisma_access @elastic/security-service-integrations /packages/prisma_cloud @elastic/security-service-integrations /packages/problemchild @elastic/ml-ui @elastic/sec-applied-ml +/packages/projectdiscovery_cloud @elastic/security-service-integrations /packages/profilingmetrics_otel @elastic/ingest-otel-data /packages/prometheus @elastic/obs-infraobs-integrations /packages/prometheus/data_stream/collector @elastic/obs-infraobs-integrations diff --git a/packages/projectdiscovery_cloud/.gitignore b/packages/projectdiscovery_cloud/.gitignore new file mode 100644 index 00000000000..e8489f5fa5c --- /dev/null +++ b/packages/projectdiscovery_cloud/.gitignore @@ -0,0 +1,3 @@ +.manifest.yml.swp +.manifest.yml.swo +.env diff --git a/packages/projectdiscovery_cloud/LICENSE.txt b/packages/projectdiscovery_cloud/LICENSE.txt new file mode 100644 index 00000000000..809108b857f --- /dev/null +++ b/packages/projectdiscovery_cloud/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark.yml b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark.yml new file mode 100644 index 00000000000..a37831a36ca --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark.yml @@ -0,0 +1,14 @@ +--- +description: Benchmark 100000 projectdiscovery_cloud.changelogs events ingested +data_stream: + name: changelogs +corpora: + generator: + total_events: 100000 + template: + type: gotext + path: ./changelogs-benchmark/template.ndjson + config: + path: ./changelogs-benchmark/config.yml + fields: + path: ./changelogs-benchmark/fields.yml diff --git a/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/config.yml b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/config.yml new file mode 100644 index 00000000000..8639780500a --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/config.yml @@ -0,0 +1,49 @@ +fields: + - name: vuln_id + cardinality: 50000 + - name: vuln_status + enum: + - open + - closed + - reopened + - false_positive + - name: created_at + period: -24h + - name: updated_at + period: -24h + - name: target + cardinality: 10000 + - name: vuln_hash + cardinality: 50000 + - name: scan_id + cardinality: 5000 + - name: template_url + cardinality: 1000 + - name: matcher_status + enum: + - matched + - unmatched + - partial + - name: change_event + enum: + - status_changed + - severity_changed + - new_detection + - resolved + - name: event.port + range: + min: 1 + max: 65535 + - name: event.info.description + cardinality: 5000 + - name: event.info.reference + cardinality: 5000 + - name: event.info.severity + enum: + - critical + - high + - medium + - low + - info + - name: timestamp + period: -24h diff --git a/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/fields.yml b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/fields.yml new file mode 100644 index 00000000000..3dbe2ed7af9 --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/fields.yml @@ -0,0 +1,30 @@ +- name: vuln_id + type: keyword +- name: vuln_status + type: keyword +- name: created_at + type: date +- name: updated_at + type: date +- name: target + type: keyword +- name: vuln_hash + type: keyword +- name: scan_id + type: keyword +- name: template_url + type: keyword +- name: matcher_status + type: keyword +- name: change_event + type: keyword +- name: event.port + type: integer +- name: event.info.description + type: text +- name: event.info.reference + type: keyword +- name: event.info.severity + type: keyword +- name: timestamp + type: date diff --git a/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/template.ndjson b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/template.ndjson new file mode 100644 index 00000000000..6c3fcdcf52f --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/benchmark/rally/changelogs-benchmark/template.ndjson @@ -0,0 +1,48 @@ +{{- $vuln_id := generate "vuln_id" }} +{{- $vuln_status := generate "vuln_status" }} +{{- $created_at := generate "created_at" | date "2006-01-02T15:04:05.000000Z" }} +{{- $updated_at := generate "updated_at" | date "2006-01-02T15:04:05.000000Z" }} +{{- $target := generate "target" }} +{{- $vuln_hash := generate "vuln_hash" }} +{{- $scan_id := generate "scan_id" }} +{{- $template_url := generate "template_url" }} +{{- $matcher_status := generate "matcher_status" }} +{{- $change_event := generate "change_event" }} +{{- $port := generate "event.port" }} +{{- $description := generate "event.info.description" }} +{{- $reference := generate "event.info.reference" }} +{{- $severity := generate "event.info.severity" }} +{{- $timestamp := generate "timestamp" }} +{ + "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}", + "agent": { + "ephemeral_id": "{{ $vuln_hash }}", + "id": "benchmark-agent", + "name": "benchmark-fleet-agent", + "type": "filebeat", + "version": "9.2.0" + }, + "data_stream": { + "dataset": "projectdiscovery_cloud.changelogs", + "namespace": "benchmark", + "type": "logs" + }, + "elastic_agent": { + "id": "benchmark-agent", + "snapshot": false, + "version": "9.2.0" + }, + "event": { + "dataset": "projectdiscovery_cloud.changelogs" + }, + "input": { + "type": "cel" + }, + "message": "{\"vuln_id\":\"{{ $vuln_id }}\",\"vuln_status\":\"{{ $vuln_status }}\",\"created_at\":\"{{ $created_at }}\",\"updated_at\":\"{{ $updated_at }}\",\"target\":\"{{ $target }}\",\"vuln_hash\":\"{{ $vuln_hash }}\",\"scan_id\":\"{{ $scan_id }}\",\"template_url\":\"https://templates.nuclei.sh/{{ $template_url }}\",\"matcher_status\":\"{{ $matcher_status }}\",\"change_event\":\"{{ $change_event }}\",\"event\":{\"info\":{\"description\":\"{{ $description }}\",\"reference\":[\"https://nvd.nist.gov/vuln/detail/{{ $reference }}\"],\"severity\":\"{{ $severity }}\"},\"port\":{{ $port }}}}", + "tags": [ + "forwarded", + "projectdiscovery-cloud", + "vulnerability", + "changelogs" + ] +} diff --git a/packages/projectdiscovery_cloud/_dev/build/build.yml b/packages/projectdiscovery_cloud/_dev/build/build.yml new file mode 100644 index 00000000000..54c17b3bd7f --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: "git@v9.2.0" diff --git a/packages/projectdiscovery_cloud/_dev/build/docs/README.md b/packages/projectdiscovery_cloud/_dev/build/docs/README.md new file mode 100644 index 00000000000..10035138b5a --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/build/docs/README.md @@ -0,0 +1,160 @@ +# ProjectDiscovery Cloud Integration + +[![Version](https://img.shields.io/badge/version-0.1.1-blue.svg)](https://github.com/elastic/integrations) +[![License](https://img.shields.io/badge/license-Elastic--2.0-green.svg)](LICENSE.txt) + +The ProjectDiscovery Cloud integration allows you to monitor and ingest vulnerability changelog events from [ProjectDiscovery Cloud](https://cloud.projectdiscovery.io) into Elasticsearch. ProjectDiscovery Cloud is an External Attack Surface Management (EASM) platform that continuously scans and monitors your external attack surface for security vulnerabilities using the Nuclei scanner. + +## Overview + +This integration collects vulnerability changelog events from the ProjectDiscovery Cloud API and ingests them into Elasticsearch with proper ECS (Elastic Common Schema) field mappings. This enables you to: + +- **Monitor** vulnerability status changes in real-time +- **Track** security posture across your attack surface +- **Visualize** vulnerability trends in Kibana +- **Alert** on critical vulnerability changes +- **Investigate** security incidents with full context + +For example, if you want to track when SSL/TLS vulnerabilities are detected or fixed on your infrastructure, this integration will automatically collect those events from ProjectDiscovery Cloud, normalize them to ECS format, and make them searchable in Elasticsearch. + +## Features + +- ✅ **Real-time ingestion** via ProjectDiscovery Cloud API +- ✅ **ECS-compliant** field mappings for standardized security data +- ✅ **Offset-based pagination** for reliable data collection +- ✅ **Vendor namespace preservation** for ProjectDiscovery-specific fields +- ✅ **Configurable collection intervals** and batch sizes +- ✅ **HTTP request tracing** for debugging +- ✅ **Comprehensive testing** with pipeline and system tests + +## Data Streams + +The ProjectDiscovery Cloud integration collects one type of data stream: **logs**. + +### Vulnerability Changelogs (`changelogs`) + +**Logs** help you keep a record of vulnerability status changes detected by ProjectDiscovery Cloud's Nuclei scanner. + +The `changelogs` data stream collects changelog events including: +- Vulnerability status transitions (open → fixed, fixed → reopened, etc.) +- Vulnerability metadata (ID, severity, description) +- Scanner information (Nuclei template details) +- Target information (host, IP, port) +- Change event details (from/to values) + +See more details in the [Logs Reference](#logs-reference) section. + +## Requirements + +### Elastic Stack + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + +**Minimum versions:** +- Kibana: `^9.1.0` +- Elasticsearch: Compatible with Kibana version +- Elastic subscription: `basic` + +### ProjectDiscovery Cloud + +- Active ProjectDiscovery Cloud account +- API Key with read access to vulnerability changelogs +- Team ID associated with your ProjectDiscovery Cloud account + +### Permissions + +The API credentials must have permissions to: +- Read vulnerability changelog events (`GET /v1/scans/vuln/changelogs`) + +## Setup + +### Step 1: Obtain ProjectDiscovery Cloud Credentials + +1. Log in to [ProjectDiscovery Cloud](https://cloud.projectdiscovery.io) +2. Navigate to **Settings** → **API Keys** +3. Create a new API key or use an existing one +4. Note your **Team ID** (found in your account settings) + +### Step 2: Install the Integration + +1. In Kibana, navigate to **Management** → **Integrations** +2. Search for "ProjectDiscovery Cloud" +3. Click **Add ProjectDiscovery Cloud** + +### Step 3: Configure the Integration + +Configure the following settings: + +| Setting | Description | Default | Required | +|---------|-------------|---------|----------| +| **API Base URL** | ProjectDiscovery Cloud API endpoint | `https://api.projectdiscovery.io` | Yes | +| **API Key** | Your ProjectDiscovery Cloud API key | - | Yes | +| **Team ID** | Your ProjectDiscovery Cloud team ID | - | Yes | +| **Collection Interval** | How often to poll for new events | `5m` | Yes | +| **Batch Size** | Number of events per API request | `100` | Yes | +| **Time Window** | Filter events by time (e.g., `24h`, `7d`) | - | No | +| **HTTP Client Timeout** | Timeout for HTTP requests | `30s` | No | +| **Enable Request Tracer** | Enable detailed HTTP logging | `false` | No | + +### Step 4: Deploy and Verify + +1. Click **Save and Continue** +2. Add the integration to an agent policy +3. Deploy the agent policy to your Elastic Agent +4. Verify data ingestion in **Discover** by searching for `data_stream.dataset: "projectdiscovery_cloud.changelogs"` + +For step-by-step instructions, see the [Getting started with Elastic Observability](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + +## Configuration Details + +### API Authentication + +The integration authenticates with ProjectDiscovery Cloud using two HTTP headers: +- `X-API-Key`: Your API key +- `X-Team-Id`: Your team ID + +Both headers are automatically set by the integration based on your configuration. + +### Pagination and Incremental Ingestion + +The `changelogs` data stream uses **offset-based incremental ingestion** to efficiently collect only new events: +- Initial request starts at `offset=0` +- Each polling cycle fetches events since the last recorded offset +- The cursor tracks progress between polling intervals +- Subsequent requests increment `offset` by the number of events received +- Continues until no more events are returned + +This incremental approach allows for aggressive polling intervals (e.g., 5 minutes) without re-ingesting duplicate data. The optional `time_window` filter further limits the time range of changelogs fetched. + +### HTTP Request Tracing + +Enable request tracing for debugging: +1. Set **Enable Request Tracer** to `true` +2. Trace files are written to: `../../logs/cel/http-request-trace-*.ndjson` +3. Up to 5 backup files are kept + +**⚠️ Security Warning:** Request tracing logs may contain sensitive data including API keys. Only enable for debugging and disable when done. + +## Logs Reference + +### Changelogs Data Stream + +The `changelogs` data stream provides changelog events from ProjectDiscovery Cloud's vulnerability scanner. + +#### Event Types + +- **event.kind**: `event` +- **event.category**: `["vulnerability"]` +- **event.type**: `["info"]` + +#### Exported Fields + +{{ fields "changelogs" }} + +## License + +This integration is licensed under the Elastic License 2.0. + +--- + +**Version:** 0.1.1 diff --git a/packages/projectdiscovery_cloud/_dev/deploy/docker/docker-compose.yml b/packages/projectdiscovery_cloud/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..e1195d393a5 --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,28 @@ +version: "2.3" +services: + projectdiscovery-changelogs: + image: docker.elastic.co/observability/stream:v0.18.0 + hostname: projectdiscovery-changelogs + ports: + - 8090 + volumes: + - ./files:/files:ro + environment: + PORT: '8090' + command: + - http-server + - --addr=:8090 + - --config=/files/config-changelogs.yml + projectdiscovery-export: + image: docker.elastic.co/observability/stream:v0.18.0 + hostname: projectdiscovery-export + ports: + - 8091 + volumes: + - ./files:/files:ro + environment: + PORT: '8091' + command: + - http-server + - --addr=:8091 + - --config=/files/config-export.yml diff --git a/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-changelogs.yml b/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-changelogs.yml new file mode 100644 index 00000000000..98f6e81a5b2 --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-changelogs.yml @@ -0,0 +1,87 @@ +rules: + - path: /v1/scans/vuln/changelogs + methods: ['GET'] + query_params: + limit: 2 + offset: 0 + event_type: vuln_status + sort_desc: created_at + responses: + - status_code: 200 + headers: + Content-Type: + - application/json + body: |- + {{ minify_json ` + { + "data": [ + { + "created_at": "2025-08-26T03:41:55.288025Z", + "updated_at": "2025-08-26T03:41:56.388431Z", + "event": { "port": 443 }, + "target": "example.com", + "vuln_id": "abc123", + "vuln_status": "open" + }, + { + "created_at": "2025-08-26T03:41:56.388431Z", + "updated_at": "2025-08-26T03:42:06.000000Z", + "event": { "port": 80 }, + "target": "def.example.com", + "vuln_id": "def456", + "vuln_status": "fixed" + } + ] + } + `}} + - path: /v1/scans/vuln/changelogs + methods: ['GET'] + query_params: + limit: 2 + offset: 2 + event_type: vuln_status + sort_desc: created_at + responses: + - status_code: 200 + headers: + Content-Type: + - application/json + body: |- + {{ minify_json ` + { + "data": [ + { + "created_at": "2025-08-26T03:42:07.000000Z", + "updated_at": "2025-08-26T03:42:08.000000Z", + "event": { "port": 8080 }, + "target": "ghi.example.com", + "vuln_id": "ghi789", + "vuln_status": "open" + }, + { + "created_at": "2025-08-26T03:42:09.000000Z", + "updated_at": "2025-08-26T03:42:10.000000Z", + "event": { "port": 22 }, + "target": "jkl.example.com", + "vuln_id": "jkl012", + "vuln_status": "fixed" + } + ] + } + `}} + - path: /v1/scans/vuln/changelogs + methods: ['GET'] + query_params: + limit: 2 + offset: 4 + event_type: vuln_status + sort_desc: created_at + responses: + - status_code: 200 + headers: + Content-Type: + - application/json + body: |- + {{ minify_json ` + { "data": [] } + `}} diff --git a/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-export.yml b/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-export.yml new file mode 100644 index 00000000000..82c2f57d82d --- /dev/null +++ b/packages/projectdiscovery_cloud/_dev/deploy/docker/files/config-export.yml @@ -0,0 +1,50 @@ +rules: + # First request returns one vulnerability + - path: /v1/scans/results/export + methods: ['POST'] + query_params: + type: json + responses: + - status_code: 200 + headers: + Content-Type: + - application/json + body: |- + {{ minify_json ` + [ + { + "id": "d340kdgviq0c990u7e5g", + "scan_id": "d2o2sn9qccvs73eecu80", + "vuln_status": "open", + "template_url": "https://cloud.projectdiscovery.io/public/CVE-2025-22457", + "template_name": "Ivanti Connect Secure - Stack-based Buffer Overflow", + "template_id": "CVE-2025-22457", + "vuln_hash": "e5b96ed6bf5c5b2c17e17c90773b3188", + "matcher_name": null, + "matcher_status": true, + "created_at": "2025-09-15T12:44:45.935924Z", + "updated_at": "2025-09-17T13:00:42.413800Z", + "severity": "critical", + "host": "test.elastic.dev", + "matched_at": "https://test.elastic.dev", + "tags": ["cve", "cve2025", "ivanti", "intrusive", "kev"], + "description": "Ivanti Connect Secure versions prior to 22.7R2.6 are vulnerable to a stack-based buffer overflow.", + "category": "buffer_overflow", + "request": null, + "response": null, + "remediation": "Update to the latest versions as per Ivanti's security advisory.", + "reference": [ + "https://labs.watchtowr.com/cve-2025-22457", + "https://www.cvedetails.com/cve/cve-2025-22457", + "https://github.com/securekomodo/cve-2025-22457" + ] + } + ] + `}} + # Subsequent requests return empty array + - status_code: 200 + headers: + Content-Type: + - application/json + body: |- + {{ minify_json `[]` }} diff --git a/packages/projectdiscovery_cloud/changelog.yml b/packages/projectdiscovery_cloud/changelog.yml new file mode 100644 index 00000000000..1e742c3f92c --- /dev/null +++ b/packages/projectdiscovery_cloud/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.1" + changes: + - description: Initial release of ProjectDiscovery Cloud integration with ECS-compliant vulnerability and changelog data streams + type: enhancement + link: https://github.com/elastic/integrations/pull/15760 diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log new file mode 100644 index 00000000000..022bfea7502 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log @@ -0,0 +1 @@ +{"vuln_id":"def456","vuln_status":"fixed","created_at":"2025-08-27T10:15:30.123456Z","updated_at":"2025-08-27T10:15:31.234567Z","target":"45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it","event":{"info":{"description":"Malformed target delimiter issue","reference":["https://example.com/issue/def456"],"severity":"medium"},"port":8080}} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-config.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-config.yml new file mode 100644 index 00000000000..74df6959753 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-config.yml @@ -0,0 +1,5 @@ +fields: + tags: + - preserve_duplicate_custom_fields + - sanitize_target + input.type: cel diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-expected.json b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-expected.json new file mode 100644 index 00000000000..98819a13875 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs-sanitized.log-expected.json @@ -0,0 +1,68 @@ +{ + "expected": [ + { + "@timestamp": "2025-08-27T10:15:30.123Z", + "destination": { + "address": "45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it", + "domain": "45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it" + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "vulnerability" + ], + "dataset": "projectdiscovery_cloud.changelogs", + "kind": "event", + "module": "projectdiscovery_cloud", + "original": "{\"vuln_id\":\"def456\",\"vuln_status\":\"fixed\",\"created_at\":\"2025-08-27T10:15:30.123456Z\",\"updated_at\":\"2025-08-27T10:15:31.234567Z\",\"target\":\"45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it\",\"event\":{\"info\":{\"description\":\"Malformed target delimiter issue\",\"reference\":[\"https://example.com/issue/def456\"],\"severity\":\"medium\"},\"port\":8080}}", + "type": [ + "info" + ] + }, + "input": { + "type": "cel" + }, + "message": "ProjectDiscovery changelog: def456 → fixed", + "projectdiscovery": { + "created_at": "2025-08-27T10:15:30.123456Z", + "event": { + "info": { + "description": "Malformed target delimiter issue", + "reference": [ + "https://example.com/issue/def456" + ], + "severity": "medium" + }, + "port": 8080 + }, + "target": "-mdns..www...dns.stokke..mdns.unix..mdns.ras.cvswww.fix10erts.fet.cqmwww.mdns.scholtenbaijings..mdns.in48ming.extrude.studiodns.thomasfeichtner..dns.comeurope-q..ukdns.nos.acccoryo.oripgitww.orgdns.leaydn..mdns.nutella..mdns.dyson.it", + "updated_at": "2025-08-27T10:15:31.234567Z", + "vuln_status": "fixed" + }, + "server": { + "port": 8080 + }, + "tags": [ + "preserve_duplicate_custom_fields", + "sanitize_target", + "projectdiscovery-cloud", + "vulnerability", + "changelogs", + "forwarded" + ], + "vulnerability": { + "description": "Malformed target delimiter issue", + "id": "def456", + "reference": [ + "https://example.com/issue/def456" + ], + "scanner": { + "vendor": "ProjectDiscovery" + }, + "severity": "medium" + } + } + ] +} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log new file mode 100644 index 00000000000..4b61256e8fb --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log @@ -0,0 +1,2 @@ +{"vuln_id":"abc123","vuln_status":"open","created_at":"2025-08-26T03:41:55.288025Z","updated_at":"2025-08-26T03:41:56.388431Z","target":"example.com","event":{"info":{"description":"Example vulnerability description","reference":["https://example.com/vuln/abc123"],"severity":"low"},"port":443}} +{"vuln_id":"def456","vuln_status":"fixed","created_at":"2025-08-27T10:15:30.123456Z","updated_at":"2025-08-27T10:15:31.234567Z","target":"45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it","event":{"info":{"description":"Malformed target delimiter issue","reference":["https://example.com/issue/def456"],"severity":"medium"},"port":8080}} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log-expected.json b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log-expected.json new file mode 100644 index 00000000000..1ebd1201d06 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-changelogs.log-expected.json @@ -0,0 +1,130 @@ +{ + "expected": [ + { + "@timestamp": "2025-08-26T03:41:55.288Z", + "destination": { + "address": "example.com", + "domain": "example.com" + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "vulnerability" + ], + "dataset": "projectdiscovery_cloud.changelogs", + "kind": "event", + "module": "projectdiscovery_cloud", + "original": "{\"vuln_id\":\"abc123\",\"vuln_status\":\"open\",\"created_at\":\"2025-08-26T03:41:55.288025Z\",\"updated_at\":\"2025-08-26T03:41:56.388431Z\",\"target\":\"example.com\",\"event\":{\"info\":{\"description\":\"Example vulnerability description\",\"reference\":[\"https://example.com/vuln/abc123\"],\"severity\":\"low\"},\"port\":443}}", + "type": [ + "info" + ] + }, + "input": { + "type": "cel" + }, + "message": "ProjectDiscovery changelog: abc123 → open", + "projectdiscovery": { + "created_at": "2025-08-26T03:41:55.288025Z", + "event": { + "info": { + "description": "Example vulnerability description", + "reference": [ + "https://example.com/vuln/abc123" + ], + "severity": "low" + }, + "port": 443 + }, + "target": "example.com", + "updated_at": "2025-08-26T03:41:56.388431Z", + "vuln_status": "open" + }, + "server": { + "port": 443 + }, + "tags": [ + "preserve_duplicate_custom_fields", + "projectdiscovery-cloud", + "vulnerability", + "changelogs", + "forwarded" + ], + "vulnerability": { + "description": "Example vulnerability description", + "id": "abc123", + "reference": [ + "https://example.com/vuln/abc123" + ], + "scanner": { + "vendor": "ProjectDiscovery" + }, + "severity": "low" + } + }, + { + "@timestamp": "2025-08-27T10:15:30.123Z", + "destination": { + "address": "45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it", + "domain": "45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it" + }, + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "vulnerability" + ], + "dataset": "projectdiscovery_cloud.changelogs", + "kind": "event", + "module": "projectdiscovery_cloud", + "original": "{\"vuln_id\":\"def456\",\"vuln_status\":\"fixed\",\"created_at\":\"2025-08-27T10:15:30.123456Z\",\"updated_at\":\"2025-08-27T10:15:31.234567Z\",\"target\":\"45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it\",\"event\":{\"info\":{\"description\":\"Malformed target delimiter issue\",\"reference\":[\"https://example.com/issue/def456\"],\"severity\":\"medium\"},\"port\":8080}}", + "type": [ + "info" + ] + }, + "input": { + "type": "cel" + }, + "message": "ProjectDiscovery changelog: def456 → fixed", + "projectdiscovery": { + "created_at": "2025-08-27T10:15:30.123456Z", + "event": { + "info": { + "description": "Malformed target delimiter issue", + "reference": [ + "https://example.com/issue/def456" + ], + "severity": "medium" + }, + "port": 8080 + }, + "target": "45mdns.48aawww.48.48dns.stokke.48mdns.unix.48mdns.ras.cvswww.fix10erts.fet.cqmwww48mdns.scholtenbaijings.48mdns.in48ming.extrude.studiodns.thomasfeichtner.48dns.comeurope-q.48.ukdns.nos.acccoryo.oripgitww.orgdns.leaydn.48mdns.nutella.48mdns.dyson.it", + "updated_at": "2025-08-27T10:15:31.234567Z", + "vuln_status": "fixed" + }, + "server": { + "port": 8080 + }, + "tags": [ + "preserve_duplicate_custom_fields", + "projectdiscovery-cloud", + "vulnerability", + "changelogs", + "forwarded" + ], + "vulnerability": { + "description": "Malformed target delimiter issue", + "id": "def456", + "reference": [ + "https://example.com/issue/def456" + ], + "scanner": { + "vendor": "ProjectDiscovery" + }, + "severity": "medium" + } + } + ] +} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-common-config.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..b52222a2c1e --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + tags: + - preserve_duplicate_custom_fields + input.type: cel diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.expected b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.expected new file mode 100644 index 00000000000..29d94481f36 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.expected @@ -0,0 +1,121 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: projectdiscovery_cloud + name: test-default-projectdiscovery_cloud + streams: + - config_version: 2 + data_stream: + dataset: projectdiscovery_cloud.changelogs + interval: 24h + program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + state.?cursor.?offset.orValue(int(state.offset)).as(current_offset, + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/vuln/changelogs", + "query": { + "limit": [string(state.batch_size)], + "offset": [string(current_offset)], + "event_type": ["vuln_status"], + "sort_desc": ["created_at"], + ?"time": (state.time_window != "") ? + optional.of([string(state.time_window)]) + : + optional.none() + } + }.as(req, + request( + "GET", + req.base + "?" + req.query.format_query() + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)] + } + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : {} + }.as(vars, + { + "items": (vars.status_ok && ("data" in vars.body) && vars.body["data"] != null) ? vars.body["data"] : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": { "offset": int(current_offset) + x.items.size() }, + "want_more": vars.status_ok && x.items.size() == int(state.batch_size), + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "batch_size": state.batch_size, + "time_window": state.time_window + } + ) + ) + ) + ) + ) + ) + redact: + fields: + - api_key + - team_id + resource.ssl: null + resource.timeout: 30s + resource.tracer: + enabled: false + filename: ../../logs/cel/http-request-trace-*.ndjson + maxbackups: 5 + resource.url: https://api.projectdiscovery.io + state: + api_key: ${SECRET_0} + batch_size: 100 + offset: 0 + team_id: default_team + time_window: 7d + url: https://api.projectdiscovery.io + tags: + - projectdiscovery-cloud + - vulnerability + - changelogs + - forwarded + type: cel + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-projectdiscovery_cloud.changelogs-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.yml new file mode 100644 index 00000000000..c26ab14f650 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-default.yml @@ -0,0 +1,13 @@ +vars: + base_url: https://api.projectdiscovery.io + api_key: default_secret + team_id: default_team +data_stream: + vars: + interval: 24h + batch_size: 100 + http_client_timeout: 30s + time_window: 7d + enable_request_tracer: false + preserve_original_event: true + preserve_duplicate_custom_fields: false diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.expected b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.expected new file mode 100644 index 00000000000..8ec4d7c5af2 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.expected @@ -0,0 +1,121 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: projectdiscovery_cloud + name: test-traced-projectdiscovery_cloud + streams: + - config_version: 2 + data_stream: + dataset: projectdiscovery_cloud.changelogs + interval: 24h + program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + state.?cursor.?offset.orValue(int(state.offset)).as(current_offset, + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/vuln/changelogs", + "query": { + "limit": [string(state.batch_size)], + "offset": [string(current_offset)], + "event_type": ["vuln_status"], + "sort_desc": ["created_at"], + ?"time": (state.time_window != "") ? + optional.of([string(state.time_window)]) + : + optional.none() + } + }.as(req, + request( + "GET", + req.base + "?" + req.query.format_query() + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)] + } + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : {} + }.as(vars, + { + "items": (vars.status_ok && ("data" in vars.body) && vars.body["data"] != null) ? vars.body["data"] : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": { "offset": int(current_offset) + x.items.size() }, + "want_more": vars.status_ok && x.items.size() == int(state.batch_size), + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "batch_size": state.batch_size, + "time_window": state.time_window + } + ) + ) + ) + ) + ) + ) + redact: + fields: + - api_key + - team_id + resource.ssl: null + resource.timeout: 10s + resource.tracer: + enabled: true + filename: ../../logs/cel/http-request-trace-*.ndjson + maxbackups: 5 + resource.url: https://api.projectdiscovery.io + state: + api_key: ${SECRET_0} + batch_size: 50 + offset: 0 + team_id: traced_team + time_window: 24h + url: https://api.projectdiscovery.io + tags: + - projectdiscovery-cloud + - vulnerability + - changelogs + - forwarded + type: cel + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-projectdiscovery_cloud.changelogs-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.yml new file mode 100644 index 00000000000..78a3cf68129 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/policy/test-traced.yml @@ -0,0 +1,13 @@ +vars: + base_url: https://api.projectdiscovery.io + api_key: traced_secret + team_id: traced_team +data_stream: + vars: + interval: 24h + batch_size: 50 + http_client_timeout: 10s + time_window: 24h + enable_request_tracer: true + preserve_original_event: true + preserve_duplicate_custom_fields: false diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-default-config.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..d039851dc56 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-default-config.yml @@ -0,0 +1,17 @@ +service: projectdiscovery-changelogs +vars: + base_url: http://{{Hostname}}:{{Port}} + api_key: "test_api_key" + team_id: "test_team" +input: cel +data_stream: + vars: + interval: 1s + batch_size: 2 + time_window: "" + http_client_timeout: 5s + enable_request_tracer: false + preserve_original_event: true + preserve_duplicate_custom_fields: false +assert: + hit_count: 4 diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-traced-config.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-traced-config.yml new file mode 100644 index 00000000000..a3778e7fbfc --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/_dev/test/system/test-traced-config.yml @@ -0,0 +1,17 @@ +service: projectdiscovery-changelogs +vars: + base_url: http://{{Hostname}}:{{Port}} + api_key: "test_api_key" + team_id: "test_team" +input: cel +data_stream: + vars: + interval: 1s + batch_size: 2 + time_window: "" + http_client_timeout: 5s + enable_request_tracer: true + preserve_original_event: true + preserve_duplicate_custom_fields: false +assert: + hit_count: 4 diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/agent/stream/cel.yml.hbs b/packages/projectdiscovery_cloud/data_stream/changelogs/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..ad0bb47c718 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/agent/stream/cel.yml.hbs @@ -0,0 +1,105 @@ +config_version: 2 +interval: {{interval}} + +# HTTP client settings (not visible inside CEL) +resource.url: {{base_url}} +resource.timeout: {{http_client_timeout}} +{{#if ssl}} +resource.ssl: {{ssl}} +{{/if}} + +# NEW — drive request tracing from var +resource.tracer: + enabled: {{enable_request_tracer}} + filename: "../../logs/cel/http-request-trace-*.ndjson" + maxbackups: 5 + +# Redact these keys in tracer logs by name +redact: + fields: + - api_key + - team_id + +# Everything the CEL program needs must be under `state` +state: + url: {{base_url}} + api_key: "{{api_key}}" + team_id: "{{team_id}}" + offset: 0 + batch_size: {{batch_size}} + # force string; when unset this becomes "" + time_window: "{{time_window}}" + +program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + state.?cursor.?offset.orValue(int(state.offset)).as(current_offset, + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/vuln/changelogs", + "query": { + "limit": [string(state.batch_size)], + "offset": [string(current_offset)], + "event_type": ["vuln_status"], + "sort_desc": ["created_at"], + ?"time": (state.time_window != "") ? + optional.of([string(state.time_window)]) + : + optional.none() + } + }.as(req, + request( + "GET", + req.base + "?" + req.query.format_query() + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)] + } + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : {} + }.as(vars, + { + "items": (vars.status_ok && ("data" in vars.body) && vars.body["data"] != null) ? vars.body["data"] : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": { "offset": int(current_offset) + x.items.size() }, + "want_more": vars.status_ok && x.items.size() == int(state.batch_size), + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "batch_size": state.batch_size, + "time_window": state.time_window + } + ) + ) + ) + ) + ) + ) +tags: + - projectdiscovery-cloud + - vulnerability + - changelogs + - forwarded diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/elasticsearch/ingest_pipeline/default.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..cbe2e441308 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,271 @@ +--- +description: ProjectDiscovery Cloud vulnerability changelogs → ECS + +processors: + - set: + field: ecs.version + value: '9.2.0' + + # Move the raw line into event.original if it's not already there + - rename: + field: message + tag: rename_message_to_event_original + target_field: event.original + ignore_missing: true + if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + if: ctx.event?.original != null + + # Parse original JSON into ctx.json + - json: + field: event.original + tag: json_event_original + target_field: json + if: ctx.event?.original != null + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + # Standard metadata (no input.type; rely on ecs@mappings) + - set: + field: event.module + value: projectdiscovery_cloud + - set: + field: event.dataset + value: projectdiscovery_cloud.changelogs + + # Default tags + - append: + field: tags + value: + - projectdiscovery-cloud + - vulnerability + - changelogs + - forwarded + allow_duplicates: false + + # ECS event framing + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - vulnerability + - set: + field: event.type + value: + - info + + # Timestamp + - date: + field: json.created_at + target_field: '@timestamp' + formats: + - ISO8601 + if: ctx.json?.created_at != null && ctx.json.created_at != '' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + # ECS vulnerability mapping + - set: + field: vulnerability.id + copy_from: json.vuln_id + ignore_empty_value: true + - set: + field: vulnerability.description + copy_from: json.event.info.description + ignore_empty_value: true + - set: + field: vulnerability.reference + copy_from: json.event.info.reference + ignore_empty_value: true + - set: + field: vulnerability.severity + copy_from: json.event.info.severity + ignore_empty_value: true + - set: + field: vulnerability.scanner.vendor + value: ProjectDiscovery + + # Ports + - convert: + field: json.event.port + type: long + target_field: json.event.port + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: server.port + copy_from: json.event.port + ignore_empty_value: true + if: ctx.json?.event?.port != null + + # Vendor namespace copies (keep vendor fidelity after ECS copies) + - rename: + field: json.target + target_field: projectdiscovery.target + ignore_missing: true + - rename: + field: json.vuln_status + target_field: projectdiscovery.vuln_status + ignore_missing: true + + # ECS destination mapping + - set: + field: destination.address + copy_from: projectdiscovery.target + ignore_empty_value: true + if: ctx.projectdiscovery?.target != null + - convert: + description: Copy destination.address to either destination.ip or destination.domain + tag: convert_destination_address + field: destination.address + target_field: destination.ip + type: ip + ignore_missing: true + on_failure: + - set: + copy_from: destination.address + field: destination.domain + override: true + + # Optional: Sanitize malformed target field (gated by tag) + # Fixes upstream API issue where delimiters are replaced with numeric ASCII codes + - script: + tag: sanitize_target_field + lang: painless + description: Cleans up malformed target field by replacing numeric artifacts with proper delimiters + if: ctx.tags != null && ctx.tags.contains('sanitize_target') && ctx.projectdiscovery?.target != null + source: | + String target = ctx.projectdiscovery.target; + // Replace common numeric artifacts with delimiters + // 45 = ASCII for '-' (hyphen) + // 48 = ASCII for '0' (zero, often appears as artifact) + target = target.replace('45mdns.', '-mdns.'); + target = target.replace('48mdns.', '.mdns.'); + target = target.replace('48dns.', '.dns.'); + target = target.replace('48aawww.', '.www.'); + target = target.replace('48.48', '.'); + target = target.replace('48.', '.'); + // Store back + ctx.projectdiscovery.target = target; + on_failure: + - append: + field: error.message + value: 'Failed to sanitize target field: {{{_ingest.on_failure_message}}}' + + - rename: + field: json.vuln_hash + target_field: projectdiscovery.vuln_hash + ignore_missing: true + - rename: + field: json.scan_id + target_field: projectdiscovery.scan_id + ignore_missing: true + - set: + field: vulnerability.report_id + copy_from: projectdiscovery.scan_id + ignore_empty_value: true + if: ctx.projectdiscovery?.scan_id != null + - rename: + field: json.template_url + target_field: projectdiscovery.template_url + ignore_missing: true + - rename: + field: json.matcher_status + target_field: projectdiscovery.matcher_status + ignore_missing: true + - rename: + field: json.created_at + target_field: projectdiscovery.created_at + ignore_missing: true + - rename: + field: json.updated_at + target_field: projectdiscovery.updated_at + ignore_missing: true + - rename: + field: json.change_event + target_field: projectdiscovery.change_event + ignore_missing: true + - rename: + field: json.event + target_field: projectdiscovery.event + ignore_missing: true + + # Final message + - set: + field: message + value: 'ProjectDiscovery changelog: {{vulnerability.id}} → {{projectdiscovery.vuln_status}}' + if: ctx.vulnerability?.id != null && ctx.projectdiscovery?.vuln_status != null + - set: + field: message + value: ProjectDiscovery vulnerability changelog event + if: ctx.vulnerability?.id == null + + # Remove duplicate custom fields if tag is not set + - remove: + field: + - projectdiscovery.created_at + - projectdiscovery.updated_at + tag: remove_custom_duplicate_fields + ignore_missing: true + if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) + + # Remove json temporary field + - remove: + field: json + tag: remove_json + ignore_missing: true + + # Drop null/empty values recursively + - script: + tag: script_to_drop_null_values + lang: painless + description: Drops null/empty values recursively. + source: |- + boolean drop(def v) { + if (v == null) return true; + if (v instanceof String) return v.length() == 0; + if (v instanceof List) { + for (int i = v.size() - 1; i >= 0; i--) { + if (drop(v.get(i))) { v.remove(i); } + } + return v.size() == 0; + } + if (v instanceof Map) { + def it = v.entrySet().iterator(); + while (it.hasNext()) { + def e = it.next(); + if (drop(e.getValue())) { it.remove(); } + } + return v.size() == 0; + } + return false; + } + drop(ctx); + + # Set pipeline error if needed + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + +on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/fields/base-fields.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/base-fields.yml new file mode 100644 index 00000000000..c931d602c71 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/base-fields.yml @@ -0,0 +1,22 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. + value: logs +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. + value: projectdiscovery_cloud.changelogs +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module. + value: projectdiscovery_cloud +- name: event.dataset + type: constant_keyword + description: Event dataset. + value: projectdiscovery_cloud.changelogs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/fields/beats.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/beats.yml new file mode 100644 index 00000000000..3c48f1f224f --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/beats.yml @@ -0,0 +1,3 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/fields/fields.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/fields.yml new file mode 100644 index 00000000000..c8ed43fee8f --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/fields/fields.yml @@ -0,0 +1,23 @@ +- name: projectdiscovery + type: group + fields: + - name: target + type: keyword + - name: vuln_hash + type: keyword + - name: scan_id + type: keyword + - name: vuln_status + type: keyword + - name: template_url + type: keyword + - name: matcher_status + type: boolean + - name: created_at + type: date + - name: updated_at + type: date + - name: change_event + type: flattened + - name: event + type: flattened diff --git a/packages/projectdiscovery_cloud/data_stream/changelogs/manifest.yml b/packages/projectdiscovery_cloud/data_stream/changelogs/manifest.yml new file mode 100644 index 00000000000..832ee0c3836 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/changelogs/manifest.yml @@ -0,0 +1,102 @@ +title: Collect Vulnerability Changelogs from ProjectDiscovery Cloud +type: logs +streams: + - input: cel + template_path: cel.yml.hbs + title: Vulnerability Changelogs + description: Collect vulnerability status change events from ProjectDiscovery Cloud. + enabled: false + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the ProjectDiscovery Cloud API. Supported units for this parameter are h/m/s. + default: 5m + multi: false + required: true + show_user: true + - name: batch_size + type: integer + title: Batch Size + description: Number of changelogs to fetch per API request. + default: 100 + multi: false + required: true + show_user: false + - name: time_window + type: text + title: Time Window + description: Optional time window filter for changelogs (e.g., 24h, 7d). Leave empty to fetch all available. + multi: false + required: false + show_user: true + - name: http_client_timeout + type: text + title: HTTP Client Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. + multi: false + required: true + show_user: false + default: 30s + - name: enable_request_tracer + type: bool + title: Enable request tracing + default: false + multi: false + required: false + show_user: false + description: >- + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. Disabling the request tracer will delete any stored traces. + - name: ssl + type: yaml + title: SSL Configuration + description: i/o timeout SSL configuration for the HTTP client. See [SSL Configuration](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html) for details. + multi: false + required: false + show_user: false + default: | + #verification_mode: full + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # ... + # -----END CERTIFICATE----- + #ca_trusted_fingerprint: "" + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - projectdiscovery-cloud + - vulnerability + - changelogs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: true + show_user: false + title: Preserve duplicate custom fields + description: Preserve projectdiscovery.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-common-config.yml b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..b52222a2c1e --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + tags: + - preserve_duplicate_custom_fields + input.type: cel diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log new file mode 100644 index 00000000000..32dde4e0ba4 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log @@ -0,0 +1 @@ +{"id":"d340kdgviq0c990u7e5g","scan_id":"d2o2sn9qccvs73eecu80","vuln_status":"open","template_url":"https://cloud.projectdiscovery.io/public/CVE-2025-22457","template_name":"Ivanti Connect Secure - Stack-based Buffer Overflow","template_id":"CVE-2025-22457","vuln_hash":"e5b96ed6bf5c5b2c17e17c90773b3188","matcher_name":null,"matcher_status":true,"created_at":"2025-09-15T12:44:45.935924Z","updated_at":"2025-09-17T13:00:42.413800Z","severity":"critical","host":"example.elastic.dev","matched_at":"https://example.elastic.dev","tags":["cve","cve2025","ivanti","intrusive","kev"],"description":"Ivanti Connect Secure versions prior to 22.7R2.6 are vulnerable to a stack-based buffer overflow.","category":"buffer_overflow","request":null,"response":null,"remediation":"Update to the latest versions as per Ivanti's security advisory.","reference":["https://labs.watchtowr.com/cve-2025-22457","https://www.cvedetails.com/cve/cve-2025-22457","https://github.com/securekomodo/cve-2025-22457"]} diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log-expected.json b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log-expected.json new file mode 100644 index 00000000000..78fe3e6f894 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/pipeline/test-export.log-expected.json @@ -0,0 +1,74 @@ +{ + "expected": [ + { + "ecs": { + "version": "9.2.0" + }, + "event": { + "category": [ + "vulnerability" + ], + "dataset": "projectdiscovery_cloud.export", + "kind": "event", + "module": "projectdiscovery_cloud", + "original": "{\"id\":\"d340kdgviq0c990u7e5g\",\"scan_id\":\"d2o2sn9qccvs73eecu80\",\"vuln_status\":\"open\",\"template_url\":\"https://cloud.projectdiscovery.io/public/CVE-2025-22457\",\"template_name\":\"Ivanti Connect Secure - Stack-based Buffer Overflow\",\"template_id\":\"CVE-2025-22457\",\"vuln_hash\":\"e5b96ed6bf5c5b2c17e17c90773b3188\",\"matcher_name\":null,\"matcher_status\":true,\"created_at\":\"2025-09-15T12:44:45.935924Z\",\"updated_at\":\"2025-09-17T13:00:42.413800Z\",\"severity\":\"critical\",\"host\":\"example.elastic.dev\",\"matched_at\":\"https://example.elastic.dev\",\"tags\":[\"cve\",\"cve2025\",\"ivanti\",\"intrusive\",\"kev\"],\"description\":\"Ivanti Connect Secure versions prior to 22.7R2.6 are vulnerable to a stack-based buffer overflow.\",\"category\":\"buffer_overflow\",\"request\":null,\"response\":null,\"remediation\":\"Update to the latest versions as per Ivanti's security advisory.\",\"reference\":[\"https://labs.watchtowr.com/cve-2025-22457\",\"https://www.cvedetails.com/cve/cve-2025-22457\",\"https://github.com/securekomodo/cve-2025-22457\"]}", + "type": [ + "info" + ] + }, + "host": { + "hostname": "example.elastic.dev" + }, + "input": { + "type": "cel" + }, + "message": "ProjectDiscovery export: d340kdgviq0c990u7e5g [critical] open", + "projectdiscovery": { + "category": "buffer_overflow", + "created_at": "2025-09-15T12:44:45.935924Z", + "matched_at": "https://example.elastic.dev", + "matcher_status": true, + "remediation": "Update to the latest versions as per Ivanti's security advisory.", + "scan_id": "d2o2sn9qccvs73eecu80", + "template_id": "CVE-2025-22457", + "template_name": "Ivanti Connect Secure - Stack-based Buffer Overflow", + "template_url": "https://cloud.projectdiscovery.io/public/CVE-2025-22457", + "updated_at": "2025-09-17T13:00:42.413800Z", + "vuln_hash": "e5b96ed6bf5c5b2c17e17c90773b3188", + "vuln_status": "open" + }, + "tags": [ + "preserve_duplicate_custom_fields", + "projectdiscovery-cloud", + "vulnerability", + "export", + "forwarded", + "cve", + "cve2025", + "ivanti", + "intrusive", + "kev" + ], + "url": { + "full": "https://example.elastic.dev" + }, + "vulnerability": { + "category": [ + "buffer_overflow" + ], + "description": "Ivanti Connect Secure versions prior to 22.7R2.6 are vulnerable to a stack-based buffer overflow.", + "id": "d340kdgviq0c990u7e5g", + "reference": [ + "https://labs.watchtowr.com/cve-2025-22457", + "https://www.cvedetails.com/cve/cve-2025-22457", + "https://github.com/securekomodo/cve-2025-22457" + ], + "report_id": "d2o2sn9qccvs73eecu80", + "scanner": { + "vendor": "ProjectDiscovery" + }, + "severity": "critical" + } + } + ] +} diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.expected b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.expected new file mode 100644 index 00000000000..5b9098b3f32 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.expected @@ -0,0 +1,131 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: projectdiscovery_cloud + name: test-default-projectdiscovery_cloud + streams: + - config_version: 2 + data_stream: + dataset: projectdiscovery_cloud.export + interval: 24h + program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + // Check if we've already completed the export + state.?cursor.?export_complete.orValue(false) ? + { + "events": [], + "want_more": false, + "cursor": state.?cursor.orValue({}), + "api_key": state.?api_key.orValue(""), + "team_id": state.?team_id.orValue(""), + "url": state.?url.orValue(""), + "vuln_status": state.?vuln_status.orValue(""), + "severity": state.?severity.orValue("") + } + : + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/results/export?type=json", + "payload": { + ?"vuln_status": (string(state.vuln_status) != "") ? + optional.of(string(state.vuln_status)) : + optional.none(), + ?"severity": (string(state.severity) != "") ? + optional.of(string(state.severity).split(",")) : + optional.none() + } + }.as(req, + request( + "POST", + req.base + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)], + "Content-Type": ["application/json"] + }, + "Body": bytes(req.payload.encode_json()) + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : [] + }.as(vars, + { + "items": (vars.status_ok && vars.body != null) ? vars.body : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": {"export_complete": true}, + "want_more": false, + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "vuln_status": state.vuln_status, + "severity": state.severity + } + ) + ) + ) + ) + ) + redact: + fields: + - api_key + - team_id + resource.ssl: null + resource.timeout: 30s + resource.tracer: + enabled: false + filename: ../../logs/cel/http-request-trace-*.ndjson + maxbackups: 5 + resource.url: https://api.projectdiscovery.io + state: + api_key: ${SECRET_0} + severity: low,medium,high,critical + team_id: default_team + url: https://api.projectdiscovery.io + vuln_status: open,triaged,fix_in_progress + tags: + - projectdiscovery-cloud + - vulnerability + - export + - forwarded + type: cel + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-projectdiscovery_cloud.export-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.yml b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.yml new file mode 100644 index 00000000000..101f6ca5148 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-default.yml @@ -0,0 +1,13 @@ +vars: + base_url: https://api.projectdiscovery.io + api_key: default_secret + team_id: default_team +data_stream: + vars: + interval: 24h + vuln_status: "open,triaged,fix_in_progress" + severity: "low,medium,high,critical" + http_client_timeout: 30s + enable_request_tracer: false + preserve_original_event: true + preserve_duplicate_custom_fields: false diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.expected b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.expected new file mode 100644 index 00000000000..4d6d7d6a92a --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.expected @@ -0,0 +1,131 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: projectdiscovery_cloud + name: test-traced-projectdiscovery_cloud + streams: + - config_version: 2 + data_stream: + dataset: projectdiscovery_cloud.export + interval: 24h + program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + // Check if we've already completed the export + state.?cursor.?export_complete.orValue(false) ? + { + "events": [], + "want_more": false, + "cursor": state.?cursor.orValue({}), + "api_key": state.?api_key.orValue(""), + "team_id": state.?team_id.orValue(""), + "url": state.?url.orValue(""), + "vuln_status": state.?vuln_status.orValue(""), + "severity": state.?severity.orValue("") + } + : + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/results/export?type=json", + "payload": { + ?"vuln_status": (string(state.vuln_status) != "") ? + optional.of(string(state.vuln_status)) : + optional.none(), + ?"severity": (string(state.severity) != "") ? + optional.of(string(state.severity).split(",")) : + optional.none() + } + }.as(req, + request( + "POST", + req.base + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)], + "Content-Type": ["application/json"] + }, + "Body": bytes(req.payload.encode_json()) + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : [] + }.as(vars, + { + "items": (vars.status_ok && vars.body != null) ? vars.body : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": {"export_complete": true}, + "want_more": false, + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "vuln_status": state.vuln_status, + "severity": state.severity + } + ) + ) + ) + ) + ) + redact: + fields: + - api_key + - team_id + resource.ssl: null + resource.timeout: 30s + resource.tracer: + enabled: true + filename: ../../logs/cel/http-request-trace-*.ndjson + maxbackups: 5 + resource.url: https://api.projectdiscovery.io + state: + api_key: ${SECRET_0} + severity: low,medium,high,critical + team_id: traced_team + url: https://api.projectdiscovery.io + vuln_status: open,triaged,fix_in_progress + tags: + - projectdiscovery-cloud + - vulnerability + - export + - forwarded + type: cel + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-projectdiscovery_cloud.export-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.yml b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.yml new file mode 100644 index 00000000000..c0a9217670d --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/policy/test-traced.yml @@ -0,0 +1,13 @@ +vars: + base_url: https://api.projectdiscovery.io + api_key: traced_secret + team_id: traced_team +data_stream: + vars: + interval: 24h + vuln_status: "open,triaged,fix_in_progress" + severity: "low,medium,high,critical" + http_client_timeout: 30s + enable_request_tracer: true + preserve_original_event: true + preserve_duplicate_custom_fields: false diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-default-config.yml b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..24f7c58c07d --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-default-config.yml @@ -0,0 +1,17 @@ +service: projectdiscovery-export +vars: + base_url: http://{{Hostname}}:{{Port}} + api_key: "test_api_key" + team_id: "test_team" +input: cel +data_stream: + vars: + interval: 1s + vuln_status: "open,triaged,fix_in_progress" + severity: "low,medium,high,critical" + http_client_timeout: 5s + enable_request_tracer: false + preserve_original_event: true + preserve_duplicate_custom_fields: false +assert: + hit_count: 1 diff --git a/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-traced-config.yml b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-traced-config.yml new file mode 100644 index 00000000000..0a7114fe6a1 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/_dev/test/system/test-traced-config.yml @@ -0,0 +1,17 @@ +service: projectdiscovery-export +vars: + base_url: http://{{Hostname}}:{{Port}} + api_key: "test_api_key" + team_id: "test_team" +input: cel +data_stream: + vars: + interval: 1s + vuln_status: "open,triaged,fix_in_progress" + severity: "low,medium,high,critical" + http_client_timeout: 5s + enable_request_tracer: true + preserve_original_event: true + preserve_duplicate_custom_fields: false +assert: + hit_count: 1 diff --git a/packages/projectdiscovery_cloud/data_stream/export/agent/stream/cel.yml.hbs b/packages/projectdiscovery_cloud/data_stream/export/agent/stream/cel.yml.hbs new file mode 100644 index 00000000000..3bcfae67db5 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/agent/stream/cel.yml.hbs @@ -0,0 +1,118 @@ +config_version: 2 +interval: {{interval}} + +# HTTP client settings +resource.url: {{base_url}} +resource.timeout: {{http_client_timeout}} +{{#if ssl}} +resource.ssl: {{ssl}} +{{/if}} + +# Request tracing for debugging +resource.tracer: + enabled: {{enable_request_tracer}} + filename: "../../logs/cel/http-request-trace-*.ndjson" + maxbackups: 5 + +# Redact sensitive fields in logs +redact: + fields: + - api_key + - team_id + +# State contains API credentials and filter configuration +state: + url: {{base_url}} + api_key: "{{api_key}}" + team_id: "{{team_id}}" + vuln_status: "{{vuln_status}}" + severity: "{{severity}}" + +# CEL program for POST /v1/scans/results/export +# This endpoint returns a snapshot of all vulnerabilities matching the filter. +# We track whether we've already fetched the export to avoid duplicate polling. +program: |- + // Early validation: fail fast if required credentials are missing + !has(state.api_key) || string(state.api_key) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "api_key is required but was not provided"} + } + : !has(state.team_id) || string(state.team_id) == "" ? + { + "events": [], + "want_more": false, + "error": {"message": "team_id is required but was not provided"} + } + : + // Check if we've already completed the export + state.?cursor.?export_complete.orValue(false) ? + { + "events": [], + "want_more": false, + "cursor": state.?cursor.orValue({}), + "api_key": state.?api_key.orValue(""), + "team_id": state.?team_id.orValue(""), + "url": state.?url.orValue(""), + "vuln_status": state.?vuln_status.orValue(""), + "severity": state.?severity.orValue("") + } + : + state.with( + { + "base": state.url.trim_right("/") + "/v1/scans/results/export?type=json", + "payload": { + ?"vuln_status": (string(state.vuln_status) != "") ? + optional.of(string(state.vuln_status)) : + optional.none(), + ?"severity": (string(state.severity) != "") ? + optional.of(string(state.severity).split(",")) : + optional.none() + } + }.as(req, + request( + "POST", + req.base + ).with( + { + "Header": { + "X-API-Key": [string(state.api_key)], + "X-Team-Id": [string(state.team_id)], + "Content-Type": ["application/json"] + }, + "Body": bytes(req.payload.encode_json()) + } + ).do_request().as(resp, + { + "status_ok": resp.StatusCode == 200, + "resp": resp, + "body": (size(resp.Body) != 0) ? bytes(resp.Body).decode_json() : [] + }.as(vars, + { + "items": (vars.status_ok && vars.body != null) ? vars.body : [] + }.as(x, + { + "events": x.items.map(e, {"message": e.encode_json()}), + "cursor": {"export_complete": true}, + "want_more": false, + ?"error": vars.status_ok ? + optional.none() : + optional.of({"message": "HTTP " + string(vars.resp.StatusCode) + " " + string(vars.resp.Body)}), + "api_key": ("api_key" in state) ? state["api_key"] : "", + "team_id": ("team_id" in state) ? state["team_id"] : "", + "url": ("url" in state) ? state["url"] : "", + "vuln_status": state.vuln_status, + "severity": state.severity + } + ) + ) + ) + ) + ) + +tags: + - projectdiscovery-cloud + - vulnerability + - export + - forwarded \ No newline at end of file diff --git a/packages/projectdiscovery_cloud/data_stream/export/elasticsearch/ingest_pipeline/default.yml b/packages/projectdiscovery_cloud/data_stream/export/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..ac2ec5cb6ba --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,255 @@ +--- +description: ProjectDiscovery Cloud vulnerability export → ECS + +processors: + - set: + field: ecs.version + value: '9.2.0' + + # Move the raw line into event.original if it's not already there + - rename: + field: message + tag: rename_message_to_event_original + target_field: event.original + ignore_missing: true + if: ctx.event?.original == null + - remove: + field: message + tag: remove_message + ignore_missing: true + if: ctx.event?.original != null + + # Parse original JSON into ctx.json + - json: + field: event.original + tag: json_event_original + target_field: json + if: ctx.event?.original != null + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + # Standard metadata + - set: + field: event.module + value: projectdiscovery_cloud + - set: + field: event.dataset + value: projectdiscovery_cloud.export + + # Default tags + - append: + field: tags + value: + - projectdiscovery-cloud + - vulnerability + - export + - forwarded + allow_duplicates: false + + # ECS event framing (export = current state snapshot) + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - vulnerability + - set: + field: event.type + value: + - info + + # @timestamp will use ingestion time for snapshot exports + # Vendor timestamps preserved in projectdiscovery.created_at and projectdiscovery.updated_at + + # ECS vulnerability mapping (copy, don't rename yet) + - set: + field: vulnerability.id + copy_from: json.id + ignore_empty_value: true + - set: + field: vulnerability.description + copy_from: json.description + ignore_empty_value: true + - set: + field: vulnerability.reference + copy_from: json.reference + ignore_empty_value: true + - set: + field: vulnerability.severity + copy_from: json.severity + ignore_empty_value: true + - append: + field: vulnerability.category + value: '{{{json.category}}}' + allow_duplicates: false + if: ctx.json?.category != null + - set: + field: vulnerability.scanner.vendor + value: ProjectDiscovery + + # ECS host/asset mapping (from 'host' field or matched_at URL) + - set: + field: host.hostname + copy_from: json.host + ignore_empty_value: true + + # URL mapping from matched_at when it looks like a URL + - set: + field: url.full + copy_from: json.matched_at + ignore_empty_value: true + if: ctx.json?.matched_at != null && ctx.json.matched_at.startsWith('http') + + # Ports (if present in export data) + - convert: + field: json.port + type: long + target_field: server.port + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + # Vendor namespace copies (keep vendor fidelity after ECS copies) + - rename: + field: json.scan_id + target_field: projectdiscovery.scan_id + ignore_missing: true + - set: + field: vulnerability.report_id + copy_from: projectdiscovery.scan_id + ignore_empty_value: true + if: ctx.projectdiscovery?.scan_id != null + - rename: + field: json.vuln_hash + target_field: projectdiscovery.vuln_hash + ignore_missing: true + - rename: + field: json.template_url + target_field: projectdiscovery.template_url + ignore_missing: true + - rename: + field: json.template_name + target_field: projectdiscovery.template_name + ignore_missing: true + - rename: + field: json.template_id + target_field: projectdiscovery.template_id + ignore_missing: true + - rename: + field: json.matcher_name + target_field: projectdiscovery.matcher_name + ignore_missing: true + - rename: + field: json.matcher_status + target_field: projectdiscovery.matcher_status + ignore_missing: true + - rename: + field: json.created_at + target_field: projectdiscovery.created_at + ignore_missing: true + - rename: + field: json.updated_at + target_field: projectdiscovery.updated_at + ignore_missing: true + - rename: + field: json.matched_at + target_field: projectdiscovery.matched_at + ignore_missing: true + - foreach: + field: json.tags + processor: + append: + field: tags + value: '{{{_ingest._value}}}' + allow_duplicates: false + ignore_missing: true + - remove: + field: json.tags + ignore_missing: true + - rename: + field: json.category + target_field: projectdiscovery.category + ignore_missing: true + - rename: + field: json.request + target_field: http.request.body.content + ignore_missing: true + - rename: + field: json.response + target_field: http.response.body.content + ignore_missing: true + - rename: + field: json.remediation + target_field: projectdiscovery.remediation + ignore_missing: true + - rename: + field: json.vuln_status + target_field: projectdiscovery.vuln_status + ignore_missing: true + + # Final message + - set: + field: message + value: 'ProjectDiscovery export: {{vulnerability.id}} [{{vulnerability.severity}}] {{projectdiscovery.vuln_status}}' + if: ctx.vulnerability?.id != null && ctx.projectdiscovery?.vuln_status != null + - set: + field: message + value: ProjectDiscovery vulnerability export event + if: ctx.vulnerability?.id == null + + # Vendor timestamps (created_at, updated_at) are preserved as they are not duplicated in ECS + # @timestamp uses ingestion time for snapshot exports + + # Remove json temporary field + - remove: + field: json + tag: remove_json + ignore_missing: true + + # Drop null/empty values recursively + - script: + tag: script_to_drop_null_values + lang: painless + description: Drops null/empty values recursively. + source: |- + boolean drop(def v) { + if (v == null) return true; + if (v instanceof String) return v.length() == 0; + if (v instanceof List) { + for (int i = v.size() - 1; i >= 0; i--) { + if (drop(v.get(i))) { v.remove(i); } + } + return v.size() == 0; + } + if (v instanceof Map) { + def it = v.entrySet().iterator(); + while (it.hasNext()) { + def e = it.next(); + if (drop(e.getValue())) { it.remove(); } + } + return v.size() == 0; + } + return false; + } + drop(ctx); + + # Set pipeline error if needed + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + +on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error diff --git a/packages/projectdiscovery_cloud/data_stream/export/fields/base-fields.yml b/packages/projectdiscovery_cloud/data_stream/export/fields/base-fields.yml new file mode 100644 index 00000000000..cc8681b93b1 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + external: ecs +- name: data_stream.dataset + external: ecs +- name: data_stream.namespace + external: ecs +- name: event.module + external: ecs +- name: event.dataset + external: ecs +- name: '@timestamp' + external: ecs diff --git a/packages/projectdiscovery_cloud/data_stream/export/fields/beats.yml b/packages/projectdiscovery_cloud/data_stream/export/fields/beats.yml new file mode 100644 index 00000000000..3c48f1f224f --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/fields/beats.yml @@ -0,0 +1,3 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. diff --git a/packages/projectdiscovery_cloud/data_stream/export/fields/fields.yml b/packages/projectdiscovery_cloud/data_stream/export/fields/fields.yml new file mode 100644 index 00000000000..f6605a012e7 --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/fields/fields.yml @@ -0,0 +1,30 @@ +- name: projectdiscovery + type: group + fields: + - name: scan_id + type: keyword + - name: vuln_hash + type: keyword + - name: template_url + type: keyword + - name: template_name + type: keyword + - name: template_id + type: keyword + - name: matcher_name + type: keyword + - name: matcher_status + type: boolean + - name: created_at + type: date + - name: updated_at + type: date + - name: matched_at + type: keyword + - name: category + type: keyword + - name: remediation + type: text + - name: vuln_status + type: keyword + description: Vulnerability status from ProjectDiscovery Cloud (e.g., open, triaged, fix_in_progress). diff --git a/packages/projectdiscovery_cloud/data_stream/export/manifest.yml b/packages/projectdiscovery_cloud/data_stream/export/manifest.yml new file mode 100644 index 00000000000..cb71c71d1dc --- /dev/null +++ b/packages/projectdiscovery_cloud/data_stream/export/manifest.yml @@ -0,0 +1,102 @@ +title: Collect Vulnerability Results from ProjectDiscovery Cloud +type: logs +streams: + - input: cel + template_path: cel.yml.hbs + title: Vulnerability Results Export + description: Collect current vulnerability state snapshots from ProjectDiscovery Cloud. + enabled: false + vars: + - name: interval + type: text + title: Interval + description: Duration between requests to the ProjectDiscovery Cloud API. Supported units for this parameter are h/m/s. + default: 24h + multi: false + required: true + show_user: true + - name: vuln_status + type: text + title: Vulnerability Status Filter + description: Comma-separated list of vulnerability statuses to export (e.g., "open,triaged,fix_in_progress"). Leave empty to export all. + multi: false + required: false + show_user: true + default: open + - name: severity + type: text + title: Severity Filter + description: Comma-separated list of severity levels to export (e.g., "low,medium,high,critical"). Leave empty to export all. + multi: false + required: false + show_user: true + - name: http_client_timeout + type: text + title: HTTP Client Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. + multi: false + required: true + show_user: false + default: 10m + - name: enable_request_tracer + type: bool + title: Enable request tracing + default: false + multi: false + required: false + show_user: false + description: >- + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. Disabling the request tracer will delete any stored traces. + - name: ssl + type: yaml + title: SSL Configuration + description: SSL configuration for the HTTP client. See [SSL Configuration](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html) for details. + multi: false + required: false + show_user: false + default: | + #verification_mode: full + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # ... + # -----END CERTIFICATE----- + #ca_trusted_fingerprint: "" + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - projectdiscovery-cloud + - vulnerability + - export + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: preserve_duplicate_custom_fields + required: true + show_user: false + title: Preserve duplicate custom fields + description: Preserve projectdiscovery.* fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. diff --git a/packages/projectdiscovery_cloud/docs/README.md b/packages/projectdiscovery_cloud/docs/README.md new file mode 100644 index 00000000000..64c12bbfb20 --- /dev/null +++ b/packages/projectdiscovery_cloud/docs/README.md @@ -0,0 +1,181 @@ +# ProjectDiscovery Cloud Integration + +[![Version](https://img.shields.io/badge/version-0.1.1-blue.svg)](https://github.com/elastic/integrations) +[![License](https://img.shields.io/badge/license-Elastic--2.0-green.svg)](LICENSE.txt) + +The ProjectDiscovery Cloud integration allows you to monitor and ingest vulnerability changelog events from [ProjectDiscovery Cloud](https://cloud.projectdiscovery.io) into Elasticsearch. ProjectDiscovery Cloud is an External Attack Surface Management (EASM) platform that continuously scans and monitors your external attack surface for security vulnerabilities using the Nuclei scanner. + +## Overview + +This integration collects vulnerability changelog events from the ProjectDiscovery Cloud API and ingests them into Elasticsearch with proper ECS (Elastic Common Schema) field mappings. This enables you to: + +- **Monitor** vulnerability status changes in real-time +- **Track** security posture across your attack surface +- **Visualize** vulnerability trends in Kibana +- **Alert** on critical vulnerability changes +- **Investigate** security incidents with full context + +For example, if you want to track when SSL/TLS vulnerabilities are detected or fixed on your infrastructure, this integration will automatically collect those events from ProjectDiscovery Cloud, normalize them to ECS format, and make them searchable in Elasticsearch. + +## Features + +- ✅ **Real-time ingestion** via ProjectDiscovery Cloud API +- ✅ **ECS-compliant** field mappings for standardized security data +- ✅ **Offset-based pagination** for reliable data collection +- ✅ **Vendor namespace preservation** for ProjectDiscovery-specific fields +- ✅ **Configurable collection intervals** and batch sizes +- ✅ **HTTP request tracing** for debugging +- ✅ **Comprehensive testing** with pipeline and system tests + +## Data Streams + +The ProjectDiscovery Cloud integration collects one type of data stream: **logs**. + +### Vulnerability Changelogs (`changelogs`) + +**Logs** help you keep a record of vulnerability status changes detected by ProjectDiscovery Cloud's Nuclei scanner. + +The `changelogs` data stream collects changelog events including: +- Vulnerability status transitions (open → fixed, fixed → reopened, etc.) +- Vulnerability metadata (ID, severity, description) +- Scanner information (Nuclei template details) +- Target information (host, IP, port) +- Change event details (from/to values) + +See more details in the [Logs Reference](#logs-reference) section. + +## Requirements + +### Elastic Stack + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + +**Minimum versions:** +- Kibana: `^9.1.0` +- Elasticsearch: Compatible with Kibana version +- Elastic subscription: `basic` + +### ProjectDiscovery Cloud + +- Active ProjectDiscovery Cloud account +- API Key with read access to vulnerability changelogs +- Team ID associated with your ProjectDiscovery Cloud account + +### Permissions + +The API credentials must have permissions to: +- Read vulnerability changelog events (`GET /v1/scans/vuln/changelogs`) + +## Setup + +### Step 1: Obtain ProjectDiscovery Cloud Credentials + +1. Log in to [ProjectDiscovery Cloud](https://cloud.projectdiscovery.io) +2. Navigate to **Settings** → **API Keys** +3. Create a new API key or use an existing one +4. Note your **Team ID** (found in your account settings) + +### Step 2: Install the Integration + +1. In Kibana, navigate to **Management** → **Integrations** +2. Search for "ProjectDiscovery Cloud" +3. Click **Add ProjectDiscovery Cloud** + +### Step 3: Configure the Integration + +Configure the following settings: + +| Setting | Description | Default | Required | +|---------|-------------|---------|----------| +| **API Base URL** | ProjectDiscovery Cloud API endpoint | `https://api.projectdiscovery.io` | Yes | +| **API Key** | Your ProjectDiscovery Cloud API key | - | Yes | +| **Team ID** | Your ProjectDiscovery Cloud team ID | - | Yes | +| **Collection Interval** | How often to poll for new events | `5m` | Yes | +| **Batch Size** | Number of events per API request | `100` | Yes | +| **Time Window** | Filter events by time (e.g., `24h`, `7d`) | - | No | +| **HTTP Client Timeout** | Timeout for HTTP requests | `30s` | No | +| **Enable Request Tracer** | Enable detailed HTTP logging | `false` | No | + +### Step 4: Deploy and Verify + +1. Click **Save and Continue** +2. Add the integration to an agent policy +3. Deploy the agent policy to your Elastic Agent +4. Verify data ingestion in **Discover** by searching for `data_stream.dataset: "projectdiscovery_cloud.changelogs"` + +For step-by-step instructions, see the [Getting started with Elastic Observability](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + +## Configuration Details + +### API Authentication + +The integration authenticates with ProjectDiscovery Cloud using two HTTP headers: +- `X-API-Key`: Your API key +- `X-Team-Id`: Your team ID + +Both headers are automatically set by the integration based on your configuration. + +### Pagination and Incremental Ingestion + +The `changelogs` data stream uses **offset-based incremental ingestion** to efficiently collect only new events: +- Initial request starts at `offset=0` +- Each polling cycle fetches events since the last recorded offset +- The cursor tracks progress between polling intervals +- Subsequent requests increment `offset` by the number of events received +- Continues until no more events are returned + +This incremental approach allows for aggressive polling intervals (e.g., 5 minutes) without re-ingesting duplicate data. The optional `time_window` filter further limits the time range of changelogs fetched. + +### HTTP Request Tracing + +Enable request tracing for debugging: +1. Set **Enable Request Tracer** to `true` +2. Trace files are written to: `../../logs/cel/http-request-trace-*.ndjson` +3. Up to 5 backup files are kept + +**⚠️ Security Warning:** Request tracing logs may contain sensitive data including API keys. Only enable for debugging and disable when done. + +## Logs Reference + +### Changelogs Data Stream + +The `changelogs` data stream provides changelog events from ProjectDiscovery Cloud's vulnerability scanner. + +#### Event Types + +- **event.kind**: `event` +- **event.category**: `["vulnerability"]` +- **event.type**: `["info"]` + +#### Exported Fields + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.dataset | Event dataset. | constant_keyword | +| event.module | Event module. | constant_keyword | +| input.type | Type of Filebeat input. | keyword | +| projectdiscovery.change_event | | flattened | +| projectdiscovery.created_at | | date | +| projectdiscovery.event | | flattened | +| projectdiscovery.matcher_status | | boolean | +| projectdiscovery.scan_id | | keyword | +| projectdiscovery.target | | keyword | +| projectdiscovery.template_url | | keyword | +| projectdiscovery.updated_at | | date | +| projectdiscovery.vuln_hash | | keyword | +| projectdiscovery.vuln_status | | keyword | + + +## License + +This integration is licensed under the Elastic License 2.0. + +--- + +**Version:** 0.1.1 diff --git a/packages/projectdiscovery_cloud/img/sample-logo.svg b/packages/projectdiscovery_cloud/img/sample-logo.svg new file mode 100644 index 00000000000..ff4624ea06d --- /dev/null +++ b/packages/projectdiscovery_cloud/img/sample-logo.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/projectdiscovery_cloud/img/sample-screenshot.png b/packages/projectdiscovery_cloud/img/sample-screenshot.png new file mode 100644 index 00000000000..a5e16dd8f80 Binary files /dev/null and b/packages/projectdiscovery_cloud/img/sample-screenshot.png differ diff --git a/packages/projectdiscovery_cloud/manifest.yml b/packages/projectdiscovery_cloud/manifest.yml new file mode 100644 index 00000000000..051b0765194 --- /dev/null +++ b/packages/projectdiscovery_cloud/manifest.yml @@ -0,0 +1,51 @@ +format_version: 3.4.1 +name: projectdiscovery_cloud +title: "ProjectDiscovery Cloud" +version: 0.1.1 +source: + license: "Elastic-2.0" +description: > + External Attack Surface Management (EASM) from ProjectDiscovery. Collects vulnerability changelogs and current vulnerability states via the ProjectDiscovery Cloud API. + +type: integration +categories: + - security + - cloud +conditions: + kibana: + version: "^8.18.0 || ^9.0.0" + elastic: + subscription: "basic" +owner: + github: elastic/security-service-integrations + type: elastic +# Drives Fleet "Add integration" form and supplies .vars.* to templates. +policy_templates: + - name: projectdiscovery_cloud + title: ProjectDiscovery Cloud + description: Collect vulnerability changelogs and export results from ProjectDiscovery Cloud + inputs: + - type: cel + title: ProjectDiscovery Cloud API + description: Collects vulnerability changelogs and export results from ProjectDiscovery Cloud + vars: + - name: base_url + type: text + title: API Base URL + description: Base URL for ProjectDiscovery Cloud API + default: https://api.projectdiscovery.io + required: true + show_user: true + - name: api_key + type: password + title: API Key + description: API key for authenticating to ProjectDiscovery Cloud + secret: true + required: true + show_user: true + - name: team_id + type: text + title: Team ID + description: Team ID for your ProjectDiscovery Cloud account + required: true + show_user: true