Port packages to TQL2

tenzir · Dec 24, 2024 · e1046b9 · e1046b9
1 parent 1eaacb5
commit e1046b9
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 123 deletions.
diff --git a/alphamountain/package.yaml b/alphamountain/package.yaml
@@ -68,11 +68,12 @@ pipelines:
     definition: |
       // tql2
       every {{ inputs.refresh-interval }} {
-        legacy "load https://batch.alphamountain.ai/threat/feed/csv
-          'version:=1'
-          'license:=\"{{ inputs.license }}\"'
-          'risk_min:={{ inputs.min-risk }}'
-          'risk_max:=10.0'"
+        from "https://batch.alphamountain.ai/threat/feed/csv", data = {
+          version: 1,
+          license: "{{ inputs.license }}",
+          risk_min: {{ inputs.min-risk }},
+          risk_max: 10.0
+        }
         read_csv
       }
       @name = "alphamountain.threat"
@@ -87,7 +88,7 @@ pipelines:
       // tql2
       subscribe "alphamountain"
       where @name == "alphamountain.threat"
-      legacy "context update alphamountain-threats --key=hostname --create-timeout {{ inputs.threat-expiry }}"
+      context::update "alphamountain-threats", key=hostname, create_timeout={{ inputs.threat-expiry }}
 
   publish-categories:
     name: Publish alphaMountain Categories Feed
@@ -97,9 +98,10 @@ pipelines:
     definition: |
       // tql2
       every {{ inputs.refresh-interval }} {
-        legacy "load https://batch.alphamountain.ai/category/feed/csv
-          'version:=1'
-          'license:=\"{{ inputs.license }}\"'
+        from "https://batch.alphamountain.ai/category/feed/csv", data = {
+          version: 1,
+          license: "{{ inputs.license }}",
+        }
         read_csv
       }
       @name = "alphamountain.category"
@@ -114,21 +116,23 @@ pipelines:
       // tql2
       subscribe "alphamountain"
       where @name == "alphamountain.category"
-      legacy "context update alphamountain-categories --key=hostname --create-timeout {{ inputs.threat-expiry }}"
+      context::update "alphamountain-categories", key=hostname, create_timeout={{ inputs.threat-expiry }}
 
 examples:
   - name: Match all string and IP fields
     description: |
       Starts a lookup that matches all historic and newly arriving events
       against the `alphamountain-threats` context.
     definition: |
-      lookup alphamountain-threats --field :string,:ip
+      // tql2
+      legacy "lookup alphamountain-threats --field :string,:ip"
 
   - name: Show top alphaMountain threats by score
     description: |
       Shows the top ten threats from alphaMountain by score.
     definition: |
-      context inspect alphamountain-threats
-      | yield value
-      | sort score desc
-      | head 10
+      // tql2
+      context::inspect "alphamountain-threats"
+      this = value
+      sort -score
+      head 10
diff --git a/demo-node/package.yaml b/demo-node/package.yaml
@@ -63,16 +63,18 @@ examples:
       Use the `where` operator with a more complex expression to identify
       long-running connections in a specific CIDR subnet.
     definition: |
+      // tql2
       export
-      | where #schema == "zeek.conn"
-      | where :ip in 10.10.5.0/25
-      | where orig_bytes > 1 Mi or duration > 30 min
+      where @name == "zeek.conn"
+      legacy "where :ip in 10.10.5.0/25"
+      where orig_bytes > 1 Mi or duration > 30 min
 
   - name: Identify top hitters
     description: |
       Use the `top` operator to see the top 5 hitters in Zeek notice logs.
     definition: |
+      // tql2
       export
-      | where #schema == "zeek.notice"
-      | top msg
-      | head 5
+      where @name == "zeek.notice"
+      top msg
+      head 5
diff --git a/feodo/package.yaml b/feodo/package.yaml
@@ -21,23 +21,24 @@ pipelines:
     name: Update Feodo Context
     description: A pipeline that periodically refreshes the Feodo lookup-table context.
     definition: |
-      every 1h
-        from https://feodotracker.abuse.ch/downloads/ipblocklist_aggressive.csv read csv --allow-comments
-        | context update feodo --key=dst_ip --clear
+      // tql2
+      every 1h {
+        load_http "https://feodotracker.abuse.ch/downloads/ipblocklist_aggressive.csv"
+        read_csv allow_comments=true
+      }
+      context::update "feodo", key="dst_ip" --clear
     restart-on-error: 1h
 
 examples:
   - name: Lookup Feodo
     description: Evaluate all stored data with a `src_ip` field against the Feodo IP Blocklist.
     definition: |
-      lookup feodo --field=dst_ip
+      // tql2
+      legacy "lookup feodo --field=dst_ip"
 
   - name: Enrich Feodo
     description: Evaluate the `src_ip` field of the input data against the Feodo IP Blocklist.
     definition: |
-      export | enrich feodo --field=dst_ip
-
-  - name: Feodo Table Entries
-    description: A gauge that shows the number of entries in the Feodo IP Blocklist lookup-table.
-    definition: |
-      show contexts | where name == "feodo" | chart gauge --value=num_entries
+      // tql2
+      export
+      context::enrich "feodo", key=dst_ip
diff --git a/fortinet-fortigate/package.yaml b/fortinet-fortigate/package.yaml
@@ -21,8 +21,10 @@ pipelines:
     name: Onboard FortiGate logs via UDP
     description: A pipeline for ingesting and processing FortiGate logs received via UDP.
     definition: |
-      from "{{ inputs.syslog-url }}" read syslog
-      | parse content kv
-      | yield content
-      | set #schema = "fortinet.fortigate"
-      | publish "fortinet"
+      // tql2
+      load_http "{{ inputs.syslog-url }}"
+      read_syslog
+      legacy "parse content kv"
+      this = content
+      @name = "fortinet.fortigate"
+      publish "fortinet"
diff --git a/foxio/package.yaml b/foxio/package.yaml
@@ -87,7 +87,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4+"
       where ja4_fingerprint != null
-      legacy "context update ja4 --key=ja4_fingerprint"
+      context::update "ja4", key=ja4_fingerprint
 
   populate-ja4s-context:
     name: Update the JA4S context
@@ -99,7 +99,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4s+"
       where ja4s_fingerprint != null
-      legacy "context update ja4s --key=ja4s_fingerprint"
+      context::update "ja4s", key=ja4s_fingerprint
 
   populate-ja4h-context:
     name: Update the JA4H context
@@ -111,7 +111,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4h+"
       where ja4h_fingerprint != null
-      legacy "context update ja4h --key=ja4h_fingerprint"
+      context::update "ja4h", key=ja4h_fingerprint
 
   populate-ja4x-context:
     name: Update the JA4X context
@@ -123,7 +123,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4x+"
       where ja4x_fingerprint != null
-      legacy "context update ja4x --key=ja4x_fingerprint"
+      context::update "ja4x", key=ja4x_fingerprint
 
   populate-ja4t-context:
     name: Update the JA4T context
@@ -135,7 +135,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4t+"
       where ja4t_fingerprint != null
-      legacy "context update ja4t --key=ja4t_fingerprint"
+      context::update "ja4t", key=ja4t_fingerprint
 
   populate-ja4ts-context:
     name: Update the JA4TS context
@@ -147,7 +147,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4ts+"
       where ja4ts_fingerprint != null
-      legacy "context update ja4ts --key=ja4ts_fingerprint"
+      context::update "ja4ts", key=ja4ts_fingerprint
 
   populate-ja4tscan-context:
     name: Update the JA4TScan context
@@ -159,7 +159,7 @@ pipelines:
       subscribe "foxio"
       where @name == "foxio.ja4tscan+"
       where ja4tscan_fingerprint != null
-      legacy "context update ja4tscan --key=ja4tscan_fingerprint"
+      context::update "ja4tscan", key=ja4tscan_fingerprint
 
 examples:
   - name: Enrich Zeek conn logs with JA4 fingerpints
@@ -168,35 +168,41 @@ examples:
       package](https://github.com/FoxIO-LLC/ja4/tree/main/zeek) to enrich your
       Zeek logs with JA4+ fingerprints.
     definition: |
-      subscribe zeek
-      | where #schema == "zeek.conn"
-      | enrich ja4t --field=ja4t
-      | enrich ja4ts --field=ja4ts
+      // tql2
+      subscribe "zeek"
+      where @name == "zeek.conn"
+      context::enrich "ja4t", key=ja4t
+      context::enrich "ja4ts", key=ja4ts
 
   - name: FoxIO database top-10 operating systems
     description: |
       Shows a bar chart of the top-10 most common operating systems in the JA4+
       database.
     definition: |
-      from "https://ja4db.com/api/read/" read json --arrays-of-objects
-      | where os != null
-      | top os
-      | head
-      | chart bar
+      // tql2
+      load_http "https://ja4db.com/api/read/"
+      read_json arrays_of_objects=true
+      where os != null
+      top os
+      head
+      legacy "chart bar"
 
   - name: FoxIO database top-10 applications
     description: |
       Shows a bar chart of the top-10 most common applications in the JA4+
       database.
     definition: |
-      from "https://ja4db.com/api/read/" read json --arrays-of-objects
-      | where application != null
-      | top application
-      | head
-      | chart bar
+      // tql2
+      load_http "https://ja4db.com/api/read/"
+      read_json arrays_of_objects=true
+      where application != null
+      top application
+      head
+      legacy "chart bar"
 
   - name: Show the contents of the JA4 context.
     description: |
       Shows all entries of the FoxIO JA+s database.
     definition: |
-      context inspect ja4
+      // tql2
+      context::inspect "ja4"
diff --git a/slack/package.yaml b/slack/package.yaml
@@ -24,16 +24,18 @@ pipelines:
     description: |
       Forwards all events arriving on the topic `slack` to Slack.
     definition: |
+      // tql2
       // Slack requires every message to arrive in a JSON body in the format
       // `{text: "<message>"}`. To achieve that, we just write the content out
       // line by line and then read it back in, which gives us a human-readable
       // dump of the events in Slack, no matter what form they were in before.
-      subscribe slack
-      | write lines
-      | read lines
-      | put text=line
-      | batch 1
-      | to http POST "{{ inputs.webhook-url }}" write json
+      subscribe "slack"
+      write_lines
+      read_lines
+      select text=line
+      batch 1
+      write_json
+      save_http "{{ inputs.webhook-url }}", method="POST"
 
 examples:
   - name: Say Hello in Slack

diff --git a/splunk/package.yaml b/splunk/package.yaml
@@ -21,17 +21,11 @@ inputs:
     type: string
     default: splunk
 
-  splunk_host:
-    name: Splunk host
-    description: The hostname or IP address of the Splunk instance.
+  splunk_url:
+    name: URL for the Splunk Indexer
+    description: URL for the Splunk Indexer
     type: string
-    default: localhost
-
-  splunk_port:
-    name: Splunk port
-    description: The port where Splunk HEC listens.
-    type: string
-    default: 8088
+    default: https://localhost:8080
 
   event_index:
     name: Splunk event index
@@ -44,12 +38,9 @@ pipelines:
     name: Send data to Splunk
     description: Send data to a Splunk instance.
     definition: |
-      subscribe {{ inputs.events_topic }}
-      | fluent-bit
-          splunk
-          host={{ inputs.splunk_host }}
-          port={{ inputs.splunk_port }}
-          tls=on
-          tls.verify=off
-          splunk_token={{ inputs.splunk_token }}
-          event_index={{ inputs.event_index }}
+      // tql2
+      subscribe "{{ inputs.events_topic }}"
+      to_splunk "{{ inputs.splunk_url }}",
+        hec_token="{{ inputs.splunk_token }}",
+        index="{{ inputs.event_index }}",
+        skip_peer_verification = true
diff --git a/sslbl/package.yaml b/sslbl/package.yaml
@@ -34,7 +34,7 @@ pipelines:
         load_http "https://sslbl.abuse.ch/blacklist/sslblacklist.csv"
         read_csv comments=true, header="timestamp,SHA1,reason"
       }
-      legacy "context update sslbl --key=SHA1"
+      context::update "sslbl", key=SHA1
     restart-on-error: 1h
 
 examples:
@@ -47,15 +47,16 @@ examples:
       subscribe "suricata"
       where @name == "suricata.tls"
       sha1 = tls.fingerprint.replace(":", "")
-      legacy "enrich sha1 sslbl"
+      context::enrich "sha1", key=sslbl
 
   - name: Display top-10 listing reasons
     description: |
       Shows a bar chart of the top-10 reasons why a certificate is in the
       dataset.
     definition: |
-      context inspect sslbl
-      | yield value
-      | top reason
-      | head
-      | chart bar
+      // tql2
+      context::inspect "sslbl"
+      this = value
+      top reason
+      head
+      legacy "chart bar"