Support Unnormalized Playbooks

Dockerfile

@@ -54,12 +54,12 @@ RUN dnf install -y --nodocs wget tcpdump unzip git gcc python3.14 python3.14-dev
     dnf clean all
 RUN ln -s /usr/bin/python3.14 /usr/local/bin/python3 && ln -s /usr/bin/pip3.14 /usr/local/bin/pip3
 RUN ln -s /usr/bin/python3.14 /usr/local/bin/python && ln -s /usr/bin/pip3.14 /usr/local/bin/pip
-RUN pip3 install pysigma==0.11.20 sigma-cli==1.0.5 pysigma-backend-elasticsearch pysigma-pipeline-windows
-ADD dep/pysigma_backend_securityonion-0.1.0-py3-none-any.whl /tmp
-RUN pip3 install /tmp/pysigma_backend_securityonion-0.1.0-py3-none-any.whl
+RUN pip3 install pysigma==1.4.0 sigma-cli==3.0.2 pysigma-backend-elasticsearch pysigma-pipeline-windows
+ADD dep/pysigma_backend_securityonion-1.0.0-py3-none-any.whl /tmp
+RUN pip3 install /tmp/pysigma_backend_securityonion-1.0.0-py3-none-any.whl
 RUN pip3 install yara-python==4.5.4
 RUN dnf remove -y gcc python3.14-devel openssl-devel && dnf autoremove -y && dnf clean all
-RUN rm /tmp/pysigma_backend_securityonion-0.1.0-py3-none-any.whl
+RUN rm /tmp/pysigma_backend_securityonion-1.0.0-py3-none-any.whl
 
 RUN update-ca-trust
 RUN groupadd --gid "$GID" socore && \

model/playbook.go

@@ -40,7 +40,7 @@ type Question struct {
 	AnswerSources []string `yaml:"answer_sources" json:"answer_sources"`
 	// A raw YAML Sigma query that can be used to find the answer to this question. May contain variables that will be replaced with values from the alert.
 	Query string `json:"query"`
-	// The query after variable substitution has been performed.
+	// The query after LEGACY {field} variable substitution (queryVariableSubstitution) has been performed.
 	FilledQuery string `json:"filledQuery,omitempty" yaml:"filledQuery,omitempty"`
 	// The results after the queries have been substituted, converted, and executed.
 	QueryResults []*EventRecord `json:"queryResults" yaml:"-"`

model/repo.go

@@ -79,6 +79,11 @@ func GetReposDefault(cfg map[string]interface{}, field string, licenseRequired b
 			r.Folder = &folder
 		}
 
+		branch, ok := obj["branch"].(string)
+		if ok {
+			r.Branch = &branch
+		}
+
 		repos = append(repos, r)
 	}
 

model/repo_test.go

@@ -42,6 +42,7 @@ func TestGetRepos(t *testing.T) {
 						"repo":      "repo2",
 						"license":   "GPL2",
 						"folder":    "sigma/stable",
+						"branch":    "published",
 						"community": 0,
 					},
 					map[string]interface{}{
@@ -66,6 +67,7 @@ func TestGetRepos(t *testing.T) {
 					RepoUrl: "repo2",
 					License: "GPL2",
 					Folder:  util.Ptr("sigma/stable"),
+					Branch:  util.Ptr("published"),
 				},
 				{
 					RepoUrl:   "repo3",

server/modules/playbook/placeholder_map.go

@@ -0,0 +1,132 @@
+// Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+// https://securityonion.net/license; you may not use this file except in compliance with the
+// Elastic License 2.0.
+
+package playbook
+
+import (
+	"regexp"
+
+	"github.com/security-onion-solutions/securityonion-soc/model"
+
+	"github.com/apex/log"
+	"gopkg.in/yaml.v3"
+)
+
+// eventDataPrefix is where a Sigma (Elastalert) alert document nests the original triggering event.
+// Alert-level metadata (rule.uuid, soc_id) sits at the top level; the fired event's own
+// fields live under event_data.*
+const eventDataPrefix = "event_data."
+
+var placeholderUseRe = regexp.MustCompile(`%([^%\s]+)%`)
+
+// extractPlaceholders returns the set of resolvable %token% names used across the given
+// query strings, skipping escaped \%name%\.
+func extractPlaceholders(queries ...string) map[string]bool {
+	used := map[string]bool{}
+	for _, q := range queries {
+		for _, idx := range placeholderUseRe.FindAllStringSubmatchIndex(q, -1) {
+			if idx[0] > 0 && q[idx[0]-1] == '\\' {
+				continue // escaped \%name% is literal text, not a placeholder
+			}
+			used[q[idx[2]:idx[3]]] = true
+		}
+	}
+	return used
+}
+
+// lookupEventValue resolves a field path against the alert document, trying the
+// event_data.-nested original-event location first, then the bare alert-level path, then
+// the document-id bridge (the SOC _id lives on EventRecord.Id, not in Payload). A
+// present-but-null field is treated as absent, so it degrades to the NODATA fallback
+// instead of injecting a null value (some sources ship explicit nulls)
+func lookupEventValue(event *model.EventRecord, key string) (interface{}, bool) {
+	if event == nil {
+		return nil, false
+	}
+	if v, ok := event.Payload[eventDataPrefix+key]; ok && v != nil {
+		return v, true
+	}
+	if v, ok := event.Payload[key]; ok && v != nil {
+		return v, true
+	}
+	if key == socIdPayloadKey && event.Id != "" {
+		return event.Id, true
+	}
+	return nil, false
+}
+
+// mergePlaceholderMaps overlays the user map on top of the global (shipped) map, returning a
+// new combined map.
+func mergePlaceholderMaps(global, user map[string]string) map[string]string {
+	merged := make(map[string]string, len(global)+len(user))
+	for token, field := range global {
+		merged[token] = field
+	}
+	for token, field := range user {
+		merged[token] = field
+	}
+	return merged
+}
+
+// missingValueFallback fills a mapped placeholder whose value is absent from
+// the event, so `sigma convert` doesn't raise on an unresolved placeholder.
+// Unmapped placeholders get no var and still fail.
+const missingValueFallback = "NODATA"
+
+// loadPlaceholderMap reads one placeholder map YAML file (Sigma placeholder name -> event field
+// path) into a map. Each file is an optional layer of the combined map (see mergePlaceholderMaps),
+// so a missing, malformed, or empty file is non-fatal and simply contributes no tokens.
+func (pdm *PlaybookDiskManager) loadPlaceholderMap(path string) map[string]string {
+	m := map[string]string{}
+
+	raw, err := pdm.ReadFile(path)
+	if err != nil {
+		log.WithError(err).WithField("path", path).Debug("no playbook placeholder map at path; treating as an empty layer")
+		return m
+	}
+
+	if err := yaml.Unmarshal(raw, &m); err != nil {
+		log.WithError(err).WithField("path", path).Warn("unable to parse playbook placeholder map; treating as an empty layer")
+		return map[string]string{}
+	}
+
+	return m
+}
+
+// socIdPayloadKey is the field name %document_id% resolves to (lookupEventValue bridges it
+// to EventRecord.Id). Matches the soc_id name the case/escalation handlers give event.Id.
+const socIdPayloadKey = "soc_id"
+
+// buildVarsFromEvent builds the `vars:` block for `sigma convert` from the alert event.
+//
+//	(1) Every declared token (the combined placeholder map = global map + user map) resolves to
+//	    its event value via lookupEventValue, else the NODATA fallback
+//	(2) A token USED in a query but undeclared is tried as a direct field name; a hit
+//	    covers "named the placeholder after a flat field", a miss is left ABSENT so
+//	    value_placeholders fails rather than silently resolving to the fallback.
+//
+// List values pass through as-is; the backend renders them as an OR-list.
+func (pdm *PlaybookDiskManager) buildVarsFromEvent(event *model.EventRecord, bindings map[string]string, used map[string]bool) map[string]interface{} {
+	vars := make(map[string]interface{}, len(bindings)+len(used))
+
+	for token, field := range bindings {
+		if v, ok := lookupEventValue(event, field); ok {
+			vars[token] = v
+		} else {
+			vars[token] = missingValueFallback
+		}
+	}
+
+	for token := range used {
+		if _, declared := bindings[token]; declared {
+			continue
+		}
+		if v, ok := lookupEventValue(event, token); ok {
+			vars[token] = v
+		}
+	}
+
+	return vars
+}

server/modules/playbook/placeholder_map_test.go

@@ -0,0 +1,135 @@
+// Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+// https://securityonion.net/license; you may not use this file except in compliance with the
+// Elastic License 2.0.
+
+package playbook
+
+import (
+	"testing"
+
+	"github.com/security-onion-solutions/securityonion-soc/model"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupEventValue(t *testing.T) {
+	ev := &model.EventRecord{
+		Id: "soc-doc-1",
+		Payload: map[string]interface{}{
+			"event_data.process.executable": "c:\\evil.exe", // original event nests under event_data.*
+			"rule.uuid":                     "abc-123",      // alert metadata at top level
+		},
+	}
+
+	// event_data.-nested field resolves when the binding path is the bare ECS name
+	v, ok := lookupEventValue(ev, "process.executable")
+	assert.True(t, ok)
+	assert.Equal(t, "c:\\evil.exe", v)
+
+	// also resolves when the path already carries the event_data. prefix (legacy global map)
+	v, ok = lookupEventValue(ev, "event_data.process.executable")
+	assert.True(t, ok)
+	assert.Equal(t, "c:\\evil.exe", v)
+
+	// alert-level (non event_data) field resolves via the bare fallback
+	v, ok = lookupEventValue(ev, "rule.uuid")
+	assert.True(t, ok)
+	assert.Equal(t, "abc-123", v)
+
+	// the document id bridges to EventRecord.Id (not a Payload field)
+	v, ok = lookupEventValue(ev, socIdPayloadKey)
+	assert.True(t, ok)
+	assert.Equal(t, "soc-doc-1", v)
+
+	// absent field
+	_, ok = lookupEventValue(ev, "no.such.field")
+	assert.False(t, ok)
+
+	// nil event -> not found, no panic
+	_, ok = lookupEventValue(nil, "anything")
+	assert.False(t, ok)
+
+	// present-but-null field is treated as absent (some sources ship explicit nulls)
+	evNull := &model.EventRecord{Payload: map[string]interface{}{
+		"event_data.some.field": nil,
+		"some.field":            nil,
+	}}
+	_, ok = lookupEventValue(evNull, "some.field")
+	assert.False(t, ok, "a present-but-null field must not resolve")
+
+	// a null event_data.X must not shadow a real bare X
+	evShadow := &model.EventRecord{Payload: map[string]interface{}{
+		"event_data.dst_ip": nil,
+		"dst_ip":            "1.2.3.4",
+	}}
+	v, ok = lookupEventValue(evShadow, "dst_ip")
+	assert.True(t, ok)
+	assert.Equal(t, "1.2.3.4", v)
+}
+
+func TestMergePlaceholderMaps(t *testing.T) {
+	global := map[string]string{"a": "field.a", "b": "field.b.global"}
+	user := map[string]string{"b": "field.b.override", "c": "field.c"} // overrides b, adds c
+
+	got := mergePlaceholderMaps(global, user)
+	assert.Equal(t, "field.a", got["a"], "global-only token survives")
+	assert.Equal(t, "field.b.override", got["b"], "user map wins on conflict")
+	assert.Equal(t, "field.c", got["c"], "user map adds custom vocabulary")
+
+	// inputs are not mutated
+	assert.Equal(t, "field.b.global", global["b"], "global input is left untouched")
+	_, globalHasC := global["c"]
+	assert.False(t, globalHasC, "global input is left untouched")
+
+	// nil inputs are treated as empty; the non-nil side passes through as a copy
+	assert.Empty(t, mergePlaceholderMaps(nil, nil))
+	assert.Equal(t, user, mergePlaceholderMaps(nil, user))
+	assert.Equal(t, global, mergePlaceholderMaps(global, nil))
+}
+
+func TestBuildVarsFromEvent(t *testing.T) {
+	pdm := &PlaybookDiskManager{}
+	ev := &model.EventRecord{Payload: map[string]interface{}{
+		"event_data.process.executable": "c:\\evil.exe",
+		"event_data.actor":              "alice", // a flat undeclared token can hit this directly
+	}}
+
+	bindings := map[string]string{
+		"Image":    "process.executable", // declared, present -> resolves
+		"hostname": "host.name",          // declared, ABSENT on this event -> NODATA
+	}
+	used := map[string]bool{
+		"Image":              true,
+		"hostname":           true,
+		"actor":              true, // undeclared but resolvable directly via event_data.actor
+		"undeclared_missing": true, // undeclared + not a field -> omitted (loud fail downstream)
+	}
+
+	vars := pdm.buildVarsFromEvent(ev, bindings, used)
+
+	assert.Equal(t, "c:\\evil.exe", vars["Image"])
+	assert.Equal(t, missingValueFallback, vars["hostname"])
+	assert.Equal(t, "alice", vars["actor"], "undeclared token resolves directly from a flat event field")
+	_, present := vars["undeclared_missing"]
+	assert.False(t, present, "undeclared+missing token must be omitted so convert fails loudly")
+}
+
+func TestExtractPlaceholders(t *testing.T) {
+	used := extractPlaceholders(
+		"Image|expand: '%Image%'",
+		"rule.uuid|expand: '%rule_uuid%'",
+		`CommandLine|contains: 'literal \%escaped\% text'`,       // escaped -> not a placeholder
+		"dotted|expand: '%rule.uuid%'",                           // dotted token IS surfaced ([^%\s]+)
+		"hyphen|expand: '%my-token%'",                            // hyphenated token IS surfaced
+		"CommandLine|contains: 'progress 100% then %actor% ran'", // literal % must not eat %actor%
+	)
+
+	assert.True(t, used["Image"])
+	assert.True(t, used["rule_uuid"])
+	assert.True(t, used["rule.uuid"], "dotted token is surfaced by the broader name class")
+	assert.True(t, used["my-token"], "hyphenated token is surfaced")
+	assert.True(t, used["actor"], "a literal percent must not cannibalize an adjacent placeholder")
+	assert.False(t, used["escaped"], "escaped placeholder is literal, not a placeholder")
+	assert.False(t, used[" then "], "whitespace is excluded, so a literal percent cannot match across it")
+}