feat: transfer 支持日志聚类清洗

Author: chenjiandongx

pkg/transfer/config/metadata_options.go

@@ -131,6 +131,12 @@ const (
 
 	// PipelineConfigIsLogCluster 是否开启日志聚类
 	PipelineConfigOptIsLogCluster = "is_log_cluster"
+
+	// PipelineConfigOptBackendFields 聚类中清洗 backend 需要配置指定的入库字段
+	PipelineConfigOptBackendFields = "backend_fields"
+
+	// PipelineConfigOptLogClusterConfig 聚类配置
+	PipelineConfigOptLogClusterConfig = "log_cluster_config"
 )
 
 // MetaResultTableConfig 专用

pkg/transfer/define/data_processor.go

@@ -10,6 +10,8 @@
 package define
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 )
 
@@ -18,6 +20,7 @@ type BaseDataProcessor struct {
 	Name           string
 	DisabledBizIDs map[string]struct{}
 	baseIndex      int
+	poll           time.Duration
 }
 
 // String : return frontend name
@@ -42,6 +45,14 @@ func (f *BaseDataProcessor) Index() int {
 	return f.baseIndex
 }
 
+func (f *BaseDataProcessor) SetPoll(poll time.Duration) {
+	f.poll = poll
+}
+
+func (f *BaseDataProcessor) Poll() time.Duration {
+	return f.poll
+}
+
 // NewBaseDataProcessor :
 func NewBaseDataProcessor(name string) *BaseDataProcessor {
 	return &BaseDataProcessor{

pkg/transfer/define/etl.go

@@ -10,10 +10,14 @@
 package define
 
 import (
+	"fmt"
+	"maps"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/pkg/errors"
 
+	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/transfer/bufferpool"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/transfer/types"
 )
 
@@ -65,6 +69,87 @@ type ETLRecord struct {
 	Exemplar map[string]interface{} `json:"exemplar"`
 }
 
+type ETLRecordFields struct {
+	KeepMetrics    []string `json:"keep_metrics" mapstructure:"keep_metrics"`
+	DropMetrics    []string `json:"drop_metrics" mapstructure:"drop_metrics"`
+	KeepDimensions []string `json:"keep_dimensions" mapstructure:"keep_dimensions"`
+	DropDimensions []string `json:"drop_dimensions" mapstructure:"drop_dimensions"`
+	GroupKeys      []string `json:"group_keys" mapstructure:"group_keys"`
+}
+
+// Filter 过滤 ELTRecord
+//
+// 白名单规则优先与黑名单规则 当且仅当没有白名单时黑名单才会生效
+func (f *ETLRecordFields) Filter(record ETLRecord) ETLRecord {
+	newRecord := ETLRecord{
+		Time:       record.Time,
+		Exemplar:   record.Exemplar,
+		Dimensions: record.Dimensions,
+		Metrics:    record.Metrics,
+	}
+
+	if len(f.KeepMetrics) > 0 {
+		// 指标白名单
+		newMetrics := make(map[string]interface{})
+		for _, k := range f.KeepMetrics {
+			if v, ok := record.Metrics[k]; ok {
+				newMetrics[k] = v
+			}
+		}
+		newRecord.Metrics = newMetrics
+	} else {
+		// 指标黑名单
+		if len(f.DropMetrics) > 0 {
+			cloned := maps.Clone(record.Metrics)
+			for _, k := range f.DropMetrics {
+				if _, ok := cloned[k]; ok {
+					delete(cloned, k)
+				}
+			}
+			newRecord.Metrics = cloned
+		}
+	}
+
+	if len(f.KeepDimensions) > 0 {
+		// 维度白名单
+		newDimensions := make(map[string]interface{})
+		for _, k := range f.KeepDimensions {
+			v, ok := record.Dimensions[k]
+			if ok {
+				newDimensions[k] = v
+			}
+		}
+		newRecord.Dimensions = newDimensions
+	} else {
+		// 维度黑名单
+		if len(f.DropDimensions) > 0 {
+			cloned := maps.Clone(record.Dimensions)
+			for _, k := range f.DropDimensions {
+				if _, ok := cloned[k]; ok {
+					delete(cloned, k)
+				}
+			}
+			newRecord.Dimensions = cloned
+		}
+	}
+	return newRecord
+}
+
+func (f *ETLRecordFields) GroupID(document map[string]interface{}) uint64 {
+	buf := bufferpool.Get()
+	defer bufferpool.Put(buf)
+
+	for _, key := range f.GroupKeys {
+		v, ok := document[key]
+		if !ok {
+			continue
+		}
+		buf.WriteString(key + "/")
+		fmt.Fprintf(buf, "%s/", v)
+	}
+	return xxhash.Sum64(buf.Bytes())
+}
+
 type GroupETLRecord struct {
 	*ETLRecord
 	GroupInfo []map[string]interface{} `json:"group_info"`

pkg/transfer/define/interface.go

@@ -75,6 +75,8 @@ type DataProcessor interface {
 	Finish(outputChan chan<- Payload, killChan chan<- error)
 	SetIndex(i int)
 	Index() int
+	Poll() time.Duration
+	SetPoll(t time.Duration)
 }
 
 // Frontend : Processor to pull data
@@ -90,7 +92,7 @@ type Backend interface {
 	Stringer
 	SavePoint
 	Push(d Payload, killChan chan<- error)
-	SetFilter(opts map[string]interface{})
+	SetETLRecordFields(f *ETLRecordFields)
 }
 
 // Pipeline : pipeline to process data

pkg/transfer/elasticsearch/backend.go

@@ -40,6 +40,8 @@ type BulkHandler struct {
 	writer        BulkWriter
 	indexRender   IndexRenderFn
 	transformers  map[string]etl.TransformFn
+
+	fields *define.ETLRecordFields
 }
 
 func (b *BulkHandler) makeRecordID(values map[string]interface{}) string {
@@ -100,6 +102,9 @@ func (b *BulkHandler) Handle(ctx context.Context, payload define.Payload, killCh
 		}
 	}
 
+	if b.fields != nil {
+		etlRecord = b.fields.Filter(etlRecord)
+	}
 	return &etlRecord, utils.ParseTimeStamp(*etlRecord.Time), true
 }
 
@@ -170,6 +175,24 @@ func (b *BulkHandler) flush(ctx context.Context, index string, records Records)
 	return count, errs.AsError()
 }
 
+func (b *BulkHandler) grouping(records Records) Records {
+	if b.fields == nil || len(b.fields.GroupKeys) <= 0 {
+		return records
+	}
+
+	uniq := make(map[uint64]*Record)
+	for _, record := range records {
+		uid := b.fields.GroupID(record.Document)
+		uniq[uid] = record
+	}
+
+	dst := make(Records, 0, len(uniq))
+	for _, item := range uniq {
+		dst = append(dst, item)
+	}
+	return dst
+}
+
 // Flush :
 func (b *BulkHandler) Flush(ctx context.Context, results []interface{}) (count int, err error) {
 	lastIndex := ""
@@ -193,9 +216,12 @@ func (b *BulkHandler) Flush(ctx context.Context, results []interface{}) (count i
 
 		logging.Debugf("backend %v ready to flush record %#v to index %s", b, record, index)
 
+		// TODO(mando): grouping 会导致实际写入数量低于 results 数量
+		// 但实际上并非写入失败
+
 		// 处理跨时间间隔
 		if index != lastIndex && lastIndex != "" {
-			cnt, err := b.flush(ctx, lastIndex, records)
+			cnt, err := b.flush(ctx, lastIndex, b.grouping(records))
 			records = records[:0]
 			count += cnt
 			errs.Add(err)
@@ -205,14 +231,18 @@ func (b *BulkHandler) Flush(ctx context.Context, results []interface{}) (count i
 	}
 
 	if len(records) > 0 {
-		cnt, err := b.flush(ctx, lastIndex, records)
+		cnt, err := b.flush(ctx, lastIndex, b.grouping(records))
 		count += cnt
 		errs.Add(err)
 	}
 
 	return count, errs.AsError()
 }
 
+func (b *BulkHandler) SetETLRecordFields(f *define.ETLRecordFields) {
+	b.fields = f
+}
+
 // Close :
 func (b *BulkHandler) Close() error {
 	return b.writer.Close()
@@ -268,7 +298,7 @@ func NewBulkHandler(cluster *config.ElasticSearchMetaClusterInfo, table *config.
 }
 
 // NewBackend :
-func NewBackend(ctx context.Context, name string, maxQps int) (define.Backend, error) {
+func NewBackend(ctx context.Context, name string, options *utils.MapHelper) (define.Backend, error) {
 	conf := config.FromContext(ctx)
 	resultTable := config.ResultTableConfigFromContext(ctx)
 
@@ -300,6 +330,7 @@ func NewBackend(ctx context.Context, name string, maxQps int) (define.Backend, e
 		return nil, err
 	}
 
+	maxQps, _ := options.GetInt(config.PipelineConfigOptMaxQps)
 	return pipeline.NewBulkBackendDefaultAdapter(ctx, name, bulk, maxQps), nil
 }
 
@@ -318,7 +349,6 @@ func init() {
 		}
 
 		options := utils.NewMapHelper(rt.Option)
-		maxQps, _ := options.GetInt(config.PipelineConfigOptMaxQps)
-		return NewBackend(ctx, rt.FormatName(name), maxQps)
+		return NewBackend(ctx, rt.FormatName(name), options)
 	})
 }

pkg/transfer/esb/client.go

@@ -19,7 +19,7 @@ import (
 type Client struct {
 	commonArgs *CommonArgs
 	agent      *sling.Sling
-	conf	   define.Configuration
+	conf       define.Configuration
 }
 
 // CommonArgs :

pkg/transfer/etl/transformer.go

@@ -316,18 +316,16 @@ func ParseDbmSlowQuery(url, content string, retry int) (*DbmResponse, error) {
 	return nil, err
 }
 
-var (
-	httpClient = &http.Client{
-		Transport: &http.Transport{
-			DialContext: (&net.Dialer{
-				Timeout: time.Minute,
-			}).DialContext,
-			MaxIdleConns:        200,
-			MaxIdleConnsPerHost: 100,
-			IdleConnTimeout:     2 * time.Minute,
-		},
-	}
-)
+var httpClient = &http.Client{
+	Transport: &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout: time.Minute,
+		}).DialContext,
+		MaxIdleConns:        200,
+		MaxIdleConnsPerHost: 100,
+		IdleConnTimeout:     2 * time.Minute,
+	},
+}
 
 func parseDbmSlowQuery(url, content string) (*DbmResponse, error) {
 	req := DbmRequest{Content: content}

pkg/transfer/filesystem/processor/backend.go

@@ -80,6 +80,8 @@ func (b *Backend) Close() error {
 	return b.file.Close()
 }
 
+func (b *Backend) SetETLRecordFields(f *define.ETLRecordFields) {}
+
 func init() {
 	define.RegisterBackend("file", func(ctx context.Context, name string) (define.Backend, error) {
 		return NewBackend(ctx, name), nil

pkg/transfer/go.mod

@@ -49,6 +49,7 @@ require (
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -114,7 +115,6 @@ require (
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
 	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 // 背景：官方的 JMESPath SDK 不支持扩展自定义函数

pkg/transfer/influxdb/backend.go

@@ -198,6 +198,8 @@ func (b *BulkHandler) Close() error {
 	return b.cli.Close()
 }
 
+func (b *BulkHandler) SetETLRecordFields(f *define.ETLRecordFields) {}
+
 // NewBulkBackend
 func NewBulkHandler(rt *config.MetaResultTableConfig, shipper *config.MetaClusterInfo) (*BulkHandler, error) {
 	cluster := shipper.AsInfluxCluster()