Skip to content

Commit

Permalink
NETOBSERV-1233: Remove L3 fields for ARP ethtype packets and update u…
Browse files Browse the repository at this point in the history 
…nit-test (#471)

* Remove L3 info for layer 2 EthType protocols like ARP

Signed-off-by: Mohamed Mahmoud <[email protected]>

* update golang packages to include ethernet package

Signed-off-by: Mohamed Mahmoud <[email protected]>

* update conntrack to skip l2 protocol flows

Signed-off-by: Mohamed Mahmoud <[email protected]>

* skip none transport protocols (icmpv4/6)

Signed-off-by: Mohamed Mahmoud <[email protected]>

* update readme with connection tracking mertics

Signed-off-by: Mohamed Mahmoud <[email protected]>

---------

Signed-off-by: Mohamed Mahmoud <[email protected]>
  • Loading branch information
@msherif1234
msherif1234 authored Aug 21, 2023
1 parent 0fd6e81 commit 914d470
Show file tree
Hide file tree
Showing 16 changed files with 944 additions and 106 deletions.
10 changes: 10 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,16 @@ A possible output would look like:
"TimeReceived": 1661430300
}
```
#### Connection tracking metrics
The following table shows the possible values of the `classification` label in `conntrack_input_records` operational metric.
| Metric | Reason |
|-----------------|--------------------------------------------------------------------------------------------|
| `discarded` | layer2 protocols like ARP, none transport protocols like ICMPv4/6 and too many connections |
| `rejected` | when Error happens calculating connection track hash |
| `duplicate` | for duplicate flows |
| `newConnection` | when new connection tracking flow is created |
Notice that all output records contain `_RecordType` and `_HashId` fields.
Output fields that set `splitAB: true` (like in `Bytes`) are split into 2 fields `Bytes_AB` and `Bytes_BA` which
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/ip2location/ip2location-go/v9 v9.2.0
github.com/json-iterator/go v1.1.12
github.com/mariomac/guara v0.0.0-20220523124851-5fc279816f1f
github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118
github.com/minio/minio-go/v7 v7.0.44
github.com/mitchellh/mapstructure v1.4.3
github.com/netobserv/gopipes v0.3.0
Expand Down
6 changes: 6 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-jsonnet v0.19.1 h1:MORxkrG0elylUqh36R4AcSPX0oZQa9hvI3lroN+kDhs=
Expand Down Expand Up @@ -569,6 +570,7 @@ github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqx
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
Expand Down Expand Up @@ -664,6 +666,10 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE=
github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og=
github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU=
github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
Expand Down
29 changes: 27 additions & 2 deletions pkg/config/generic_map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,18 @@

package config

import "github.com/netobserv/flowlogs-pipeline/pkg/utils"
import (
"syscall"

"github.com/netobserv/flowlogs-pipeline/pkg/utils"
)

type GenericMap map[string]interface{}

const duplicateFieldName = "Duplicate"
const (
duplicateFieldName = "Duplicate"
protoFieldName = "Proto"
)

// Copy will create a flat copy of GenericMap
func (m GenericMap) Copy() GenericMap {
Expand All @@ -42,3 +49,21 @@ func (m GenericMap) IsDuplicate() bool {
}
return false
}

func (m GenericMap) IsValidProtocol() bool {
if _, ok := m[protoFieldName]; ok {
return true
}
return false
}

func (m GenericMap) IsTransportProtocol() bool {
if v, ok := m[protoFieldName]; ok {
if proto, err := utils.ConvertToFloat64(v); err == nil {
if proto == float64(syscall.IPPROTO_TCP) || proto == float64(syscall.IPPROTO_UDP) || proto == float64(syscall.IPPROTO_SCTP) {
return true
}
}
}
return false
}
36 changes: 20 additions & 16 deletions pkg/pipeline/decode/decode_protobuf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/netobserv/flowlogs-pipeline/pkg/config"
"github.com/netobserv/netobserv-ebpf-agent/pkg/pbflow"

"github.com/mdlayher/ethernet"
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
)
Expand Down Expand Up @@ -46,13 +47,10 @@ func PBFlowToMap(flow *pbflow.Record) config.GenericMap {
}
out := config.GenericMap{
"FlowDirection": int(flow.Direction.Number()),
"SrcAddr": ipToStr(flow.Network.GetSrcAddr()),
"DstAddr": ipToStr(flow.Network.GetDstAddr()),
"SrcMac": macToStr(flow.DataLink.GetSrcMac()),
"DstMac": macToStr(flow.DataLink.GetDstMac()),
"Etype": flow.EthProtocol,
"Duplicate": flow.Duplicate,
"Proto": flow.Transport.GetProtocol(),
"TimeFlowStartMs": flow.TimeFlowStart.AsTime().UnixMilli(),
"TimeFlowEndMs": flow.TimeFlowEnd.AsTime().UnixMilli(),
"TimeReceived": time.Now().Unix(),
Expand All @@ -68,20 +66,26 @@ func PBFlowToMap(flow *pbflow.Record) config.GenericMap {
out["Packets"] = flow.Packets
}

proto := flow.Transport.GetProtocol()
if proto == syscall.IPPROTO_ICMP || proto == syscall.IPPROTO_ICMPV6 {
out["IcmpType"] = flow.GetIcmpType()
out["IcmpCode"] = flow.GetIcmpCode()
}
ethType := ethernet.EtherType(flow.EthProtocol)
if ethType == ethernet.EtherTypeIPv4 || ethType == ethernet.EtherTypeIPv6 {
out["SrcAddr"] = ipToStr(flow.Network.GetSrcAddr())
out["DstAddr"] = ipToStr(flow.Network.GetDstAddr())
out["Proto"] = flow.Transport.GetProtocol()
proto := flow.Transport.GetProtocol()
if proto == syscall.IPPROTO_ICMP || proto == syscall.IPPROTO_ICMPV6 {
out["IcmpType"] = flow.GetIcmpType()
out["IcmpCode"] = flow.GetIcmpCode()
}

if proto == syscall.IPPROTO_TCP || proto == syscall.IPPROTO_UDP || proto == syscall.IPPROTO_SCTP {
if proto == syscall.IPPROTO_TCP {
out["SrcPort"] = flow.Transport.GetSrcPort()
out["DstPort"] = flow.Transport.GetDstPort()
out["Flags"] = flow.Flags
} else {
out["SrcPort"] = flow.Transport.GetSrcPort()
out["DstPort"] = flow.Transport.GetDstPort()
if proto == syscall.IPPROTO_TCP || proto == syscall.IPPROTO_UDP || proto == syscall.IPPROTO_SCTP {
if proto == syscall.IPPROTO_TCP {
out["SrcPort"] = flow.Transport.GetSrcPort()
out["DstPort"] = flow.Transport.GetDstPort()
out["Flags"] = flow.Flags
} else {
out["SrcPort"] = flow.Transport.GetSrcPort()
out["DstPort"] = flow.Transport.GetDstPort()
}
}
}

Expand Down
Loading

0 comments on commit 914d470

Please sign in to comment.