From 7ac39cbd8bec74eadb99f160d05e20590d56b66a Mon Sep 17 00:00:00 2001 From: "libokang.lbk" Date: Wed, 13 Jul 2022 14:41:48 +0800 Subject: [PATCH] feat: add network priority --- daemon/daemon.go | 40 ++-- daemon/k8s.go | 10 + docs/qos.md | 36 ++++ pkg/tc/u32.go | 116 +++++++++++ pkg/tc/u32_test.go | 135 ++++++++++++ plugin/datapath/consts_linux.go | 10 + plugin/datapath/consts_linux_test.go | 1 - plugin/datapath/exclusive_eni_linux_test.go | 1 - plugin/datapath/ipvlan_linux.go | 21 ++ plugin/datapath/ipvlan_linux_test.go | 1 - plugin/datapath/policy_router_linux.go | 23 +++ plugin/datapath/policy_router_linux_test.go | 1 - plugin/datapath/vpc_router_linux_test.go | 1 - plugin/driver/types/types.go | 10 + plugin/driver/utils/netlink_linux.go | 13 +- plugin/driver/utils/utils_linux.go | 197 ++++++++++++++++++ plugin/driver/utils/utils_linux_test.go | 121 ++++++++++- plugin/driver/vlan/vlan_test.go | 1 - plugin/terway/cni.go | 60 +++--- plugin/terway/cni_linux.go | 9 +- rpc/rpc.pb.go | 218 ++++++++++---------- rpc/rpc.proto | 1 + rpc/tracing.pb.go | 2 +- types/k8s.go | 13 ++ types/res.go | 25 +-- 25 files changed, 900 insertions(+), 166 deletions(-) create mode 100644 pkg/tc/u32.go create mode 100644 pkg/tc/u32_test.go diff --git a/daemon/daemon.go b/daemon/daemon.go index f790f437..1f8fafd0 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -331,8 +331,9 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r Trunk: false, }, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, IfName: "", ExtraRoutes: nil, @@ -395,8 +396,9 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r Trunk: false, }, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, IfName: "", ExtraRoutes: nil, @@ -432,8 +434,9 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r }, ENIInfo: nil, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, IfName: "", ExtraRoutes: nil, @@ -610,8 +613,9 @@ func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) ( Trunk: false, }, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, IfName: "", ExtraRoutes: nil, @@ -636,8 +640,9 @@ func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) ( ServiceCIDR: n.k8s.GetServiceCIDR().ToRPC(), }, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, DefaultRoute: true, }) @@ -672,8 +677,9 @@ func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) ( Trunk: podinfo.PodENI && n.enableTrunk && eni.Trunk, }, Pod: &rpc.Pod{ - Ingress: podinfo.TcIngress, - Egress: podinfo.TcEgress, + Ingress: podinfo.TcIngress, + Egress: podinfo.TcEgress, + NetworkPriority: podinfo.NetworkPriority, }, IfName: "", ExtraRoutes: nil, @@ -1018,8 +1024,9 @@ func (n *networkService) multiIPFromCRD(podInfo *types.PodInfo, waitReady bool) }, ENIInfo: eniInfo, Pod: &rpc.Pod{ - Ingress: podInfo.TcIngress, - Egress: podInfo.TcEgress, + Ingress: podInfo.TcIngress, + Egress: podInfo.TcEgress, + NetworkPriority: podInfo.NetworkPriority, }, IfName: alloc.Interface, ExtraRoutes: parseExtraRoute(alloc.ExtraRoutes), @@ -1092,8 +1099,9 @@ func (n *networkService) exclusiveENIFromCRD(podInfo *types.PodInfo, waitReady b }, ENIInfo: eniInfo, Pod: &rpc.Pod{ - Ingress: podInfo.TcIngress, - Egress: podInfo.TcEgress, + Ingress: podInfo.TcIngress, + Egress: podInfo.TcEgress, + NetworkPriority: podInfo.NetworkPriority, }, IfName: alloc.Interface, ExtraRoutes: parseExtraRoute(alloc.ExtraRoutes), diff --git a/daemon/k8s.go b/daemon/k8s.go index 1548fe75..80a10bf3 100644 --- a/daemon/k8s.go +++ b/daemon/k8s.go @@ -563,6 +563,16 @@ func convertPod(daemonMode string, statefulWorkloadKindSet sets.String, pod *cor } } + if prio, ok := podAnnotation[types.NetworkPriority]; ok { + switch types.NetworkPrio(prio) { + case types.NetworkPrioBestEffort, types.NetworkPrioBurstable, types.NetworkPrioGuaranteed: + pi.NetworkPriority = prio + default: + _ = tracing.RecordPodEvent(pod.Name, pod.Namespace, eventTypeWarning, + "ParseFailed", fmt.Sprintf("Parse pod annotation %s failed.", types.NetworkPriority)) + } + } + // determine whether pod's IP will stick 5 minutes for a reuse, priorities as below, // 1. pod has a positive pod-ip-reservation annotation // 2. pod is owned by a known stateful workload diff --git a/docs/qos.md b/docs/qos.md index 636f37e3..2d98ef0e 100644 --- a/docs/qos.md +++ b/docs/qos.md @@ -20,6 +20,23 @@ Traffic control mainly have two aspect | `kubernetes.io/ingress-bandwidth: 10M` | ingress banwidth | | `kubernetes.io/egress-bandwidth: 10M` | egress banwidth | +### config shaping + +to enable shaping, follow config need to add in `eni-config` + +```yaml +# kubectl edit cm -n kube-system eni-config +apiVersion: v1 +data: + 10-terway.conf: | + { + "cniVersion": "0.3.1", + "name": "terway", + "capabilities": {"bandwidth": true}, # add + "type": "terway" + } +``` + ## priority We have three annotations available for pod, to control different priority. @@ -34,3 +51,22 @@ When priority is set, a `priority qdisc` is set to the `eni` related to pod. Pod egress traffic will classify into different bands based on the config. For more info about how priority works , please refer to [tc-prio](https://man7.org/linux/man-pages/man8/tc-prio.8.html) . + +> note: default qdisc will be replaced + +### config priority + +to enable priority, follow config need to add in `eni-config` + +```yaml +# kubectl edit cm -n kube-system eni-config +apiVersion: v1 +data: + 10-terway.conf: | + { + "cniVersion": "0.3.1", + "name": "terway", + "enable_network_priority": true, # add + "type": "terway" + } +``` diff --git a/pkg/tc/u32.go b/pkg/tc/u32.go new file mode 100644 index 00000000..d924a3b2 --- /dev/null +++ b/pkg/tc/u32.go @@ -0,0 +1,116 @@ +/* +Copyright 2022 The Terway Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tc + +import ( + "encoding/binary" + "net" + + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" +) + +// FilterBySrcIP found u32 filter by pod ip +// used for prio only +func FilterBySrcIP(link netlink.Link, parent uint32, ipNet *net.IPNet) (*netlink.U32, error) { + filters, err := netlink.FilterList(link, parent) + if err != nil { + return nil, err + } + + matches := U32MatchSrc(ipNet) + if len(matches) == 0 { + return nil, nil + } +OUT: + for _, f := range filters { + u32, ok := f.(*netlink.U32) + if !ok { + continue + } + if u32.Attrs().LinkIndex != link.Attrs().Index || + u32.Protocol != unix.ETH_P_IP || + u32.Sel == nil { + continue + } + + // check all matches is satisfied with current + for _, m := range matches { + found := false + for _, key := range u32.Sel.Keys { + if key.Off != m.Off || key.Val != m.Val || key.Mask != m.Mask { + continue + } + found = true + break + } + if !found { + continue OUT + } + } + return u32, nil + } + return nil, nil +} + +// MatchSrc add match for source ip +func MatchSrc(u32 *netlink.U32, ipNet *net.IPNet) { + if u32.Sel == nil { + u32.Sel = &netlink.TcU32Sel{ + Flags: nl.TC_U32_TERMINAL, + } + } + + u32.Sel.Keys = append(u32.Sel.Keys, U32MatchSrc(ipNet)...) + u32.Sel.Nkeys = uint8(len(u32.Sel.Keys)) +} + +// U32MatchSrc return u32 match key by src ip +func U32MatchSrc(ipNet *net.IPNet) []netlink.TcU32Key { + if ipNet.IP.To4() == nil { + return U32IPv6Src(ipNet) + } + return []netlink.TcU32Key{U32IPv4Src(ipNet)} +} + +func U32IPv4Src(ipNet *net.IPNet) netlink.TcU32Key { + mask := net.IP(ipNet.Mask).To4() + val := ipNet.IP.Mask(ipNet.Mask).To4() + return netlink.TcU32Key{ + Mask: binary.BigEndian.Uint32(mask), + Val: binary.BigEndian.Uint32(val), + Off: 12, + } +} + +func U32IPv6Src(ipNet *net.IPNet) []netlink.TcU32Key { + mask := ipNet.Mask + val := ipNet.IP.Mask(ipNet.Mask) + + r := make([]netlink.TcU32Key, 0, 4) + for i := 0; i < 4; i++ { + m := binary.BigEndian.Uint32(mask) + if m != 0 { + r = append(r, netlink.TcU32Key{ + Mask: m, + Val: binary.BigEndian.Uint32(val), + Off: int32(8 + 4*i), + }) + } + mask = mask[4:] + val = val[4:] + } + return r +} diff --git a/pkg/tc/u32_test.go b/pkg/tc/u32_test.go new file mode 100644 index 00000000..6128f2a4 --- /dev/null +++ b/pkg/tc/u32_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 The Terway Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tc + +import ( + "net" + "reflect" + "testing" + + "github.com/vishvananda/netlink" +) + +func TestU32IPv4Src(t *testing.T) { + type args struct { + ipNet *net.IPNet + } + tests := []struct { + name string + args args + want netlink.TcU32Key + }{ + { + name: "127.0.0.1/32", + args: args{ipNet: &net.IPNet{ + IP: net.ParseIP("127.0.0.1"), + Mask: net.CIDRMask(32, 32), + }}, + want: netlink.TcU32Key{ + Mask: 0xffffffff, + Val: 0x7f000001, + Off: 12, + OffMask: 0, + }, + }, + { + name: "127.0.0.0/8", + args: args{ipNet: &net.IPNet{ + IP: net.ParseIP("127.0.0.1"), + Mask: net.CIDRMask(8, 32), + }}, + want: netlink.TcU32Key{ + Mask: 0xff000000, + Val: 0x7f000000, + Off: 12, + OffMask: 0, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := U32IPv4Src(tt.args.ipNet); !reflect.DeepEqual(got, tt.want) { + t.Errorf("U32IPv4Src() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestU32IPv6Src(t *testing.T) { + type args struct { + ipNet *net.IPNet + } + tests := []struct { + name string + args args + want []netlink.TcU32Key + }{ + { + name: "fd00::1/128", + args: args{ipNet: &net.IPNet{ + IP: net.ParseIP("fd00::1"), + Mask: net.CIDRMask(128, 128), + }}, + want: []netlink.TcU32Key{ + { + Mask: 0xffffffff, + Val: 0xfd000000, + Off: 8, + OffMask: 0, + }, { + Mask: 0xffffffff, + Val: 0x00000000, + Off: 12, + OffMask: 0, + }, { + Mask: 0xffffffff, + Val: 0x00000000, + Off: 16, + OffMask: 0, + }, { + Mask: 0xffffffff, + Val: 0x00000001, + Off: 20, + OffMask: 0, + }, + }, + }, { + name: "fd:aaaa::/64", + args: args{ipNet: &net.IPNet{ + IP: net.ParseIP("fd:aaaa::"), + Mask: net.CIDRMask(64, 128), + }}, + want: []netlink.TcU32Key{ + { + Mask: 0xffffffff, + Val: 0x00fdaaaa, + Off: 8, + OffMask: 0, + }, { + Mask: 0xffffffff, + Val: 0x00000000, + Off: 12, + OffMask: 0, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := U32IPv6Src(tt.args.ipNet); !reflect.DeepEqual(got, tt.want) { + t.Errorf("U32IPv6Src() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/plugin/datapath/consts_linux.go b/plugin/datapath/consts_linux.go index 774905e2..a6cf237f 100644 --- a/plugin/datapath/consts_linux.go +++ b/plugin/datapath/consts_linux.go @@ -2,6 +2,9 @@ package datapath import ( "net" + + "github.com/AliyunContainerService/terway/types" + "github.com/vishvananda/netlink" ) const ( @@ -24,3 +27,10 @@ var ( Mask: net.CIDRMask(128, 128), } ) + +var PrioMap = map[string]uint32{ + string(types.NetworkPrioBestEffort): netlink.MakeHandle(1, 2), // band 1 + string(types.NetworkPrioBurstable): netlink.MakeHandle(1, 3), // band 2 + string(types.NetworkPrioGuaranteed): netlink.MakeHandle(1, 1), // band 0 + "": netlink.MakeHandle(1, 2), +} diff --git a/plugin/datapath/consts_linux_test.go b/plugin/datapath/consts_linux_test.go index 66b583c8..aef766c5 100644 --- a/plugin/datapath/consts_linux_test.go +++ b/plugin/datapath/consts_linux_test.go @@ -1,5 +1,4 @@ //go:build privileged -// +build privileged package datapath diff --git a/plugin/datapath/exclusive_eni_linux_test.go b/plugin/datapath/exclusive_eni_linux_test.go index 686032b8..cd854921 100644 --- a/plugin/datapath/exclusive_eni_linux_test.go +++ b/plugin/datapath/exclusive_eni_linux_test.go @@ -1,5 +1,4 @@ //go:build privileged -// +build privileged package datapath diff --git a/plugin/datapath/ipvlan_linux.go b/plugin/datapath/ipvlan_linux.go index 1d5bf9da..17a99167 100644 --- a/plugin/datapath/ipvlan_linux.go +++ b/plugin/datapath/ipvlan_linux.go @@ -262,6 +262,13 @@ func (d *IPvlanDriver) Setup(cfg *types.SetupConfig, netNS ns.NetNS) error { return err } + if cfg.EnableNetworkPriority { + err = utils.SetEgressPriority(parentLink, cfg.NetworkPriority, cfg.ContainerIPNet) + if err != nil { + return err + } + } + err = ipvlan.Setup(&ipvlan.IPVlan{ Parent: parentLink.Attrs().Name, PreName: cfg.HostVETHName, @@ -308,6 +315,20 @@ func (d *IPvlanDriver) Teardown(cfg *types.TeardownCfg, netNS ns.NetNS) error { return err } + if cfg.EnableNetworkPriority { + link, err := netlink.LinkByIndex(cfg.ENIIndex) + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); !ok { + return err + } + } else { + err = utils.DelEgressPriority(link, cfg.ContainerIPNet) + if err != nil { + return err + } + } + } + // del route to container return d.teardownInitNamespace(cfg.ContainerIPNet) } diff --git a/plugin/datapath/ipvlan_linux_test.go b/plugin/datapath/ipvlan_linux_test.go index 6af4999f..4ab202f5 100644 --- a/plugin/datapath/ipvlan_linux_test.go +++ b/plugin/datapath/ipvlan_linux_test.go @@ -1,5 +1,4 @@ //go:build privileged -// +build privileged package datapath diff --git a/plugin/datapath/policy_router_linux.go b/plugin/datapath/policy_router_linux.go index bb951510..b817a5a2 100644 --- a/plugin/datapath/policy_router_linux.go +++ b/plugin/datapath/policy_router_linux.go @@ -345,6 +345,14 @@ func (d *PolicyRoute) Setup(cfg *types.SetupConfig, netNS ns.NetNS) error { if err != nil { return err } + + if cfg.EnableNetworkPriority { + err = utils.SetEgressPriority(eni, cfg.NetworkPriority, cfg.ContainerIPNet) + if err != nil { + return err + } + } + table := utils.GetRouteTableID(eni.Attrs().Index) eniCfg := generateENICfgForPolicy(cfg, eni, table) @@ -390,3 +398,18 @@ func (d *PolicyRoute) Check(cfg *types.CheckConfig) error { }) return err } + +func (d *PolicyRoute) Teardown(cfg *types.TeardownCfg, netNS ns.NetNS) error { + if !cfg.EnableNetworkPriority { + return nil + } + + link, err := netlink.LinkByIndex(cfg.ENIIndex) + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); !ok { + return err + } + return nil + } + return utils.DelEgressPriority(link, cfg.ContainerIPNet) +} diff --git a/plugin/datapath/policy_router_linux_test.go b/plugin/datapath/policy_router_linux_test.go index 52a223b1..f67f6e40 100644 --- a/plugin/datapath/policy_router_linux_test.go +++ b/plugin/datapath/policy_router_linux_test.go @@ -1,5 +1,4 @@ //go:build privileged -// +build privileged package datapath diff --git a/plugin/datapath/vpc_router_linux_test.go b/plugin/datapath/vpc_router_linux_test.go index 830dc9ba..740f1a76 100644 --- a/plugin/datapath/vpc_router_linux_test.go +++ b/plugin/datapath/vpc_router_linux_test.go @@ -1,5 +1,4 @@ //go:build privileged -// +build privileged package datapath diff --git a/plugin/driver/types/types.go b/plugin/driver/types/types.go index 878aea14..f9e3ff98 100644 --- a/plugin/driver/types/types.go +++ b/plugin/driver/types/types.go @@ -35,6 +35,9 @@ type CNIConf struct { BandwidthMode string `json:"bandwidth_mode"` + // EnableNetworkPriority by enable priority control, eni qdisc is replaced with tc_prio + EnableNetworkPriority bool `json:"enable_network_priority"` + // Debug Debug bool `json:"debug"` } @@ -108,6 +111,9 @@ type SetupConfig struct { Ingress uint64 Egress uint64 + EnableNetworkPriority bool + NetworkPriority uint32 + RuntimeConfig cni.RuntimeConfig // for windows @@ -120,8 +126,12 @@ type TeardownCfg struct { HostVETHName string + ENIIndex int + ContainerIfName string ContainerIPNet *terwayTypes.IPNetSet ServiceCIDR *terwayTypes.IPNetSet + + EnableNetworkPriority bool } diff --git a/plugin/driver/utils/netlink_linux.go b/plugin/driver/utils/netlink_linux.go index 8dd96fe0..9bf26e70 100644 --- a/plugin/driver/utils/netlink_linux.go +++ b/plugin/driver/utils/netlink_linux.go @@ -180,8 +180,8 @@ func LinkSetNsFd(link netlink.Link, netNS ns.NetNS) error { return nil } -func QdiscReplace(qdisc *netlink.GenericQdisc) error { - cmd := fmt.Sprintf("tc qdisc replace %s", qdisc.String()) +func QdiscReplace(qdisc netlink.Qdisc) error { + cmd := fmt.Sprintf("tc qdisc replace %s", qdisc.Attrs().String()) Log.Infof(cmd) err := netlink.QdiscReplace(qdisc) if err != nil { @@ -189,3 +189,12 @@ func QdiscReplace(qdisc *netlink.GenericQdisc) error { } return nil } +func QdiscDel(qdisc netlink.Qdisc) error { + cmd := fmt.Sprintf("tc qdisc del %s", qdisc.Attrs().String()) + Log.Infof(cmd) + err := netlink.QdiscDel(qdisc) + if err != nil { + return fmt.Errorf("error %s, %w", cmd, err) + } + return nil +} diff --git a/plugin/driver/utils/utils_linux.go b/plugin/driver/utils/utils_linux.go index 58854a8d..78ba10a5 100644 --- a/plugin/driver/utils/utils_linux.go +++ b/plugin/driver/utils/utils_linux.go @@ -480,6 +480,203 @@ func EnsureClsActQdsic(link netlink.Link) error { return nil } +// EnsurePrioQdiscAt10 write qdisc attach under mq +func EnsurePrioQdiscAt10(link netlink.Link) error { + qds, err := netlink.QdiscList(link) + if err != nil { + return fmt.Errorf("list qdisc for dev %s error, %w", link.Attrs().Name, err) + } + for _, q := range qds { + // only handle qd under mq at 1: + major, minor := netlink.MajorMinor(q.Attrs().Parent) + if major != 1 || minor == 0 { + continue + } + + _, ok := q.(*netlink.Prio) + if !ok { + err = QdiscReplace(netlink.NewPrio(netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: q.Attrs().Parent, + })) + if err != nil { + return err + } + } + } + return nil +} + +// EnsureMQQdisc write qdisc +func EnsureMQQdisc(link netlink.Link) error { + qds, err := netlink.QdiscList(link) + if err != nil { + return fmt.Errorf("list qdisc for dev %s error, %w", link.Attrs().Name, err) + } + var prev netlink.Qdisc + for _, q := range qds { + _, minor := netlink.MajorMinor(q.Attrs().Handle) + if q.Attrs().Parent == netlink.HANDLE_ROOT && minor == 0 { + prev = q + } + + if q.Type() == "mq" && q.Attrs().Parent == netlink.HANDLE_ROOT && q.Attrs().Handle == netlink.MakeHandle(1, 0) { + return nil + } + } + if prev != nil { + _ = QdiscDel(prev) + } + + return QdiscReplace(&netlink.GenericQdisc{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: netlink.HANDLE_ROOT, + Handle: netlink.MakeHandle(1, 0), + }, + QdiscType: "mq", + }) +} + +func FilterAdd(filter *netlink.U32) error { + cmd := fmt.Sprintf("tc filter add %s", filter.String()) + Log.Info(cmd) + err := netlink.FilterAdd(filter) + if err != nil { + return fmt.Errorf("error %s, %w", cmd, err) + } + return nil +} + +func FilterDel(filter *netlink.U32) error { + cmd := fmt.Sprintf("tc filter del %s", filter.String()) + Log.Info(cmd) + err := netlink.FilterDel(filter) + if err != nil { + return fmt.Errorf("error %s, %w", cmd, err) + } + return nil +} + +// SetFilter write u32 filter +func SetFilter(link netlink.Link, parentID, classID uint32, ipNetSet *terwayTypes.IPNetSet) error { + exec := func(ipNet *net.IPNet) error { + found, err := tc.FilterBySrcIP(link, parentID, ipNet) + if err != nil { + return err + } + if found != nil && found.ClassId == classID { + return nil + } + + u32 := &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: link.Attrs().Index, + Parent: parentID, + Priority: 1, + Protocol: unix.ETH_P_IP, + }, + ClassId: classID, + } + tc.MatchSrc(u32, ipNet) + + return FilterAdd(u32) + } + + if ipNetSet.IPv4 != nil { + err := exec(NewIPNetWithMaxMask(ipNetSet.IPv4)) + if err != nil { + return err + } + } + if ipNetSet.IPv6 != nil { + err := exec(NewIPNetWithMaxMask(ipNetSet.IPv6)) + if err != nil { + return err + } + } + return nil +} + +// DelFilter del u32 filter by pod ip +func DelFilter(link netlink.Link, parentID uint32, ipNetSet *terwayTypes.IPNetSet) error { + exec := func(ipNet *net.IPNet) error { + found, err := tc.FilterBySrcIP(link, parentID, ipNet) + if err != nil { + return err + } + if found == nil { + return nil + } + err = FilterDel(found) + if err != nil { + return err + } + return nil + } + + if ipNetSet.IPv4 != nil { + err := exec(NewIPNetWithMaxMask(ipNetSet.IPv4)) + if err != nil { + return err + } + } + if ipNetSet.IPv6 != nil { + err := exec(NewIPNetWithMaxMask(ipNetSet.IPv6)) + if err != nil { + return err + } + } + + return nil +} + +// SetEgressPriority write egress priority rule for pod +func SetEgressPriority(link netlink.Link, classID uint32, ipNetSet *terwayTypes.IPNetSet) error { + err := EnsureMQQdisc(link) + if err != nil { + return err + } + err = EnsurePrioQdiscAt10(link) + if err != nil { + return err + } + qds, err := netlink.QdiscList(link) + if err != nil { + return fmt.Errorf("list qdisc for dev %s error, %w", link.Attrs().Name, err) + } + for _, q := range qds { + _, ok := q.(*netlink.Prio) + if !ok { + continue + } + + err = SetFilter(link, q.Attrs().Handle, classID, ipNetSet) + if err != nil { + return err + } + } + return nil +} + +func DelEgressPriority(link netlink.Link, ipNetSet *terwayTypes.IPNetSet) error { + qds, err := netlink.QdiscList(link) + if err != nil { + return err + } + for _, q := range qds { + _, ok := q.(*netlink.Prio) + if !ok { + continue + } + err = DelFilter(link, q.Attrs().Handle, ipNetSet) + if err != nil { + return err + } + } + return nil +} + func SetupTC(link netlink.Link, bandwidthInBytes uint64) error { rule := &tc.TrafficShapingRule{ Rate: bandwidthInBytes, diff --git a/plugin/driver/utils/utils_linux_test.go b/plugin/driver/utils/utils_linux_test.go index 5a32384f..31351f3b 100644 --- a/plugin/driver/utils/utils_linux_test.go +++ b/plugin/driver/utils/utils_linux_test.go @@ -1,13 +1,19 @@ //go:build privileged -// +build privileged package utils import ( + "fmt" + "net" "runtime" "testing" + "github.com/AliyunContainerService/terway/pkg/tc" + terwayTypes "github.com/AliyunContainerService/terway/types" + "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" ) @@ -53,3 +59,116 @@ func TestEnsureVlanUntagger(t *testing.T) { t.Fail() } } + +var _ = Describe("Test TC filter", func() { + var hostNS ns.NetNS + const nicName = "eni" + BeforeEach(func() { + var err error + hostNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + err = hostNS.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + fmt.Printf("LinkAdd ---------- \n") + return netlink.LinkAdd(&netlink.GenericLink{ + LinkAttrs: netlink.LinkAttrs{ + Name: nicName, + TxQLen: 1000, + NumTxQueues: 2, + NumRxQueues: 2, + }, + LinkType: "netdevsim", + }) + }) + Expect(err).NotTo(HaveOccurred()) + }) + AfterEach(func() { + _ = hostNS.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + eni, err := netlink.LinkByName(nicName) + Expect(err).NotTo(HaveOccurred()) + err = netlink.LinkDel(eni) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + Expect(hostNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(hostNS)).To(Succeed()) + }) + + It("add qdisc mq", func() { + var err error + err = hostNS.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + eni, err := netlink.LinkByName(nicName) + Expect(err).NotTo(HaveOccurred()) + + err = EnsureMQQdisc(eni) + Expect(err).NotTo(HaveOccurred()) + + qds, err := netlink.QdiscList(eni) + Expect(err).NotTo(HaveOccurred()) + + found := false + for _, qd := range qds { + if qd.Type() != "mq" { + continue + } + Expect(qd.Attrs().Parent).Should(Equal(uint32(netlink.HANDLE_ROOT))) + Expect(qd.Attrs().Handle).Should(Equal(netlink.MakeHandle(1, 0))) + found = true + break + } + Expect(found).Should(BeTrue()) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) + It("set egress filter", func() { + var err error + err = hostNS.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + eni, err := netlink.LinkByName(nicName) + Expect(err).NotTo(HaveOccurred()) + + err = SetEgressPriority(eni, netlink.MakeHandle(1, 1), &terwayTypes.IPNetSet{ + IPv4: &net.IPNet{ + IP: net.ParseIP("192.168.1.1"), + Mask: net.CIDRMask(32, 32), + }, + IPv6: &net.IPNet{ + IP: net.ParseIP("fd00::1"), + Mask: net.CIDRMask(128, 128), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + qds, err := netlink.QdiscList(eni) + Expect(err).NotTo(HaveOccurred()) + + for _, qd := range qds { + _, ok := qd.(*netlink.Prio) + if !ok { + continue + } + + u32, err := tc.FilterBySrcIP(eni, qd.Attrs().Handle, &net.IPNet{ + IP: net.ParseIP("192.168.1.1"), + Mask: net.CIDRMask(32, 32), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(u32).NotTo(BeNil(), "tc filter with src ipv4 should be found. Qdisc %#v", qd) + + u32v6, err := tc.FilterBySrcIP(eni, qd.Attrs().Handle, &net.IPNet{ + IP: net.ParseIP("fd00::1"), + Mask: net.CIDRMask(128, 128), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(u32v6).NotTo(BeNil(), "tc filter with src ipv6 should be found") + } + return nil + }) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/plugin/driver/vlan/vlan_test.go b/plugin/driver/vlan/vlan_test.go index dab4e8ad..79a4ff84 100644 --- a/plugin/driver/vlan/vlan_test.go +++ b/plugin/driver/vlan/vlan_test.go @@ -1,4 +1,3 @@ //go:build privileged -// +build privileged package vlan diff --git a/plugin/terway/cni.go b/plugin/terway/cni.go index 40117bd9..62b3529f 100644 --- a/plugin/terway/cni.go +++ b/plugin/terway/cni.go @@ -8,6 +8,7 @@ import ( "time" "github.com/AliyunContainerService/terway/pkg/link" + "github.com/AliyunContainerService/terway/plugin/datapath" "github.com/AliyunContainerService/terway/plugin/driver/types" "github.com/AliyunContainerService/terway/plugin/driver/utils" "github.com/AliyunContainerService/terway/rpc" @@ -240,8 +241,9 @@ func parseSetupConf(args *skel.CmdArgs, alloc *rpc.NetConf, conf *types.CNIConf, trunkENI bool vid uint32 - ingress uint64 - egress uint64 + ingress uint64 + egress uint64 + networkPriority uint32 routes []cniTypes.Route @@ -299,6 +301,7 @@ func parseSetupConf(args *skel.CmdArgs, alloc *rpc.NetConf, conf *types.CNIConf, if alloc.GetPod() != nil { ingress = alloc.GetPod().GetIngress() egress = alloc.GetPod().GetEgress() + networkPriority = datapath.PrioMap[alloc.GetPod().GetNetworkPriority()] } if conf.RuntimeConfig.Bandwidth.EgressRate > 0 { egress = uint64(conf.RuntimeConfig.Bandwidth.EgressRate / 8) @@ -337,24 +340,26 @@ func parseSetupConf(args *skel.CmdArgs, alloc *rpc.NetConf, conf *types.CNIConf, dp := getDatePath(ipType, conf.VlanStripType, trunkENI) return &types.SetupConfig{ - DP: dp, - ContainerIfName: name, - ContainerIPNet: containerIPNet, - GatewayIP: gatewayIP, - MTU: conf.MTU, - ENIIndex: int(deviceID), - ENIGatewayIP: eniGatewayIP, - ServiceCIDR: serviceCIDR, - HostStackCIDRs: hostStackCIDRs, - BandwidthMode: conf.BandwidthMode, - Ingress: ingress, - Egress: egress, - StripVlan: trunkENI, - Vid: int(vid), - DefaultRoute: alloc.GetDefaultRoute(), - ExtraRoutes: routes, - DisableCreatePeer: disableCreatePeer, - RuntimeConfig: conf.RuntimeConfig, + DP: dp, + ContainerIfName: name, + ContainerIPNet: containerIPNet, + GatewayIP: gatewayIP, + MTU: conf.MTU, + ENIIndex: int(deviceID), + ENIGatewayIP: eniGatewayIP, + ServiceCIDR: serviceCIDR, + HostStackCIDRs: hostStackCIDRs, + BandwidthMode: conf.BandwidthMode, + EnableNetworkPriority: conf.EnableNetworkPriority, + Ingress: ingress, + Egress: egress, + StripVlan: trunkENI, + Vid: int(vid), + DefaultRoute: alloc.GetDefaultRoute(), + ExtraRoutes: routes, + DisableCreatePeer: disableCreatePeer, + RuntimeConfig: conf.RuntimeConfig, + NetworkPriority: networkPriority, }, nil } @@ -367,6 +372,7 @@ func parseTearDownConf(alloc *rpc.NetConf, conf *types.CNIConf, ipType rpc.IPTyp err error containerIPNet *terwayTypes.IPNetSet serviceCIDR *terwayTypes.IPNetSet + eniIndex int32 ) serviceCIDR, err = terwayTypes.ToIPNetSet(alloc.GetBasicInfo().GetServiceCIDR()) @@ -393,12 +399,20 @@ func parseTearDownConf(alloc *rpc.NetConf, conf *types.CNIConf, ipType rpc.IPTyp return nil, err } } + if alloc.GetENIInfo() != nil { + mac := alloc.GetENIInfo().GetMAC() + if mac != "" { + eniIndex, _ = link.GetDeviceNumber(mac) + } + } dp := getDatePath(ipType, conf.VlanStripType, false) return &types.TeardownCfg{ - DP: dp, - ContainerIPNet: containerIPNet, - ServiceCIDR: serviceCIDR, + DP: dp, + ContainerIPNet: containerIPNet, + ServiceCIDR: serviceCIDR, + ENIIndex: int(eniIndex), + EnableNetworkPriority: conf.EnableNetworkPriority, }, nil } diff --git a/plugin/terway/cni_linux.go b/plugin/terway/cni_linux.go index c4ed684d..f3459d9b 100644 --- a/plugin/terway/cni_linux.go +++ b/plugin/terway/cni_linux.go @@ -374,9 +374,16 @@ func doCmdDel(ctx context.Context, logger *logrus.Entry, client rpc.TerwayBacken if err != nil { return err } - break + continue } } + fallthrough + case types.PolicyRoute: + utils.Hook.AddExtraInfo("dp", "policyRoute") + err = datapath.NewPolicyRoute().Teardown(teardownCfg, cniNetns) + if err != nil { + return err + } } } return nil diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index c7a8e2d2..f64e2592 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 +// protoc-gen-go v1.28.0 // protoc v3.19.4 // source: rpc.proto @@ -711,8 +711,9 @@ type Pod struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Ingress uint64 `protobuf:"varint,1,opt,name=Ingress,proto3" json:"Ingress,omitempty"` - Egress uint64 `protobuf:"varint,2,opt,name=Egress,proto3" json:"Egress,omitempty"` + Ingress uint64 `protobuf:"varint,1,opt,name=Ingress,proto3" json:"Ingress,omitempty"` + Egress uint64 `protobuf:"varint,2,opt,name=Egress,proto3" json:"Egress,omitempty"` + NetworkPriority string `protobuf:"bytes,3,opt,name=NetworkPriority,proto3" json:"NetworkPriority,omitempty"` } func (x *Pod) Reset() { @@ -761,6 +762,13 @@ func (x *Pod) GetEgress() uint64 { return 0 } +func (x *Pod) GetNetworkPriority() string { + if x != nil { + return x.NetworkPriority + } + return "" +} + type ReleaseIPRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1300,111 +1308,113 @@ var file_rpc_proto_rawDesc = []byte{ 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x09, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x49, 0x50, 0x22, 0x19, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x44, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x44, 0x73, 0x74, 0x22, 0x37, + 0x44, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x44, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x03, 0x50, 0x6f, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x93, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, - 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, - 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, - 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, - 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, - 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, - 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, + 0x06, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, + 0x79, 0x22, 0x93, 0x02, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, + 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, + 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, + 0x50, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x26, 0x0a, + 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, + 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, + 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x9e, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x12, 0x26, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, + 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, + 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, + 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x22, 0x92, 0x01, 0x0a, 0x0e, 0x47, 0x65, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, + 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, + 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, + 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, + 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0xe9, 0x01, + 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x23, + 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x26, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, - 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x4d, - 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x61, - 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x9e, 0x01, - 0x0a, 0x0e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x26, 0x0a, 0x08, 0x49, 0x50, - 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x49, 0x50, 0x53, 0x65, 0x74, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, - 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, - 0x76, 0x36, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x22, 0x92, - 0x01, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, - 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x4b, - 0x38, 0x73, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x73, - 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x49, 0x64, 0x22, 0xe9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x12, 0x23, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, 0x36, 0x12, 0x28, 0x0a, 0x08, 0x4e, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, - 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, - 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x20, 0x0a, - 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0a, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xec, 0x01, 0x0a, 0x0c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x32, 0x0a, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, - 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2c, - 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x0e, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3c, - 0x0a, 0x0a, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, - 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, - 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x3b, 0x0a, 0x06, - 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x79, 0x70, 0x65, 0x56, 0x50, - 0x43, 0x49, 0x50, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x79, 0x70, 0x65, 0x56, 0x50, 0x43, - 0x45, 0x4e, 0x49, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x54, 0x79, 0x70, 0x65, 0x45, 0x4e, 0x49, - 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x49, 0x50, 0x10, 0x02, 0x2a, 0x29, 0x0a, 0x05, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x12, 0x0c, 0x0a, 0x08, 0x45, 0x72, 0x72, 0x4e, 0x6f, 0x45, 0x72, 0x72, 0x10, 0x00, - 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x72, 0x72, 0x43, 0x52, 0x44, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, - 0x6e, 0x64, 0x10, 0x01, 0x2a, 0x36, 0x0a, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x6f, 0x64, 0x10, 0x01, 0x2a, 0x36, 0x0a, 0x09, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x10, 0x00, 0x12, 0x14, - 0x0a, 0x10, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x57, 0x61, 0x72, 0x6e, 0x69, - 0x6e, 0x67, 0x10, 0x01, 0x32, 0xeb, 0x01, 0x0a, 0x0d, 0x54, 0x65, 0x72, 0x77, 0x61, 0x79, 0x42, - 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x33, 0x0a, 0x07, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, - 0x50, 0x12, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x50, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x6c, 0x6c, - 0x6f, 0x63, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x09, 0x52, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, - 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x49, 0x50, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, - 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x33, 0x0a, - 0x0b, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x11, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x0f, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x2f, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, + 0x04, 0x49, 0x50, 0x76, 0x34, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x49, 0x50, 0x76, + 0x34, 0x12, 0x12, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x04, 0x49, 0x50, 0x76, 0x36, 0x12, 0x28, 0x0a, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x08, 0x4e, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x73, 0x12, + 0x26, 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, + 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x54, + 0x72, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x20, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xec, 0x01, 0x0a, 0x0c, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, 0x0b, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1e, + 0x0a, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, + 0x0a, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2c, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, + 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3c, 0x0a, 0x0a, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, + 0x12, 0x14, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x3b, 0x0a, 0x06, 0x49, 0x50, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x79, 0x70, 0x65, 0x56, 0x50, 0x43, 0x49, 0x50, 0x10, 0x00, 0x12, + 0x0e, 0x0a, 0x0a, 0x54, 0x79, 0x70, 0x65, 0x56, 0x50, 0x43, 0x45, 0x4e, 0x49, 0x10, 0x01, 0x12, + 0x12, 0x0a, 0x0e, 0x54, 0x79, 0x70, 0x65, 0x45, 0x4e, 0x49, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x49, + 0x50, 0x10, 0x02, 0x2a, 0x29, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x0c, 0x0a, 0x08, + 0x45, 0x72, 0x72, 0x4e, 0x6f, 0x45, 0x72, 0x72, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x72, + 0x72, 0x43, 0x52, 0x44, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x10, 0x01, 0x2a, 0x36, + 0x0a, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x13, 0x0a, + 0x0f, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, + 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x50, 0x6f, 0x64, 0x10, 0x01, 0x2a, 0x36, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x32, 0xeb, + 0x01, 0x0a, 0x0d, 0x54, 0x65, 0x72, 0x77, 0x61, 0x79, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, + 0x12, 0x33, 0x0a, 0x07, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x50, 0x12, 0x13, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x50, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x11, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x49, 0x50, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x09, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x49, 0x50, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x49, 0x50, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x50, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x49, 0x50, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x13, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x0b, 0x52, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x11, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, + 0x2e, 0x2f, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/rpc/rpc.proto b/rpc/rpc.proto index ab2ffa46..dd765968 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -73,6 +73,7 @@ enum IPType { message Pod { uint64 Ingress = 1; uint64 Egress = 2; + string NetworkPriority = 3; } message ReleaseIPRequest { diff --git a/rpc/tracing.pb.go b/rpc/tracing.pb.go index 52199a4a..49a53f04 100644 --- a/rpc/tracing.pb.go +++ b/rpc/tracing.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 +// protoc-gen-go v1.28.0 // protoc v3.19.4 // source: tracing.proto diff --git a/types/k8s.go b/types/k8s.go index 2daf900f..0974cb8f 100644 --- a/types/k8s.go +++ b/types/k8s.go @@ -45,6 +45,9 @@ const ( // PodUID store pod uid PodUID = AnnotationPrefix + "pod-uid" + + // NetworkPriority set pod network priority + NetworkPriority = AnnotationPrefix + "network-priority" ) // FinalizerPodENI finalizer for podENI resource @@ -80,3 +83,13 @@ func PodUseENI(pod *corev1.Pod) bool { } return v } + +// NetworkPrio network priority for pod +type NetworkPrio string + +// NetworkPrio val +const ( + NetworkPrioBestEffort NetworkPrio = "best-effort" + NetworkPrioBurstable NetworkPrio = "burstable" + NetworkPrioGuaranteed NetworkPrio = "guaranteed" +) diff --git a/types/res.go b/types/res.go index a8c7b51d..aedc249e 100644 --- a/types/res.go +++ b/types/res.go @@ -22,18 +22,19 @@ type PodEipInfo struct { // NOTE: this is the type store in db type PodInfo struct { //K8sPod *v1.Pod - Name string - Namespace string - TcIngress uint64 - TcEgress uint64 - PodNetworkType string - PodIP string // used for eip and mip - PodIPs IPSet // used for eip and mip - SandboxExited bool - EipInfo PodEipInfo - IPStickTime time.Duration - PodENI bool - PodUID string + Name string + Namespace string + TcIngress uint64 + TcEgress uint64 + PodNetworkType string + PodIP string // used for eip and mip + PodIPs IPSet // used for eip and mip + SandboxExited bool + EipInfo PodEipInfo + IPStickTime time.Duration + PodENI bool + PodUID string + NetworkPriority string } // ExtraEipInfo store extra eip info