From f7b02b47f8021b4c361b371d365c726e6e072706 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Sun, 22 Mar 2020 13:08:11 +0530 Subject: [PATCH] Add namespace in function name for metrics This commit adds namespace in function names while logging metrics to prometheus, irrespective of the function is invoked with namespace suffix or not. This is also required to add multiple namespace support to faas-idler https://github.com/openfaas-incubator/faas-idler/issues/37 which is part of https://github.com/openfaas/faas-netes/issues/511 Signed-off-by: Vivek Singh --- gateway/handlers/notifiers.go | 13 ++- gateway/main.go | 3 +- gateway/metrics/exporter.go | 146 +++++++++++++++++++++++++--------- 3 files changed, 119 insertions(+), 43 deletions(-) diff --git a/gateway/handlers/notifiers.go b/gateway/handlers/notifiers.go index ce581cf50..f7a2c956a 100644 --- a/gateway/handlers/notifiers.go +++ b/gateway/handlers/notifiers.go @@ -43,15 +43,21 @@ func urlToLabel(path string) string { // PrometheusFunctionNotifier records metrics to Prometheus type PrometheusFunctionNotifier struct { Metrics *metrics.MetricOptions + //FunctionNamespace default namespace of the function + FunctionNamespace string } // Notify records metrics in Prometheus func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) { - if event == "completed" { + serviceName := getServiceName(originalURL) + if len(p.FunctionNamespace) > 0 { + if index := strings.Index(serviceName, "."); index == -1 { + serviceName = fmt.Sprintf("%s.%s", serviceName, p.FunctionNamespace) + } + } + if event == "completed" { seconds := duration.Seconds() - serviceName := getServiceName(originalURL) - p.Metrics.GatewayFunctionsHistogram. WithLabelValues(serviceName). Observe(seconds) @@ -62,7 +68,6 @@ func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalUR With(prometheus.Labels{"function_name": serviceName, "code": code}). Inc() } else if event == "started" { - serviceName := getServiceName(originalURL) p.Metrics.GatewayFunctionInvocationStarted.WithLabelValues(serviceName).Inc() } diff --git a/gateway/main.go b/gateway/main.go index 2be7a11d6..5f9716845 100644 --- a/gateway/main.go +++ b/gateway/main.go @@ -77,7 +77,8 @@ func main() { loggingNotifier := handlers.LoggingNotifier{} prometheusNotifier := handlers.PrometheusFunctionNotifier{ - Metrics: &metricsOptions, + Metrics: &metricsOptions, + FunctionNamespace: config.Namespace, } prometheusServiceNotifier := handlers.PrometheusServiceNotifier{ diff --git a/gateway/metrics/exporter.go b/gateway/metrics/exporter.go index 5c662fdd6..e16c43928 100644 --- a/gateway/metrics/exporter.go +++ b/gateway/metrics/exporter.go @@ -6,10 +6,12 @@ package metrics import ( "encoding/json" + "fmt" "io/ioutil" "net" "net/http" "net/url" + "path" "time" "log" @@ -56,8 +58,14 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.metricOptions.ServiceReplicasGauge.Reset() for _, service := range e.services { + var serviceName string + if len(service.Namespace) > 0 { + serviceName = fmt.Sprintf("%s.%s", service.Name, service.Namespace) + } else { + serviceName = service.Name + } e.metricOptions.ServiceReplicasGauge. - WithLabelValues(service.Name). + WithLabelValues(serviceName). Set(float64(service.Replicas)) } @@ -72,9 +80,46 @@ func (e *Exporter) StartServiceWatcher(endpointURL url.URL, metricsOptions Metri ticker := time.NewTicker(interval) quit := make(chan struct{}) - timeout := 3 * time.Second + go func() { + for { + select { + case <-ticker.C: + + namespaces, err := e.getNamespaces(endpointURL) + if err != nil { + log.Println(err) + } + + if len(namespaces) == 0 { + emptyNamespace := "" + services, err := e.getFunctions(endpointURL, emptyNamespace) + if err != nil { + log.Println(err) + continue + } + e.services = services + } else { + for _, namespace := range namespaces { + services, err := e.getFunctions(endpointURL, namespace) + if err != nil { + log.Println(err) + continue + } + e.services = append(e.services, services...) + } + } + + break + case <-quit: + return + } + } + }() +} - proxyClient := http.Client{ +func (e *Exporter) getHTTPClient(timeout time.Duration) http.Client { + + return http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ @@ -87,45 +132,70 @@ func (e *Exporter) StartServiceWatcher(endpointURL url.URL, metricsOptions Metri ExpectContinueTimeout: 1500 * time.Millisecond, }, } +} - go func() { - for { - select { - case <-ticker.C: +func (e *Exporter) getFunctions(endpointURL url.URL, namespace string) ([]types.FunctionStatus, error) { + timeout := 3 * time.Second + proxyClient := e.getHTTPClient(timeout) - get, err := http.NewRequest(http.MethodGet, endpointURL.String()+"system/functions", nil) - if err != nil { - log.Println(err) - return - } + endpointURL.Path = path.Join(endpointURL.Path, "/system/functions") + if len(namespace) > 0 { + q := endpointURL.Query() + q.Set("namespace", namespace) + endpointURL.RawQuery = q.Encode() + } - if e.credentials != nil { - get.SetBasicAuth(e.credentials.User, e.credentials.Password) - } + get, _ := http.NewRequest(http.MethodGet, endpointURL.String(), nil) + if e.credentials != nil { + get.SetBasicAuth(e.credentials.User, e.credentials.Password) + } - services := []types.FunctionStatus{} - res, err := proxyClient.Do(get) - if err != nil { - log.Println(err) - continue - } - bytesOut, readErr := ioutil.ReadAll(res.Body) - if readErr != nil { - log.Println(err) - continue - } - unmarshalErr := json.Unmarshal(bytesOut, &services) - if unmarshalErr != nil { - log.Println(err) - continue - } + services := []types.FunctionStatus{} + res, err := proxyClient.Do(get) + if err != nil { + return services, err + } - e.services = services + bytesOut, readErr := ioutil.ReadAll(res.Body) + if readErr != nil { + return services, readErr + } - break - case <-quit: - return - } - } - }() + unmarshalErr := json.Unmarshal(bytesOut, &services) + if unmarshalErr != nil { + return services, unmarshalErr + } + return services, nil +} + +func (e *Exporter) getNamespaces(endpointURL url.URL) ([]string, error) { + namespaces := []string{} + + get, _ := http.NewRequest(http.MethodGet, endpointURL.String()+"system/namespaces", nil) + if e.credentials != nil { + get.SetBasicAuth(e.credentials.User, e.credentials.Password) + } + + timeout := 3 * time.Second + proxyClient := e.getHTTPClient(timeout) + + res, err := proxyClient.Do(get) + if err != nil { + return namespaces, err + } + + if res.StatusCode == http.StatusNotFound { + return namespaces, nil + } + + bytesOut, readErr := ioutil.ReadAll(res.Body) + if readErr != nil { + return namespaces, readErr + } + + unmarshalErr := json.Unmarshal(bytesOut, &namespaces) + if unmarshalErr != nil { + return namespaces, unmarshalErr + } + return namespaces, nil }