Merge pull request #173 from prometheus/superq/multi-target

Add multi-target scrape support

README.md

@@ -188,6 +188,48 @@ using the `--web.config.file` parameter. The format of the file is described
 To use TLS for connections to memcached, use the `--memcached.tls.*` flags.
 See `memcached_exporter --help` for details.
 
+## Multi-target
+
+The exporter also supports the [multi-target](https://prometheus.io/docs/guides/multi-target-exporter/) pattern on the `/scrape` endpoint. Example:
+```
+curl `localhost:9150/scrape?target=memcached-host.company.com:11211
+```
+
+An example configuration using [prometheus-elasticache-sd](https://github.com/maxbrunet/prometheus-elasticache-sd):
+
+```
+scrape_configs:
+  - job_name: "memcached_exporter_targets"
+    file_sd_configs:
+    - files:
+        - /path/to/elasticache.json  # File created by service discovery
+    metrics_path: /scrape
+    relabel_configs:
+      # Filter for memcached cache nodes
+      - source_labels: [__meta_elasticache_engine]
+        regex: memcached
+        action: keep
+      # Build Memcached URL to use as target parameter for the exporter
+      - source_labels:
+          - __meta_elasticache_endpoint_address
+          - __meta_elasticache_endpoint_port
+        replacement: $1
+        separator: ':'
+        target_label: __param_target
+      # Use Redis URL as instance label
+      - source_labels: [__param_target]
+        target_label: instance
+      # Set exporter address
+      - target_label: __address__
+        replacement: memcached-exporter-service.company.com:9151
+```
+
+If you are running solely for `multi-target` start the exporter with `--memcached.address=""` to avoid attempting to connect to a non existing memcached host, example:
+
+```
+./memcached-exporter --memcached.address=""
+```
+
 [buildstatus]: https://circleci.com/gh/prometheus/memcached_exporter/tree/master.svg?style=shield
 [circleci]: https://circleci.com/gh/prometheus/memcached_exporter
 [hub]: https://hub.docker.com/r/prom/memcached-exporter/

cmd/memcached_exporter/main.go

@@ -33,6 +33,7 @@ import (
 	webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag"
 
 	"github.com/prometheus/memcached_exporter/pkg/exporter"
+	"github.com/prometheus/memcached_exporter/scraper"
 )
 
 func main() {
@@ -48,7 +49,9 @@ func main() {
 		serverName         = kingpin.Flag("memcached.tls.server-name", "Memcached TLS certificate servername").Default("").String()
 		webConfig          = webflag.AddFlags(kingpin.CommandLine, ":9150")
 		metricsPath        = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
+		scrapePath         = kingpin.Flag("web.scrape-path", "Path under which to receive scrape requests.").Default("/scrape").String()
 	)
+
 	promlogConfig := &promlog.Config{}
 	flag.AddFlags(kingpin.CommandLine, promlogConfig)
 	kingpin.HelpFlag.Short('h')
@@ -91,7 +94,10 @@ func main() {
 	}
 
 	prometheus.MustRegister(version.NewCollector("memcached_exporter"))
-	prometheus.MustRegister(exporter.New(*address, *timeout, logger, tlsConfig))
+
+	if *address != "" {
+		prometheus.MustRegister(exporter.New(*address, *timeout, logger, tlsConfig))
+	}
 
 	if *pidFile != "" {
 		procExporter := collectors.NewProcessCollector(collectors.ProcessCollectorOpts{
@@ -102,6 +108,9 @@ func main() {
 	}
 
 	http.Handle(*metricsPath, promhttp.Handler())
+	scraper := scraper.New(*timeout, logger, tlsConfig)
+	http.Handle(*scrapePath, scraper.Handler())
+
 	if *metricsPath != "/" && *metricsPath != "" {
 		landingConfig := web.LandingConfig{
 			Name:        "memcached_exporter",

cmd/memcached_exporter/main_test.go

@@ -28,7 +28,59 @@ import (
 	"github.com/grobie/gomemcache/memcache"
 )
 
-func TestAcceptance(t *testing.T) {
+func waitExporterReady(t *testing.T, errorChannel chan error, address string) {
+	for {
+		timer := time.NewTimer(100 * time.Millisecond)
+		select {
+		case <-timer.C:
+			resp, err := http.Get(address)
+			if err != nil {
+				t.Logf("error requesting the exporter: %v", err)
+				continue
+			}
+			if resp.StatusCode != http.StatusOK {
+				resp.Body.Close()
+				t.Logf("unexpected exporter status code: %d", resp.StatusCode)
+				continue
+			} else {
+				return
+			}
+
+		case err := <-errorChannel:
+			t.Fatal("error running the exporter:", err)
+		}
+	}
+}
+
+func warmUpMemcached(t *testing.T, client *memcache.Client) {
+	item := &memcache.Item{Key: "foo", Value: []byte("bar")}
+	if err := client.Set(item); err != nil {
+		t.Fatal(err)
+	}
+	if err := client.Set(item); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := client.Get("foo"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := client.Get("qux"); err != memcache.ErrCacheMiss {
+		t.Fatal(err)
+	}
+	last, err := client.Get("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	last.Value = []byte("banana")
+	if err := client.CompareAndSwap(last); err != nil {
+		t.Fatal(err)
+	}
+	large := &memcache.Item{Key: "large", Value: bytes.Repeat([]byte("."), 130)}
+	if err := client.Set(large); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAcceptanceSingleInstance(t *testing.T) {
 	errc := make(chan error)
 
 	addr := "localhost:11211"
@@ -37,6 +89,7 @@ func TestAcceptance(t *testing.T) {
 		addr = strings.TrimPrefix(env, "tcp://")
 	}
 
+	t.Logf("starting exporter")
 	ctx, cancel := context.WithCancel(context.Background())
 	exporter := exec.CommandContext(ctx, "../../memcached_exporter", "--memcached.address", addr)
 	go func() {
@@ -50,22 +103,8 @@ func TestAcceptance(t *testing.T) {
 	defer cancel()
 
 	// Wait for the exporter to be up and running.
-OUTER:
-	for {
-		timer := time.NewTimer(100 * time.Millisecond)
-		select {
-		case <-timer.C:
-			resp, err := http.Get("http://localhost:9150/")
-			if err == nil {
-				resp.Body.Close()
-				if resp.StatusCode == http.StatusOK {
-					break OUTER
-				}
-			}
-		case err := <-errc:
-			t.Fatal("error running the exporter:", err)
-		}
-	}
+	t.Logf("waiting exporter initialization")
+	waitExporterReady(t, errc, "http://localhost:9150")
 
 	client, err := memcache.New(addr)
 	if err != nil {
@@ -75,32 +114,146 @@ OUTER:
 		t.Fatal(err)
 	}
 
-	item := &memcache.Item{Key: "foo", Value: []byte("bar")}
-	if err := client.Set(item); err != nil {
+	warmUpMemcached(t, client)
+
+	stats_settings, err := client.StatsSettings()
+	if err != nil {
 		t.Fatal(err)
 	}
-	if err := client.Set(item); err != nil {
+
+	use_temp_lru := false
+	for _, t := range stats_settings {
+		if t["temp_lru"] == "true" {
+			use_temp_lru = true
+		}
+	}
+
+	stats, err := client.Stats()
+	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err := client.Get("foo"); err != nil {
+
+	memcache_version := ""
+	for _, t := range stats {
+		memcache_version = t.Stats["version"]
+	}
+
+	resp, err := http.Get("http://localhost:9150/metrics")
+	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err := client.Get("qux"); err != memcache.ErrCacheMiss {
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
 		t.Fatal(err)
 	}
-	last, err := client.Get("foo")
+
+	tests := []string{
+		// memcached_current_connections varies depending on memcached versions
+		// so it isn't practical to check for an exact value.
+		`memcached_current_connections `,
+		`memcached_up 1`,
+		`memcached_commands_total{command="get",status="hit"} 2`,
+		`memcached_commands_total{command="get",status="miss"} 1`,
+		`memcached_commands_total{command="set",status="hit"} 3`,
+		`memcached_commands_total{command="cas",status="hit"} 1`,
+		`memcached_current_bytes 262`,
+		`memcached_max_connections 1024`,
+		`memcached_current_items 2`,
+		`memcached_items_total 4`,
+		`memcached_slab_current_items{slab="1"} 1`,
+		`memcached_slab_current_items{slab="5"} 1`,
+		`memcached_slab_commands_total{command="set",slab="1",status="hit"} 2`,
+		`memcached_slab_commands_total{command="cas",slab="1",status="hit"} 1`,
+		`memcached_slab_commands_total{command="set",slab="5",status="hit"} 1`,
+		`memcached_slab_commands_total{command="cas",slab="5",status="hit"} 0`,
+		`memcached_slab_current_chunks{slab="1"} 10922`,
+		`memcached_slab_current_chunks{slab="5"} 4369`,
+		`memcached_slab_mem_requested_bytes{slab="1"} 68`,
+		`memcached_slab_mem_requested_bytes{slab="5"} 194`,
+	}
+
+	if use_temp_lru == true {
+		tests = append(tests,
+			`memcached_slab_temporary_items{slab="1"}`,
+			`memcached_slab_lru_hits_total{lru="temporary",slab="5"}`,
+			`memcached_slab_temporary_items{slab="5"}`)
+	}
+
+	memcache_version_major_minor, err := strconv.ParseFloat(memcache_version[0:3], 64)
 	if err != nil {
 		t.Fatal(err)
 	}
-	last.Value = []byte("banana")
-	if err := client.CompareAndSwap(last); err != nil {
+
+	if memcache_version_major_minor >= 1.5 {
+		tests = append(tests,
+			`memcached_slab_lru_hits_total{lru="hot",slab="1"}`,
+			`memcached_slab_lru_hits_total{lru="cold",slab="1"}`,
+			`memcached_slab_lru_hits_total{lru="warm",slab="1"}`,
+			`memcached_slab_lru_hits_total{lru="temporary",slab="1"}`,
+			`memcached_slab_hot_items{slab="1"}`,
+			`memcached_slab_warm_items{slab="1"}`,
+			`memcached_slab_cold_items{slab="1"}`,
+			`memcached_slab_hot_age_seconds{slab="1"}`,
+			`memcached_slab_warm_age_seconds{slab="1"}`,
+			`memcached_slab_lru_hits_total{lru="hot",slab="5"}`,
+			`memcached_slab_lru_hits_total{lru="cold",slab="5"}`,
+			`memcached_slab_lru_hits_total{lru="warm",slab="5"}`,
+			`memcached_slab_hot_items{slab="5"}`,
+			`memcached_slab_warm_items{slab="5"}`,
+			`memcached_slab_cold_items{slab="5"}`,
+			`memcached_slab_hot_age_seconds{slab="5"}`,
+			`memcached_slab_warm_age_seconds{slab="5"}`)
+	}
+
+	for _, test := range tests {
+		if !bytes.Contains(body, []byte(test)) {
+			t.Errorf("want metrics to include %q, have:\n%s", test, body)
+		}
+	}
+
+	cancel()
+
+	<-errc
+}
+
+func TestAcceptanceScrapper(t *testing.T) {
+	errc := make(chan error)
+
+	addr := "localhost:11211"
+	// MEMCACHED_PORT might be set by a linked memcached docker container.
+	if env := os.Getenv("MEMCACHED_PORT"); env != "" {
+		addr = strings.TrimPrefix(env, "tcp://")
+	}
+
+	t.Logf("starting exporter")
+	ctx, cancel := context.WithCancel(context.Background())
+	exporter := exec.CommandContext(ctx, "../../memcached_exporter", "--memcached.address", "")
+	go func() {
+		defer close(errc)
+
+		if err := exporter.Run(); err != nil && errc != nil {
+			errc <- err
+		}
+	}()
+
+	defer cancel()
+
+	// Wait for the exporter to be up and running.
+	t.Logf("waiting exporter initialization")
+	waitExporterReady(t, errc, "http://localhost:9150")
+
+	client, err := memcache.New(addr)
+	if err != nil {
 		t.Fatal(err)
 	}
-	large := &memcache.Item{Key: "large", Value: bytes.Repeat([]byte("."), 130)}
-	if err := client.Set(large); err != nil {
+	if err := client.StatsReset(); err != nil {
 		t.Fatal(err)
 	}
 
+	warmUpMemcached(t, client)
+
 	stats_settings, err := client.StatsSettings()
 	if err != nil {
 		t.Fatal(err)
@@ -123,7 +276,7 @@ OUTER:
 		memcache_version = t.Stats["version"]
 	}
 
-	resp, err := http.Get("http://localhost:9150/metrics")
+	resp, err := http.Get("http://localhost:9150/scrape?target=" + addr)
 	if err != nil {
 		t.Fatal(err)
 	}