Adding profiling / instrumenting mode to mdox.

README.md

@@ -83,12 +83,13 @@ Requirements to build this tool:
 * Linux or MacOS
 
 ```shell
-go get github.com/bwplotka/mdox && go mod tidy
+go install github.com/bwplotka/mdox
 ```
 
 or via [bingo](https://github.com/bwplotka/bingo) if want to pin it:
 
 ```shell
+go install github.com/bwplotka/bingo
 bingo get -u github.com/bwplotka/mdox
 ```
 

go.mod

@@ -5,18 +5,23 @@ go 1.14
 require (
 	github.com/Kunde21/markdownfmt/v2 v2.0.4-0.20201214081534-353201c4cdce
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
-	github.com/efficientgo/tools v0.0.0-20201227131601-4dde9fba2562
+	github.com/efficientgo/tools/core v0.0.0-20201228212819-69909db83cda
 	github.com/go-kit/kit v0.10.0
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gohugoio/hugo v0.74.3
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mitchellh/mapstructure v1.2.2 // indirect
 	github.com/oklog/run v1.1.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.3.0
+	github.com/prometheus/common v0.7.0
 	github.com/sergi/go-diff v1.0.0
 	github.com/yuin/goldmark v1.2.1
 	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )
 
-replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201225192631-f2e7830d9793
+replace (
+	github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201225192631-f2e7830d9793
+	github.com/efficientgo/tools/core => ../tools/core
+)

main.go

@@ -6,28 +6,34 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
 	"strings"
 	"syscall"
 
-	"github.com/bwplotka/mdox/pkg/clilog"
 	"github.com/bwplotka/mdox/pkg/extkingpin"
 	"github.com/bwplotka/mdox/pkg/mdformatter"
 	"github.com/bwplotka/mdox/pkg/mdformatter/linktransformer"
 	"github.com/bwplotka/mdox/pkg/mdformatter/mdgen"
 	"github.com/bwplotka/mdox/pkg/version"
+	"github.com/efficientgo/tools/core/pkg/clilog"
+	"github.com/efficientgo/tools/core/pkg/errcapture"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/oklog/run"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/expfmt"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
 const (
 	logFormatLogfmt = "logfmt"
-	logFormatJson   = "json"
+	logFormatJSON   = "json"
 	logFormatCLILog = "clilog"
 )
 
@@ -46,7 +52,7 @@ func setupLogger(logLevel, logFormat string) log.Logger {
 		panic("unexpected log level")
 	}
 	switch logFormat {
-	case logFormatJson:
+	case logFormatJSON:
 		return level.NewFilter(log.NewJSONLogger(log.NewSyncWriter(os.Stderr)), lvl)
 	case logFormatLogfmt:
 		return level.NewFilter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), lvl)
@@ -62,7 +68,7 @@ func main() {
 	logLevel := app.Flag("log.level", "Log filtering level.").
 		Default("info").Enum("error", "warn", "info", "debug")
 	logFormat := app.Flag("log.format", "Log format to use.").
-		Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog)
+		Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJSON, logFormatCLILog)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	registerFmt(ctx, app)
@@ -150,11 +156,39 @@ This directive runs executable with arguments and put its stderr and stdout outp
 		}
 
 		var linkTr []mdformatter.LinkTransformer
+		var m *httpMetrics
 		if *linksValidateEnabled {
 			v, err := linktransformer.NewValidator(logger, *linksValidateExceptDomains, anchorDir)
 			if err != nil {
 				return err
 			}
+
+			m = &httpMetrics{
+				reg: prometheus.NewRegistry(),
+			}
+			requests := prometheus.NewCounterVec(
+				prometheus.CounterOpts{Name: "does_not_matter"},
+				[]string{"domain", "code", "method"},
+			)
+			perDomainLatency := prometheus.NewHistogramVec(
+				prometheus.HistogramOpts{Name: "does_not_matter1", Buckets: prometheus.DefBuckets},
+				[]string{"domain", "code", "method"},
+			)
+			m.reg.MustRegister(requests, perDomainLatency)
+			defer errcapture.Close(&err, m.Print, "print")
+
+			v.SetTransportFunc(func(u string) http.RoundTripper {
+				parsed, err := url.Parse(u)
+				if err != nil {
+					panic(err)
+				}
+				return promhttp.InstrumentRoundTripperCounter(
+					requests,
+					promhttp.InstrumentRoundTripperDuration(perDomainLatency.MustCurryWith(prometheus.Labels{
+						"domain": parsed.Host,
+					}), http.DefaultTransport),
+				)
+			})
 			linkTr = append(linkTr, v)
 		}
 		if *linksLocalizeForAddress != nil {
@@ -179,6 +213,26 @@ This directive runs executable with arguments and put its stderr and stdout outp
 	})
 }
 
+type httpMetrics struct {
+	reg *prometheus.Registry
+}
+
+func (h *httpMetrics) Print() error {
+	mfs, err := h.reg.Gather()
+	if err != nil {
+		return err
+	}
+
+	enc := expfmt.NewEncoder(os.Stdout, expfmt.FmtProtoText)
+
+	for _, mf := range mfs {
+		if err := enc.Encode(mf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // validateAnchorDir returns validated anchor dir against files provided.
 func validateAnchorDir(anchorDir string, files []string) (_ string, err error) {
 	if anchorDir == "" {

main_test.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/efficientgo/tools/pkg/testutil"
+	"github.com/efficientgo/tools/core/pkg/testutil"
 )
 
 func TestValidateAnchorDir(t *testing.T) {