first commit

Author: apetruhin

.gitignore

@@ -0,0 +1 @@
+.idea

clean.go

@@ -0,0 +1,37 @@
+package logparser
+
+import (
+	"regexp"
+	"strings"
+)
+
+var (
+	timestampRegexes = []*regexp.Regexp{
+		regexp.MustCompile(`(^|\s)\d{2}:\d{2}(:\d{2}[^\s"']*)?`),
+		regexp.MustCompile(`\d{2} [A-Z][a-z]{2} \d{4}`),
+		regexp.MustCompile(`\d{4}-\d{2}-\d{2}`),
+		regexp.MustCompile(`\d{4}/\d{2}/\d{2}`),
+		regexp.MustCompile(`\d{4}\.\d{2}\.\d{2}`),
+		regexp.MustCompile(`[A-Z][a-z]{2} \d{2}`),
+		regexp.MustCompile(`\d{2}-\d{2}-\d{4}`),
+		regexp.MustCompile(`\d{2}/\d{2}/\d{4}`),
+		regexp.MustCompile(`\d{2}\.\d{2}\.\d{4}`),
+	}
+	extraSpaces = regexp.MustCompile(`\s+`)
+)
+
+func clean(line string) string {
+	for _, r := range timestampRegexes {
+		line = r.ReplaceAllString(line, "")
+	}
+	return strings.TrimSpace(extraSpaces.ReplaceAllString(line, " "))
+}
+
+func containsTimestamp(line string) bool {
+	for _, re := range timestampRegexes {
+		if re.MatchString(line) {
+			return true
+		}
+	}
+	return false
+}

clean_test.go

@@ -0,0 +1,27 @@
+package logparser
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_containsTimestamp(t *testing.T) {
+	assert.True(t, containsTimestamp("2005-08-09"))
+	assert.True(t, containsTimestamp("2020/06/26"))
+	assert.True(t, containsTimestamp("02/17/2009"))
+	assert.True(t, containsTimestamp("25.02.2013"))
+	assert.True(t, containsTimestamp("2013.25.02"))
+	assert.True(t, containsTimestamp("18:31"))
+	assert.True(t, containsTimestamp("18:31:42"))
+	assert.True(t, containsTimestamp("18:31:42+03"))
+	assert.True(t, containsTimestamp("18:31:42-03"))
+	assert.True(t, containsTimestamp("18:31:42+03:30"))
+	assert.True(t, containsTimestamp("18:31:42-03:30"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42+03"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42-03"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42+03:30"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42-03:30"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42"))
+	assert.True(t, containsTimestamp("2005-08-09T18:31:42.201"))
+}

decoder.go

@@ -0,0 +1,41 @@
+package logparser
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type DockerLogJson struct {
+	Log string
+}
+
+type Decoder interface {
+	Decode(string) (string, error)
+}
+
+type DockerJsonDecoder struct{}
+
+func (d DockerJsonDecoder) Decode(src string) (string, error) {
+	obj := DockerLogJson{}
+	if err := json.Unmarshal([]byte(src), &obj); err != nil {
+		return "", fmt.Errorf(`failed to unmarshal docker log entry "%s": %s`, src, err)
+	}
+	return obj.Log, nil
+}
+
+type CriDecoder struct{}
+
+func (d CriDecoder) Decode(src string) (string, error) {
+	c := 0
+	i := strings.IndexFunc(src, func(r rune) bool {
+		if r == ' ' {
+			c++
+		}
+		return c == 3
+	})
+	if i < 0 {
+		return "", fmt.Errorf("unexpected entry format: %s", src)
+	}
+	return src[i+1:], nil
+}

go.mod

@@ -0,0 +1,5 @@
+module github.com/coroot/logparser
+
+go 1.16
+
+require github.com/stretchr/testify v1.7.0

go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

level.go

@@ -0,0 +1,164 @@
+package logparser
+
+import (
+	"strings"
+	"unicode"
+)
+
+type Level string
+
+const (
+	LevelUnknown  Level = "unknown"
+	LevelDebug    Level = "debug"
+	LevelInfo     Level = "info"
+	LevelWarning  Level = "warning"
+	LevelError    Level = "error"
+	LevelCritical Level = "critical"
+
+	maxLineLenForGuessingLevel = 255
+	guessLevelInFields         = 5
+)
+
+var (
+	glogLevelsMapping = map[byte]Level{
+		'I': LevelInfo,
+		'W': LevelWarning,
+		'E': LevelError,
+		'F': LevelCritical,
+	}
+	priority2Levels = map[string]Level{
+		"0": LevelCritical,
+		"1": LevelCritical,
+		"2": LevelCritical,
+		"3": LevelError,
+		"4": LevelWarning,
+		"5": LevelInfo,
+		"6": LevelInfo,
+		"7": LevelDebug,
+	}
+)
+
+func LevelByPriority(priority string) Level {
+	if level, ok := priority2Levels[priority]; ok {
+		return level
+	}
+	return LevelUnknown
+}
+
+func GuessLevel(line string) Level {
+	if len(line) > maxLineLenForGuessingLevel {
+		line = line[:maxLineLenForGuessingLevel]
+	}
+	fields := strings.Fields(line)
+	if len(fields) == 0 {
+		return LevelUnknown
+	}
+	limit := len(fields)
+	if limit > guessLevelInFields {
+		limit = guessLevelInFields
+	}
+
+	if l := tryGlog(fields); l != LevelUnknown {
+		return l
+	}
+
+	for _, f := range fields[:limit] {
+		subfields := strings.FieldsFunc(f, func(r rune) bool {
+			return r == ']' || r == ')' || r == ';' || r == '|' || r == ':' || r == ','
+		})
+		for _, sf := range subfields {
+			sf = strings.TrimLeft(strings.ToLower(sf), "\"[('")
+			sf = strings.TrimPrefix(sf, "level=")
+			if len(sf) < 4 {
+				continue
+			}
+			switch sf[:4] {
+			case "debu":
+				return LevelDebug
+			case "info":
+				return LevelInfo
+			case "warn":
+				return LevelWarning
+			case "erro":
+				return LevelError
+			case "crit", "fata":
+				return LevelCritical
+			}
+		}
+	}
+	if l := guessRedisLevel(fields); l != LevelUnknown {
+		return l
+	}
+	return LevelUnknown
+}
+
+func tryGlog(fields []string) Level {
+	firstField := fields[0]
+	if len(firstField) != 5 {
+		return LevelUnknown
+	}
+	level, ok := glogLevelsMapping[firstField[0]]
+	if !ok {
+		return LevelUnknown
+	}
+	for _, r := range firstField[1:] {
+		if !unicode.IsDigit(r) {
+			return LevelUnknown
+		}
+	}
+	return level
+}
+
+// redis 2.x
+// [pid] date loglevel message
+// [4018] 14 Nov 07:01:22.119 * Background saving terminated with success
+
+// redis 3.x+
+// pid:role timestamp loglevel message
+// 1:S 12 Nov 07:52:11.999 * FAIL message received from X about Y
+
+// redis 5.x: the year was added
+// 1:S 12 Nov 2019 07:52:11.999 * FAIL message received from X about Y
+func guessRedisLevel(fields []string) Level {
+	if len(fields) < 6 {
+		return LevelUnknown
+	}
+	if strings.HasPrefix(fields[0], "[") && strings.HasSuffix(fields[0], "]") {
+		return redisCharToLevel(fields[4])
+	}
+	if len(strings.Split(fields[0], ":")) == 2 {
+		if len(fields[3]) == 4 { //redis 5.x+
+			return redisCharToLevel(fields[5])
+		} else {
+			return redisCharToLevel(fields[4])
+		}
+	}
+	return LevelUnknown
+}
+
+func redisCharToLevel(level string) Level {
+	switch level {
+	case ".":
+		return LevelDebug
+	case "-":
+		return LevelInfo
+	case "*", "#":
+		return LevelWarning
+	}
+	return LevelUnknown
+}
+
+// todo
+//
+// python
+//
+//   loglevels: DEBUG, INFO, WARNING, ERROR,CRITICAL
+// pylogging:
+//  default "%(levelname)s:%(name)s:%(message)s" https://github.com/python/cpython/blob/master/Lib/logging/__init__.py#L502
+// django:
+//   {levelname} {message}
+//   {asctime} {module} [{levelname}] okserver
+//   [%(asctime)s] %(levelname)s
+//   %(levelname)s %(asctime)s %(module)s: %(message)s
+// asctime: %(asctime)s Human-readable time when the LogRecord was created. By default this is of the form ‘2003-07-08 16:49:45,896’ (the numbers after the comma are millisecond portion of the time).
+//

level_test.go

@@ -0,0 +1,29 @@
+package logparser
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestGuessLevelGlog(t *testing.T) {
+	//glog & klog
+	assert.Equal(t, LevelUnknown, GuessLevel(`11002 a msg`))
+	assert.Equal(t, LevelUnknown, GuessLevel(`WHAT1 a msg`))
+	assert.Equal(t, LevelInfo, GuessLevel(`I0430 11:58:31.792717       1 cluster.go:337] memberlist 2020/04/30 11:58:31 [DEBUG] memberlist: Initiating push/pull sync with: 127.0.0.1:4000`))
+	assert.Equal(t, LevelWarning, GuessLevel(`W0430 11:29:23.177635       1 nanny.go:120] Got EOF from stdout`))
+	assert.Equal(t, LevelError, GuessLevel(`E0504 07:38:36.184861       1 replica_set.go:450] Sync "monitoring/prometheus-operator-5cfbdc9b67" failed with pods "prometheus-operator-5cfbdc9b67-" is forbidden: error looking up service account monitoring/prometheus-operator: serviceaccount "prometheus-operator" not found`))
+	assert.Equal(t, LevelCritical, GuessLevel(`F0825 185142 test.cc:22] Check failed: write(1, NULL, 2) >= 0 Write NULL failed: Bad address [14]`))
+}
+
+func TestGuessLevelRedis(t *testing.T) {
+	assert.Equal(t, LevelWarning, GuessLevel(`[4018] 14 Nov 07:01:22.119 * Background saving terminated with success`))
+	assert.Equal(t, LevelInfo, GuessLevel(`1:S 12 Nov 07:52:11.999 - some msg`))
+	assert.Equal(t, LevelDebug, GuessLevel(`1:S 12 Nov 2019 07:52:11.999 . verbosed`))
+}
+
+func TestGuessLevel(t *testing.T) {
+	assert.Equal(t, LevelInfo, GuessLevel(`[info:2016-02-16T16:04:05.930-08:00] Some log text here`))
+	assert.Equal(t, LevelInfo, GuessLevel(`2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json`))
+	assert.Equal(t, LevelError, GuessLevel(`2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404`))
+	assert.Equal(t, LevelDebug, GuessLevel(`[2020-06-25 17:35:37,609][DEBUG][action.search            ] [srv] [tweets-100][6]`))
+}