diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 82c035f..8ab4c7d 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -14,7 +14,7 @@ jobs: - name: golangci-lint uses: reviewdog/action-golangci-lint@v1 with: - golangci_lint_flags: "--enable-all --timeout=10m --exclude-use-default=false --tests=false --disable=gochecknoinits,gochecknoglobals,exhaustive,nakedret,wrapcheck" + golangci_lint_flags: "--enable-all --timeout=10m --exclude-use-default=false --tests=false --disable=gochecknoinits,gochecknoglobals,exhaustive,nakedret,wrapcheck -D G302" test: name: test diff --git a/log/logger.go b/log/logger.go index c004df8..ce9a2c6 100644 --- a/log/logger.go +++ b/log/logger.go @@ -18,12 +18,16 @@ package log import ( + "compress/gzip" "errors" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime/debug" + "sort" + "strconv" "strings" "sync" "time" @@ -205,6 +209,7 @@ func (l *Logger) start() error { l.create = time.Now() } l.writer = file + l.mill() l.once.Do(l.startRotate) // start rotate, only once } } @@ -247,6 +252,7 @@ func (l *Logger) handler() { } } case buf := <-l.writeBufferChan: + l.Write(buf.Bytes()) PutLogBuffer(buf) } @@ -276,6 +282,7 @@ func (l *Logger) reopen() error { if err := closer.Close(); err != nil { fmt.Fprintf(os.Stderr, "logger %s close error when restart, error: %v", l.output, err) } + l.mill() return l.start() } return ErrReopenUnsupported @@ -471,3 +478,203 @@ func parseSyslogAddress(location string) *syslogAddress { return nil } + +const ( + compressSuffix = ".gz" +) + +// millRunOnce performs compression and removal of stale log files. +// Log files are compressed if enabled via configuration and old log +// files are removed, keeping at most l.MaxBackups files, as long as +// none of them are older than MaxAge. +func (l *Logger) millRunOnce() error { + files, err := l.oldLogFiles() + if err != nil { + return err + } + + compress, remove := l.screeningCompressFile(files) + + for _, f := range remove { + _ = os.Remove(filepath.Join(l.dir(), f.FileName)) + } + + for _, f := range compress { + var fnCompress string + fnCompress, err = l.findCompressFile(f.FileName) + if err != nil { + return err + } + errCompress := l.compressLogFile(f.FileName, fnCompress) + if err != nil && errCompress != nil { + err = errCompress + } + } + + return err +} + +func (l *Logger) screeningCompressFile(files []LoggerInfo) (compress, remove []LoggerInfo) { + resFiles, removeByMaxAge := l.screeningCompressFileByMaxAge(files) + resFiles, remove = l.screeningCompressFileByMaxBackups(resFiles, removeByMaxAge) + + if l.roller.Compress { + for i := range resFiles { + if !strings.HasSuffix(resFiles[i].FileName, compressSuffix) { + compress = append(compress, resFiles[i]) + } + } + } + return +} + +func (l *Logger) screeningCompressFileByMaxAge(files []LoggerInfo) (resFiles, remove []LoggerInfo) { + if l.roller.MaxAge > 0 { + diff := time.Duration(int64(maxRotateHour*time.Hour) * int64(l.roller.MaxAge)) + cutoff := time.Now().Add(-1 * diff) + + for i := range files { + if files[i].CreateTime.Before(cutoff) { + remove = append(remove, files[i]) + } else { + resFiles = append(resFiles, files[i]) + } + } + } else { + resFiles = files + } + return +} + +func (l *Logger) screeningCompressFileByMaxBackups(files, remove []LoggerInfo) (resFiles, resRemove []LoggerInfo) { + if l.roller.MaxBackups > 0 && l.roller.MaxBackups < len(files) { + preserved := make(map[string]bool) + + for i := range files { + // Only count the uncompressed log file or the + // compressed log file, not both. + fn := files[i].FileName + + preserved[strings.TrimSuffix(fn, compressSuffix)] = true + + if len(preserved) > l.roller.MaxBackups { + remove = append(remove, files[i]) + } else { + resFiles = append(resFiles, files[i]) + } + } + } else { + resFiles = files + } + resRemove = remove + return +} + +func (l *Logger) findCompressFile(fileName string) (string, error) { + num := 1 + statName := fileName + + for i := 0; i < 10; i++ { + if _, err := os.Stat(l.dir() + statName + compressSuffix); os.IsNotExist(err) { + return statName + compressSuffix, nil + } + statName = fileName + "." + strconv.Itoa(num) + num++ + } + return fileName, errors.New("findCompressFile failed") +} + +func (l *Logger) mill() { + if l.roller.MaxBackups != 0 || l.roller.MaxAge != 0 || l.roller.Compress { + _ = l.millRunOnce() + } +} + +// oldLogFiles returns the list of backup log files stored in the same +// directory as the current log file, sorted by ModTime +func (l *Logger) oldLogFiles() ([]LoggerInfo, error) { + files, err := ioutil.ReadDir(l.dir()) + if err != nil { + return nil, err + } + logFiles := []LoggerInfo{} + + for _, f := range files { + if f.IsDir() { + continue + } + if !strings.HasPrefix(f.Name(), filepath.Base(l.output)+".") { + continue + } + //use modTime replace createTime + logFiles = append(logFiles, LoggerInfo{*l.roller, f.Name(), f.ModTime()}) + } + sort.Sort(byFormatTime(logFiles)) + + return logFiles, nil +} + +// dir returns the directory for the current filename. +func (l *Logger) dir() string { + return filepath.Dir(l.output) +} + +// compressLogFile compresses the given log file, removing the +// uncompressed log file if successful. +func (l *Logger) compressLogFile(srcFile, dstFile string) error { + f, err := os.Open(filepath.Join(l.dir(), filepath.Clean(srcFile))) + if err != nil { + return err + } + + defer func() { + _ = f.Close() + }() + + gzf, err := os.OpenFile(filepath.Join(l.dir(), filepath.Clean(dstFile)), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return err + } + + defer func() { + _ = gzf.Close() + if err != nil { + _ = os.Remove(filepath.Join(l.dir(), filepath.Clean(dstFile))) + } + }() + + gz := gzip.NewWriter(gzf) + + if _, err = io.Copy(gz, f); err != nil { + return err + } + + if err = gz.Close(); err != nil { + return err + } + + if err = gzf.Close(); err != nil { + return err + } + + if err = f.Close(); err != nil { + return err + } + + return os.Remove(filepath.Join(l.dir(), filepath.Clean(srcFile))) +} + +// byFormatTime sorts by newest time formatted in the name. +type byFormatTime []LoggerInfo + +func (b byFormatTime) Less(i, j int) bool { + return b[i].CreateTime.After(b[j].CreateTime) +} + +func (b byFormatTime) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b byFormatTime) Len() int { + return len(b) +} diff --git a/log/logger_test.go b/log/logger_test.go index c15cc1a..932db34 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -400,3 +400,59 @@ WAIT: t.Logf("received %d reopens", reopens) close(l.stopRotate) } + +func TestLogRollerTimeAndCompress(t *testing.T) { + logName := "/tmp/mosn_bench/defaultCompress.log" + rollerName := logName + "." + time.Now().Format("2006-01-02_15") + os.Remove(logName) + os.Remove(rollerName) + // replace rotate interval for test + doRotate = testRotateByKeep + defer func() { + doRotate = testRotateByKeep + }() + logger, err := GetOrCreateLogger(logName, &Roller{MaxTime: 2, Handler: rollerHandler, Compress: true}) + if err != nil { + t.Fatal(err) + } + // 1111 will be rotated to rollerName + logger.Print(newLogBufferString("1111111"), false) + time.Sleep(2 * time.Second) + // 2222 will be writed in logName + logger.Print(newLogBufferString("2222222"), false) + time.Sleep(1 * time.Second) + logger.Close() // stop the rotate + + if !exists(rollerName + compressSuffix) { + t.Fatalf("compress is failed") + } +} + +func testRotateByKeep(l *Logger, interval time.Duration) { + doRotateFunc(l, 1*time.Second) +} +func TestLogRollerTimeAndKeep(t *testing.T) { + logName := "/tmp/mosn_bench/defaultKeep.log" + rollerName := logName + "." + time.Now().Format("2006-01-02_15") + os.Remove(logName) + os.Remove(rollerName) + // replace rotate interval for test + doRotate = testRotateByKeep + defer func() { + doRotate = testRotateByKeep + }() + logger, err := GetOrCreateLogger(logName, &Roller{MaxTime: 2, Handler: rollerHandler, MaxBackups: 1}) + if err != nil { + t.Fatal(err) + } + logger.Print(newLogBufferString("1111111"), false) + time.Sleep(2 * time.Second) + logger.Print(newLogBufferString("2222222"), false) + time.Sleep(2 * time.Second) + logger.Print(newLogBufferString("3333333"), false) + time.Sleep(1 * time.Second) + logger.Close() // stop the rotate + if exists(rollerName) { + t.Fatalf(" %s is exists", rollerName) + } +}