From 6ab30ea68520d8c25e6aa03006279a98c5789c07 Mon Sep 17 00:00:00 2001 From: yzj Date: Wed, 29 Jun 2022 10:28:07 +0800 Subject: [PATCH] first log --- log/logger.go | 272 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 271 insertions(+), 1 deletion(-) diff --git a/log/logger.go b/log/logger.go index c004df8..e5136a9 100644 --- a/log/logger.go +++ b/log/logger.go @@ -18,12 +18,15 @@ package log import ( + "compress/gzip" "errors" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime/debug" + "strconv" "strings" "sync" "time" @@ -73,6 +76,9 @@ type Logger struct { reopenChan chan struct{} closeChan chan struct{} writeBufferChan chan LogBuffer + size int64 + millCh chan struct{} + mu sync.Mutex } type LoggerInfo struct { @@ -181,6 +187,7 @@ func (l *Logger) start() error { } l.writer = writer } else { // write to file + fmt.Println(l.output) if err := os.MkdirAll(filepath.Dir(l.output), 0755); err != nil { return err } @@ -188,6 +195,28 @@ func (l *Logger) start() error { if err != nil { return err } + + if fileInfo, err := os.Stat(l.output); err == nil { + l.size = fileInfo.Size() + } + + //if l.roller.MaxBackups != 0 { + // fileInfoList, err := ioutil.ReadDir(filepath.Dir(l.output)) + // if err != nil { + // return err + // } + // logFilesNum := 0 + // for i := range fileInfoList { + // if strings.HasPrefix(fileInfoList[i].Name(), strings.TrimSuffix(path.Base(l.output), path.Ext(path.Base(l.output)))) { + // logFilesNum++ + // } + // } + // if l.roller.MaxBackups <= logFilesNum { + // l.writer = file + // break + // } + //} + if l.roller.MaxTime == 0 { file.Close() l.roller.Filename = l.output @@ -247,6 +276,7 @@ func (l *Logger) handler() { } } case buf := <-l.writeBufferChan: + l.Write(buf.Bytes()) PutLogBuffer(buf) } @@ -276,6 +306,7 @@ func (l *Logger) reopen() error { if err := closer.Close(); err != nil { fmt.Fprintf(os.Stderr, "logger %s close error when restart, error: %v", l.output, err) } + l.mill() return l.start() } return ErrReopenUnsupported @@ -427,7 +458,37 @@ func doRotateFunc(l *Logger, interval time.Duration) { } func (l *Logger) Write(p []byte) (n int, err error) { - return l.writer.Write(p) + l.mu.Lock() + defer l.mu.Unlock() + writeLen := int64(len(p)) + if writeLen > l.max() { + return 0, fmt.Errorf( + "write length %d exceeds maximum file size %d", writeLen, l.roller.MaxSize, + ) + } + + if l.writer == nil { + if err = l.start(); err != nil { + return 0, err + } + } + + if l.size+writeLen > l.max() { + rollerHandler(&LoggerInfo{ + LogRoller: *l.roller, + FileName: l.output, + CreateTime: time.Now(), + }) + if err := l.reopen(); err != nil { + return 0, err + } + } + + n, err = l.writer.Write(p) + l.size += int64(n) + + return n, err + //return l.writer.Write(p) } func (l *Logger) Close() error { @@ -471,3 +532,212 @@ func parseSyslogAddress(location string) *syslogAddress { return nil } + +//------------------------------------------- +const ( + compressSuffix = ".gz" + megabyte = 1024 * 1024 +) + +// max returns the maximum size in bytes of log files before rolling. +func (l *Logger) max() int64 { + if l.roller.MaxSize == 0 { + return int64(defaultRotateSize * megabyte) + } + return int64(l.roller.MaxSize) * int64(megabyte) +} + +// millRunOnce performs compression and removal of stale log files. +// Log files are compressed if enabled via configuration and old log +// files are removed, keeping at most l.MaxBackups files, as long as +// none of them are older than MaxAge. +func (l *Logger) millRunOnce() error { + if l.roller.MaxBackups == 0 && l.roller.MaxAge == 0 && !l.roller.Compress { + return nil + } + + files, err := l.oldLogFiles() + if err != nil { + return err + } + + var compress, remove []LoggerInfo + + if l.roller.MaxBackups > 0 && l.roller.MaxBackups < len(files) { + preserved := make(map[string]bool) + var remaining []LoggerInfo + for _, f := range files { + // Only count the uncompressed log file or the + // compressed log file, not both. + fn := f.FileName + if strings.HasSuffix(fn, compressSuffix) { + fn = fn[:len(fn)-len(compressSuffix)] + } + preserved[fn] = true + + if len(preserved) > l.roller.MaxBackups { + remove = append(remove, f) + } else { + remaining = append(remaining, f) + } + } + files = remaining + } + if l.roller.MaxAge > 0 { + diff := time.Duration(int64(24*time.Hour) * int64(l.roller.MaxAge)) + cutoff := time.Now().Add(-1 * diff) + + var remaining []LoggerInfo + for _, f := range files { + if f.CreateTime.Before(cutoff) { + remove = append(remove, f) + } else { + remaining = append(remaining, f) + } + } + files = remaining + } + + if l.roller.Compress { + for _, f := range files { + if !strings.HasSuffix(f.FileName, compressSuffix) { + compress = append(compress, f) + } + } + } + + for _, f := range remove { + errRemove := os.Remove(filepath.Join(l.dir(), f.FileName)) + if err == nil && errRemove != nil { + err = errRemove + } + } + for _, f := range compress { + fn := filepath.Join(l.dir(), f.FileName) + fnCompress := compressSuffixFile(fn + compressSuffix) + errCompress := compressLogFile(fn, fnCompress) + if err == nil && errCompress != nil { + err = errCompress + } + } + + return err +} + +func compressSuffixFile(fileName string) string { + var num int + for { + if _, err := os.Stat(fileName); !os.IsNotExist(err) { + return fileName + } + fileName = fileName + "." + strconv.Itoa(num) + num++ + } +} + +// mill performs post-rotation compression and removal of stale log files, +// starting the mill goroutine if necessary. +func (l *Logger) mill() { + l.millRunOnce() +} + +// oldLogFiles returns the list of backup log files stored in the same +// directory as the current log file, sorted by ModTime +func (l *Logger) oldLogFiles() ([]LoggerInfo, error) { + files, err := ioutil.ReadDir(l.dir()) + if err != nil { + return nil, fmt.Errorf("can't read log file directory: %s", err) + } + logFiles := []LoggerInfo{} + + for _, f := range files { + if f.IsDir() { + continue + } + if !strings.HasPrefix(f.Name(), filepath.Base(l.output)+".") { + continue + } + //use modTime replace createTime + logFiles = append(logFiles, LoggerInfo{*l.roller, f.Name(), f.ModTime()}) + } + + return logFiles, nil +} + +// timeFromName extracts the formatted time from the filename by stripping off +// the filename's prefix and extension. This prevents someone's filename from +// confusing time.parse. +func (l *Logger) timeFromName(filename, prefix, ext string) (time.Time, error) { + if !strings.HasPrefix(filename, prefix) { + return time.Time{}, errors.New("mismatched prefix") + } + if !strings.HasSuffix(filename, ext) { + return time.Time{}, errors.New("mismatched extension") + } + return time.Now(), nil +} + +// prefixAndExt returns the filename part and extension part from the Logger's +// filename. +func (l *Logger) prefixAndExt() (prefix, ext string) { + filename := filepath.Base(l.output) + + prefix = filename + "." + + return prefix, ext +} + +// dir returns the directory for the current filename. +func (l *Logger) dir() string { + return filepath.Dir(l.output) +} + +// compressLogFile compresses the given log file, removing the +// uncompressed log file if successful. +func compressLogFile(src, dst string) (err error) { + f, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open log file: %v", err) + } + defer f.Close() + + fi, err := os.Stat(src) + if err != nil { + return fmt.Errorf("failed to stat log file: %v", err) + } + // If this file already exists, we presume it was created by + // a previous attempt to compress the log file. + gzf, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode()) + if err != nil { + return fmt.Errorf("failed to open compressed log file: %v", err) + } + defer gzf.Close() + + gz := gzip.NewWriter(gzf) + + defer func() { + if err != nil { + os.Remove(dst) + err = fmt.Errorf("failed to compress log file: %v", err) + } + }() + + if _, err := io.Copy(gz, f); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + if err := gzf.Close(); err != nil { + return err + } + + if err := f.Close(); err != nil { + return err + } + if err := os.Remove(src); err != nil { + return err + } + + return nil +}