From 000c29cafcbbed0977926648b9fc6d27085e0cf5 Mon Sep 17 00:00:00 2001 From: Krasi Georgiev Date: Wed, 9 May 2018 16:02:33 +0300 Subject: [PATCH] first draft Signed-off-by: Krasi Georgiev --- cmd/tsdb/main.go | 61 ++++++++++++++++++++++++++++++++-- db.go | 13 +++----- scan.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 11 deletions(-) create mode 100644 scan.go diff --git a/cmd/tsdb/main.go b/cmd/tsdb/main.go index 269fa5fb..d2082658 100644 --- a/cmd/tsdb/main.go +++ b/cmd/tsdb/main.go @@ -65,15 +65,72 @@ func main() { } printBlocks(db.Blocks()) case scanCmd.FullCommand(): - corrupted, err := tsdb.Scan(*scanPath) + useless, err := tsdb.Scan(*scanPath) if err != nil { exitWithError(err) } - // list corrupted blocks with date ranges and prompt: "Do you want to delete corrupted blocks N,y ?" + + if len(useless) == 0 { + fmt.Println("Hooray! The db has no corrupted blocks (or the scan tool is broken).\U0001f638") + return + } + + for err, paths := range useless { + fmt.Println(err) + for _, path := range paths { + fmt.Println(path) + } + switch err.(type) { + case tsdb.ErrorTmp: + fmt.Println(` +Scanning found some temporary files! +These are usually caused by a crash or incomplete compaction and +are safe to delete as long as you are sure that no other application is currently using this database folder.`) + case tsdb.ErrorOpen: + fmt.Println(` +Scanning found some blocks that cannot be opened! +Deleting these will allow a clean startup, but you will loose all data in the listed time ranges.`) + case tsdb.ErrorSequence: + fmt.Println(` +Scanning found some overlapping blocks! +Deleting these will allow a clean startup, but you will loose all data in the listed time ranges.`) + } + + if yes := prompt(); yes { + for _, path := range paths { + if err := os.Remove(path); err != nil { + fmt.Printf("error deleting: %v, %v", path, err) + } + fmt.Printf("%v \n", path) + } + } + } } flag.CommandLine.Set("log.level", "debug") } +func prompt() bool { + fmt.Printf("Do you want to delete these (y/N)? ") + var s string + _, err := fmt.Scanln(&s) + if err != nil { + exitWithError(err) + } + + s = strings.TrimSpace(s) + s = strings.ToLower(s) + + if s == "y" || s == "yes" { + return true + } + if s == "n" || s == "no" { + return false + } + fmt.Printf("%v is not a valid answer \n", s) + prompt() + return false +} + type writeBenchmark struct { outPath string samplesFile string diff --git a/db.go b/db.go index 25e38dc5..407750b5 100644 --- a/db.go +++ b/db.go @@ -250,11 +250,6 @@ func (db *DB) Dir() string { return db.dir } -// Scan checks the integrity of the db for a given directory and returns a list of unreadable blocks that can be deleted to allow a clean startup. -func Scan(dir string) ([]string, error) { - return nil, nil -} - func (db *DB) run() { defer close(db.donec) @@ -528,8 +523,8 @@ func (db *DB) reload(deleteable ...string) (err error) { exist[meta.ULID] = struct{}{} } - if err := validateBlockSequence(blocks); err != nil { - return errors.Wrap(err, "invalid block sequence") + if overlaps := validateBlockSequence(blocks); overlaps != nil { + return errors.Errorf("invalid block sequence , block time ranges overlap: %s", overlaps) } // Swap in new blocks first for subsequently created readers to be seen. @@ -562,7 +557,7 @@ func (db *DB) reload(deleteable ...string) (err error) { } // validateBlockSequence returns error if given block meta files indicate that some blocks overlaps within sequence. -func validateBlockSequence(bs []*Block) error { +func validateBlockSequence(bs []*Block) Overlaps { if len(bs) <= 1 { return nil } @@ -574,7 +569,7 @@ func validateBlockSequence(bs []*Block) error { overlaps := OverlappingBlocks(metas) if len(overlaps) > 0 { - return errors.Errorf("block time ranges overlap: %s", overlaps) + return overlaps } return nil diff --git a/scan.go b/scan.go new file mode 100644 index 00000000..c1fe3953 --- /dev/null +++ b/scan.go @@ -0,0 +1,85 @@ +package tsdb + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +type ErrorOpen string + +func (e ErrorOpen) Error() string { + return fmt.Sprintf("block open error: %v", e) +} + +type ErrorSequence string + +func (e ErrorSequence) Error() string { + return fmt.Sprintf("block sequence error %v", e) +} + +type ErrorTmp struct{} + +func (e ErrorTmp) Error() string { + return fmt.Sprintf("abandoned tmp files") +} + +// Scan checks the integrity of the tsdb for a given directory and returns a list of files and folders that can be deleted. +// It returns a map of error to explain the reason and slice of dirs for further processing. +func Scan(dir string) (map[error][]string, error) { + useless := make(map[error][]string) + + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil, err + } + + db := &DB{ + dir: dir, + logger: nil, + opts: &Options{}, + } + + dirs, err := blockDirs(db.dir) + if err != nil { + return nil, errors.Wrap(err, "find blocks") + } + + // Scan for blocks that can't be opened. + var errOpen ErrorOpen + for _, dir := range dirs { + b, err := OpenBlock(dir, nil) + if err != nil { + errOpen = ErrorOpen(err.Error()) + useless[errOpen] = append(useless[errOpen], dir) + break + } + + db.blocks = append(db.blocks, b) + } + + // Scan for overlaping blocks. + if overlaps := validateBlockSequence(db.blocks); overlaps != nil { + var dirs []string + for _, b := range db.blocks { + dirs = append(dirs, b.Dir()) + } + useless[ErrorSequence(overlaps.String())] = dirs + } + + // Scan for temporary files. + var tmpFiles []string + filepath.Walk(db.dir, func(path string, f os.FileInfo, _ error) error { + if !f.IsDir() { + if filepath.Ext(path) == ".tmp" { + tmpFiles = append(tmpFiles, path) + } + } + return nil + }) + if len(tmpFiles) > 0 { + useless[ErrorTmp{}] = tmpFiles + } + return useless, nil +}