From 0b2c5c881f1b7ce29b99a43455749c2c5e3caadb Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Fri, 25 Apr 2025 10:01:29 +0300 Subject: [PATCH 01/15] add simple rules for parsing report output --- cmd/inspect.go | 205 +++++++++++++++++++++++++++++++++++++-- tools/inspect_rules.toml | 25 +++++ 2 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 tools/inspect_rules.toml diff --git a/cmd/inspect.go b/cmd/inspect.go index 2b55c3876..b5cd3f5d0 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -1,10 +1,16 @@ package cmd import ( + "encoding/csv" "fmt" + "io" "os" "os/signal" "path/filepath" + "regexp" + "strconv" + "strings" + "time" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -14,6 +20,7 @@ import ( "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" + "github.com/pelletier/go-toml/v2" "github.com/supabase/cli/internal/inspect" "github.com/supabase/cli/internal/inspect/calls" "github.com/supabase/cli/internal/inspect/index_sizes" @@ -218,20 +225,33 @@ var ( Short: "Generate a CSV output for all inspect commands", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - if len(outputDir) == 0 { - defaultPath := filepath.Join(utils.CurrentDirAbs, "report") - title := fmt.Sprintf("Enter a directory to save output files (or leave blank to use %s): ", utils.Bold(defaultPath)) - if dir, err := utils.NewConsole().PromptText(ctx, title); err != nil { - return err - } else if len(dir) == 0 { - outputDir = defaultPath - } + if err := inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()); err != nil { + return err } - return inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()) + return printReportSummary(outputDir) }, } ) +// Load rules file at runtime (tools/inspect_rules.toml) + +// Rule defines a validation rule for a CSV file +type Rule struct { + PatternRegex string `toml:"pattern_regex,omitempty"` + Regex *regexp.Regexp `toml:"-"` + Name string `toml:"name"` + Type string `toml:"type"` + Column string `toml:"column,omitempty"` + Threshold string `toml:"threshold,omitempty"` + Pass string `toml:"pass,omitempty"` + Fail string `toml:"fail,omitempty"` +} + +// Config holds all rules +type Config struct { + Rules []Rule `toml:"rule"` +} + func init() { inspectFlags := inspectCmd.PersistentFlags() inspectFlags.String("db-url", "", "Inspect the database specified by the connection string (must be percent-encoded).") @@ -263,3 +283,170 @@ func init() { inspectCmd.AddCommand(reportCmd) rootCmd.AddCommand(inspectCmd) } + +func printReportSummary(outDir string) error { + // Load rules from tools/inspect_rules.toml + data, err := os.ReadFile(filepath.Join(utils.CurrentDirAbs, "tools", "inspect_rules.toml")) + if err != nil { + return err + } + var cfg Config + if err := toml.Unmarshal(data, &cfg); err != nil { + return err + } + // Compile regex for each rule + for i := range cfg.Rules { + raw := cfg.Rules[i].PatternRegex + if raw != "" { + re, err := regexp.Compile(raw) + if err != nil { + return fmt.Errorf("invalid regex %q: %w", raw, err) + } + cfg.Rules[i].Regex = re + } + } + fmt.Println("Report Summary:") + entries, err := os.ReadDir(outDir) + if err != nil { + return err + } + fmt.Printf("%-30s %s\n", "File", "Status") + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".csv") { + continue + } + name := entry.Name() + path := filepath.Join(outDir, name) + f, err := os.Open(path) + if err != nil { + return err + } + reader := csv.NewReader(f) + headers, err := reader.Read() + if err != nil && err != io.EOF { + f.Close() + return err + } + // find rule + var matched *Rule + for i := range cfg.Rules { + r := &cfg.Rules[i] + if r.Regex != nil && r.Regex.MatchString(name) { + matched = r + break + } + } + status := "--" + if matched != nil { + status = matched.Fail + switch matched.Name { + case "empty": + filelen, err := reader.ReadAll() + if err != nil { + f.Close() + return err + } + if len(filelen) <= 1 { + status = matched.Pass + } + case "above_threshold": + idx := -1 + for i, h := range headers { + if h == matched.Column { + idx = i + break + } + } + + for { + rec, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + f.Close() + return err + } + if matched.Type == "time" { + thr, _ := time.ParseDuration(matched.Threshold) + parts := strings.Split(rec[idx], ":") + if len(parts) == 3 { + h, _ := strconv.Atoi(parts[0]) + m, _ := strconv.Atoi(parts[1]) + s, _ := strconv.Atoi(parts[2]) + dur := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second + if dur > thr { + status = matched.Pass + break + } + } + } else { + var val, thr float64 + if val, err = strconv.ParseFloat(rec[idx], 64); err != nil { + f.Close() + return err + } + if thr, err = strconv.ParseFloat(matched.Threshold, 64); err != nil { + f.Close() + return err + } + if val > thr { + status = matched.Pass + break + } + } + } + + case "below_threshold": + idx := -1 + for i, h := range headers { + if h == matched.Column { + idx = i + break + } + } + for { + rec, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + f.Close() + return err + } + if matched.Type == "time" { + thr, _ := time.ParseDuration(matched.Threshold) + parts := strings.Split(rec[idx], ":") + if len(parts) == 3 { + h, _ := strconv.Atoi(parts[0]) + m, _ := strconv.Atoi(parts[1]) + s, _ := strconv.Atoi(parts[2]) + dur := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second + if dur < thr { + status = matched.Pass + break + } + } + } else { + var val, thr float64 + if val, err = strconv.ParseFloat(rec[idx], 64); err != nil { + f.Close() + return err + } + if thr, err = strconv.ParseFloat(matched.Threshold, 64); err != nil { + f.Close() + return err + } + if val < thr { + status = matched.Pass + break + } + } + } + } + } + f.Close() + fmt.Printf("%-30s %s\n", name, status) + } + return nil +} diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml new file mode 100644 index 000000000..dec369707 --- /dev/null +++ b/tools/inspect_rules.toml @@ -0,0 +1,25 @@ +# Rules to validate CSV report files + +[[rule]] +pattern_regex = "^locks_.*$" +name = "above_threshold" +type = "time" +column = "age" +threshold = "2m" +pass = "??" +fail = "✔" + +[[rule]] +pattern_regex = "^(blocking|calls|unused_indexes|long_running_queries)_.*\\.csv$" +type = "empty" +pass = "✔" +fail = "??" + + +[[rule]] +pattern_regex = "^cache_.*$" +name = "below_threshold" +threshold = "0.94" +column = "ratio" +pass = "✔" +fail = "??" \ No newline at end of file From c2136bd8cced14ebc563545d846cfb360ede19b2 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Fri, 25 Apr 2025 11:59:14 +0300 Subject: [PATCH 02/15] added csvq driver and normalised column names across queries --- cmd/inspect.go | 158 ++++-------------- go.mod | 7 +- go.sum | 10 ++ .../table_index_sizes/table_index_sizes.go | 4 +- .../table_index_sizes/table_index_sizes.sql | 2 +- .../table_index_sizes_test.go | 2 +- .../table_record_counts.go | 5 +- .../table_record_counts.sql | 3 +- .../table_record_counts_test.go | 3 +- internal/inspect/table_sizes/table_sizes.go | 9 +- internal/inspect/table_sizes/table_sizes.sql | 3 +- .../inspect/table_sizes/table_sizes_test.go | 5 +- .../total_table_sizes/total_table_sizes.go | 9 +- .../total_table_sizes/total_table_sizes.sql | 3 +- .../total_table_sizes_test.go | 5 +- .../inspect/unused_indexes/unused_indexes.go | 4 +- .../inspect/unused_indexes/unused_indexes.sql | 2 +- .../unused_indexes/unused_indexes_test.go | 2 +- tools/inspect_rules.toml | 22 ++- 19 files changed, 88 insertions(+), 170 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index b5cd3f5d0..51c199759 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -1,16 +1,13 @@ package cmd import ( - "encoding/csv" + "database/sql" "fmt" - "io" "os" "os/signal" "path/filepath" "regexp" - "strconv" "strings" - "time" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -20,6 +17,7 @@ import ( "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" + _ "github.com/mithrandie/csvq-driver" "github.com/pelletier/go-toml/v2" "github.com/supabase/cli/internal/inspect" "github.com/supabase/cli/internal/inspect/calls" @@ -239,12 +237,12 @@ var ( type Rule struct { PatternRegex string `toml:"pattern_regex,omitempty"` Regex *regexp.Regexp `toml:"-"` - Name string `toml:"name"` + Query string `toml:"query"` + Pass string `toml:"pass"` + Fail string `toml:"fail"` + Column string `toml:"column"` + Threshold string `toml:"threshold"` Type string `toml:"type"` - Column string `toml:"column,omitempty"` - Threshold string `toml:"threshold,omitempty"` - Pass string `toml:"pass,omitempty"` - Fail string `toml:"fail,omitempty"` } // Config holds all rules @@ -305,29 +303,24 @@ func printReportSummary(outDir string) error { cfg.Rules[i].Regex = re } } + // Open csvq database rooted at the output directory + db, err := sql.Open("csvq", outDir) + if err != nil { + return err + } + defer db.Close() + fmt.Println("Report Summary:") entries, err := os.ReadDir(outDir) if err != nil { return err } - fmt.Printf("%-30s %s\n", "File", "Status") for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".csv") { continue } name := entry.Name() - path := filepath.Join(outDir, name) - f, err := os.Open(path) - if err != nil { - return err - } - reader := csv.NewReader(f) - headers, err := reader.Read() - if err != nil && err != io.EOF { - f.Close() - return err - } - // find rule + // find matching rule var matched *Rule for i := range cfg.Rules { r := &cfg.Rules[i] @@ -337,115 +330,22 @@ func printReportSummary(outDir string) error { } } status := "--" - if matched != nil { - status = matched.Fail - switch matched.Name { - case "empty": - filelen, err := reader.ReadAll() - if err != nil { - f.Close() - return err - } - if len(filelen) <= 1 { - status = matched.Pass - } - case "above_threshold": - idx := -1 - for i, h := range headers { - if h == matched.Column { - idx = i - break - } - } - - for { - rec, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - f.Close() - return err - } - if matched.Type == "time" { - thr, _ := time.ParseDuration(matched.Threshold) - parts := strings.Split(rec[idx], ":") - if len(parts) == 3 { - h, _ := strconv.Atoi(parts[0]) - m, _ := strconv.Atoi(parts[1]) - s, _ := strconv.Atoi(parts[2]) - dur := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second - if dur > thr { - status = matched.Pass - break - } - } - } else { - var val, thr float64 - if val, err = strconv.ParseFloat(rec[idx], 64); err != nil { - f.Close() - return err - } - if thr, err = strconv.ParseFloat(matched.Threshold, 64); err != nil { - f.Close() - return err - } - if val > thr { - status = matched.Pass - break - } - } - } - - case "below_threshold": - idx := -1 - for i, h := range headers { - if h == matched.Column { - idx = i - break - } - } - for { - rec, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - f.Close() - return err - } - if matched.Type == "time" { - thr, _ := time.ParseDuration(matched.Threshold) - parts := strings.Split(rec[idx], ":") - if len(parts) == 3 { - h, _ := strconv.Atoi(parts[0]) - m, _ := strconv.Atoi(parts[1]) - s, _ := strconv.Atoi(parts[2]) - dur := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second - if dur < thr { - status = matched.Pass - break - } - } - } else { - var val, thr float64 - if val, err = strconv.ParseFloat(rec[idx], 64); err != nil { - f.Close() - return err - } - if thr, err = strconv.ParseFloat(matched.Threshold, 64); err != nil { - f.Close() - return err - } - if val < thr { - status = matched.Pass - break - } - } - } + if matched != nil && matched.Query != "" { + // Run the rule query against the CSV via csvq + // table := strings.TrimSuffix(name, ".csv") + q := fmt.Sprintf(matched.Query, name) + row := db.QueryRow(q) + var ok bool + + if err := row.Scan(&ok); err != nil { + println(err.Error()) + status = "ERR" + } else if ok { + status = matched.Pass + } else { + status = matched.Fail } } - f.Close() fmt.Printf("%-30s %s\n", name, status) } return nil diff --git a/go.mod b/go.mod index 96c41c447..1cd907add 100644 --- a/go.mod +++ b/go.mod @@ -35,9 +35,11 @@ require ( github.com/jackc/pgtype v1.14.4 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 + github.com/mithrandie/csvq-driver v1.7.0 github.com/muesli/reflow v0.3.0 github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 github.com/oapi-codegen/runtime v1.1.1 + github.com/pelletier/go-toml/v2 v2.2.4 github.com/slack-go/slack v0.16.0 github.com/spf13/afero v1.14.0 github.com/spf13/cobra v1.9.1 @@ -222,6 +224,10 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/mithrandie/csvq v1.18.1 // indirect + github.com/mithrandie/go-file/v2 v2.1.0 // indirect + github.com/mithrandie/go-text v1.6.0 // indirect + github.com/mithrandie/ternary v1.1.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect @@ -239,7 +245,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index b0a84118b..391c145fa 100644 --- a/go.sum +++ b/go.sum @@ -697,6 +697,16 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mithrandie/csvq v1.18.1 h1:f7NB2scbb7xx2ffPduJ2VtZ85RpWXfvanYskAkGlCBU= +github.com/mithrandie/csvq v1.18.1/go.mod h1:MRJj7AtcXfk7jhNGxLuJGP3LORmh4lpiPWxQ7VyCRn8= +github.com/mithrandie/csvq-driver v1.7.0 h1:ejiavXNWwTPMyr3fJFnhcqd1L1cYudA0foQy9cZrqhw= +github.com/mithrandie/csvq-driver v1.7.0/go.mod h1:HcN3xL9UCJnBYA/AIQOOB/KlyfXAiYr5yxDmiwrGk5o= +github.com/mithrandie/go-file/v2 v2.1.0 h1:XA5Tl+73GXMDvgwSE3Sg0uC5FkLr3hnXs8SpUas0hyg= +github.com/mithrandie/go-file/v2 v2.1.0/go.mod h1:9YtTF3Xo59GqC1Pxw6KyGVcM/qubAMlxVsqI/u9r++c= +github.com/mithrandie/go-text v1.6.0 h1:8gOXTMPbMY8DJbKMTv8kHhADcJlDWXqS/YQH4SyWO6s= +github.com/mithrandie/go-text v1.6.0/go.mod h1:xCgj1xiNbI/d4xA9sLVvXkjh5B2tNx2ZT2/3rpmh8to= +github.com/mithrandie/ternary v1.1.1 h1:k/joD6UGVYxHixYmSR8EGgDFNONBMqyD373xT4QRdC4= +github.com/mithrandie/ternary v1.1.1/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= diff --git a/internal/inspect/table_index_sizes/table_index_sizes.go b/internal/inspect/table_index_sizes/table_index_sizes.go index e61f23361..5b3c87c99 100644 --- a/internal/inspect/table_index_sizes/table_index_sizes.go +++ b/internal/inspect/table_index_sizes/table_index_sizes.go @@ -19,7 +19,7 @@ import ( var TableIndexSizesQuery string type Result struct { - Table string + Name string Index_size string } @@ -40,7 +40,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu table := "|Table|Index size|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Table, r.Index_size) + table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Index_size) } return list.RenderTable(table) } diff --git a/internal/inspect/table_index_sizes/table_index_sizes.sql b/internal/inspect/table_index_sizes/table_index_sizes.sql index 0c9bc6bfc..8899b67b3 100644 --- a/internal/inspect/table_index_sizes/table_index_sizes.sql +++ b/internal/inspect/table_index_sizes/table_index_sizes.sql @@ -1,5 +1,5 @@ SELECT - n.nspname || '.' || c.relname AS table, + n.nspname || '.' || c.relname AS name, pg_size_pretty(pg_indexes_size(c.oid)) AS index_size FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) diff --git a/internal/inspect/table_index_sizes/table_index_sizes_test.go b/internal/inspect/table_index_sizes/table_index_sizes_test.go index 20ad80fc9..0666a8a89 100644 --- a/internal/inspect/table_index_sizes/table_index_sizes_test.go +++ b/internal/inspect/table_index_sizes/table_index_sizes_test.go @@ -29,7 +29,7 @@ func TestTableIndexSizesCommand(t *testing.T) { defer conn.Close(t) conn.Query(TableIndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Table: "public.test_table", + Name: "public.test_table", Index_size: "3GB", }) // Run test diff --git a/internal/inspect/table_record_counts/table_record_counts.go b/internal/inspect/table_record_counts/table_record_counts.go index e0b394374..15cac8950 100644 --- a/internal/inspect/table_record_counts/table_record_counts.go +++ b/internal/inspect/table_record_counts/table_record_counts.go @@ -19,7 +19,6 @@ import ( var TableRecordCountsQuery string type Result struct { - Schema string Name string Estimated_count int64 } @@ -39,9 +38,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "Schema|Table|Estimated count|\n|-|-|-|\n" + table := "|Table|Estimated count|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%d`|\n", r.Schema, r.Name, r.Estimated_count) + table += fmt.Sprintf("|`%s`|`%d`|\n", r.Name, r.Estimated_count) } return list.RenderTable(table) } diff --git a/internal/inspect/table_record_counts/table_record_counts.sql b/internal/inspect/table_record_counts/table_record_counts.sql index 1b24f04a4..86b75b0e5 100644 --- a/internal/inspect/table_record_counts/table_record_counts.sql +++ b/internal/inspect/table_record_counts/table_record_counts.sql @@ -1,6 +1,5 @@ SELECT - schemaname AS schema, - relname AS name, + schemaname || '.' || relname AS name, n_live_tup AS estimated_count FROM pg_stat_user_tables WHERE NOT schemaname LIKE ANY($1) diff --git a/internal/inspect/table_record_counts/table_record_counts_test.go b/internal/inspect/table_record_counts/table_record_counts_test.go index a03714d97..e2b5d0be8 100644 --- a/internal/inspect/table_record_counts/table_record_counts_test.go +++ b/internal/inspect/table_record_counts/table_record_counts_test.go @@ -29,8 +29,7 @@ func TestTableRecordCountsCommand(t *testing.T) { defer conn.Close(t) conn.Query(TableRecordCountsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", + Name: "public.test_table", Estimated_count: 100, }) // Run test diff --git a/internal/inspect/table_sizes/table_sizes.go b/internal/inspect/table_sizes/table_sizes.go index 7741f0119..750d655af 100644 --- a/internal/inspect/table_sizes/table_sizes.go +++ b/internal/inspect/table_sizes/table_sizes.go @@ -19,9 +19,8 @@ import ( var TableSizesQuery string type Result struct { - Schema string - Name string - Size string + Name string + Size string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -39,9 +38,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "Schema|Table|size|\n|-|-|-|\n" + table := "|Table|size|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) + table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Size) } return list.RenderTable(table) } diff --git a/internal/inspect/table_sizes/table_sizes.sql b/internal/inspect/table_sizes/table_sizes.sql index 2c8fb3064..30bcc1414 100644 --- a/internal/inspect/table_sizes/table_sizes.sql +++ b/internal/inspect/table_sizes/table_sizes.sql @@ -1,6 +1,5 @@ SELECT - n.nspname AS schema, - c.relname AS name, + n.nspname || '.' || c.relname AS name, pg_size_pretty(pg_table_size(c.oid)) AS size FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) diff --git a/internal/inspect/table_sizes/table_sizes_test.go b/internal/inspect/table_sizes/table_sizes_test.go index 5cc6426ad..4d449c529 100644 --- a/internal/inspect/table_sizes/table_sizes_test.go +++ b/internal/inspect/table_sizes/table_sizes_test.go @@ -29,9 +29,8 @@ func TestTableSizesCommand(t *testing.T) { defer conn.Close(t) conn.Query(TableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). Reply("SELECT 1", Result{ - Schema: "schema", - Name: "test_table", - Size: "3GB", + Name: "public.test_table", + Size: "3GB", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/total_table_sizes/total_table_sizes.go b/internal/inspect/total_table_sizes/total_table_sizes.go index 80b1c89a8..7c150df61 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.go +++ b/internal/inspect/total_table_sizes/total_table_sizes.go @@ -19,9 +19,8 @@ import ( var TotalTableSizesQuery string type Result struct { - Schema string - Name string - Size string + Name string + Size string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -39,9 +38,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "Schema|Table|Size|\n|-|-|-|\n" + table := "|Table|Size|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) + table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Size) } return list.RenderTable(table) } diff --git a/internal/inspect/total_table_sizes/total_table_sizes.sql b/internal/inspect/total_table_sizes/total_table_sizes.sql index 471d0655f..3e8df07f8 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.sql +++ b/internal/inspect/total_table_sizes/total_table_sizes.sql @@ -1,6 +1,5 @@ SELECT - n.nspname AS schema, - c.relname AS name, + n.nspname || '.' || c.relname AS name, pg_size_pretty(pg_total_relation_size(c.oid)) AS size FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) diff --git a/internal/inspect/total_table_sizes/total_table_sizes_test.go b/internal/inspect/total_table_sizes/total_table_sizes_test.go index bc548af60..b62b0c66e 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes_test.go +++ b/internal/inspect/total_table_sizes/total_table_sizes_test.go @@ -29,9 +29,8 @@ func TestTotalTableSizesCommand(t *testing.T) { defer conn.Close(t) conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", - Size: "3GB", + Name: "public.test_table", + Size: "3GB", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/unused_indexes/unused_indexes.go b/internal/inspect/unused_indexes/unused_indexes.go index 2a30a46d7..bfbbf523a 100644 --- a/internal/inspect/unused_indexes/unused_indexes.go +++ b/internal/inspect/unused_indexes/unused_indexes.go @@ -19,7 +19,7 @@ import ( var UnusedIndexesQuery string type Result struct { - Table string + Name string Index string Index_size string Index_scans int64 @@ -42,7 +42,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu table := "|Table|Index|Index Size|Index Scans\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Table, r.Index, r.Index_size, r.Index_scans) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Name, r.Index, r.Index_size, r.Index_scans) } return list.RenderTable(table) } diff --git a/internal/inspect/unused_indexes/unused_indexes.sql b/internal/inspect/unused_indexes/unused_indexes.sql index 6e775967d..c47a1f666 100644 --- a/internal/inspect/unused_indexes/unused_indexes.sql +++ b/internal/inspect/unused_indexes/unused_indexes.sql @@ -1,5 +1,5 @@ SELECT - schemaname || '.' || relname AS table, + schemaname || '.' || relname AS name, indexrelname AS index, pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, idx_scan as index_scans diff --git a/internal/inspect/unused_indexes/unused_indexes_test.go b/internal/inspect/unused_indexes/unused_indexes_test.go index ee4182094..517f88c61 100644 --- a/internal/inspect/unused_indexes/unused_indexes_test.go +++ b/internal/inspect/unused_indexes/unused_indexes_test.go @@ -29,7 +29,7 @@ func TestUnusedIndexesCommand(t *testing.T) { defer conn.Close(t) conn.Query(UnusedIndexesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Table: "test_table", + Name: "public.test_table", Index: "test_table_idx", Index_size: "3GB", Index_scans: 2, diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index dec369707..c7e2b1dd1 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -2,16 +2,28 @@ [[rule]] pattern_regex = "^locks_.*$" +query = "SELECT count(*) > 0 AS ok FROM `%s` WHERE age > '00:02:00'" name = "above_threshold" type = "time" -column = "age" -threshold = "2m" +column = "ok" +pass = "✔" +fail = "??" + +[[rule]] +pattern_regex = "^locks_.*$" +query = "SELECT count(granted) > 0 AS ok FROM `%s` WHERE granted = 'f'" +name = "above_threshold" +type = "time" +column = "ok" pass = "??" fail = "✔" + [[rule]] -pattern_regex = "^(blocking|calls|unused_indexes|long_running_queries)_.*\\.csv$" +pattern_regex = "^(blocking|calls|unused_indexes|long_running_queries|outliers)_.*\\.csv$" +query = "select count(*) = 0 as ok from `%s`" type = "empty" +column = "ok" pass = "✔" fail = "??" @@ -19,7 +31,7 @@ fail = "??" [[rule]] pattern_regex = "^cache_.*$" name = "below_threshold" -threshold = "0.94" -column = "ratio" +query = "select count(*) > 0 AS ok from `%s` where ratio < 0.94" +column = "ok" pass = "✔" fail = "??" \ No newline at end of file From 92494297e30c7d4e802d61cec35247089859fac8 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:41:17 +0300 Subject: [PATCH 03/15] modify to use table output --- cmd/inspect.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index 51c199759..970d02fee 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -37,6 +37,7 @@ import ( "github.com/supabase/cli/internal/inspect/total_table_sizes" "github.com/supabase/cli/internal/inspect/unused_indexes" "github.com/supabase/cli/internal/inspect/vacuum_stats" + "github.com/supabase/cli/internal/migration/list" ) var ( @@ -310,7 +311,8 @@ func printReportSummary(outDir string) error { } defer db.Close() - fmt.Println("Report Summary:") + // Build report summary table + table := "NAME|STATUS\n|-|-|\n" entries, err := os.ReadDir(outDir) if err != nil { return err @@ -346,7 +348,7 @@ func printReportSummary(outDir string) error { status = matched.Fail } } - fmt.Printf("%-30s %s\n", name, status) + table += fmt.Sprintf("|`%s`|`%s`|\n", name, status) } - return nil + return list.RenderTable(table) } From 5edbff22ce49185df7bd8a3502c0227ce3470499 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:09:40 +0300 Subject: [PATCH 04/15] modified seq_scans output to remove reserved words, added some rules --- cmd/inspect.go | 80 ++++++-------------- internal/inspect/report.go | 14 +++- internal/inspect/seq_scans/seq_scans.go | 8 +- internal/inspect/seq_scans/seq_scans.sql | 2 +- internal/inspect/seq_scans/seq_scans_test.go | 4 +- tools/inspect_rules.toml | 55 ++++++++------ 6 files changed, 74 insertions(+), 89 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index 970d02fee..ea30c50a7 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -6,8 +6,7 @@ import ( "os" "os/signal" "path/filepath" - "regexp" - "strings" + "time" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -236,14 +235,10 @@ var ( // Rule defines a validation rule for a CSV file type Rule struct { - PatternRegex string `toml:"pattern_regex,omitempty"` - Regex *regexp.Regexp `toml:"-"` - Query string `toml:"query"` - Pass string `toml:"pass"` - Fail string `toml:"fail"` - Column string `toml:"column"` - Threshold string `toml:"threshold"` - Type string `toml:"type"` + Query string `toml:"query"` + Pass string `toml:"pass"` + Fail string `toml:"fail"` + Name string `toml:"name"` } // Config holds all rules @@ -284,6 +279,9 @@ func init() { } func printReportSummary(outDir string) error { + // point to the date-based subdirectory + date := time.Now().Format("2006-01-02") + outDir = filepath.Join(outDir, date) // Load rules from tools/inspect_rules.toml data, err := os.ReadFile(filepath.Join(utils.CurrentDirAbs, "tools", "inspect_rules.toml")) if err != nil { @@ -293,17 +291,6 @@ func printReportSummary(outDir string) error { if err := toml.Unmarshal(data, &cfg); err != nil { return err } - // Compile regex for each rule - for i := range cfg.Rules { - raw := cfg.Rules[i].PatternRegex - if raw != "" { - re, err := regexp.Compile(raw) - if err != nil { - return fmt.Errorf("invalid regex %q: %w", raw, err) - } - cfg.Rules[i].Regex = re - } - } // Open csvq database rooted at the output directory db, err := sql.Open("csvq", outDir) if err != nil { @@ -312,41 +299,24 @@ func printReportSummary(outDir string) error { defer db.Close() // Build report summary table - table := "NAME|STATUS\n|-|-|\n" - entries, err := os.ReadDir(outDir) - if err != nil { - return err - } - for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".csv") { - continue - } - name := entry.Name() - // find matching rule - var matched *Rule - for i := range cfg.Rules { - r := &cfg.Rules[i] - if r.Regex != nil && r.Regex.MatchString(name) { - matched = r - break - } - } + table := "RULE|STATUS\n|-|-|\n" + + // find matching rule + + for i := range cfg.Rules { + r := &cfg.Rules[i] + name := r.Name status := "--" - if matched != nil && matched.Query != "" { - // Run the rule query against the CSV via csvq - // table := strings.TrimSuffix(name, ".csv") - q := fmt.Sprintf(matched.Query, name) - row := db.QueryRow(q) - var ok bool - - if err := row.Scan(&ok); err != nil { - println(err.Error()) - status = "ERR" - } else if ok { - status = matched.Pass - } else { - status = matched.Fail - } + row := db.QueryRow(r.Query) + var ok bool + + if err := row.Scan(&ok); err != nil { + println(err.Error()) + status = "ERR" + } else if ok { + status = r.Pass + } else { + status = r.Fail } table += fmt.Sprintf("|`%s`|`%s`|\n", name, status) } diff --git a/internal/inspect/report.go b/internal/inspect/report.go index de721d954..b09302175 100644 --- a/internal/inspect/report.go +++ b/internal/inspect/report.go @@ -26,6 +26,11 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs if err := utils.MkdirIfNotExistFS(fsys, out); err != nil { return err } + // create a date-based subdirectory + dateDir := filepath.Join(out, date) + if err := utils.MkdirIfNotExistFS(fsys, dateDir); err != nil { + return err + } conn, err := utils.ConnectByConfig(ctx, config, options...) if err != nil { return err @@ -44,15 +49,16 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs return errors.Errorf("failed to read query: %w", err) } name := strings.Split(d.Name(), ".")[0] - outPath := filepath.Join(out, fmt.Sprintf("%s_%s.csv", name, date)) + outPath := filepath.Join(dateDir, fmt.Sprintf("%s.csv", name)) return copyToCSV(ctx, string(query), outPath, conn.PgConn(), fsys) }); err != nil { return err } - if !filepath.IsAbs(out) { - out, _ = filepath.Abs(out) + // print the actual save location + if !filepath.IsAbs(dateDir) { + dateDir, _ = filepath.Abs(dateDir) } - fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(out)) + fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(dateDir)) return nil } diff --git a/internal/inspect/seq_scans/seq_scans.go b/internal/inspect/seq_scans/seq_scans.go index 6b52538ee..830f30ea8 100644 --- a/internal/inspect/seq_scans/seq_scans.go +++ b/internal/inspect/seq_scans/seq_scans.go @@ -19,8 +19,8 @@ import ( var SeqScansQuery string type Result struct { - Name string - Count int64 + Name string + SeqScans int64 } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -38,9 +38,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Name|Count|\n|-|-|\n" + table := "|Name|Sequential_Scans|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%d`|\n", r.Name, r.Count) + table += fmt.Sprintf("|`%s`|`%d`|\n", r.Name, r.SeqScans) } return list.RenderTable(table) } diff --git a/internal/inspect/seq_scans/seq_scans.sql b/internal/inspect/seq_scans/seq_scans.sql index c8edfc8e3..975604845 100644 --- a/internal/inspect/seq_scans/seq_scans.sql +++ b/internal/inspect/seq_scans/seq_scans.sql @@ -1,6 +1,6 @@ SELECT schemaname || '.' || relname AS name, - seq_scan as count + seq_scan as seq_scans FROM pg_stat_user_tables WHERE NOT schemaname LIKE ANY($1) ORDER BY seq_scan DESC diff --git a/internal/inspect/seq_scans/seq_scans_test.go b/internal/inspect/seq_scans/seq_scans_test.go index 3db6caee5..6aff49484 100644 --- a/internal/inspect/seq_scans/seq_scans_test.go +++ b/internal/inspect/seq_scans/seq_scans_test.go @@ -29,8 +29,8 @@ func TestSequentialScansCommand(t *testing.T) { defer conn.Close(t) conn.Query(SeqScansQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Name: "test_table", - Count: 99999, + Name: "test_table", + SeqScans: 99999, }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index c7e2b1dd1..5d9e38a21 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -1,37 +1,46 @@ # Rules to validate CSV report files [[rule]] -pattern_regex = "^locks_.*$" -query = "SELECT count(*) > 0 AS ok FROM `%s` WHERE age > '00:02:00'" -name = "above_threshold" -type = "time" -column = "ok" +query = "SELECT count(*) = 0 AS ok FROM `locks.csv` WHERE age > '00:02:00'" +name = "No old locks" pass = "✔" -fail = "??" +fail = "There is at least one lock older than 2 minutes" [[rule]] -pattern_regex = "^locks_.*$" -query = "SELECT count(granted) > 0 AS ok FROM `%s` WHERE granted = 'f'" -name = "above_threshold" -type = "time" -column = "ok" -pass = "??" -fail = "✔" +query = "SELECT count(*) = 0 AS ok FROM `locks.csv` WHERE granted = 'f'" +name = "No ungranted locks" +pass = "✔" +fail = "There is at least one ungranted lock" [[rule]] -pattern_regex = "^(blocking|calls|unused_indexes|long_running_queries|outliers)_.*\\.csv$" -query = "select count(*) = 0 as ok from `%s`" -type = "empty" -column = "ok" +query = "select count(*) = 0 as ok from `unused_indexes.csv`" pass = "✔" -fail = "??" +fail = "There is at least one unused index" +name = "No unused indexes" [[rule]] -pattern_regex = "^cache_.*$" -name = "below_threshold" -query = "select count(*) > 0 AS ok from `%s` where ratio < 0.94" -column = "ok" +name = "Check cache hit is within acceptable bounds" +query = "select count(*) = 0 AS ok from `cache.csv` where ratio < 0.94" pass = "✔" -fail = "??" \ No newline at end of file +fail = "There is at least one table with a cache hit ratio below 94%" + +[[rule]] +query = "SELECT count(*) = 0 AS ok FROM `seq_scans.csv` s JOIN `table_record_counts.csv` t ON s.name = t.name WHERE t.estimated_count > 1000 AND s.seq_scans > t.estimated_count * 0.1;" +name = "No large tables with sequential scans more than 10% of rows" +pass = "✔" +fail = "At least one table is showing sequential scans more than 10% of total row count" + + +[[rule]] +query = "SELECT count(*) = 0 AS ok FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" +name = "No large tables waiting on autovacuum" +pass = "✔" +fail = "At least one table is waiting on autovacuum" + +[[rule]] +query = "SELECT count(*) = 0 AS ok FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum IS NULL OR s.last_vacuum IS NULL);" +name = "No tables yet to be vacuumed" +pass = "✔" +fail = "At least one table has never had autovacuum or vacuum run on it" From 86ba7c6128c5f74268fe3118edb2dcfc92b62b35 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Wed, 30 Apr 2025 10:12:31 +0300 Subject: [PATCH 05/15] remove more keywords from queries and improve rule output --- cmd/inspect.go | 17 +++++++++-------- internal/inspect/locks/locks.go | 10 +++++----- internal/inspect/locks/locks.sql | 2 +- internal/inspect/locks/locks_test.go | 2 +- internal/inspect/vacuum_stats/vacuum_stats.go | 6 +++--- internal/inspect/vacuum_stats/vacuum_stats.sql | 2 +- .../inspect/vacuum_stats/vacuum_stats_test.go | 2 +- tools/inspect_rules.toml | 14 +++++++------- 8 files changed, 28 insertions(+), 27 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index ea30c50a7..bebda4721 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -299,7 +299,7 @@ func printReportSummary(outDir string) error { defer db.Close() // Build report summary table - table := "RULE|STATUS\n|-|-|\n" + table := "RULE|STATUS|MATCHES\n|-|-|-|\n" // find matching rule @@ -308,17 +308,18 @@ func printReportSummary(outDir string) error { name := r.Name status := "--" row := db.QueryRow(r.Query) - var ok bool + var match string - if err := row.Scan(&ok); err != nil { - println(err.Error()) - status = "ERR" - } else if ok { - status = r.Pass + if err := row.Scan(&match); err != nil { + if err == sql.ErrNoRows { + status = r.Pass + } else { + status = err.Error() + } } else { status = r.Fail } - table += fmt.Sprintf("|`%s`|`%s`|\n", name, status) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, match) } return list.RenderTable(table) } diff --git a/internal/inspect/locks/locks.go b/internal/inspect/locks/locks.go index 0b2db71c6..be5c34d3a 100644 --- a/internal/inspect/locks/locks.go +++ b/internal/inspect/locks/locks.go @@ -23,7 +23,7 @@ type Result struct { Relname string Transactionid string Granted bool - Query string + Stmt string Age string } @@ -42,16 +42,16 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|pid|relname|transaction id|granted|query|age|\n|-|-|-|-|-|-|\n" + table := "|pid|relname|transaction id|granted|stmt|age|\n|-|-|-|-|-|-|\n" for _, r := range result { // remove whitespace from query re := regexp.MustCompile(`\s+|\r+|\n+|\t+|\v`) - query := re.ReplaceAllString(r.Query, " ") + stmt := re.ReplaceAllString(r.Stmt, " ") // escape pipes in query re = regexp.MustCompile(`\|`) - query = re.ReplaceAllString(query, `\|`) - table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, query, r.Age) + stmt = re.ReplaceAllString(stmt, `\|`) + table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, stmt, r.Age) } return list.RenderTable(table) } diff --git a/internal/inspect/locks/locks.sql b/internal/inspect/locks/locks.sql index e140f4786..8a19d788d 100644 --- a/internal/inspect/locks/locks.sql +++ b/internal/inspect/locks/locks.sql @@ -3,7 +3,7 @@ SELECT COALESCE(pg_class.relname, 'null') AS relname, COALESCE(pg_locks.transactionid, 'null') AS transactionid, pg_locks.granted, - pg_stat_activity.query, + pg_stat_activity.query AS stmt, age(now(), pg_stat_activity.query_start)::text AS age FROM pg_stat_activity, pg_locks LEFT OUTER JOIN pg_class ON (pg_locks.relation = pg_class.oid) WHERE pg_stat_activity.query <> '' diff --git a/internal/inspect/locks/locks_test.go b/internal/inspect/locks/locks_test.go index e4c55c6bd..0964ad122 100644 --- a/internal/inspect/locks/locks_test.go +++ b/internal/inspect/locks/locks_test.go @@ -31,7 +31,7 @@ func TestLocksCommand(t *testing.T) { Relname: "rel", Transactionid: "9301", Granted: true, - Query: "select 1", + Stmt: "select 1", Age: "300ms", }) // Run test diff --git a/internal/inspect/vacuum_stats/vacuum_stats.go b/internal/inspect/vacuum_stats/vacuum_stats.go index dc9326d79..7f1cbc0ae 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.go +++ b/internal/inspect/vacuum_stats/vacuum_stats.go @@ -21,7 +21,7 @@ var VacuumStatsQuery string type Result struct { Schema string - Table string + Tbl string Last_vacuum string Last_autovacuum string Rowcount string @@ -45,10 +45,10 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Schema|Table|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|-|\n" + table := "|Schema|Tbl|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|-|\n" for _, r := range result { rowcount := strings.Replace(r.Rowcount, "-1", "No stats", 1) - table += fmt.Sprintf("|`%s`|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Schema, r.Table, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) + table += fmt.Sprintf("|`%s`|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Schema, r.Tbl, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) } return list.RenderTable(table) } diff --git a/internal/inspect/vacuum_stats/vacuum_stats.sql b/internal/inspect/vacuum_stats/vacuum_stats.sql index 707847497..4ce3b3bdc 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.sql +++ b/internal/inspect/vacuum_stats/vacuum_stats.sql @@ -21,7 +21,7 @@ WITH table_opts AS ( ) SELECT vacuum_settings.nspname AS schema, - vacuum_settings.relname AS table, + vacuum_settings.relname AS tbl, coalesce(to_char(psut.last_vacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_vacuum, coalesce(to_char(psut.last_autovacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_autovacuum, to_char(pg_class.reltuples, '9G999G999G999') AS rowcount, diff --git a/internal/inspect/vacuum_stats/vacuum_stats_test.go b/internal/inspect/vacuum_stats/vacuum_stats_test.go index 0d3cbec10..49dafc84e 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats_test.go +++ b/internal/inspect/vacuum_stats/vacuum_stats_test.go @@ -30,7 +30,7 @@ func TestVacuumCommand(t *testing.T) { conn.Query(VacuumStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ Schema: "test_schema", - Table: "test_table", + Tbl: "test_table", Last_vacuum: "2021-01-01 00:00:00", Last_autovacuum: "2021-01-01 00:00:00", Rowcount: "1000", diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index 5d9e38a21..5b5034983 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -1,20 +1,20 @@ # Rules to validate CSV report files [[rule]] -query = "SELECT count(*) = 0 AS ok FROM `locks.csv` WHERE age > '00:02:00'" +query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE age > '00:02:00'" name = "No old locks" pass = "✔" fail = "There is at least one lock older than 2 minutes" [[rule]] -query = "SELECT count(*) = 0 AS ok FROM `locks.csv` WHERE granted = 'f'" +query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE granted = 'f'" name = "No ungranted locks" pass = "✔" fail = "There is at least one ungranted lock" [[rule]] -query = "select count(*) = 0 as ok from `unused_indexes.csv`" +query = "SELECT LISTAGG(index, ',') AS match FROM `unused_indexes.csv`" pass = "✔" fail = "There is at least one unused index" name = "No unused indexes" @@ -22,25 +22,25 @@ name = "No unused indexes" [[rule]] name = "Check cache hit is within acceptable bounds" -query = "select count(*) = 0 AS ok from `cache.csv` where ratio < 0.94" +query = "SELECT LISTAGG(name, ',') AS match FROM `cache.csv` WHERE ratio < 0.94" pass = "✔" fail = "There is at least one table with a cache hit ratio below 94%" [[rule]] -query = "SELECT count(*) = 0 AS ok FROM `seq_scans.csv` s JOIN `table_record_counts.csv` t ON s.name = t.name WHERE t.estimated_count > 1000 AND s.seq_scans > t.estimated_count * 0.1;" +query = "SELECT LISTAGG(s.name, ',') AS match FROM `seq_scans.csv` s JOIN `table_record_counts.csv` t ON s.name = t.name WHERE t.estimated_count > 1000 AND s.seq_scans > t.estimated_count * 0.1;" name = "No large tables with sequential scans more than 10% of rows" pass = "✔" fail = "At least one table is showing sequential scans more than 10% of total row count" [[rule]] -query = "SELECT count(*) = 0 AS ok FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" +query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" name = "No large tables waiting on autovacuum" pass = "✔" fail = "At least one table is waiting on autovacuum" [[rule]] -query = "SELECT count(*) = 0 AS ok FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum IS NULL OR s.last_vacuum IS NULL);" +query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum IS NULL OR s.last_vacuum IS NULL);" name = "No tables yet to be vacuumed" pass = "✔" fail = "At least one table has never had autovacuum or vacuum run on it" From c370a6188b329f608e8174647a075ac7ea232aa5 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:02:48 +0300 Subject: [PATCH 06/15] use null string for empty responses --- cmd/inspect.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index bebda4721..4fd389cc3 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -308,7 +308,7 @@ func printReportSummary(outDir string) error { name := r.Name status := "--" row := db.QueryRow(r.Query) - var match string + var match sql.NullString if err := row.Scan(&match); err != nil { if err == sql.ErrNoRows { @@ -317,9 +317,17 @@ func printReportSummary(outDir string) error { status = err.Error() } } else { - status = r.Fail + if !match.Valid || match.String == "" { + status = r.Pass + } else { + status = r.Fail + } + } + matchStr := "" + if match.Valid { + matchStr = match.String } - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, match) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, matchStr) } return list.RenderTable(table) } From bd064bf7655ae9a3c42d77e1261027518d99e249 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Tue, 6 May 2025 21:51:46 +0300 Subject: [PATCH 07/15] update bloat query --- internal/inspect/bloat/bloat.go | 13 ++++++------- internal/inspect/bloat/bloat_test.go | 9 ++++----- internal/inspect/vacuum_stats/vacuum_stats.go | 7 +++---- internal/inspect/vacuum_stats/vacuum_stats.sql | 3 +-- tools/inspect_rules.toml | 4 ++-- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/internal/inspect/bloat/bloat.go b/internal/inspect/bloat/bloat.go index 6a97e41c8..3e4aec5da 100644 --- a/internal/inspect/bloat/bloat.go +++ b/internal/inspect/bloat/bloat.go @@ -19,11 +19,10 @@ import ( var BloatQuery string type Result struct { - Type string - Schemaname string - Object_name string - Bloat string - Waste string + Type string + Name string + Bloat string + Waste string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -41,9 +40,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Type|Schema name|Object name|Bloat|Waste\n|-|-|-|-|-|\n" + table := "|Type|Name|Bloat|Waste\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Type, r.Schemaname, r.Object_name, r.Bloat, r.Waste) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|\n", r.Type, r.Schemaname+"."+r.Object_name, r.Bloat, r.Waste) } return list.RenderTable(table) } diff --git a/internal/inspect/bloat/bloat_test.go b/internal/inspect/bloat/bloat_test.go index 8646565ce..d172e4220 100644 --- a/internal/inspect/bloat/bloat_test.go +++ b/internal/inspect/bloat/bloat_test.go @@ -29,11 +29,10 @@ func TestBloat(t *testing.T) { defer conn.Close(t) conn.Query(BloatQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Type: "index hit rate", - Schemaname: "public", - Object_name: "table", - Bloat: "0.9", - Waste: "0.1", + Type: "index hit rate", + Name: "public.table", + Bloat: "0.9", + Waste: "0.1", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/vacuum_stats/vacuum_stats.go b/internal/inspect/vacuum_stats/vacuum_stats.go index 7f1cbc0ae..b0ce400f8 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.go +++ b/internal/inspect/vacuum_stats/vacuum_stats.go @@ -20,8 +20,7 @@ import ( var VacuumStatsQuery string type Result struct { - Schema string - Tbl string + Name string Last_vacuum string Last_autovacuum string Rowcount string @@ -45,10 +44,10 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Schema|Tbl|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|-|\n" + table := "|Name|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|\n" for _, r := range result { rowcount := strings.Replace(r.Rowcount, "-1", "No stats", 1) - table += fmt.Sprintf("|`%s`|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Schema, r.Tbl, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) + table += fmt.Sprintf("|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Name, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) } return list.RenderTable(table) } diff --git a/internal/inspect/vacuum_stats/vacuum_stats.sql b/internal/inspect/vacuum_stats/vacuum_stats.sql index 4ce3b3bdc..12161b762 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.sql +++ b/internal/inspect/vacuum_stats/vacuum_stats.sql @@ -20,8 +20,7 @@ WITH table_opts AS ( table_opts ) SELECT - vacuum_settings.nspname AS schema, - vacuum_settings.relname AS tbl, + vacuum_settings.nspname || '.' || vacuum_settings.relname AS name, coalesce(to_char(psut.last_vacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_vacuum, coalesce(to_char(psut.last_autovacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_autovacuum, to_char(pg_class.reltuples, '9G999G999G999') AS rowcount, diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index 5b5034983..3192cebd9 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -34,13 +34,13 @@ fail = "At least one table is showing sequential scans more than 10% of total ro [[rule]] -query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" +query = "SELECT LISTAGG(s.tbl, ',') AS match FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" name = "No large tables waiting on autovacuum" pass = "✔" fail = "At least one table is waiting on autovacuum" [[rule]] -query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum IS NULL OR s.last_vacuum IS NULL);" +query = "SELECT LISTAGG(s.tbl, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum = '' OR s.last_vacuum = '');" name = "No tables yet to be vacuumed" pass = "✔" fail = "At least one table has never had autovacuum or vacuum run on it" From 75385377d7e28a9eefc3a4e8ed46a91b8a20fd90 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Tue, 6 May 2025 23:32:18 +0300 Subject: [PATCH 08/15] combined table and index queries to index_stats and table_stats --- cmd/inspect.go | 107 +++--------------- internal/inspect/calls/calls.sql | 7 +- internal/inspect/index_sizes/index_sizes.go | 46 -------- internal/inspect/index_sizes/index_sizes.sql | 9 -- .../inspect/index_sizes/index_sizes_test.go | 40 ------- .../index_stats.go} | 20 ++-- internal/inspect/index_stats/index_stats.sql | 49 ++++++++ .../index_stats_test.go} | 24 ++-- internal/inspect/index_usage/index_usage.go | 47 -------- internal/inspect/index_usage/index_usage.sql | 17 --- .../inspect/index_usage/index_usage_test.go | 41 ------- internal/inspect/locks/locks.go | 10 +- internal/inspect/locks/locks.sql | 4 +- internal/inspect/locks/locks_test.go | 2 +- internal/inspect/outliers/outliers.sql | 7 +- internal/inspect/report.go | 14 ++- .../inspect/role_configs/role_configs.sql | 5 - .../inspect/role_configs/role_configs_test.go | 38 ------- .../role_connections/role_connections.go | 62 ---------- .../role_stats.go} | 18 +-- .../role_stats.sql} | 10 +- .../role_stats_test.go} | 7 +- internal/inspect/seq_scans/seq_scans.go | 46 -------- internal/inspect/seq_scans/seq_scans.sql | 6 - .../table_index_sizes/table_index_sizes.sql | 8 -- .../table_index_sizes_test.go | 40 ------- .../table_record_counts.go | 47 -------- .../table_record_counts.sql | 7 -- .../table_record_counts_test.go | 41 ------- internal/inspect/table_sizes/table_sizes.sql | 9 -- .../table_stats.go} | 19 ++-- internal/inspect/table_stats/table_stats.sql | 27 +++++ .../table_stats_test.go} | 21 ++-- .../total_table_sizes/total_table_sizes.go | 47 -------- .../total_table_sizes/total_table_sizes.sql | 9 -- .../total_table_sizes_test.go | 41 ------- .../inspect/unused_indexes/unused_indexes.go | 48 -------- .../inspect/unused_indexes/unused_indexes.sql | 13 --- .../unused_indexes/unused_indexes_test.go | 42 ------- internal/inspect/vacuum_stats/vacuum_stats.go | 7 +- .../inspect/vacuum_stats/vacuum_stats.sql | 3 +- .../inspect/vacuum_stats/vacuum_stats_test.go | 2 +- 42 files changed, 190 insertions(+), 877 deletions(-) delete mode 100644 internal/inspect/index_sizes/index_sizes.go delete mode 100644 internal/inspect/index_sizes/index_sizes.sql delete mode 100644 internal/inspect/index_sizes/index_sizes_test.go rename internal/inspect/{table_index_sizes/table_index_sizes.go => index_stats/index_stats.go} (62%) create mode 100644 internal/inspect/index_stats/index_stats.sql rename internal/inspect/{table_sizes/table_sizes_test.go => index_stats/index_stats_test.go} (63%) delete mode 100644 internal/inspect/index_usage/index_usage.go delete mode 100644 internal/inspect/index_usage/index_usage.sql delete mode 100644 internal/inspect/index_usage/index_usage_test.go delete mode 100644 internal/inspect/role_configs/role_configs.sql delete mode 100644 internal/inspect/role_configs/role_configs_test.go delete mode 100644 internal/inspect/role_connections/role_connections.go rename internal/inspect/{role_configs/role_configs.go => role_stats/role_stats.go} (64%) rename internal/inspect/{role_connections/role_connections.sql => role_stats/role_stats.sql} (64%) rename internal/inspect/{role_connections/role_connections_test.go => role_stats/role_stats_test.go} (84%) delete mode 100644 internal/inspect/seq_scans/seq_scans.go delete mode 100644 internal/inspect/seq_scans/seq_scans.sql delete mode 100644 internal/inspect/table_index_sizes/table_index_sizes.sql delete mode 100644 internal/inspect/table_index_sizes/table_index_sizes_test.go delete mode 100644 internal/inspect/table_record_counts/table_record_counts.go delete mode 100644 internal/inspect/table_record_counts/table_record_counts.sql delete mode 100644 internal/inspect/table_record_counts/table_record_counts_test.go delete mode 100644 internal/inspect/table_sizes/table_sizes.sql rename internal/inspect/{table_sizes/table_sizes.go => table_stats/table_stats.go} (66%) create mode 100644 internal/inspect/table_stats/table_stats.sql rename internal/inspect/{seq_scans/seq_scans_test.go => table_stats/table_stats_test.go} (62%) delete mode 100644 internal/inspect/total_table_sizes/total_table_sizes.go delete mode 100644 internal/inspect/total_table_sizes/total_table_sizes.sql delete mode 100644 internal/inspect/total_table_sizes/total_table_sizes_test.go delete mode 100644 internal/inspect/unused_indexes/unused_indexes.go delete mode 100644 internal/inspect/unused_indexes/unused_indexes.sql delete mode 100644 internal/inspect/unused_indexes/unused_indexes_test.go diff --git a/cmd/inspect.go b/cmd/inspect.go index 2b55c3876..7e615be34 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -11,26 +11,21 @@ import ( "github.com/supabase/cli/internal/inspect/bloat" "github.com/supabase/cli/internal/inspect/blocking" "github.com/supabase/cli/internal/inspect/cache" + "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/internal/inspect" "github.com/supabase/cli/internal/inspect/calls" - "github.com/supabase/cli/internal/inspect/index_sizes" - "github.com/supabase/cli/internal/inspect/index_usage" + "github.com/supabase/cli/internal/inspect/index_stats" "github.com/supabase/cli/internal/inspect/locks" "github.com/supabase/cli/internal/inspect/long_running_queries" "github.com/supabase/cli/internal/inspect/outliers" "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/role_configs" - "github.com/supabase/cli/internal/inspect/role_connections" - "github.com/supabase/cli/internal/inspect/seq_scans" - "github.com/supabase/cli/internal/inspect/table_index_sizes" - "github.com/supabase/cli/internal/inspect/table_record_counts" - "github.com/supabase/cli/internal/inspect/table_sizes" + + "github.com/supabase/cli/internal/inspect/table_stats" "github.com/supabase/cli/internal/inspect/total_index_size" - "github.com/supabase/cli/internal/inspect/total_table_sizes" - "github.com/supabase/cli/internal/inspect/unused_indexes" + "github.com/supabase/cli/internal/inspect/vacuum_stats" ) @@ -67,11 +62,11 @@ var ( }, } - inspectIndexUsageCmd = &cobra.Command{ - Use: "index-usage", - Short: "Show information about the efficiency of indexes", + inspectIndexStatsCmd = &cobra.Command{ + Use: "index-stats", + Short: "Show combined index size, usage percent, scan counts, and unused status", RunE: func(cmd *cobra.Command, args []string) error { - return index_usage.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -115,54 +110,6 @@ var ( }, } - inspectIndexSizesCmd = &cobra.Command{ - Use: "index-sizes", - Short: "Show index sizes of individual indexes", - RunE: func(cmd *cobra.Command, args []string) error { - return index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectTableSizesCmd = &cobra.Command{ - Use: "table-sizes", - Short: "Show table sizes of individual tables without their index sizes", - RunE: func(cmd *cobra.Command, args []string) error { - return table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectTableIndexSizesCmd = &cobra.Command{ - Use: "table-index-sizes", - Short: "Show index sizes of individual tables", - RunE: func(cmd *cobra.Command, args []string) error { - return table_index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectTotalTableSizesCmd = &cobra.Command{ - Use: "total-table-sizes", - Short: "Show total table sizes, including table index sizes", - RunE: func(cmd *cobra.Command, args []string) error { - return total_table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectUnusedIndexesCmd = &cobra.Command{ - Use: "unused-indexes", - Short: "Show indexes with low usage", - RunE: func(cmd *cobra.Command, args []string) error { - return unused_indexes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectSeqScansCmd = &cobra.Command{ - Use: "seq-scans", - Short: "Show number of sequential scans recorded against all tables", - RunE: func(cmd *cobra.Command, args []string) error { - return seq_scans.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - inspectLongRunningQueriesCmd = &cobra.Command{ Use: "long-running-queries", Short: "Show currently running queries running for longer than 5 minutes", @@ -171,14 +118,6 @@ var ( }, } - inspectTableRecordCountsCmd = &cobra.Command{ - Use: "table-record-counts", - Short: "Show estimated number of rows per table", - RunE: func(cmd *cobra.Command, args []string) error { - return table_record_counts.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - inspectBloatCmd = &cobra.Command{ Use: "bloat", Short: "Estimates space allocated to a relation that is full of dead tuples", @@ -195,19 +134,11 @@ var ( }, } - inspectRoleConfigsCmd = &cobra.Command{ - Use: "role-configs", - Short: "Show configuration settings for database roles when they have been modified", - RunE: func(cmd *cobra.Command, args []string) error { - return role_configs.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - - inspectRoleConnectionsCmd = &cobra.Command{ - Use: "role-connections", - Short: "Show number of active connections for all database roles", + inspectTableStatsCmd = &cobra.Command{ + Use: "table-stats", + Short: "Show combined table size, index size, and estimated row count", RunE: func(cmd *cobra.Command, args []string) error { - return role_connections.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -240,24 +171,16 @@ func init() { inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local") inspectDBCmd.AddCommand(inspectCacheHitCmd) inspectDBCmd.AddCommand(inspectReplicationSlotsCmd) - inspectDBCmd.AddCommand(inspectIndexUsageCmd) + inspectDBCmd.AddCommand(inspectIndexStatsCmd) inspectDBCmd.AddCommand(inspectLocksCmd) inspectDBCmd.AddCommand(inspectBlockingCmd) inspectDBCmd.AddCommand(inspectOutliersCmd) inspectDBCmd.AddCommand(inspectCallsCmd) inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd) - inspectDBCmd.AddCommand(inspectIndexSizesCmd) - inspectDBCmd.AddCommand(inspectTableSizesCmd) - inspectDBCmd.AddCommand(inspectTableIndexSizesCmd) - inspectDBCmd.AddCommand(inspectTotalTableSizesCmd) - inspectDBCmd.AddCommand(inspectUnusedIndexesCmd) - inspectDBCmd.AddCommand(inspectSeqScansCmd) inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd) - inspectDBCmd.AddCommand(inspectTableRecordCountsCmd) inspectDBCmd.AddCommand(inspectBloatCmd) inspectDBCmd.AddCommand(inspectVacuumStatsCmd) - inspectDBCmd.AddCommand(inspectRoleConfigsCmd) - inspectDBCmd.AddCommand(inspectRoleConnectionsCmd) + inspectDBCmd.AddCommand(inspectTableStatsCmd) inspectCmd.AddCommand(inspectDBCmd) reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in") inspectCmd.AddCommand(reportCmd) diff --git a/internal/inspect/calls/calls.sql b/internal/inspect/calls/calls.sql index 57c3ecee2..d0b2c623c 100644 --- a/internal/inspect/calls/calls.sql +++ b/internal/inspect/calls/calls.sql @@ -3,7 +3,10 @@ SELECT (interval '1 millisecond' * total_exec_time)::text AS total_exec_time, to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%' AS prop_exec_time, to_char(calls, 'FM999G999G990') AS ncalls, - (interval '1 millisecond' * (blk_read_time + blk_write_time))::text AS sync_io_time -FROM pg_stat_statements + (interval '1 millisecond' * ( + COALESCE((row_to_json(s)->>'blk_read_time')::numeric, 0) + + COALESCE((row_to_json(s)->>'blk_write_time')::numeric, 0) + ))::text AS sync_io_time +FROM pg_stat_statements AS s ORDER BY calls DESC LIMIT 10 diff --git a/internal/inspect/index_sizes/index_sizes.go b/internal/inspect/index_sizes/index_sizes.go deleted file mode 100644 index 2cfc06e8d..000000000 --- a/internal/inspect/index_sizes/index_sizes.go +++ /dev/null @@ -1,46 +0,0 @@ -package index_sizes - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed index_sizes.sql -var IndexSizesQuery string - -type Result struct { - Name string - Size string -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, IndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Name|size|\n|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Size) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/index_sizes/index_sizes.sql b/internal/inspect/index_sizes/index_sizes.sql deleted file mode 100644 index 25b6f96d5..000000000 --- a/internal/inspect/index_sizes/index_sizes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS name, - pg_size_pretty(sum(c.relpages::bigint*8192)::bigint) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'i' -GROUP BY n.nspname, c.relname -ORDER BY sum(c.relpages) DESC diff --git a/internal/inspect/index_sizes/index_sizes_test.go b/internal/inspect/index_sizes/index_sizes_test.go deleted file mode 100644 index 9071c5710..000000000 --- a/internal/inspect/index_sizes/index_sizes_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package index_sizes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestIndexSizes(t *testing.T) { - t.Run("inspects index sizes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(IndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Name: "test_table_idx", - Size: "100GB", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_index_sizes/table_index_sizes.go b/internal/inspect/index_stats/index_stats.go similarity index 62% rename from internal/inspect/table_index_sizes/table_index_sizes.go rename to internal/inspect/index_stats/index_stats.go index e61f23361..89f0bccc8 100644 --- a/internal/inspect/table_index_sizes/table_index_sizes.go +++ b/internal/inspect/index_stats/index_stats.go @@ -1,4 +1,4 @@ -package table_index_sizes +package index_stats import ( "context" @@ -15,12 +15,16 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed table_index_sizes.sql -var TableIndexSizesQuery string +//go:embed index_stats.sql +var IndexStatsQuery string type Result struct { - Table string - Index_size string + Name string + Size string + Percent_used string + Index_scans int64 + Seq_scans int64 + Unused bool } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -29,7 +33,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableIndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) + rows, err := conn.Query(ctx, IndexStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -38,9 +42,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Table|Index size|\n|-|-|\n" + table := "|Index|Size|Percent used|Index scans|Seq scans|Unused|\n|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Table, r.Index_size) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|`%d`|`%t`|\n", r.Name, r.Size, r.Percent_used, r.Index_scans, r.Seq_scans, r.Unused) } return list.RenderTable(table) } diff --git a/internal/inspect/index_stats/index_stats.sql b/internal/inspect/index_stats/index_stats.sql new file mode 100644 index 000000000..26337fdde --- /dev/null +++ b/internal/inspect/index_stats/index_stats.sql @@ -0,0 +1,49 @@ +-- Combined index statistics: size, usage percent, seq scans, and mark unused +WITH idx_sizes AS ( + SELECT + i.indexrelid AS oid, + n.nspname || '.' || c.relname AS name, + pg_relation_size(i.indexrelid) AS index_size_bytes + FROM pg_stat_user_indexes ui + JOIN pg_index i ON ui.indexrelid = i.indexrelid + JOIN pg_class c ON ui.indexrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE NOT n.nspname LIKE ANY($1) +), +idx_usage AS ( + SELECT + indexrelid AS oid, + idx_scan::bigint AS idx_scans + FROM pg_stat_user_indexes ui + WHERE NOT schemaname LIKE ANY($1) +), +seq_usage AS ( + SELECT + relid AS oid, + seq_scan::bigint AS seq_scans + FROM pg_stat_user_tables + WHERE NOT schemaname LIKE ANY($1) +), +usage_pct AS ( + SELECT + u.oid, + CASE + WHEN u.idx_scans IS NULL OR u.idx_scans = 0 THEN 0 + WHEN s.seq_scans IS NULL THEN 100 + ELSE ROUND(100.0 * u.idx_scans / (s.seq_scans + u.idx_scans), 1) + END AS percent_used + FROM idx_usage u + LEFT JOIN seq_usage s ON s.oid = u.oid +) +SELECT + s.name, + pg_size_pretty(s.index_size_bytes) AS size, + COALESCE(up.percent_used, 0)::text || '%' AS percent_used, + COALESCE(u.idx_scans, 0) AS index_scans, + COALESCE(sq.seq_scans, 0) AS seq_scans, + CASE WHEN COALESCE(u.idx_scans, 0) = 0 THEN true ELSE false END AS unused +FROM idx_sizes s +LEFT JOIN idx_usage u ON u.oid = s.oid +LEFT JOIN seq_usage sq ON sq.oid = s.oid +LEFT JOIN usage_pct up ON up.oid = s.oid +ORDER BY s.index_size_bytes DESC diff --git a/internal/inspect/table_sizes/table_sizes_test.go b/internal/inspect/index_stats/index_stats_test.go similarity index 63% rename from internal/inspect/table_sizes/table_sizes_test.go rename to internal/inspect/index_stats/index_stats_test.go index 5cc6426ad..657242738 100644 --- a/internal/inspect/table_sizes/table_sizes_test.go +++ b/internal/inspect/index_stats/index_stats_test.go @@ -1,4 +1,4 @@ -package table_sizes +package index_stats import ( "context" @@ -20,22 +20,24 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestTableSizesCommand(t *testing.T) { - t.Run("inspects table sizes", func(t *testing.T) { - // Setup in-memory fs +func TestIndexStatsCommand(t *testing.T) { + t.Run("inspects index stats", func(t *testing.T) { fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(TableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). + + // Mock index stats + conn.Query(IndexStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). Reply("SELECT 1", Result{ - Schema: "schema", - Name: "test_table", - Size: "3GB", + Name: "public.test_idx", + Size: "1GB", + Percent_used: "50%", + Index_scans: 5, + Seq_scans: 5, + Unused: false, }) - // Run test + err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error assert.NoError(t, err) }) } diff --git a/internal/inspect/index_usage/index_usage.go b/internal/inspect/index_usage/index_usage.go deleted file mode 100644 index cd8875f79..000000000 --- a/internal/inspect/index_usage/index_usage.go +++ /dev/null @@ -1,47 +0,0 @@ -package index_usage - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed index_usage.sql -var IndexUsageQuery string - -type Result struct { - Name string - Percent_of_times_index_used string - Rows_in_table int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, IndexUsageQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - // TODO: implement a markdown table marshaller - table := "|Table name|Percentage of times index used|Rows in table|\n|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%d`|\n", r.Name, r.Percent_of_times_index_used, r.Rows_in_table) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/index_usage/index_usage.sql b/internal/inspect/index_usage/index_usage.sql deleted file mode 100644 index fa83e97db..000000000 --- a/internal/inspect/index_usage/index_usage.sql +++ /dev/null @@ -1,17 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - CASE - WHEN idx_scan IS NULL THEN 'Insufficient data' - WHEN idx_scan = 0 THEN 'Insufficient data' - ELSE ROUND(100.0 * idx_scan / (seq_scan + idx_scan), 1) || '%' - END percent_of_times_index_used, - n_live_tup rows_in_table -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY - CASE - WHEN idx_scan is null then 1 - WHEN idx_scan = 0 then 1 - ELSE 0 - END, - n_live_tup DESC diff --git a/internal/inspect/index_usage/index_usage_test.go b/internal/inspect/index_usage/index_usage_test.go deleted file mode 100644 index 5b735bb60..000000000 --- a/internal/inspect/index_usage/index_usage_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package index_usage - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestIndexUsage(t *testing.T) { - t.Run("inspects index usage", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(IndexUsageQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Name: "test_table_idx", - Percent_of_times_index_used: "0.9", - Rows_in_table: 300, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/locks/locks.go b/internal/inspect/locks/locks.go index 0b2db71c6..be5c34d3a 100644 --- a/internal/inspect/locks/locks.go +++ b/internal/inspect/locks/locks.go @@ -23,7 +23,7 @@ type Result struct { Relname string Transactionid string Granted bool - Query string + Stmt string Age string } @@ -42,16 +42,16 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|pid|relname|transaction id|granted|query|age|\n|-|-|-|-|-|-|\n" + table := "|pid|relname|transaction id|granted|stmt|age|\n|-|-|-|-|-|-|\n" for _, r := range result { // remove whitespace from query re := regexp.MustCompile(`\s+|\r+|\n+|\t+|\v`) - query := re.ReplaceAllString(r.Query, " ") + stmt := re.ReplaceAllString(r.Stmt, " ") // escape pipes in query re = regexp.MustCompile(`\|`) - query = re.ReplaceAllString(query, `\|`) - table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, query, r.Age) + stmt = re.ReplaceAllString(stmt, `\|`) + table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, stmt, r.Age) } return list.RenderTable(table) } diff --git a/internal/inspect/locks/locks.sql b/internal/inspect/locks/locks.sql index e140f4786..9369057b5 100644 --- a/internal/inspect/locks/locks.sql +++ b/internal/inspect/locks/locks.sql @@ -1,9 +1,9 @@ SELECT pg_stat_activity.pid, COALESCE(pg_class.relname, 'null') AS relname, - COALESCE(pg_locks.transactionid, 'null') AS transactionid, + COALESCE(pg_locks.transactionid::text, 'null') AS transactionid, pg_locks.granted, - pg_stat_activity.query, + pg_stat_activity.query AS stmt, age(now(), pg_stat_activity.query_start)::text AS age FROM pg_stat_activity, pg_locks LEFT OUTER JOIN pg_class ON (pg_locks.relation = pg_class.oid) WHERE pg_stat_activity.query <> '' diff --git a/internal/inspect/locks/locks_test.go b/internal/inspect/locks/locks_test.go index e4c55c6bd..0964ad122 100644 --- a/internal/inspect/locks/locks_test.go +++ b/internal/inspect/locks/locks_test.go @@ -31,7 +31,7 @@ func TestLocksCommand(t *testing.T) { Relname: "rel", Transactionid: "9301", Granted: true, - Query: "select 1", + Stmt: "select 1", Age: "300ms", }) // Run test diff --git a/internal/inspect/outliers/outliers.sql b/internal/inspect/outliers/outliers.sql index d222cc8de..69336eef6 100644 --- a/internal/inspect/outliers/outliers.sql +++ b/internal/inspect/outliers/outliers.sql @@ -2,8 +2,11 @@ SELECT (interval '1 millisecond' * total_exec_time)::text AS total_exec_time, to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%' AS prop_exec_time, to_char(calls, 'FM999G999G999G990') AS ncalls, - (interval '1 millisecond' * (blk_read_time + blk_write_time))::text AS sync_io_time, + (interval '1 millisecond' * ( + COALESCE((row_to_json(s)->>'blk_read_time')::numeric, 0) + + COALESCE((row_to_json(s)->>'blk_write_time')::numeric, 0) + ))::text AS sync_io_time, query -FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) +FROM pg_stat_statements AS s WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) ORDER BY total_exec_time DESC LIMIT 10 diff --git a/internal/inspect/report.go b/internal/inspect/report.go index de721d954..b09302175 100644 --- a/internal/inspect/report.go +++ b/internal/inspect/report.go @@ -26,6 +26,11 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs if err := utils.MkdirIfNotExistFS(fsys, out); err != nil { return err } + // create a date-based subdirectory + dateDir := filepath.Join(out, date) + if err := utils.MkdirIfNotExistFS(fsys, dateDir); err != nil { + return err + } conn, err := utils.ConnectByConfig(ctx, config, options...) if err != nil { return err @@ -44,15 +49,16 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs return errors.Errorf("failed to read query: %w", err) } name := strings.Split(d.Name(), ".")[0] - outPath := filepath.Join(out, fmt.Sprintf("%s_%s.csv", name, date)) + outPath := filepath.Join(dateDir, fmt.Sprintf("%s.csv", name)) return copyToCSV(ctx, string(query), outPath, conn.PgConn(), fsys) }); err != nil { return err } - if !filepath.IsAbs(out) { - out, _ = filepath.Abs(out) + // print the actual save location + if !filepath.IsAbs(dateDir) { + dateDir, _ = filepath.Abs(dateDir) } - fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(out)) + fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(dateDir)) return nil } diff --git a/internal/inspect/role_configs/role_configs.sql b/internal/inspect/role_configs/role_configs.sql deleted file mode 100644 index fd7f964d0..000000000 --- a/internal/inspect/role_configs/role_configs.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - rolname as role_name, - array_to_string(rolconfig, ',', '*') as custom_config -from - pg_roles where rolconfig is not null diff --git a/internal/inspect/role_configs/role_configs_test.go b/internal/inspect/role_configs/role_configs_test.go deleted file mode 100644 index 554a12526..000000000 --- a/internal/inspect/role_configs/role_configs_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package role_configs - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestRoleCommand(t *testing.T) { - t.Run("inspects role connections", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(RoleConfigsQuery). - Reply("SELECT 1", Result{ - Role_name: "postgres", - Custom_config: "statement_timeout=3s", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/role_connections/role_connections.go b/internal/inspect/role_connections/role_connections.go deleted file mode 100644 index 5b0a56539..000000000 --- a/internal/inspect/role_connections/role_connections.go +++ /dev/null @@ -1,62 +0,0 @@ -package role_connections - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed role_connections.sql -var RoleConnectionsQuery string - -type Result struct { - Rolname string - Active_connections int - Connection_limit int -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, RoleConnectionsQuery) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Role Name|Active connction|\n|-|-|\n" - sum := 0 - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%d`|\n", r.Rolname, r.Active_connections) - sum += r.Active_connections - } - - if err := list.RenderTable(table); err != nil { - return err - } - - if len(result) > 0 { - fmt.Printf("\nActive connections %d/%d\n\n", sum, result[0].Connection_limit) - } - - if matches := utils.ProjectHostPattern.FindStringSubmatch(config.Host); len(matches) == 4 { - fmt.Println("Go to the dashboard for more here:") - fmt.Printf("https://app.supabase.com/project/%s/database/roles\n", matches[2]) - } - - return nil -} diff --git a/internal/inspect/role_configs/role_configs.go b/internal/inspect/role_stats/role_stats.go similarity index 64% rename from internal/inspect/role_configs/role_configs.go rename to internal/inspect/role_stats/role_stats.go index f4fb79382..ee5e724f2 100644 --- a/internal/inspect/role_configs/role_configs.go +++ b/internal/inspect/role_stats/role_stats.go @@ -1,4 +1,4 @@ -package role_configs +package role_stats import ( "context" @@ -14,12 +14,14 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed role_configs.sql -var RoleConfigsQuery string +//go:embed role_stats.sql +var RoleStatsQuery string type Result struct { - Role_name string - Custom_config string + Role_name string + Active_connections int + Connection_limit int + Custom_config string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -28,7 +30,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, RoleConfigsQuery) + rows, err := conn.Query(ctx, RoleStatsQuery) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -37,9 +39,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Role name|Custom config|\n|-|-|\n" + table := "|Role name|Active connections|Connection limit|Custom config|\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Role_name, r.Custom_config) + table += fmt.Sprintf("|`%s`|`%d`|`%d`|`%s`|\n", r.Role_name, r.Active_connections, r.Connection_limit, r.Custom_config) } return list.RenderTable(table) diff --git a/internal/inspect/role_connections/role_connections.sql b/internal/inspect/role_stats/role_stats.sql similarity index 64% rename from internal/inspect/role_connections/role_connections.sql rename to internal/inspect/role_stats/role_stats.sql index 40b103669..09972c2d0 100644 --- a/internal/inspect/role_connections/role_connections.sql +++ b/internal/inspect/role_stats/role_stats.sql @@ -1,5 +1,5 @@ SELECT - rolname, + rolname as role_name, ( SELECT count(*) @@ -11,6 +11,8 @@ SELECT CASE WHEN rolconnlimit = -1 THEN current_setting('max_connections')::int8 ELSE rolconnlimit - END AS connection_limit -FROM pg_roles -ORDER BY 2 DESC + END AS connection_limit, + array_to_string(rolconfig, ',', '*') as custom_config +FROM + pg_roles +ORDER BY 3 DESC diff --git a/internal/inspect/role_connections/role_connections_test.go b/internal/inspect/role_stats/role_stats_test.go similarity index 84% rename from internal/inspect/role_connections/role_connections_test.go rename to internal/inspect/role_stats/role_stats_test.go index 32ecab768..0539c594f 100644 --- a/internal/inspect/role_connections/role_connections_test.go +++ b/internal/inspect/role_stats/role_stats_test.go @@ -1,4 +1,4 @@ -package role_connections +package role_stats import ( "context" @@ -25,9 +25,10 @@ func TestRoleCommand(t *testing.T) { // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(RoleConnectionsQuery). + conn.Query(RoleStatsQuery). Reply("SELECT 1", Result{ - Rolname: "postgres", + Role_name: "postgres", + Custom_config: "statement_timeout=3s", Active_connections: 1, Connection_limit: 10, }) diff --git a/internal/inspect/seq_scans/seq_scans.go b/internal/inspect/seq_scans/seq_scans.go deleted file mode 100644 index 6b52538ee..000000000 --- a/internal/inspect/seq_scans/seq_scans.go +++ /dev/null @@ -1,46 +0,0 @@ -package seq_scans - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed seq_scans.sql -var SeqScansQuery string - -type Result struct { - Name string - Count int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, SeqScansQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Name|Count|\n|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%d`|\n", r.Name, r.Count) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/seq_scans/seq_scans.sql b/internal/inspect/seq_scans/seq_scans.sql deleted file mode 100644 index c8edfc8e3..000000000 --- a/internal/inspect/seq_scans/seq_scans.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - seq_scan as count -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY seq_scan DESC diff --git a/internal/inspect/table_index_sizes/table_index_sizes.sql b/internal/inspect/table_index_sizes/table_index_sizes.sql deleted file mode 100644 index 0c9bc6bfc..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS table, - pg_size_pretty(pg_indexes_size(c.oid)) AS index_size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_indexes_size(c.oid) DESC diff --git a/internal/inspect/table_index_sizes/table_index_sizes_test.go b/internal/inspect/table_index_sizes/table_index_sizes_test.go deleted file mode 100644 index 20ad80fc9..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package table_index_sizes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTableIndexSizesCommand(t *testing.T) { - t.Run("inspects table index sizes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TableIndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Table: "public.test_table", - Index_size: "3GB", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_record_counts/table_record_counts.go b/internal/inspect/table_record_counts/table_record_counts.go deleted file mode 100644 index e0b394374..000000000 --- a/internal/inspect/table_record_counts/table_record_counts.go +++ /dev/null @@ -1,47 +0,0 @@ -package table_record_counts - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed table_record_counts.sql -var TableRecordCountsQuery string - -type Result struct { - Schema string - Name string - Estimated_count int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableRecordCountsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "Schema|Table|Estimated count|\n|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%d`|\n", r.Schema, r.Name, r.Estimated_count) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/table_record_counts/table_record_counts.sql b/internal/inspect/table_record_counts/table_record_counts.sql deleted file mode 100644 index 1b24f04a4..000000000 --- a/internal/inspect/table_record_counts/table_record_counts.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - schemaname AS schema, - relname AS name, - n_live_tup AS estimated_count -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY n_live_tup DESC diff --git a/internal/inspect/table_record_counts/table_record_counts_test.go b/internal/inspect/table_record_counts/table_record_counts_test.go deleted file mode 100644 index a03714d97..000000000 --- a/internal/inspect/table_record_counts/table_record_counts_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package table_record_counts - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTableRecordCountsCommand(t *testing.T) { - t.Run("inspects table record counts", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TableRecordCountsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). - Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", - Estimated_count: 100, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_sizes/table_sizes.sql b/internal/inspect/table_sizes/table_sizes.sql deleted file mode 100644 index 2c8fb3064..000000000 --- a/internal/inspect/table_sizes/table_sizes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - n.nspname AS schema, - c.relname AS name, - pg_size_pretty(pg_table_size(c.oid)) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_table_size(c.oid) DESC diff --git a/internal/inspect/table_sizes/table_sizes.go b/internal/inspect/table_stats/table_stats.go similarity index 66% rename from internal/inspect/table_sizes/table_sizes.go rename to internal/inspect/table_stats/table_stats.go index 7741f0119..e66a5dce9 100644 --- a/internal/inspect/table_sizes/table_sizes.go +++ b/internal/inspect/table_stats/table_stats.go @@ -1,4 +1,4 @@ -package table_sizes +package table_stats import ( "context" @@ -15,13 +15,14 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed table_sizes.sql -var TableSizesQuery string +//go:embed table_stats.sql +var TableStatsQuery string type Result struct { - Schema string - Name string - Size string + Name string + Table_size string + Index_size string + Estimated_row_count int64 } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -30,7 +31,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)) + rows, err := conn.Query(ctx, TableStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -39,9 +40,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "Schema|Table|size|\n|-|-|-|\n" + table := "|Table|Size|Index size|Estimated row count|\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Name, r.Table_size, r.Index_size, r.Estimated_row_count) } return list.RenderTable(table) } diff --git a/internal/inspect/table_stats/table_stats.sql b/internal/inspect/table_stats/table_stats.sql new file mode 100644 index 000000000..5a16e554f --- /dev/null +++ b/internal/inspect/table_stats/table_stats.sql @@ -0,0 +1,27 @@ +SELECT + ts.name, + pg_size_pretty(ts.table_size_bytes) AS table_size, + pg_size_pretty(ts.index_size_bytes) AS index_size, + pg_size_pretty(ts.total_size_bytes) AS total_size, + COALESCE(rc.estimated_row_count, 0) AS estimated_row_count, + COALESCE(rc.seq_scans, 0) AS seq_scans +FROM ( + SELECT + n.nspname || '.' || c.relname AS name, + pg_table_size(c.oid) AS table_size_bytes, + pg_indexes_size(c.oid) AS index_size_bytes, + pg_total_relation_size(c.oid) AS total_size_bytes + FROM pg_class c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE NOT n.nspname LIKE ANY($1) + AND c.relkind = 'r' +) ts +LEFT JOIN ( + SELECT + schemaname || '.' || relname AS name, + n_live_tup AS estimated_row_count, + seq_scan AS seq_scans + FROM pg_stat_user_tables + WHERE NOT schemaname LIKE ANY($1) +) rc ON rc.name = ts.name +ORDER BY ts.total_size_bytes DESC \ No newline at end of file diff --git a/internal/inspect/seq_scans/seq_scans_test.go b/internal/inspect/table_stats/table_stats_test.go similarity index 62% rename from internal/inspect/seq_scans/seq_scans_test.go rename to internal/inspect/table_stats/table_stats_test.go index 3db6caee5..ddd4e4f29 100644 --- a/internal/inspect/seq_scans/seq_scans_test.go +++ b/internal/inspect/table_stats/table_stats_test.go @@ -1,4 +1,4 @@ -package seq_scans +package table_stats import ( "context" @@ -20,21 +20,22 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestSequentialScansCommand(t *testing.T) { - t.Run("inspects sequential scans", func(t *testing.T) { - // Setup in-memory fs +func TestTableStatsCommand(t *testing.T) { + t.Run("inspects table stats", func(t *testing.T) { fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(SeqScansQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + + // Mock table sizes and index sizes + conn.Query(TableStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). Reply("SELECT 1", Result{ - Name: "test_table", - Count: 99999, + Name: "public.test_table", + Table_size: "3GB", + Index_size: "1GB", + Estimated_row_count: 100, }) - // Run test + err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error assert.NoError(t, err) }) } diff --git a/internal/inspect/total_table_sizes/total_table_sizes.go b/internal/inspect/total_table_sizes/total_table_sizes.go deleted file mode 100644 index 80b1c89a8..000000000 --- a/internal/inspect/total_table_sizes/total_table_sizes.go +++ /dev/null @@ -1,47 +0,0 @@ -package total_table_sizes - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed total_table_sizes.sql -var TotalTableSizesQuery string - -type Result struct { - Schema string - Name string - Size string -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "Schema|Table|Size|\n|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/total_table_sizes/total_table_sizes.sql b/internal/inspect/total_table_sizes/total_table_sizes.sql deleted file mode 100644 index 471d0655f..000000000 --- a/internal/inspect/total_table_sizes/total_table_sizes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - n.nspname AS schema, - c.relname AS name, - pg_size_pretty(pg_total_relation_size(c.oid)) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_total_relation_size(c.oid) DESC diff --git a/internal/inspect/total_table_sizes/total_table_sizes_test.go b/internal/inspect/total_table_sizes/total_table_sizes_test.go deleted file mode 100644 index bc548af60..000000000 --- a/internal/inspect/total_table_sizes/total_table_sizes_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package total_table_sizes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTotalTableSizesCommand(t *testing.T) { - t.Run("inspects total table sizes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). - Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", - Size: "3GB", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/unused_indexes/unused_indexes.go b/internal/inspect/unused_indexes/unused_indexes.go deleted file mode 100644 index 2a30a46d7..000000000 --- a/internal/inspect/unused_indexes/unused_indexes.go +++ /dev/null @@ -1,48 +0,0 @@ -package unused_indexes - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed unused_indexes.sql -var UnusedIndexesQuery string - -type Result struct { - Table string - Index string - Index_size string - Index_scans int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, UnusedIndexesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Table|Index|Index Size|Index Scans\n|-|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Table, r.Index, r.Index_size, r.Index_scans) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/unused_indexes/unused_indexes.sql b/internal/inspect/unused_indexes/unused_indexes.sql deleted file mode 100644 index 6e775967d..000000000 --- a/internal/inspect/unused_indexes/unused_indexes.sql +++ /dev/null @@ -1,13 +0,0 @@ -SELECT - schemaname || '.' || relname AS table, - indexrelname AS index, - pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, - idx_scan as index_scans -FROM pg_stat_user_indexes ui -JOIN pg_index i ON ui.indexrelid = i.indexrelid -WHERE - NOT indisunique AND idx_scan < 50 AND pg_relation_size(relid) > 5 * 8192 - AND NOT schemaname LIKE ANY($1) -ORDER BY - pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST, - pg_relation_size(i.indexrelid) DESC diff --git a/internal/inspect/unused_indexes/unused_indexes_test.go b/internal/inspect/unused_indexes/unused_indexes_test.go deleted file mode 100644 index ee4182094..000000000 --- a/internal/inspect/unused_indexes/unused_indexes_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package unused_indexes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestUnusedIndexesCommand(t *testing.T) { - t.Run("inspects unused indexes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(UnusedIndexesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Table: "test_table", - Index: "test_table_idx", - Index_size: "3GB", - Index_scans: 2, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/vacuum_stats/vacuum_stats.go b/internal/inspect/vacuum_stats/vacuum_stats.go index dc9326d79..b0ce400f8 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.go +++ b/internal/inspect/vacuum_stats/vacuum_stats.go @@ -20,8 +20,7 @@ import ( var VacuumStatsQuery string type Result struct { - Schema string - Table string + Name string Last_vacuum string Last_autovacuum string Rowcount string @@ -45,10 +44,10 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Schema|Table|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|-|\n" + table := "|Name|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|\n" for _, r := range result { rowcount := strings.Replace(r.Rowcount, "-1", "No stats", 1) - table += fmt.Sprintf("|`%s`|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Schema, r.Table, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) + table += fmt.Sprintf("|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Name, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) } return list.RenderTable(table) } diff --git a/internal/inspect/vacuum_stats/vacuum_stats.sql b/internal/inspect/vacuum_stats/vacuum_stats.sql index 707847497..12161b762 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.sql +++ b/internal/inspect/vacuum_stats/vacuum_stats.sql @@ -20,8 +20,7 @@ WITH table_opts AS ( table_opts ) SELECT - vacuum_settings.nspname AS schema, - vacuum_settings.relname AS table, + vacuum_settings.nspname || '.' || vacuum_settings.relname AS name, coalesce(to_char(psut.last_vacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_vacuum, coalesce(to_char(psut.last_autovacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_autovacuum, to_char(pg_class.reltuples, '9G999G999G999') AS rowcount, diff --git a/internal/inspect/vacuum_stats/vacuum_stats_test.go b/internal/inspect/vacuum_stats/vacuum_stats_test.go index 0d3cbec10..49dafc84e 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats_test.go +++ b/internal/inspect/vacuum_stats/vacuum_stats_test.go @@ -30,7 +30,7 @@ func TestVacuumCommand(t *testing.T) { conn.Query(VacuumStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ Schema: "test_schema", - Table: "test_table", + Tbl: "test_table", Last_vacuum: "2021-01-01 00:00:00", Last_autovacuum: "2021-01-01 00:00:00", Rowcount: "1000", From d89117536d1391aaa5224b7518e42e1999dd58ad Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Wed, 7 May 2025 14:09:42 +0300 Subject: [PATCH 09/15] combine report queries to creats rollup for tables, indexes, roles and db overall --- cmd/inspect.go | 39 +++++++------ internal/inspect/cache/cache.go | 57 ------------------- internal/inspect/cache/cache.sql | 9 --- internal/inspect/cache/cache_test.go | 52 ----------------- .../db_stats.go} | 22 ++++--- internal/inspect/db_stats/db_stats.sql | 14 +++++ .../db_stats_test.go} | 15 +++-- internal/inspect/index_stats/index_stats.go | 4 +- internal/inspect/report_test.go | 38 ++----------- internal/inspect/table_stats/table_stats.go | 8 ++- .../inspect/table_stats/table_stats_test.go | 2 + .../total_index_size/total_index_size.sql | 6 -- .../inspect/vacuum_stats/vacuum_stats_test.go | 3 +- 13 files changed, 72 insertions(+), 197 deletions(-) delete mode 100644 internal/inspect/cache/cache.go delete mode 100644 internal/inspect/cache/cache.sql delete mode 100644 internal/inspect/cache/cache_test.go rename internal/inspect/{total_index_size/total_index_size.go => db_stats/db_stats.go} (51%) create mode 100644 internal/inspect/db_stats/db_stats.sql rename internal/inspect/{total_index_size/total_index_size_test.go => db_stats/db_stats_test.go} (64%) delete mode 100644 internal/inspect/total_index_size/total_index_size.sql diff --git a/cmd/inspect.go b/cmd/inspect.go index 7e615be34..88c4cba9b 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -10,21 +10,20 @@ import ( "github.com/spf13/cobra" "github.com/supabase/cli/internal/inspect/bloat" "github.com/supabase/cli/internal/inspect/blocking" - "github.com/supabase/cli/internal/inspect/cache" + "github.com/supabase/cli/internal/inspect/role_stats" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" "github.com/supabase/cli/internal/inspect" "github.com/supabase/cli/internal/inspect/calls" + "github.com/supabase/cli/internal/inspect/db_stats" "github.com/supabase/cli/internal/inspect/index_stats" "github.com/supabase/cli/internal/inspect/locks" "github.com/supabase/cli/internal/inspect/long_running_queries" "github.com/supabase/cli/internal/inspect/outliers" "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/table_stats" - "github.com/supabase/cli/internal/inspect/total_index_size" "github.com/supabase/cli/internal/inspect/vacuum_stats" ) @@ -46,11 +45,11 @@ var ( }, } - inspectCacheHitCmd = &cobra.Command{ - Use: "cache-hit", + inspectDBStatsCmd = &cobra.Command{ + Use: "db-stats", Short: "Show cache hit rates for tables and indices", RunE: func(cmd *cobra.Command, args []string) error { - return cache.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -62,14 +61,6 @@ var ( }, } - inspectIndexStatsCmd = &cobra.Command{ - Use: "index-stats", - Short: "Show combined index size, usage percent, scan counts, and unused status", - RunE: func(cmd *cobra.Command, args []string) error { - return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - inspectLocksCmd = &cobra.Command{ Use: "locks", Short: "Show queries which have taken out an exclusive lock on a relation", @@ -102,11 +93,11 @@ var ( }, } - inspectTotalIndexSizeCmd = &cobra.Command{ - Use: "total-index-size", - Short: "Show total size of all indexes", + inspectIndexStatsCmd = &cobra.Command{ + Use: "index-stats", + Short: "Show combined index size, usage percent, scan counts, and unused status", RunE: func(cmd *cobra.Command, args []string) error { - return total_index_size.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -126,6 +117,14 @@ var ( }, } + inspectRoleStatsCmd = &cobra.Command{ + Use: "role-stats", + Short: "Show information about roles on the database", + RunE: func(cmd *cobra.Command, args []string) error { + return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + inspectVacuumStatsCmd = &cobra.Command{ Use: "vacuum-stats", Short: "Show statistics related to vacuum operations per table", @@ -169,18 +168,18 @@ func init() { inspectFlags.Bool("linked", true, "Inspect the linked project.") inspectFlags.Bool("local", false, "Inspect the local database.") inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local") - inspectDBCmd.AddCommand(inspectCacheHitCmd) inspectDBCmd.AddCommand(inspectReplicationSlotsCmd) inspectDBCmd.AddCommand(inspectIndexStatsCmd) inspectDBCmd.AddCommand(inspectLocksCmd) inspectDBCmd.AddCommand(inspectBlockingCmd) inspectDBCmd.AddCommand(inspectOutliersCmd) inspectDBCmd.AddCommand(inspectCallsCmd) - inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd) inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd) inspectDBCmd.AddCommand(inspectBloatCmd) inspectDBCmd.AddCommand(inspectVacuumStatsCmd) inspectDBCmd.AddCommand(inspectTableStatsCmd) + inspectDBCmd.AddCommand(inspectRoleStatsCmd) + inspectDBCmd.AddCommand(inspectDBStatsCmd) inspectCmd.AddCommand(inspectDBCmd) reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in") inspectCmd.AddCommand(reportCmd) diff --git a/internal/inspect/cache/cache.go b/internal/inspect/cache/cache.go deleted file mode 100644 index 2b7ad5f25..000000000 --- a/internal/inspect/cache/cache.go +++ /dev/null @@ -1,57 +0,0 @@ -package cache - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed cache.sql -var CacheQuery string - -type Result struct { - Name string - Ratio float64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - // Ref: https://github.com/heroku/heroku-pg-extras/blob/main/commands/cache_hit.js#L7 - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, CacheQuery) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - // TODO: implement a markdown table marshaller - table := "|Name|Ratio|OK?|Explanation|\n|-|-|-|-|\n" - for _, r := range result { - ok := "Yup!" - if r.Ratio < 0.94 { - ok = "Maybe not..." - } - var explanation string - switch r.Name { - case "index hit rate": - explanation = "This is the ratio of index hits to index scans. If this ratio is low, it means that the database is not using indexes effectively. Check the `index-usage` command for more info." - case "table hit rate": - explanation = "This is the ratio of table hits to table scans. If this ratio is low, it means that your queries are not finding the data effectively. Check your query performance and it might be worth increasing your compute." - } - table += fmt.Sprintf("|`%s`|`%.6f`|`%s`|`%s`|\n", r.Name, r.Ratio, ok, explanation) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/cache/cache.sql b/internal/inspect/cache/cache.sql deleted file mode 100644 index fc14b288c..000000000 --- a/internal/inspect/cache/cache.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - 'index hit rate' AS name, - (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read),0) AS ratio -FROM pg_statio_user_indexes -UNION ALL -SELECT - 'table hit rate' AS name, - sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read),0) AS ratio -FROM pg_statio_user_tables diff --git a/internal/inspect/cache/cache_test.go b/internal/inspect/cache/cache_test.go deleted file mode 100644 index 28fa1cb73..000000000 --- a/internal/inspect/cache/cache_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package cache - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestCacheCommand(t *testing.T) { - t.Run("inspects cache hit rate", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(CacheQuery). - Reply("SELECT 1", Result{ - Name: "index hit rate", - Ratio: 0.9, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) - - t.Run("throws error on empty result", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(CacheQuery). - Reply("SELECT 1", []interface{}{}) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.ErrorContains(t, err, "cannot find field Name in returned row") - }) -} diff --git a/internal/inspect/total_index_size/total_index_size.go b/internal/inspect/db_stats/db_stats.go similarity index 51% rename from internal/inspect/total_index_size/total_index_size.go rename to internal/inspect/db_stats/db_stats.go index fbc66b259..0a4e2043b 100644 --- a/internal/inspect/total_index_size/total_index_size.go +++ b/internal/inspect/db_stats/db_stats.go @@ -1,4 +1,4 @@ -package total_index_size +package db_stats import ( "context" @@ -9,17 +9,23 @@ import ( "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" "github.com/supabase/cli/internal/migration/list" "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed total_index_size.sql -var TotalIndexSizeQuery string +//go:embed db_stats.sql +var DBStatsQuery string type Result struct { - Size string + Database_size string + Total_index_size string + Total_table_size string + Total_toast_size string + Time_since_stats_reset string + Index_hit_rate string + Table_hit_rate string + WAL_size string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -28,7 +34,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TotalIndexSizeQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) + rows, err := conn.Query(ctx, DBStatsQuery) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -37,9 +43,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Size|\n|-|\n" + table := "|Database Size|Total Index Size|Total Table Size|Total Toast Size|Time Since Stats Reset|Index Hit Rate|Table Hit Rate|WAL Size|\n|-|-|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|\n", r.Size) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Database_size, r.Total_index_size, r.Total_table_size, r.Total_toast_size, r.Time_since_stats_reset, r.Index_hit_rate, r.Table_hit_rate, r.WAL_size) } return list.RenderTable(table) } diff --git a/internal/inspect/db_stats/db_stats.sql b/internal/inspect/db_stats/db_stats.sql new file mode 100644 index 000000000..dd0950aab --- /dev/null +++ b/internal/inspect/db_stats/db_stats.sql @@ -0,0 +1,14 @@ +SELECT + pg_size_pretty(pg_database_size('postgres')) AS database_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'i') AS total_index_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'r') AS total_table_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 't') AS total_toast_size, + (SELECT (now() - stats_reset)::text FROM pg_stat_statements_info) AS time_since_stats_reset, + (SELECT round(SUM(idx_blks_hit)::numeric / nullif(SUM(idx_blks_hit + idx_blks_read),0) * 100,2)::text || '%' + FROM pg_statio_user_indexes) AS index_hit_rate, + (SELECT round(SUM(heap_blks_hit)::numeric / nullif(SUM(heap_blks_hit + heap_blks_read),0) * 100,2)::text || '%' + FROM pg_statio_user_tables) AS table_hit_rate, + (SELECT pg_size_pretty(SUM(size)) FROM pg_ls_waldir()) AS wal_size \ No newline at end of file diff --git a/internal/inspect/total_index_size/total_index_size_test.go b/internal/inspect/db_stats/db_stats_test.go similarity index 64% rename from internal/inspect/total_index_size/total_index_size_test.go rename to internal/inspect/db_stats/db_stats_test.go index 8eb0e0aa9..dbeadf983 100644 --- a/internal/inspect/total_index_size/total_index_size_test.go +++ b/internal/inspect/db_stats/db_stats_test.go @@ -1,4 +1,4 @@ -package total_index_size +package db_stats import ( "context" @@ -20,16 +20,23 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestTotalIndexSizeCommand(t *testing.T) { +func TestDBStatsCommand(t *testing.T) { t.Run("inspects size of all indexes", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(TotalIndexSizeQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + conn.Query(DBStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Size: "8GB", + Database_size: "8GB", + Total_index_size: "8GB", + Total_table_size: "8GB", + Total_toast_size: "8GB", + Time_since_stats_reset: "8GB", + Index_hit_rate: "8GB", + Table_hit_rate: "8GB", + WAL_size: "8GB", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/index_stats/index_stats.go b/internal/inspect/index_stats/index_stats.go index 89f0bccc8..8f781c688 100644 --- a/internal/inspect/index_stats/index_stats.go +++ b/internal/inspect/index_stats/index_stats.go @@ -33,7 +33,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, IndexStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) + rows, err := conn.Query(ctx, IndexStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -42,7 +42,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Index|Size|Percent used|Index scans|Seq scans|Unused|\n|-|-|-|-|-|-|\n" + table := "|Name|Size|Percent used|Index scans|Seq scans|Unused|\n|-|-|-|-|-|-|\n" for _, r := range result { table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|`%d`|`%t`|\n", r.Name, r.Size, r.Percent_used, r.Index_scans, r.Seq_scans, r.Unused) } diff --git a/internal/inspect/report_test.go b/internal/inspect/report_test.go index 6b4220451..cd4dca610 100644 --- a/internal/inspect/report_test.go +++ b/internal/inspect/report_test.go @@ -10,23 +10,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/supabase/cli/internal/inspect/bloat" "github.com/supabase/cli/internal/inspect/blocking" - "github.com/supabase/cli/internal/inspect/cache" "github.com/supabase/cli/internal/inspect/calls" - "github.com/supabase/cli/internal/inspect/index_sizes" - "github.com/supabase/cli/internal/inspect/index_usage" + "github.com/supabase/cli/internal/inspect/db_stats" + "github.com/supabase/cli/internal/inspect/index_stats" "github.com/supabase/cli/internal/inspect/locks" "github.com/supabase/cli/internal/inspect/long_running_queries" "github.com/supabase/cli/internal/inspect/outliers" "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/role_configs" - "github.com/supabase/cli/internal/inspect/role_connections" - "github.com/supabase/cli/internal/inspect/seq_scans" - "github.com/supabase/cli/internal/inspect/table_index_sizes" - "github.com/supabase/cli/internal/inspect/table_record_counts" - "github.com/supabase/cli/internal/inspect/table_sizes" - "github.com/supabase/cli/internal/inspect/total_index_size" - "github.com/supabase/cli/internal/inspect/total_table_sizes" - "github.com/supabase/cli/internal/inspect/unused_indexes" "github.com/supabase/cli/internal/inspect/vacuum_stats" "github.com/supabase/cli/pkg/pgtest" ) @@ -50,14 +40,8 @@ func TestReportCommand(t *testing.T) { Reply("COPY 0"). Query(wrapQuery(blocking.BlockingQuery)). Reply("COPY 0"). - Query(wrapQuery(cache.CacheQuery)). - Reply("COPY 0"). Query(wrapQuery(calls.CallsQuery)). Reply("COPY 0"). - Query(wrapQuery(index_sizes.IndexSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(index_usage.IndexUsageQuery)). - Reply("COPY 0"). Query(wrapQuery(locks.LocksQuery)). Reply("COPY 0"). Query(wrapQuery(long_running_queries.LongRunningQueriesQuery)). @@ -66,23 +50,9 @@ func TestReportCommand(t *testing.T) { Reply("COPY 0"). Query(wrapQuery(replication_slots.ReplicationSlotsQuery)). Reply("COPY 0"). - Query(wrapQuery(role_configs.RoleConfigsQuery)). - Reply("COPY 0"). - Query(wrapQuery(role_connections.RoleConnectionsQuery)). - Reply("COPY 0"). - Query(wrapQuery(seq_scans.SeqScansQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_index_sizes.TableIndexSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_record_counts.TableRecordCountsQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_sizes.TableSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(total_index_size.TotalIndexSizeQuery)). - Reply("COPY 0"). - Query(wrapQuery(total_table_sizes.TotalTableSizesQuery)). + Query(wrapQuery(db_stats.DBStatsQuery)). Reply("COPY 0"). - Query(wrapQuery(unused_indexes.UnusedIndexesQuery)). + Query(wrapQuery(index_stats.IndexStatsQuery)). Reply("COPY 0"). Query(wrapQuery(vacuum_stats.VacuumStatsQuery)). Reply("COPY 0") diff --git a/internal/inspect/table_stats/table_stats.go b/internal/inspect/table_stats/table_stats.go index e66a5dce9..afc3b193b 100644 --- a/internal/inspect/table_stats/table_stats.go +++ b/internal/inspect/table_stats/table_stats.go @@ -22,7 +22,9 @@ type Result struct { Name string Table_size string Index_size string + Total_size string Estimated_row_count int64 + Seq_scans int64 } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -31,7 +33,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) + rows, err := conn.Query(ctx, TableStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -40,9 +42,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Table|Size|Index size|Estimated row count|\n|-|-|-|-|\n" + table := "|Name|Table size|Index size|Total size|Estimated row count|Seq scans|\n|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Name, r.Table_size, r.Index_size, r.Estimated_row_count) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%d`|`%d`|\n", r.Name, r.Table_size, r.Index_size, r.Total_size, r.Estimated_row_count, r.Seq_scans) } return list.RenderTable(table) } diff --git a/internal/inspect/table_stats/table_stats_test.go b/internal/inspect/table_stats/table_stats_test.go index ddd4e4f29..12f50bd72 100644 --- a/internal/inspect/table_stats/table_stats_test.go +++ b/internal/inspect/table_stats/table_stats_test.go @@ -32,7 +32,9 @@ func TestTableStatsCommand(t *testing.T) { Name: "public.test_table", Table_size: "3GB", Index_size: "1GB", + Total_size: "4GB", Estimated_row_count: 100, + Seq_scans: 1, }) err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/total_index_size/total_index_size.sql b/internal/inspect/total_index_size/total_index_size.sql deleted file mode 100644 index d1e8ab3d8..000000000 --- a/internal/inspect/total_index_size/total_index_size.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - pg_size_pretty(sum(c.relpages::bigint*8192)::bigint) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'i' diff --git a/internal/inspect/vacuum_stats/vacuum_stats_test.go b/internal/inspect/vacuum_stats/vacuum_stats_test.go index 49dafc84e..7872567b3 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats_test.go +++ b/internal/inspect/vacuum_stats/vacuum_stats_test.go @@ -29,8 +29,7 @@ func TestVacuumCommand(t *testing.T) { defer conn.Close(t) conn.Query(VacuumStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Schema: "test_schema", - Tbl: "test_table", + Name: "test_schema.test_table", Last_vacuum: "2021-01-01 00:00:00", Last_autovacuum: "2021-01-01 00:00:00", Rowcount: "1000", From baad4a119ded1bea3d7d9d69b15b2c8d4d047fcd Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Fri, 9 May 2025 09:10:05 +0300 Subject: [PATCH 10/15] cleaned up bloat query output --- internal/inspect/bloat/bloat.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/inspect/bloat/bloat.sql b/internal/inspect/bloat/bloat.sql index 25917d874..638139987 100644 --- a/internal/inspect/bloat/bloat.sql +++ b/internal/inspect/bloat/bloat.sql @@ -38,12 +38,12 @@ WITH constants AS ( JOIN pg_class c2 ON c2.oid = i.indexrelid ) SELECT - type, schemaname, object_name, bloat, pg_size_pretty(raw_waste) as waste + type, name, bloat, pg_size_pretty(raw_waste) as waste FROM (SELECT 'table' as type, schemaname, - tablename as object_name, + schemaname || '.' || tablename as name, ROUND(CASE WHEN otta=0 THEN 0.0 ELSE table_bloat.relpages/otta::numeric END,1) AS bloat, CASE WHEN relpages < otta THEN '0' ELSE (bs*(table_bloat.relpages-otta)::bigint)::bigint END AS raw_waste FROM @@ -52,7 +52,7 @@ FROM SELECT 'index' as type, schemaname, - tablename || '::' || iname as object_name, + schemaname || '.' || tablename || '::' || iname as name, ROUND(CASE WHEN iotta=0 OR ipages=0 THEN 0.0 ELSE ipages/iotta::numeric END,1) AS bloat, CASE WHEN ipages < iotta THEN '0' ELSE (bs*(ipages-iotta))::bigint END AS raw_waste FROM From 0ba7cb04d334f0a4f745c56699687dc1d1db33b0 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Mon, 12 May 2025 12:31:29 +0300 Subject: [PATCH 11/15] update rules for new inspect commands --- cmd/inspect.go | 2 +- internal/inspect/db_stats/db_stats.sql | 4 ++-- tools/inspect_rules.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index d5d39a265..bdd196229 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -245,7 +245,7 @@ func printReportSummary(outDir string) error { status = r.Fail } } - matchStr := "" + matchStr := "-" if match.Valid { matchStr = match.String } diff --git a/internal/inspect/db_stats/db_stats.sql b/internal/inspect/db_stats/db_stats.sql index dd0950aab..d77c7583f 100644 --- a/internal/inspect/db_stats/db_stats.sql +++ b/internal/inspect/db_stats/db_stats.sql @@ -7,8 +7,8 @@ SELECT (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 't') AS total_toast_size, (SELECT (now() - stats_reset)::text FROM pg_stat_statements_info) AS time_since_stats_reset, - (SELECT round(SUM(idx_blks_hit)::numeric / nullif(SUM(idx_blks_hit + idx_blks_read),0) * 100,2)::text || '%' + (SELECT ROUND(SUM(idx_blks_hit)::numeric / nullif(SUM(idx_blks_hit + idx_blks_read),0),2) FROM pg_statio_user_indexes) AS index_hit_rate, - (SELECT round(SUM(heap_blks_hit)::numeric / nullif(SUM(heap_blks_hit + heap_blks_read),0) * 100,2)::text || '%' + (SELECT ROUND(SUM(heap_blks_hit)::numeric / nullif(SUM(heap_blks_hit + heap_blks_read),0),2) FROM pg_statio_user_tables) AS table_hit_rate, (SELECT pg_size_pretty(SUM(size)) FROM pg_ls_waldir()) AS wal_size \ No newline at end of file diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index 3192cebd9..a9b7b6b22 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -22,7 +22,7 @@ name = "No unused indexes" [[rule]] name = "Check cache hit is within acceptable bounds" -query = "SELECT LISTAGG(name, ',') AS match FROM `cache.csv` WHERE ratio < 0.94" +query = "SELECT LISTAGG(name, ',') AS match FROM `db_stats.csv` WHERE index_hit_rate < 0.94 OR table_hit_rate < 0.94" pass = "✔" fail = "There is at least one table with a cache hit ratio below 94%" @@ -40,7 +40,7 @@ pass = "✔" fail = "At least one table is waiting on autovacuum" [[rule]] -query = "SELECT LISTAGG(s.tbl, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum = '' OR s.last_vacuum = '');" +query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum = '' OR s.last_vacuum = '');" name = "No tables yet to be vacuumed" pass = "✔" fail = "At least one table has never had autovacuum or vacuum run on it" From 4c8824433ac9769ea669b7097f7e7d6e73a43ca2 Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Mon, 12 May 2025 16:20:44 +0300 Subject: [PATCH 12/15] update tests for all queries in inspect dir --- internal/inspect/report_test.go | 59 ++++++++++++--------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/internal/inspect/report_test.go b/internal/inspect/report_test.go index cd4dca610..43968dbcb 100644 --- a/internal/inspect/report_test.go +++ b/internal/inspect/report_test.go @@ -3,21 +3,12 @@ package inspect import ( "context" "fmt" + "io/fs" "testing" "github.com/jackc/pgconn" "github.com/spf13/afero" "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/inspect/bloat" - "github.com/supabase/cli/internal/inspect/blocking" - "github.com/supabase/cli/internal/inspect/calls" - "github.com/supabase/cli/internal/inspect/db_stats" - "github.com/supabase/cli/internal/inspect/index_stats" - "github.com/supabase/cli/internal/inspect/locks" - "github.com/supabase/cli/internal/inspect/long_running_queries" - "github.com/supabase/cli/internal/inspect/outliers" - "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/vacuum_stats" "github.com/supabase/cli/pkg/pgtest" ) @@ -31,38 +22,32 @@ var dbConfig = pgconn.Config{ func TestReportCommand(t *testing.T) { t.Run("runs all queries", func(t *testing.T) { - // Setup in-memory fs fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(wrapQuery(bloat.BloatQuery)). - Reply("COPY 0"). - Query(wrapQuery(blocking.BlockingQuery)). - Reply("COPY 0"). - Query(wrapQuery(calls.CallsQuery)). - Reply("COPY 0"). - Query(wrapQuery(locks.LocksQuery)). - Reply("COPY 0"). - Query(wrapQuery(long_running_queries.LongRunningQueriesQuery)). - Reply("COPY 0"). - Query(wrapQuery(outliers.OutliersQuery)). - Reply("COPY 0"). - Query(wrapQuery(replication_slots.ReplicationSlotsQuery)). - Reply("COPY 0"). - Query(wrapQuery(db_stats.DBStatsQuery)). - Reply("COPY 0"). - Query(wrapQuery(index_stats.IndexStatsQuery)). - Reply("COPY 0"). - Query(wrapQuery(vacuum_stats.VacuumStatsQuery)). - Reply("COPY 0") - // Run test - err := Report(context.Background(), ".", dbConfig, fsys, conn.Intercept) - // Check error + // Iterate over all embedded SQL files + var sqlCount int + err := fs.WalkDir(queries, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + data, err := queries.ReadFile(path) + if err != nil { + return err + } + conn.Query(wrapQuery(string(data))).Reply("COPY 0") + sqlCount++ + return nil + }) assert.NoError(t, err) - matches, err := afero.Glob(fsys, "*.csv") + err = Report(context.Background(), ".", dbConfig, fsys, conn.Intercept) assert.NoError(t, err) - assert.Len(t, matches, 20) + matches, err := afero.Glob(fsys, "*/*.csv") + assert.NoError(t, err) + assert.Len(t, matches, sqlCount) }) } From e266458e679bea044f4548ad3f513a59d676b48a Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Mon, 26 May 2025 11:46:37 +0300 Subject: [PATCH 13/15] updated tests to ignore internal schemas and removed unused sql queries --- cmd/inspect.go | 4 ++-- internal/inspect/db_stats/db_stats_test.go | 4 +--- internal/inspect/index_stats/index_stats_test.go | 2 +- internal/inspect/seq_scans/seq_scans.sql | 6 ------ internal/inspect/table_index_sizes/table_index_sizes.sql | 8 -------- .../inspect/table_record_counts/table_record_counts.sql | 6 ------ internal/inspect/table_sizes/table_sizes.sql | 8 -------- internal/inspect/table_stats/table_stats_test.go | 2 +- internal/inspect/total_table_sizes/total_table_sizes.go | 2 +- .../inspect/total_table_sizes/total_table_sizes_test.go | 2 +- 10 files changed, 7 insertions(+), 37 deletions(-) delete mode 100644 internal/inspect/seq_scans/seq_scans.sql delete mode 100644 internal/inspect/table_index_sizes/table_index_sizes.sql delete mode 100644 internal/inspect/table_record_counts/table_record_counts.sql delete mode 100644 internal/inspect/table_sizes/table_sizes.sql diff --git a/cmd/inspect.go b/cmd/inspect.go index bdd196229..e5b786124 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -224,11 +224,11 @@ func printReportSummary(outDir string) error { table := "RULE|STATUS|MATCHES\n|-|-|-|\n" // find matching rule - + var status string for i := range cfg.Rules { r := &cfg.Rules[i] name := r.Name - status := "--" + row := db.QueryRow(r.Query) var match sql.NullString diff --git a/internal/inspect/db_stats/db_stats_test.go b/internal/inspect/db_stats/db_stats_test.go index dbeadf983..523b3f5ed 100644 --- a/internal/inspect/db_stats/db_stats_test.go +++ b/internal/inspect/db_stats/db_stats_test.go @@ -7,8 +7,6 @@ import ( "github.com/jackc/pgconn" "github.com/spf13/afero" "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/pgtest" ) @@ -27,7 +25,7 @@ func TestDBStatsCommand(t *testing.T) { // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(DBStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + conn.Query(DBStatsQuery). Reply("SELECT 1", Result{ Database_size: "8GB", Total_index_size: "8GB", diff --git a/internal/inspect/index_stats/index_stats_test.go b/internal/inspect/index_stats/index_stats_test.go index 657242738..abc3cd654 100644 --- a/internal/inspect/index_stats/index_stats_test.go +++ b/internal/inspect/index_stats/index_stats_test.go @@ -27,7 +27,7 @@ func TestIndexStatsCommand(t *testing.T) { defer conn.Close(t) // Mock index stats - conn.Query(IndexStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). + conn.Query(IndexStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ Name: "public.test_idx", Size: "1GB", diff --git a/internal/inspect/seq_scans/seq_scans.sql b/internal/inspect/seq_scans/seq_scans.sql deleted file mode 100644 index 975604845..000000000 --- a/internal/inspect/seq_scans/seq_scans.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - seq_scan as seq_scans -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY seq_scan DESC diff --git a/internal/inspect/table_index_sizes/table_index_sizes.sql b/internal/inspect/table_index_sizes/table_index_sizes.sql deleted file mode 100644 index 8899b67b3..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS name, - pg_size_pretty(pg_indexes_size(c.oid)) AS index_size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_indexes_size(c.oid) DESC diff --git a/internal/inspect/table_record_counts/table_record_counts.sql b/internal/inspect/table_record_counts/table_record_counts.sql deleted file mode 100644 index 86b75b0e5..000000000 --- a/internal/inspect/table_record_counts/table_record_counts.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - n_live_tup AS estimated_count -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY n_live_tup DESC diff --git a/internal/inspect/table_sizes/table_sizes.sql b/internal/inspect/table_sizes/table_sizes.sql deleted file mode 100644 index 30bcc1414..000000000 --- a/internal/inspect/table_sizes/table_sizes.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS name, - pg_size_pretty(pg_table_size(c.oid)) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_table_size(c.oid) DESC diff --git a/internal/inspect/table_stats/table_stats_test.go b/internal/inspect/table_stats/table_stats_test.go index 12f50bd72..087641029 100644 --- a/internal/inspect/table_stats/table_stats_test.go +++ b/internal/inspect/table_stats/table_stats_test.go @@ -27,7 +27,7 @@ func TestTableStatsCommand(t *testing.T) { defer conn.Close(t) // Mock table sizes and index sizes - conn.Query(TableStatsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). + conn.Query(TableStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ Name: "public.test_table", Table_size: "3GB", diff --git a/internal/inspect/total_table_sizes/total_table_sizes.go b/internal/inspect/total_table_sizes/total_table_sizes.go index 7c150df61..7c7d40cf3 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.go +++ b/internal/inspect/total_table_sizes/total_table_sizes.go @@ -29,7 +29,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)) + rows, err := conn.Query(ctx, TotalTableSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } diff --git a/internal/inspect/total_table_sizes/total_table_sizes_test.go b/internal/inspect/total_table_sizes/total_table_sizes_test.go index b62b0c66e..ae16989bd 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes_test.go +++ b/internal/inspect/total_table_sizes/total_table_sizes_test.go @@ -27,7 +27,7 @@ func TestTotalTableSizesCommand(t *testing.T) { // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). + conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ Name: "public.test_table", Size: "3GB", From 9f4a035d1e4430aff9d73cf7b99dc8eeecce8ebc Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Thu, 29 May 2025 15:36:36 +0300 Subject: [PATCH 14/15] comments addressed: fix in rules for field names and formatted. errors thrown when report directory cannot be made --- internal/inspect/db_stats/db_stats.go | 5 ++-- internal/inspect/db_stats/db_stats.sql | 1 + internal/inspect/report.go | 6 ++-- .../total_table_sizes/total_table_sizes.sql | 2 +- .../inspect/unused_indexes/unused_indexes.sql | 2 +- .../inspect/vacuum_stats/vacuum_stats.sql | 2 +- tools/inspect_rules.toml | 29 +++++++++---------- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/internal/inspect/db_stats/db_stats.go b/internal/inspect/db_stats/db_stats.go index 0a4e2043b..bb9882733 100644 --- a/internal/inspect/db_stats/db_stats.go +++ b/internal/inspect/db_stats/db_stats.go @@ -18,6 +18,7 @@ import ( var DBStatsQuery string type Result struct { + Name string Database_size string Total_index_size string Total_table_size string @@ -43,9 +44,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Database Size|Total Index Size|Total Table Size|Total Toast Size|Time Since Stats Reset|Index Hit Rate|Table Hit Rate|WAL Size|\n|-|-|-|-|-|-|-|-|\n" + table := "|Name|Database Size|Total Index Size|Total Table Size|Total Toast Size|Time Since Stats Reset|Index Hit Rate|Table Hit Rate|WAL Size|\n|-|-|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Database_size, r.Total_index_size, r.Total_table_size, r.Total_toast_size, r.Time_since_stats_reset, r.Index_hit_rate, r.Table_hit_rate, r.WAL_size) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Name, r.Database_size, r.Total_index_size, r.Total_table_size, r.Total_toast_size, r.Time_since_stats_reset, r.Index_hit_rate, r.Table_hit_rate, r.WAL_size) } return list.RenderTable(table) } diff --git a/internal/inspect/db_stats/db_stats.sql b/internal/inspect/db_stats/db_stats.sql index d77c7583f..f9df0a477 100644 --- a/internal/inspect/db_stats/db_stats.sql +++ b/internal/inspect/db_stats/db_stats.sql @@ -1,4 +1,5 @@ SELECT + 'postgres' AS name, pg_size_pretty(pg_database_size('postgres')) AS database_size, (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'i') AS total_index_size, diff --git a/internal/inspect/report.go b/internal/inspect/report.go index b09302175..ca61409aa 100644 --- a/internal/inspect/report.go +++ b/internal/inspect/report.go @@ -26,7 +26,6 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs if err := utils.MkdirIfNotExistFS(fsys, out); err != nil { return err } - // create a date-based subdirectory dateDir := filepath.Join(out, date) if err := utils.MkdirIfNotExistFS(fsys, dateDir); err != nil { return err @@ -56,7 +55,10 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs } // print the actual save location if !filepath.IsAbs(dateDir) { - dateDir, _ = filepath.Abs(dateDir) + dateDir, err = filepath.Abs(dateDir) + if err != nil { + return err + } } fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(dateDir)) return nil diff --git a/internal/inspect/total_table_sizes/total_table_sizes.sql b/internal/inspect/total_table_sizes/total_table_sizes.sql index 3e8df07f8..b3e7df657 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.sql +++ b/internal/inspect/total_table_sizes/total_table_sizes.sql @@ -1,5 +1,5 @@ SELECT - n.nspname || '.' || c.relname AS name, + FORMAT('%I.%I', n.nspname, c.relname) AS name, pg_size_pretty(pg_total_relation_size(c.oid)) AS size FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) diff --git a/internal/inspect/unused_indexes/unused_indexes.sql b/internal/inspect/unused_indexes/unused_indexes.sql index c47a1f666..aa429b9ed 100644 --- a/internal/inspect/unused_indexes/unused_indexes.sql +++ b/internal/inspect/unused_indexes/unused_indexes.sql @@ -1,5 +1,5 @@ SELECT - schemaname || '.' || relname AS name, + FORMAT('%I.%I', schemaname, relname) AS name, indexrelname AS index, pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, idx_scan as index_scans diff --git a/internal/inspect/vacuum_stats/vacuum_stats.sql b/internal/inspect/vacuum_stats/vacuum_stats.sql index 12161b762..534fa4e64 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.sql +++ b/internal/inspect/vacuum_stats/vacuum_stats.sql @@ -20,7 +20,7 @@ WITH table_opts AS ( table_opts ) SELECT - vacuum_settings.nspname || '.' || vacuum_settings.relname AS name, + FORMAT('%I.%I', vacuum_settings.nspname, vacuum_settings.relname) AS name, coalesce(to_char(psut.last_vacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_vacuum, coalesce(to_char(psut.last_autovacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_autovacuum, to_char(pg_class.reltuples, '9G999G999G999') AS rowcount, diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml index a9b7b6b22..5f47c6e25 100644 --- a/tools/inspect_rules.toml +++ b/tools/inspect_rules.toml @@ -2,37 +2,34 @@ [[rule]] query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE age > '00:02:00'" -name = "No old locks" -pass = "✔" -fail = "There is at least one lock older than 2 minutes" +name = "No old locks" +pass = "✔" +fail = "There is at least one lock older than 2 minutes" [[rule]] query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE granted = 'f'" -name = "No ungranted locks" -pass = "✔" -fail = "There is at least one ungranted lock" - +name = "No ungranted locks" +pass = "✔" +fail = "There is at least one ungranted lock" [[rule]] query = "SELECT LISTAGG(index, ',') AS match FROM `unused_indexes.csv`" -pass = "✔" -fail = "There is at least one unused index" -name = "No unused indexes" - +pass = "✔" +fail = "There is at least one unused index" +name = "No unused indexes" [[rule]] -name = "Check cache hit is within acceptable bounds" +name = "Check cache hit is within acceptable bounds" query = "SELECT LISTAGG(name, ',') AS match FROM `db_stats.csv` WHERE index_hit_rate < 0.94 OR table_hit_rate < 0.94" -pass = "✔" -fail = "There is at least one table with a cache hit ratio below 94%" +pass = "✔" +fail = "There is a cache hit ratio (table or index) below 94%" [[rule]] -query = "SELECT LISTAGG(s.name, ',') AS match FROM `seq_scans.csv` s JOIN `table_record_counts.csv` t ON s.name = t.name WHERE t.estimated_count > 1000 AND s.seq_scans > t.estimated_count * 0.1;" +query = "SELECT LISTAGG(t.name, ',') AS match FROM `table_stats.csv` t WHERE t.seq_scans > t.estimated_row_count * 0.1 AND t.estimated_row_count > 1000;" name = "No large tables with sequential scans more than 10% of rows" pass = "✔" fail = "At least one table is showing sequential scans more than 10% of total row count" - [[rule]] query = "SELECT LISTAGG(s.tbl, ',') AS match FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" name = "No large tables waiting on autovacuum" From 42ac8ffac71bbd987f7a53bcfe22e45a0fe900fe Mon Sep 17 00:00:00 2001 From: Chris Gwilliams <517923+encima@users.noreply.github.com> Date: Mon, 2 Jun 2025 09:25:39 +0300 Subject: [PATCH 15/15] address review comments, re-add deprecated commands and add notice --- cmd/inspect.go | 76 +++++++++++++++++++++- internal/inspect/role_stats/role_stats.sql | 2 +- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/cmd/inspect.go b/cmd/inspect.go index e5b786124..0b0b8b988 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -52,7 +52,7 @@ var ( inspectDBStatsCmd = &cobra.Command{ Use: "db-stats", - Short: "Show cache hit rates for tables and indices", + Short: "Show stats such as cache hit rates, total sizes, and WAL size", RunE: func(cmd *cobra.Command, args []string) error { return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, @@ -146,6 +146,72 @@ var ( }, } + // DEPRECATED + + inspectCacheHitCmd = &cobra.Command{ + Use: "cache-hit", + Short: "DEPRECATED: use db-stats instead. Show cache hit rates for tables and indices", + RunE: func(cmd *cobra.Command, args []string) error { + return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectIndexUsageCmd = &cobra.Command{ + Use: "index-usage", + Short: "DEPRECATED: use index-stats instead. Show information about the efficiency of indexes", + RunE: func(cmd *cobra.Command, args []string) error { + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectTotalIndexSizeCmd = &cobra.Command{ + Use: "total-index-size", + Short: "DEPRECATED: use index-stats instead. Show total size of all indexes", + RunE: func(cmd *cobra.Command, args []string) error { + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectTableSizesCmd = &cobra.Command{ + Use: "table-sizes", + Short: "DEPRECATED: use table-stats instead. Show table sizes of individual tables without their index sizes", + RunE: func(cmd *cobra.Command, args []string) error { + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectTableIndexSizesCmd = &cobra.Command{ + Use: "table-index-sizes", + Short: "DEPRECATED: use table-stats instead. Show index sizes of individual tables", + RunE: func(cmd *cobra.Command, args []string) error { + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectTotalTableSizesCmd = &cobra.Command{ + Use: "total-table-sizes", + Short: "DEPRECATED: use table-stats instead. Show total table sizes, including table index sizes", + RunE: func(cmd *cobra.Command, args []string) error { + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectUnusedIndexesCmd = &cobra.Command{ + Use: "unused-indexes", + Short: "DEPRECATED: use index-stats instead. Show indexes with low usage", + RunE: func(cmd *cobra.Command, args []string) error { + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectSeqScansCmd = &cobra.Command{ + Use: "seq-scans", + Short: "DEPRECATED: use index-stats instead. Show number of sequential scans recorded against all tables", + RunE: func(cmd *cobra.Command, args []string) error { + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + outputDir string reportCmd = &cobra.Command{ @@ -194,6 +260,14 @@ func init() { inspectDBCmd.AddCommand(inspectTableStatsCmd) inspectDBCmd.AddCommand(inspectRoleStatsCmd) inspectDBCmd.AddCommand(inspectDBStatsCmd) + inspectDBCmd.AddCommand(inspectCacheHitCmd) + inspectDBCmd.AddCommand(inspectIndexUsageCmd) + inspectDBCmd.AddCommand(inspectSeqScansCmd) + inspectDBCmd.AddCommand(inspectUnusedIndexesCmd) + inspectDBCmd.AddCommand(inspectTotalTableSizesCmd) + inspectDBCmd.AddCommand(inspectTableIndexSizesCmd) + inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd) + inspectDBCmd.AddCommand(inspectTableSizesCmd) inspectCmd.AddCommand(inspectDBCmd) reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in") inspectCmd.AddCommand(reportCmd) diff --git a/internal/inspect/role_stats/role_stats.sql b/internal/inspect/role_stats/role_stats.sql index 09972c2d0..92f32c9e5 100644 --- a/internal/inspect/role_stats/role_stats.sql +++ b/internal/inspect/role_stats/role_stats.sql @@ -15,4 +15,4 @@ SELECT array_to_string(rolconfig, ',', '*') as custom_config FROM pg_roles -ORDER BY 3 DESC +ORDER BY 1 DESC