diff --git a/cmd/inspect.go b/cmd/inspect.go index 2b55c3876..0b0b8b988 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -1,37 +1,36 @@ package cmd import ( + "database/sql" "fmt" "os" "os/signal" "path/filepath" + "time" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/supabase/cli/internal/inspect/bloat" "github.com/supabase/cli/internal/inspect/blocking" - "github.com/supabase/cli/internal/inspect/cache" + "github.com/supabase/cli/internal/inspect/role_stats" + "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/internal/utils/flags" + _ "github.com/mithrandie/csvq-driver" + "github.com/pelletier/go-toml/v2" "github.com/supabase/cli/internal/inspect" "github.com/supabase/cli/internal/inspect/calls" - "github.com/supabase/cli/internal/inspect/index_sizes" - "github.com/supabase/cli/internal/inspect/index_usage" + "github.com/supabase/cli/internal/inspect/db_stats" + "github.com/supabase/cli/internal/inspect/index_stats" "github.com/supabase/cli/internal/inspect/locks" "github.com/supabase/cli/internal/inspect/long_running_queries" "github.com/supabase/cli/internal/inspect/outliers" "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/role_configs" - "github.com/supabase/cli/internal/inspect/role_connections" - "github.com/supabase/cli/internal/inspect/seq_scans" - "github.com/supabase/cli/internal/inspect/table_index_sizes" - "github.com/supabase/cli/internal/inspect/table_record_counts" - "github.com/supabase/cli/internal/inspect/table_sizes" - "github.com/supabase/cli/internal/inspect/total_index_size" - "github.com/supabase/cli/internal/inspect/total_table_sizes" - "github.com/supabase/cli/internal/inspect/unused_indexes" + "github.com/supabase/cli/internal/inspect/table_stats" + "github.com/supabase/cli/internal/inspect/vacuum_stats" + "github.com/supabase/cli/internal/migration/list" ) var ( @@ -51,11 +50,11 @@ var ( }, } - inspectCacheHitCmd = &cobra.Command{ - Use: "cache-hit", - Short: "Show cache hit rates for tables and indices", + inspectDBStatsCmd = &cobra.Command{ + Use: "db-stats", + Short: "Show stats such as cache hit rates, total sizes, and WAL size", RunE: func(cmd *cobra.Command, args []string) error { - return cache.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -67,14 +66,6 @@ var ( }, } - inspectIndexUsageCmd = &cobra.Command{ - Use: "index-usage", - Short: "Show information about the efficiency of indexes", - RunE: func(cmd *cobra.Command, args []string) error { - return index_usage.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) - }, - } - inspectLocksCmd = &cobra.Command{ Use: "locks", Short: "Show queries which have taken out an exclusive lock on a relation", @@ -107,107 +98,117 @@ var ( }, } - inspectTotalIndexSizeCmd = &cobra.Command{ - Use: "total-index-size", - Short: "Show total size of all indexes", + inspectIndexStatsCmd = &cobra.Command{ + Use: "index-stats", + Short: "Show combined index size, usage percent, scan counts, and unused status", RunE: func(cmd *cobra.Command, args []string) error { - return total_index_size.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectIndexSizesCmd = &cobra.Command{ - Use: "index-sizes", - Short: "Show index sizes of individual indexes", + inspectLongRunningQueriesCmd = &cobra.Command{ + Use: "long-running-queries", + Short: "Show currently running queries running for longer than 5 minutes", RunE: func(cmd *cobra.Command, args []string) error { - return index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectTableSizesCmd = &cobra.Command{ - Use: "table-sizes", - Short: "Show table sizes of individual tables without their index sizes", + inspectBloatCmd = &cobra.Command{ + Use: "bloat", + Short: "Estimates space allocated to a relation that is full of dead tuples", RunE: func(cmd *cobra.Command, args []string) error { - return table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectTableIndexSizesCmd = &cobra.Command{ - Use: "table-index-sizes", - Short: "Show index sizes of individual tables", + inspectRoleStatsCmd = &cobra.Command{ + Use: "role-stats", + Short: "Show information about roles on the database", RunE: func(cmd *cobra.Command, args []string) error { - return table_index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectTotalTableSizesCmd = &cobra.Command{ - Use: "total-table-sizes", - Short: "Show total table sizes, including table index sizes", + inspectVacuumStatsCmd = &cobra.Command{ + Use: "vacuum-stats", + Short: "Show statistics related to vacuum operations per table", RunE: func(cmd *cobra.Command, args []string) error { - return total_table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectUnusedIndexesCmd = &cobra.Command{ - Use: "unused-indexes", - Short: "Show indexes with low usage", + inspectTableStatsCmd = &cobra.Command{ + Use: "table-stats", + Short: "Show combined table size, index size, and estimated row count", RunE: func(cmd *cobra.Command, args []string) error { - return unused_indexes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectSeqScansCmd = &cobra.Command{ - Use: "seq-scans", - Short: "Show number of sequential scans recorded against all tables", + // DEPRECATED + + inspectCacheHitCmd = &cobra.Command{ + Use: "cache-hit", + Short: "DEPRECATED: use db-stats instead. Show cache hit rates for tables and indices", RunE: func(cmd *cobra.Command, args []string) error { - return seq_scans.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectLongRunningQueriesCmd = &cobra.Command{ - Use: "long-running-queries", - Short: "Show currently running queries running for longer than 5 minutes", + inspectIndexUsageCmd = &cobra.Command{ + Use: "index-usage", + Short: "DEPRECATED: use index-stats instead. Show information about the efficiency of indexes", RunE: func(cmd *cobra.Command, args []string) error { - return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectTableRecordCountsCmd = &cobra.Command{ - Use: "table-record-counts", - Short: "Show estimated number of rows per table", + inspectTotalIndexSizeCmd = &cobra.Command{ + Use: "total-index-size", + Short: "DEPRECATED: use index-stats instead. Show total size of all indexes", RunE: func(cmd *cobra.Command, args []string) error { - return table_record_counts.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectBloatCmd = &cobra.Command{ - Use: "bloat", - Short: "Estimates space allocated to a relation that is full of dead tuples", + inspectTableSizesCmd = &cobra.Command{ + Use: "table-sizes", + Short: "DEPRECATED: use table-stats instead. Show table sizes of individual tables without their index sizes", RunE: func(cmd *cobra.Command, args []string) error { - return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectVacuumStatsCmd = &cobra.Command{ - Use: "vacuum-stats", - Short: "Show statistics related to vacuum operations per table", + inspectTableIndexSizesCmd = &cobra.Command{ + Use: "table-index-sizes", + Short: "DEPRECATED: use table-stats instead. Show index sizes of individual tables", RunE: func(cmd *cobra.Command, args []string) error { - return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectRoleConfigsCmd = &cobra.Command{ - Use: "role-configs", - Short: "Show configuration settings for database roles when they have been modified", + inspectTotalTableSizesCmd = &cobra.Command{ + Use: "total-table-sizes", + Short: "DEPRECATED: use table-stats instead. Show total table sizes, including table index sizes", RunE: func(cmd *cobra.Command, args []string) error { - return role_configs.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } - inspectRoleConnectionsCmd = &cobra.Command{ - Use: "role-connections", - Short: "Show number of active connections for all database roles", + inspectUnusedIndexesCmd = &cobra.Command{ + Use: "unused-indexes", + Short: "DEPRECATED: use index-stats instead. Show indexes with low usage", RunE: func(cmd *cobra.Command, args []string) error { - return role_connections.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) + }, + } + + inspectSeqScansCmd = &cobra.Command{ + Use: "seq-scans", + Short: "DEPRECATED: use index-stats instead. Show number of sequential scans recorded against all tables", + RunE: func(cmd *cobra.Command, args []string) error { + return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs()) }, } @@ -218,48 +219,111 @@ var ( Short: "Generate a CSV output for all inspect commands", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - if len(outputDir) == 0 { - defaultPath := filepath.Join(utils.CurrentDirAbs, "report") - title := fmt.Sprintf("Enter a directory to save output files (or leave blank to use %s): ", utils.Bold(defaultPath)) - if dir, err := utils.NewConsole().PromptText(ctx, title); err != nil { - return err - } else if len(dir) == 0 { - outputDir = defaultPath - } + if err := inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()); err != nil { + return err } - return inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()) + return printReportSummary(outputDir) }, } ) +// Load rules file at runtime (tools/inspect_rules.toml) + +// Rule defines a validation rule for a CSV file +type Rule struct { + Query string `toml:"query"` + Pass string `toml:"pass"` + Fail string `toml:"fail"` + Name string `toml:"name"` +} + +// Config holds all rules +type Config struct { + Rules []Rule `toml:"rule"` +} + func init() { inspectFlags := inspectCmd.PersistentFlags() inspectFlags.String("db-url", "", "Inspect the database specified by the connection string (must be percent-encoded).") inspectFlags.Bool("linked", true, "Inspect the linked project.") inspectFlags.Bool("local", false, "Inspect the local database.") inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local") - inspectDBCmd.AddCommand(inspectCacheHitCmd) inspectDBCmd.AddCommand(inspectReplicationSlotsCmd) - inspectDBCmd.AddCommand(inspectIndexUsageCmd) + inspectDBCmd.AddCommand(inspectIndexStatsCmd) inspectDBCmd.AddCommand(inspectLocksCmd) inspectDBCmd.AddCommand(inspectBlockingCmd) inspectDBCmd.AddCommand(inspectOutliersCmd) inspectDBCmd.AddCommand(inspectCallsCmd) - inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd) - inspectDBCmd.AddCommand(inspectIndexSizesCmd) - inspectDBCmd.AddCommand(inspectTableSizesCmd) - inspectDBCmd.AddCommand(inspectTableIndexSizesCmd) - inspectDBCmd.AddCommand(inspectTotalTableSizesCmd) - inspectDBCmd.AddCommand(inspectUnusedIndexesCmd) - inspectDBCmd.AddCommand(inspectSeqScansCmd) inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd) - inspectDBCmd.AddCommand(inspectTableRecordCountsCmd) inspectDBCmd.AddCommand(inspectBloatCmd) inspectDBCmd.AddCommand(inspectVacuumStatsCmd) - inspectDBCmd.AddCommand(inspectRoleConfigsCmd) - inspectDBCmd.AddCommand(inspectRoleConnectionsCmd) + inspectDBCmd.AddCommand(inspectTableStatsCmd) + inspectDBCmd.AddCommand(inspectRoleStatsCmd) + inspectDBCmd.AddCommand(inspectDBStatsCmd) + inspectDBCmd.AddCommand(inspectCacheHitCmd) + inspectDBCmd.AddCommand(inspectIndexUsageCmd) + inspectDBCmd.AddCommand(inspectSeqScansCmd) + inspectDBCmd.AddCommand(inspectUnusedIndexesCmd) + inspectDBCmd.AddCommand(inspectTotalTableSizesCmd) + inspectDBCmd.AddCommand(inspectTableIndexSizesCmd) + inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd) + inspectDBCmd.AddCommand(inspectTableSizesCmd) inspectCmd.AddCommand(inspectDBCmd) reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in") inspectCmd.AddCommand(reportCmd) rootCmd.AddCommand(inspectCmd) } + +func printReportSummary(outDir string) error { + // point to the date-based subdirectory + date := time.Now().Format("2006-01-02") + outDir = filepath.Join(outDir, date) + // Load rules from tools/inspect_rules.toml + data, err := os.ReadFile(filepath.Join(utils.CurrentDirAbs, "tools", "inspect_rules.toml")) + if err != nil { + return err + } + var cfg Config + if err := toml.Unmarshal(data, &cfg); err != nil { + return err + } + // Open csvq database rooted at the output directory + db, err := sql.Open("csvq", outDir) + if err != nil { + return err + } + defer db.Close() + + // Build report summary table + table := "RULE|STATUS|MATCHES\n|-|-|-|\n" + + // find matching rule + var status string + for i := range cfg.Rules { + r := &cfg.Rules[i] + name := r.Name + + row := db.QueryRow(r.Query) + var match sql.NullString + + if err := row.Scan(&match); err != nil { + if err == sql.ErrNoRows { + status = r.Pass + } else { + status = err.Error() + } + } else { + if !match.Valid || match.String == "" { + status = r.Pass + } else { + status = r.Fail + } + } + matchStr := "-" + if match.Valid { + matchStr = match.String + } + table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, matchStr) + } + return list.RenderTable(table) +} diff --git a/go.mod b/go.mod index 193d1f07b..50ea3ab8f 100644 --- a/go.mod +++ b/go.mod @@ -35,9 +35,11 @@ require ( github.com/jackc/pgtype v1.14.4 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 + github.com/mithrandie/csvq-driver v1.7.0 github.com/muesli/reflow v0.3.0 github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 github.com/oapi-codegen/runtime v1.1.1 + github.com/pelletier/go-toml/v2 v2.2.4 github.com/slack-go/slack v0.16.0 github.com/spf13/afero v1.14.0 github.com/spf13/cobra v1.9.1 @@ -222,6 +224,10 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/mithrandie/csvq v1.18.1 // indirect + github.com/mithrandie/go-file/v2 v2.1.0 // indirect + github.com/mithrandie/go-text v1.6.0 // indirect + github.com/mithrandie/ternary v1.1.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect @@ -239,7 +245,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 62d3f9c73..b7e8751ea 100644 --- a/go.sum +++ b/go.sum @@ -697,6 +697,16 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mithrandie/csvq v1.18.1 h1:f7NB2scbb7xx2ffPduJ2VtZ85RpWXfvanYskAkGlCBU= +github.com/mithrandie/csvq v1.18.1/go.mod h1:MRJj7AtcXfk7jhNGxLuJGP3LORmh4lpiPWxQ7VyCRn8= +github.com/mithrandie/csvq-driver v1.7.0 h1:ejiavXNWwTPMyr3fJFnhcqd1L1cYudA0foQy9cZrqhw= +github.com/mithrandie/csvq-driver v1.7.0/go.mod h1:HcN3xL9UCJnBYA/AIQOOB/KlyfXAiYr5yxDmiwrGk5o= +github.com/mithrandie/go-file/v2 v2.1.0 h1:XA5Tl+73GXMDvgwSE3Sg0uC5FkLr3hnXs8SpUas0hyg= +github.com/mithrandie/go-file/v2 v2.1.0/go.mod h1:9YtTF3Xo59GqC1Pxw6KyGVcM/qubAMlxVsqI/u9r++c= +github.com/mithrandie/go-text v1.6.0 h1:8gOXTMPbMY8DJbKMTv8kHhADcJlDWXqS/YQH4SyWO6s= +github.com/mithrandie/go-text v1.6.0/go.mod h1:xCgj1xiNbI/d4xA9sLVvXkjh5B2tNx2ZT2/3rpmh8to= +github.com/mithrandie/ternary v1.1.1 h1:k/joD6UGVYxHixYmSR8EGgDFNONBMqyD373xT4QRdC4= +github.com/mithrandie/ternary v1.1.1/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= diff --git a/internal/inspect/bloat/bloat.go b/internal/inspect/bloat/bloat.go index 6a97e41c8..a339cd205 100644 --- a/internal/inspect/bloat/bloat.go +++ b/internal/inspect/bloat/bloat.go @@ -19,11 +19,10 @@ import ( var BloatQuery string type Result struct { - Type string - Schemaname string - Object_name string - Bloat string - Waste string + Type string + Name string + Bloat string + Waste string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -41,9 +40,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Type|Schema name|Object name|Bloat|Waste\n|-|-|-|-|-|\n" + table := "|Type|Name|Bloat|Waste\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Type, r.Schemaname, r.Object_name, r.Bloat, r.Waste) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|\n", r.Type, r.Name, r.Bloat, r.Waste) } return list.RenderTable(table) } diff --git a/internal/inspect/bloat/bloat.sql b/internal/inspect/bloat/bloat.sql index 25917d874..638139987 100644 --- a/internal/inspect/bloat/bloat.sql +++ b/internal/inspect/bloat/bloat.sql @@ -38,12 +38,12 @@ WITH constants AS ( JOIN pg_class c2 ON c2.oid = i.indexrelid ) SELECT - type, schemaname, object_name, bloat, pg_size_pretty(raw_waste) as waste + type, name, bloat, pg_size_pretty(raw_waste) as waste FROM (SELECT 'table' as type, schemaname, - tablename as object_name, + schemaname || '.' || tablename as name, ROUND(CASE WHEN otta=0 THEN 0.0 ELSE table_bloat.relpages/otta::numeric END,1) AS bloat, CASE WHEN relpages < otta THEN '0' ELSE (bs*(table_bloat.relpages-otta)::bigint)::bigint END AS raw_waste FROM @@ -52,7 +52,7 @@ FROM SELECT 'index' as type, schemaname, - tablename || '::' || iname as object_name, + schemaname || '.' || tablename || '::' || iname as name, ROUND(CASE WHEN iotta=0 OR ipages=0 THEN 0.0 ELSE ipages/iotta::numeric END,1) AS bloat, CASE WHEN ipages < iotta THEN '0' ELSE (bs*(ipages-iotta))::bigint END AS raw_waste FROM diff --git a/internal/inspect/bloat/bloat_test.go b/internal/inspect/bloat/bloat_test.go index 8646565ce..d172e4220 100644 --- a/internal/inspect/bloat/bloat_test.go +++ b/internal/inspect/bloat/bloat_test.go @@ -29,11 +29,10 @@ func TestBloat(t *testing.T) { defer conn.Close(t) conn.Query(BloatQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Type: "index hit rate", - Schemaname: "public", - Object_name: "table", - Bloat: "0.9", - Waste: "0.1", + Type: "index hit rate", + Name: "public.table", + Bloat: "0.9", + Waste: "0.1", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/cache/cache.go b/internal/inspect/cache/cache.go deleted file mode 100644 index 2b7ad5f25..000000000 --- a/internal/inspect/cache/cache.go +++ /dev/null @@ -1,57 +0,0 @@ -package cache - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed cache.sql -var CacheQuery string - -type Result struct { - Name string - Ratio float64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - // Ref: https://github.com/heroku/heroku-pg-extras/blob/main/commands/cache_hit.js#L7 - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, CacheQuery) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - // TODO: implement a markdown table marshaller - table := "|Name|Ratio|OK?|Explanation|\n|-|-|-|-|\n" - for _, r := range result { - ok := "Yup!" - if r.Ratio < 0.94 { - ok = "Maybe not..." - } - var explanation string - switch r.Name { - case "index hit rate": - explanation = "This is the ratio of index hits to index scans. If this ratio is low, it means that the database is not using indexes effectively. Check the `index-usage` command for more info." - case "table hit rate": - explanation = "This is the ratio of table hits to table scans. If this ratio is low, it means that your queries are not finding the data effectively. Check your query performance and it might be worth increasing your compute." - } - table += fmt.Sprintf("|`%s`|`%.6f`|`%s`|`%s`|\n", r.Name, r.Ratio, ok, explanation) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/cache/cache.sql b/internal/inspect/cache/cache.sql deleted file mode 100644 index fc14b288c..000000000 --- a/internal/inspect/cache/cache.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - 'index hit rate' AS name, - (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read),0) AS ratio -FROM pg_statio_user_indexes -UNION ALL -SELECT - 'table hit rate' AS name, - sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read),0) AS ratio -FROM pg_statio_user_tables diff --git a/internal/inspect/cache/cache_test.go b/internal/inspect/cache/cache_test.go deleted file mode 100644 index 28fa1cb73..000000000 --- a/internal/inspect/cache/cache_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package cache - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestCacheCommand(t *testing.T) { - t.Run("inspects cache hit rate", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(CacheQuery). - Reply("SELECT 1", Result{ - Name: "index hit rate", - Ratio: 0.9, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) - - t.Run("throws error on empty result", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(CacheQuery). - Reply("SELECT 1", []interface{}{}) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.ErrorContains(t, err, "cannot find field Name in returned row") - }) -} diff --git a/internal/inspect/calls/calls.sql b/internal/inspect/calls/calls.sql index 57c3ecee2..d0b2c623c 100644 --- a/internal/inspect/calls/calls.sql +++ b/internal/inspect/calls/calls.sql @@ -3,7 +3,10 @@ SELECT (interval '1 millisecond' * total_exec_time)::text AS total_exec_time, to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%' AS prop_exec_time, to_char(calls, 'FM999G999G990') AS ncalls, - (interval '1 millisecond' * (blk_read_time + blk_write_time))::text AS sync_io_time -FROM pg_stat_statements + (interval '1 millisecond' * ( + COALESCE((row_to_json(s)->>'blk_read_time')::numeric, 0) + + COALESCE((row_to_json(s)->>'blk_write_time')::numeric, 0) + ))::text AS sync_io_time +FROM pg_stat_statements AS s ORDER BY calls DESC LIMIT 10 diff --git a/internal/inspect/db_stats/db_stats.go b/internal/inspect/db_stats/db_stats.go new file mode 100644 index 000000000..bb9882733 --- /dev/null +++ b/internal/inspect/db_stats/db_stats.go @@ -0,0 +1,52 @@ +package db_stats + +import ( + "context" + _ "embed" + "fmt" + + "github.com/go-errors/errors" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/spf13/afero" + "github.com/supabase/cli/internal/migration/list" + "github.com/supabase/cli/internal/utils" + "github.com/supabase/cli/pkg/pgxv5" +) + +//go:embed db_stats.sql +var DBStatsQuery string + +type Result struct { + Name string + Database_size string + Total_index_size string + Total_table_size string + Total_toast_size string + Time_since_stats_reset string + Index_hit_rate string + Table_hit_rate string + WAL_size string +} + +func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { + conn, err := utils.ConnectByConfig(ctx, config, options...) + if err != nil { + return err + } + defer conn.Close(context.Background()) + rows, err := conn.Query(ctx, DBStatsQuery) + if err != nil { + return errors.Errorf("failed to query rows: %w", err) + } + result, err := pgxv5.CollectRows[Result](rows) + if err != nil { + return err + } + + table := "|Name|Database Size|Total Index Size|Total Table Size|Total Toast Size|Time Since Stats Reset|Index Hit Rate|Table Hit Rate|WAL Size|\n|-|-|-|-|-|-|-|-|\n" + for _, r := range result { + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|`%s`|\n", r.Name, r.Database_size, r.Total_index_size, r.Total_table_size, r.Total_toast_size, r.Time_since_stats_reset, r.Index_hit_rate, r.Table_hit_rate, r.WAL_size) + } + return list.RenderTable(table) +} diff --git a/internal/inspect/db_stats/db_stats.sql b/internal/inspect/db_stats/db_stats.sql new file mode 100644 index 000000000..f9df0a477 --- /dev/null +++ b/internal/inspect/db_stats/db_stats.sql @@ -0,0 +1,15 @@ +SELECT + 'postgres' AS name, + pg_size_pretty(pg_database_size('postgres')) AS database_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'i') AS total_index_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'r') AS total_table_size, + (SELECT pg_size_pretty(SUM(pg_relation_size(c.oid))) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 't') AS total_toast_size, + (SELECT (now() - stats_reset)::text FROM pg_stat_statements_info) AS time_since_stats_reset, + (SELECT ROUND(SUM(idx_blks_hit)::numeric / nullif(SUM(idx_blks_hit + idx_blks_read),0),2) + FROM pg_statio_user_indexes) AS index_hit_rate, + (SELECT ROUND(SUM(heap_blks_hit)::numeric / nullif(SUM(heap_blks_hit + heap_blks_read),0),2) + FROM pg_statio_user_tables) AS table_hit_rate, + (SELECT pg_size_pretty(SUM(size)) FROM pg_ls_waldir()) AS wal_size \ No newline at end of file diff --git a/internal/inspect/total_index_size/total_index_size_test.go b/internal/inspect/db_stats/db_stats_test.go similarity index 64% rename from internal/inspect/total_index_size/total_index_size_test.go rename to internal/inspect/db_stats/db_stats_test.go index 8eb0e0aa9..523b3f5ed 100644 --- a/internal/inspect/total_index_size/total_index_size_test.go +++ b/internal/inspect/db_stats/db_stats_test.go @@ -1,4 +1,4 @@ -package total_index_size +package db_stats import ( "context" @@ -7,8 +7,6 @@ import ( "github.com/jackc/pgconn" "github.com/spf13/afero" "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" "github.com/supabase/cli/pkg/pgtest" ) @@ -20,16 +18,23 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestTotalIndexSizeCommand(t *testing.T) { +func TestDBStatsCommand(t *testing.T) { t.Run("inspects size of all indexes", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(TotalIndexSizeQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + conn.Query(DBStatsQuery). Reply("SELECT 1", Result{ - Size: "8GB", + Database_size: "8GB", + Total_index_size: "8GB", + Total_table_size: "8GB", + Total_toast_size: "8GB", + Time_since_stats_reset: "8GB", + Index_hit_rate: "8GB", + Table_hit_rate: "8GB", + WAL_size: "8GB", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/index_sizes/index_sizes.sql b/internal/inspect/index_sizes/index_sizes.sql deleted file mode 100644 index 25b6f96d5..000000000 --- a/internal/inspect/index_sizes/index_sizes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS name, - pg_size_pretty(sum(c.relpages::bigint*8192)::bigint) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'i' -GROUP BY n.nspname, c.relname -ORDER BY sum(c.relpages) DESC diff --git a/internal/inspect/index_sizes/index_sizes.go b/internal/inspect/index_stats/index_stats.go similarity index 64% rename from internal/inspect/index_sizes/index_sizes.go rename to internal/inspect/index_stats/index_stats.go index 2cfc06e8d..8f781c688 100644 --- a/internal/inspect/index_sizes/index_sizes.go +++ b/internal/inspect/index_stats/index_stats.go @@ -1,4 +1,4 @@ -package index_sizes +package index_stats import ( "context" @@ -15,12 +15,16 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed index_sizes.sql -var IndexSizesQuery string +//go:embed index_stats.sql +var IndexStatsQuery string type Result struct { - Name string - Size string + Name string + Size string + Percent_used string + Index_scans int64 + Seq_scans int64 + Unused bool } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -29,7 +33,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, IndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) + rows, err := conn.Query(ctx, IndexStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -38,9 +42,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Name|size|\n|-|-|\n" + table := "|Name|Size|Percent used|Index scans|Seq scans|Unused|\n|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Size) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|`%d`|`%t`|\n", r.Name, r.Size, r.Percent_used, r.Index_scans, r.Seq_scans, r.Unused) } return list.RenderTable(table) } diff --git a/internal/inspect/index_stats/index_stats.sql b/internal/inspect/index_stats/index_stats.sql new file mode 100644 index 000000000..26337fdde --- /dev/null +++ b/internal/inspect/index_stats/index_stats.sql @@ -0,0 +1,49 @@ +-- Combined index statistics: size, usage percent, seq scans, and mark unused +WITH idx_sizes AS ( + SELECT + i.indexrelid AS oid, + n.nspname || '.' || c.relname AS name, + pg_relation_size(i.indexrelid) AS index_size_bytes + FROM pg_stat_user_indexes ui + JOIN pg_index i ON ui.indexrelid = i.indexrelid + JOIN pg_class c ON ui.indexrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE NOT n.nspname LIKE ANY($1) +), +idx_usage AS ( + SELECT + indexrelid AS oid, + idx_scan::bigint AS idx_scans + FROM pg_stat_user_indexes ui + WHERE NOT schemaname LIKE ANY($1) +), +seq_usage AS ( + SELECT + relid AS oid, + seq_scan::bigint AS seq_scans + FROM pg_stat_user_tables + WHERE NOT schemaname LIKE ANY($1) +), +usage_pct AS ( + SELECT + u.oid, + CASE + WHEN u.idx_scans IS NULL OR u.idx_scans = 0 THEN 0 + WHEN s.seq_scans IS NULL THEN 100 + ELSE ROUND(100.0 * u.idx_scans / (s.seq_scans + u.idx_scans), 1) + END AS percent_used + FROM idx_usage u + LEFT JOIN seq_usage s ON s.oid = u.oid +) +SELECT + s.name, + pg_size_pretty(s.index_size_bytes) AS size, + COALESCE(up.percent_used, 0)::text || '%' AS percent_used, + COALESCE(u.idx_scans, 0) AS index_scans, + COALESCE(sq.seq_scans, 0) AS seq_scans, + CASE WHEN COALESCE(u.idx_scans, 0) = 0 THEN true ELSE false END AS unused +FROM idx_sizes s +LEFT JOIN idx_usage u ON u.oid = s.oid +LEFT JOIN seq_usage sq ON sq.oid = s.oid +LEFT JOIN usage_pct up ON up.oid = s.oid +ORDER BY s.index_size_bytes DESC diff --git a/internal/inspect/index_sizes/index_sizes_test.go b/internal/inspect/index_stats/index_stats_test.go similarity index 63% rename from internal/inspect/index_sizes/index_sizes_test.go rename to internal/inspect/index_stats/index_stats_test.go index 9071c5710..abc3cd654 100644 --- a/internal/inspect/index_sizes/index_sizes_test.go +++ b/internal/inspect/index_stats/index_stats_test.go @@ -1,4 +1,4 @@ -package index_sizes +package index_stats import ( "context" @@ -20,21 +20,24 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestIndexSizes(t *testing.T) { - t.Run("inspects index sizes", func(t *testing.T) { - // Setup in-memory fs +func TestIndexStatsCommand(t *testing.T) { + t.Run("inspects index stats", func(t *testing.T) { fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(IndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + + // Mock index stats + conn.Query(IndexStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Name: "test_table_idx", - Size: "100GB", + Name: "public.test_idx", + Size: "1GB", + Percent_used: "50%", + Index_scans: 5, + Seq_scans: 5, + Unused: false, }) - // Run test + err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error assert.NoError(t, err) }) } diff --git a/internal/inspect/index_usage/index_usage.sql b/internal/inspect/index_usage/index_usage.sql deleted file mode 100644 index fa83e97db..000000000 --- a/internal/inspect/index_usage/index_usage.sql +++ /dev/null @@ -1,17 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - CASE - WHEN idx_scan IS NULL THEN 'Insufficient data' - WHEN idx_scan = 0 THEN 'Insufficient data' - ELSE ROUND(100.0 * idx_scan / (seq_scan + idx_scan), 1) || '%' - END percent_of_times_index_used, - n_live_tup rows_in_table -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY - CASE - WHEN idx_scan is null then 1 - WHEN idx_scan = 0 then 1 - ELSE 0 - END, - n_live_tup DESC diff --git a/internal/inspect/locks/locks.go b/internal/inspect/locks/locks.go index 0b2db71c6..be5c34d3a 100644 --- a/internal/inspect/locks/locks.go +++ b/internal/inspect/locks/locks.go @@ -23,7 +23,7 @@ type Result struct { Relname string Transactionid string Granted bool - Query string + Stmt string Age string } @@ -42,16 +42,16 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|pid|relname|transaction id|granted|query|age|\n|-|-|-|-|-|-|\n" + table := "|pid|relname|transaction id|granted|stmt|age|\n|-|-|-|-|-|-|\n" for _, r := range result { // remove whitespace from query re := regexp.MustCompile(`\s+|\r+|\n+|\t+|\v`) - query := re.ReplaceAllString(r.Query, " ") + stmt := re.ReplaceAllString(r.Stmt, " ") // escape pipes in query re = regexp.MustCompile(`\|`) - query = re.ReplaceAllString(query, `\|`) - table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, query, r.Age) + stmt = re.ReplaceAllString(stmt, `\|`) + table += fmt.Sprintf("|`%d`|`%s`|`%s`|`%t`|%s|`%s`|\n", r.Pid, r.Relname, r.Transactionid, r.Granted, stmt, r.Age) } return list.RenderTable(table) } diff --git a/internal/inspect/locks/locks.sql b/internal/inspect/locks/locks.sql index e140f4786..9369057b5 100644 --- a/internal/inspect/locks/locks.sql +++ b/internal/inspect/locks/locks.sql @@ -1,9 +1,9 @@ SELECT pg_stat_activity.pid, COALESCE(pg_class.relname, 'null') AS relname, - COALESCE(pg_locks.transactionid, 'null') AS transactionid, + COALESCE(pg_locks.transactionid::text, 'null') AS transactionid, pg_locks.granted, - pg_stat_activity.query, + pg_stat_activity.query AS stmt, age(now(), pg_stat_activity.query_start)::text AS age FROM pg_stat_activity, pg_locks LEFT OUTER JOIN pg_class ON (pg_locks.relation = pg_class.oid) WHERE pg_stat_activity.query <> '' diff --git a/internal/inspect/locks/locks_test.go b/internal/inspect/locks/locks_test.go index e4c55c6bd..0964ad122 100644 --- a/internal/inspect/locks/locks_test.go +++ b/internal/inspect/locks/locks_test.go @@ -31,7 +31,7 @@ func TestLocksCommand(t *testing.T) { Relname: "rel", Transactionid: "9301", Granted: true, - Query: "select 1", + Stmt: "select 1", Age: "300ms", }) // Run test diff --git a/internal/inspect/outliers/outliers.sql b/internal/inspect/outliers/outliers.sql index d222cc8de..69336eef6 100644 --- a/internal/inspect/outliers/outliers.sql +++ b/internal/inspect/outliers/outliers.sql @@ -2,8 +2,11 @@ SELECT (interval '1 millisecond' * total_exec_time)::text AS total_exec_time, to_char((total_exec_time/sum(total_exec_time) OVER()) * 100, 'FM90D0') || '%' AS prop_exec_time, to_char(calls, 'FM999G999G999G990') AS ncalls, - (interval '1 millisecond' * (blk_read_time + blk_write_time))::text AS sync_io_time, + (interval '1 millisecond' * ( + COALESCE((row_to_json(s)->>'blk_read_time')::numeric, 0) + + COALESCE((row_to_json(s)->>'blk_write_time')::numeric, 0) + ))::text AS sync_io_time, query -FROM pg_stat_statements WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) +FROM pg_stat_statements AS s WHERE userid = (SELECT usesysid FROM pg_user WHERE usename = current_user LIMIT 1) ORDER BY total_exec_time DESC LIMIT 10 diff --git a/internal/inspect/report.go b/internal/inspect/report.go index de721d954..ca61409aa 100644 --- a/internal/inspect/report.go +++ b/internal/inspect/report.go @@ -26,6 +26,10 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs if err := utils.MkdirIfNotExistFS(fsys, out); err != nil { return err } + dateDir := filepath.Join(out, date) + if err := utils.MkdirIfNotExistFS(fsys, dateDir); err != nil { + return err + } conn, err := utils.ConnectByConfig(ctx, config, options...) if err != nil { return err @@ -44,15 +48,19 @@ func Report(ctx context.Context, out string, config pgconn.Config, fsys afero.Fs return errors.Errorf("failed to read query: %w", err) } name := strings.Split(d.Name(), ".")[0] - outPath := filepath.Join(out, fmt.Sprintf("%s_%s.csv", name, date)) + outPath := filepath.Join(dateDir, fmt.Sprintf("%s.csv", name)) return copyToCSV(ctx, string(query), outPath, conn.PgConn(), fsys) }); err != nil { return err } - if !filepath.IsAbs(out) { - out, _ = filepath.Abs(out) + // print the actual save location + if !filepath.IsAbs(dateDir) { + dateDir, err = filepath.Abs(dateDir) + if err != nil { + return err + } } - fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(out)) + fmt.Fprintln(os.Stderr, "Reports saved to "+utils.Bold(dateDir)) return nil } diff --git a/internal/inspect/report_test.go b/internal/inspect/report_test.go index 6b4220451..43968dbcb 100644 --- a/internal/inspect/report_test.go +++ b/internal/inspect/report_test.go @@ -3,31 +3,12 @@ package inspect import ( "context" "fmt" + "io/fs" "testing" "github.com/jackc/pgconn" "github.com/spf13/afero" "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/inspect/bloat" - "github.com/supabase/cli/internal/inspect/blocking" - "github.com/supabase/cli/internal/inspect/cache" - "github.com/supabase/cli/internal/inspect/calls" - "github.com/supabase/cli/internal/inspect/index_sizes" - "github.com/supabase/cli/internal/inspect/index_usage" - "github.com/supabase/cli/internal/inspect/locks" - "github.com/supabase/cli/internal/inspect/long_running_queries" - "github.com/supabase/cli/internal/inspect/outliers" - "github.com/supabase/cli/internal/inspect/replication_slots" - "github.com/supabase/cli/internal/inspect/role_configs" - "github.com/supabase/cli/internal/inspect/role_connections" - "github.com/supabase/cli/internal/inspect/seq_scans" - "github.com/supabase/cli/internal/inspect/table_index_sizes" - "github.com/supabase/cli/internal/inspect/table_record_counts" - "github.com/supabase/cli/internal/inspect/table_sizes" - "github.com/supabase/cli/internal/inspect/total_index_size" - "github.com/supabase/cli/internal/inspect/total_table_sizes" - "github.com/supabase/cli/internal/inspect/unused_indexes" - "github.com/supabase/cli/internal/inspect/vacuum_stats" "github.com/supabase/cli/pkg/pgtest" ) @@ -41,58 +22,32 @@ var dbConfig = pgconn.Config{ func TestReportCommand(t *testing.T) { t.Run("runs all queries", func(t *testing.T) { - // Setup in-memory fs fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(wrapQuery(bloat.BloatQuery)). - Reply("COPY 0"). - Query(wrapQuery(blocking.BlockingQuery)). - Reply("COPY 0"). - Query(wrapQuery(cache.CacheQuery)). - Reply("COPY 0"). - Query(wrapQuery(calls.CallsQuery)). - Reply("COPY 0"). - Query(wrapQuery(index_sizes.IndexSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(index_usage.IndexUsageQuery)). - Reply("COPY 0"). - Query(wrapQuery(locks.LocksQuery)). - Reply("COPY 0"). - Query(wrapQuery(long_running_queries.LongRunningQueriesQuery)). - Reply("COPY 0"). - Query(wrapQuery(outliers.OutliersQuery)). - Reply("COPY 0"). - Query(wrapQuery(replication_slots.ReplicationSlotsQuery)). - Reply("COPY 0"). - Query(wrapQuery(role_configs.RoleConfigsQuery)). - Reply("COPY 0"). - Query(wrapQuery(role_connections.RoleConnectionsQuery)). - Reply("COPY 0"). - Query(wrapQuery(seq_scans.SeqScansQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_index_sizes.TableIndexSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_record_counts.TableRecordCountsQuery)). - Reply("COPY 0"). - Query(wrapQuery(table_sizes.TableSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(total_index_size.TotalIndexSizeQuery)). - Reply("COPY 0"). - Query(wrapQuery(total_table_sizes.TotalTableSizesQuery)). - Reply("COPY 0"). - Query(wrapQuery(unused_indexes.UnusedIndexesQuery)). - Reply("COPY 0"). - Query(wrapQuery(vacuum_stats.VacuumStatsQuery)). - Reply("COPY 0") - // Run test - err := Report(context.Background(), ".", dbConfig, fsys, conn.Intercept) - // Check error + // Iterate over all embedded SQL files + var sqlCount int + err := fs.WalkDir(queries, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + data, err := queries.ReadFile(path) + if err != nil { + return err + } + conn.Query(wrapQuery(string(data))).Reply("COPY 0") + sqlCount++ + return nil + }) assert.NoError(t, err) - matches, err := afero.Glob(fsys, "*.csv") + err = Report(context.Background(), ".", dbConfig, fsys, conn.Intercept) assert.NoError(t, err) - assert.Len(t, matches, 20) + matches, err := afero.Glob(fsys, "*/*.csv") + assert.NoError(t, err) + assert.Len(t, matches, sqlCount) }) } diff --git a/internal/inspect/role_configs/role_configs.sql b/internal/inspect/role_configs/role_configs.sql deleted file mode 100644 index fd7f964d0..000000000 --- a/internal/inspect/role_configs/role_configs.sql +++ /dev/null @@ -1,5 +0,0 @@ -select - rolname as role_name, - array_to_string(rolconfig, ',', '*') as custom_config -from - pg_roles where rolconfig is not null diff --git a/internal/inspect/role_configs/role_configs_test.go b/internal/inspect/role_configs/role_configs_test.go deleted file mode 100644 index 554a12526..000000000 --- a/internal/inspect/role_configs/role_configs_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package role_configs - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestRoleCommand(t *testing.T) { - t.Run("inspects role connections", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(RoleConfigsQuery). - Reply("SELECT 1", Result{ - Role_name: "postgres", - Custom_config: "statement_timeout=3s", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/role_connections/role_connections.go b/internal/inspect/role_connections/role_connections.go deleted file mode 100644 index 5b0a56539..000000000 --- a/internal/inspect/role_connections/role_connections.go +++ /dev/null @@ -1,62 +0,0 @@ -package role_connections - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed role_connections.sql -var RoleConnectionsQuery string - -type Result struct { - Rolname string - Active_connections int - Connection_limit int -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, RoleConnectionsQuery) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Role Name|Active connction|\n|-|-|\n" - sum := 0 - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%d`|\n", r.Rolname, r.Active_connections) - sum += r.Active_connections - } - - if err := list.RenderTable(table); err != nil { - return err - } - - if len(result) > 0 { - fmt.Printf("\nActive connections %d/%d\n\n", sum, result[0].Connection_limit) - } - - if matches := utils.ProjectHostPattern.FindStringSubmatch(config.Host); len(matches) == 4 { - fmt.Println("Go to the dashboard for more here:") - fmt.Printf("https://app.supabase.com/project/%s/database/roles\n", matches[2]) - } - - return nil -} diff --git a/internal/inspect/role_configs/role_configs.go b/internal/inspect/role_stats/role_stats.go similarity index 64% rename from internal/inspect/role_configs/role_configs.go rename to internal/inspect/role_stats/role_stats.go index f4fb79382..ee5e724f2 100644 --- a/internal/inspect/role_configs/role_configs.go +++ b/internal/inspect/role_stats/role_stats.go @@ -1,4 +1,4 @@ -package role_configs +package role_stats import ( "context" @@ -14,12 +14,14 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed role_configs.sql -var RoleConfigsQuery string +//go:embed role_stats.sql +var RoleStatsQuery string type Result struct { - Role_name string - Custom_config string + Role_name string + Active_connections int + Connection_limit int + Custom_config string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -28,7 +30,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, RoleConfigsQuery) + rows, err := conn.Query(ctx, RoleStatsQuery) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -37,9 +39,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Role name|Custom config|\n|-|-|\n" + table := "|Role name|Active connections|Connection limit|Custom config|\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Role_name, r.Custom_config) + table += fmt.Sprintf("|`%s`|`%d`|`%d`|`%s`|\n", r.Role_name, r.Active_connections, r.Connection_limit, r.Custom_config) } return list.RenderTable(table) diff --git a/internal/inspect/role_connections/role_connections.sql b/internal/inspect/role_stats/role_stats.sql similarity index 64% rename from internal/inspect/role_connections/role_connections.sql rename to internal/inspect/role_stats/role_stats.sql index 40b103669..92f32c9e5 100644 --- a/internal/inspect/role_connections/role_connections.sql +++ b/internal/inspect/role_stats/role_stats.sql @@ -1,5 +1,5 @@ SELECT - rolname, + rolname as role_name, ( SELECT count(*) @@ -11,6 +11,8 @@ SELECT CASE WHEN rolconnlimit = -1 THEN current_setting('max_connections')::int8 ELSE rolconnlimit - END AS connection_limit -FROM pg_roles -ORDER BY 2 DESC + END AS connection_limit, + array_to_string(rolconfig, ',', '*') as custom_config +FROM + pg_roles +ORDER BY 1 DESC diff --git a/internal/inspect/role_connections/role_connections_test.go b/internal/inspect/role_stats/role_stats_test.go similarity index 84% rename from internal/inspect/role_connections/role_connections_test.go rename to internal/inspect/role_stats/role_stats_test.go index 32ecab768..0539c594f 100644 --- a/internal/inspect/role_connections/role_connections_test.go +++ b/internal/inspect/role_stats/role_stats_test.go @@ -1,4 +1,4 @@ -package role_connections +package role_stats import ( "context" @@ -25,9 +25,10 @@ func TestRoleCommand(t *testing.T) { // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(RoleConnectionsQuery). + conn.Query(RoleStatsQuery). Reply("SELECT 1", Result{ - Rolname: "postgres", + Role_name: "postgres", + Custom_config: "statement_timeout=3s", Active_connections: 1, Connection_limit: 10, }) diff --git a/internal/inspect/seq_scans/seq_scans.go b/internal/inspect/seq_scans/seq_scans.go deleted file mode 100644 index 6b52538ee..000000000 --- a/internal/inspect/seq_scans/seq_scans.go +++ /dev/null @@ -1,46 +0,0 @@ -package seq_scans - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed seq_scans.sql -var SeqScansQuery string - -type Result struct { - Name string - Count int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, SeqScansQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Name|Count|\n|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%d`|\n", r.Name, r.Count) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/seq_scans/seq_scans.sql b/internal/inspect/seq_scans/seq_scans.sql deleted file mode 100644 index c8edfc8e3..000000000 --- a/internal/inspect/seq_scans/seq_scans.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - schemaname || '.' || relname AS name, - seq_scan as count -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY seq_scan DESC diff --git a/internal/inspect/seq_scans/seq_scans_test.go b/internal/inspect/seq_scans/seq_scans_test.go deleted file mode 100644 index 3db6caee5..000000000 --- a/internal/inspect/seq_scans/seq_scans_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package seq_scans - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestSequentialScansCommand(t *testing.T) { - t.Run("inspects sequential scans", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(SeqScansQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Name: "test_table", - Count: 99999, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_index_sizes/table_index_sizes.go b/internal/inspect/table_index_sizes/table_index_sizes.go deleted file mode 100644 index e61f23361..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes.go +++ /dev/null @@ -1,46 +0,0 @@ -package table_index_sizes - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed table_index_sizes.sql -var TableIndexSizesQuery string - -type Result struct { - Table string - Index_size string -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableIndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Table|Index size|\n|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|\n", r.Table, r.Index_size) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/table_index_sizes/table_index_sizes.sql b/internal/inspect/table_index_sizes/table_index_sizes.sql deleted file mode 100644 index 0c9bc6bfc..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - n.nspname || '.' || c.relname AS table, - pg_size_pretty(pg_indexes_size(c.oid)) AS index_size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_indexes_size(c.oid) DESC diff --git a/internal/inspect/table_index_sizes/table_index_sizes_test.go b/internal/inspect/table_index_sizes/table_index_sizes_test.go deleted file mode 100644 index 20ad80fc9..000000000 --- a/internal/inspect/table_index_sizes/table_index_sizes_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package table_index_sizes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTableIndexSizesCommand(t *testing.T) { - t.Run("inspects table index sizes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TableIndexSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). - Reply("SELECT 1", Result{ - Table: "public.test_table", - Index_size: "3GB", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_record_counts/table_record_counts.go b/internal/inspect/table_record_counts/table_record_counts.go deleted file mode 100644 index e0b394374..000000000 --- a/internal/inspect/table_record_counts/table_record_counts.go +++ /dev/null @@ -1,47 +0,0 @@ -package table_record_counts - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed table_record_counts.sql -var TableRecordCountsQuery string - -type Result struct { - Schema string - Name string - Estimated_count int64 -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableRecordCountsQuery, reset.LikeEscapeSchema(utils.PgSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "Schema|Table|Estimated count|\n|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%d`|\n", r.Schema, r.Name, r.Estimated_count) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/table_record_counts/table_record_counts.sql b/internal/inspect/table_record_counts/table_record_counts.sql deleted file mode 100644 index 1b24f04a4..000000000 --- a/internal/inspect/table_record_counts/table_record_counts.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - schemaname AS schema, - relname AS name, - n_live_tup AS estimated_count -FROM pg_stat_user_tables -WHERE NOT schemaname LIKE ANY($1) -ORDER BY n_live_tup DESC diff --git a/internal/inspect/table_record_counts/table_record_counts_test.go b/internal/inspect/table_record_counts/table_record_counts_test.go deleted file mode 100644 index a03714d97..000000000 --- a/internal/inspect/table_record_counts/table_record_counts_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package table_record_counts - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTableRecordCountsCommand(t *testing.T) { - t.Run("inspects table record counts", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TableRecordCountsQuery, reset.LikeEscapeSchema(utils.PgSchemas)). - Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", - Estimated_count: 100, - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/table_sizes/table_sizes.go b/internal/inspect/table_sizes/table_sizes.go deleted file mode 100644 index 7741f0119..000000000 --- a/internal/inspect/table_sizes/table_sizes.go +++ /dev/null @@ -1,47 +0,0 @@ -package table_sizes - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed table_sizes.sql -var TableSizesQuery string - -type Result struct { - Schema string - Name string - Size string -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "Schema|Table|size|\n|-|-|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/table_sizes/table_sizes.sql b/internal/inspect/table_sizes/table_sizes.sql deleted file mode 100644 index 2c8fb3064..000000000 --- a/internal/inspect/table_sizes/table_sizes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - n.nspname AS schema, - c.relname AS name, - pg_size_pretty(pg_table_size(c.oid)) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'r' -ORDER BY pg_table_size(c.oid) DESC diff --git a/internal/inspect/table_sizes/table_sizes_test.go b/internal/inspect/table_sizes/table_sizes_test.go deleted file mode 100644 index 5cc6426ad..000000000 --- a/internal/inspect/table_sizes/table_sizes_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package table_sizes - -import ( - "context" - "testing" - - "github.com/jackc/pgconn" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgtest" -) - -var dbConfig = pgconn.Config{ - Host: "127.0.0.1", - Port: 5432, - User: "admin", - Password: "password", - Database: "postgres", -} - -func TestTableSizesCommand(t *testing.T) { - t.Run("inspects table sizes", func(t *testing.T) { - // Setup in-memory fs - fsys := afero.NewMemMapFs() - // Setup mock postgres - conn := pgtest.NewConn() - defer conn.Close(t) - conn.Query(TableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). - Reply("SELECT 1", Result{ - Schema: "schema", - Name: "test_table", - Size: "3GB", - }) - // Run test - err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error - assert.NoError(t, err) - }) -} diff --git a/internal/inspect/index_usage/index_usage.go b/internal/inspect/table_stats/table_stats.go similarity index 60% rename from internal/inspect/index_usage/index_usage.go rename to internal/inspect/table_stats/table_stats.go index cd8875f79..afc3b193b 100644 --- a/internal/inspect/index_usage/index_usage.go +++ b/internal/inspect/table_stats/table_stats.go @@ -1,4 +1,4 @@ -package index_usage +package table_stats import ( "context" @@ -15,13 +15,16 @@ import ( "github.com/supabase/cli/pkg/pgxv5" ) -//go:embed index_usage.sql -var IndexUsageQuery string +//go:embed table_stats.sql +var TableStatsQuery string type Result struct { - Name string - Percent_of_times_index_used string - Rows_in_table int64 + Name string + Table_size string + Index_size string + Total_size string + Estimated_row_count int64 + Seq_scans int64 } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -30,7 +33,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, IndexUsageQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) + rows, err := conn.Query(ctx, TableStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -38,10 +41,10 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu if err != nil { return err } - // TODO: implement a markdown table marshaller - table := "|Table name|Percentage of times index used|Rows in table|\n|-|-|-|\n" + + table := "|Name|Table size|Index size|Total size|Estimated row count|Seq scans|\n|-|-|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%d`|\n", r.Name, r.Percent_of_times_index_used, r.Rows_in_table) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%s`|`%d`|`%d`|\n", r.Name, r.Table_size, r.Index_size, r.Total_size, r.Estimated_row_count, r.Seq_scans) } return list.RenderTable(table) } diff --git a/internal/inspect/table_stats/table_stats.sql b/internal/inspect/table_stats/table_stats.sql new file mode 100644 index 000000000..5a16e554f --- /dev/null +++ b/internal/inspect/table_stats/table_stats.sql @@ -0,0 +1,27 @@ +SELECT + ts.name, + pg_size_pretty(ts.table_size_bytes) AS table_size, + pg_size_pretty(ts.index_size_bytes) AS index_size, + pg_size_pretty(ts.total_size_bytes) AS total_size, + COALESCE(rc.estimated_row_count, 0) AS estimated_row_count, + COALESCE(rc.seq_scans, 0) AS seq_scans +FROM ( + SELECT + n.nspname || '.' || c.relname AS name, + pg_table_size(c.oid) AS table_size_bytes, + pg_indexes_size(c.oid) AS index_size_bytes, + pg_total_relation_size(c.oid) AS total_size_bytes + FROM pg_class c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE NOT n.nspname LIKE ANY($1) + AND c.relkind = 'r' +) ts +LEFT JOIN ( + SELECT + schemaname || '.' || relname AS name, + n_live_tup AS estimated_row_count, + seq_scan AS seq_scans + FROM pg_stat_user_tables + WHERE NOT schemaname LIKE ANY($1) +) rc ON rc.name = ts.name +ORDER BY ts.total_size_bytes DESC \ No newline at end of file diff --git a/internal/inspect/index_usage/index_usage_test.go b/internal/inspect/table_stats/table_stats_test.go similarity index 59% rename from internal/inspect/index_usage/index_usage_test.go rename to internal/inspect/table_stats/table_stats_test.go index 5b735bb60..087641029 100644 --- a/internal/inspect/index_usage/index_usage_test.go +++ b/internal/inspect/table_stats/table_stats_test.go @@ -1,4 +1,4 @@ -package index_usage +package table_stats import ( "context" @@ -20,22 +20,24 @@ var dbConfig = pgconn.Config{ Database: "postgres", } -func TestIndexUsage(t *testing.T) { - t.Run("inspects index usage", func(t *testing.T) { - // Setup in-memory fs +func TestTableStatsCommand(t *testing.T) { + t.Run("inspects table stats", func(t *testing.T) { fsys := afero.NewMemMapFs() - // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(IndexUsageQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). + + // Mock table sizes and index sizes + conn.Query(TableStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Name: "test_table_idx", - Percent_of_times_index_used: "0.9", - Rows_in_table: 300, + Name: "public.test_table", + Table_size: "3GB", + Index_size: "1GB", + Total_size: "4GB", + Estimated_row_count: 100, + Seq_scans: 1, }) - // Run test + err := Run(context.Background(), dbConfig, fsys, conn.Intercept) - // Check error assert.NoError(t, err) }) } diff --git a/internal/inspect/total_index_size/total_index_size.go b/internal/inspect/total_index_size/total_index_size.go deleted file mode 100644 index fbc66b259..000000000 --- a/internal/inspect/total_index_size/total_index_size.go +++ /dev/null @@ -1,45 +0,0 @@ -package total_index_size - -import ( - "context" - _ "embed" - "fmt" - - "github.com/go-errors/errors" - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" - "github.com/spf13/afero" - "github.com/supabase/cli/internal/db/reset" - "github.com/supabase/cli/internal/migration/list" - "github.com/supabase/cli/internal/utils" - "github.com/supabase/cli/pkg/pgxv5" -) - -//go:embed total_index_size.sql -var TotalIndexSizeQuery string - -type Result struct { - Size string -} - -func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { - conn, err := utils.ConnectByConfig(ctx, config, options...) - if err != nil { - return err - } - defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TotalIndexSizeQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) - if err != nil { - return errors.Errorf("failed to query rows: %w", err) - } - result, err := pgxv5.CollectRows[Result](rows) - if err != nil { - return err - } - - table := "|Size|\n|-|\n" - for _, r := range result { - table += fmt.Sprintf("|`%s`|\n", r.Size) - } - return list.RenderTable(table) -} diff --git a/internal/inspect/total_index_size/total_index_size.sql b/internal/inspect/total_index_size/total_index_size.sql deleted file mode 100644 index d1e8ab3d8..000000000 --- a/internal/inspect/total_index_size/total_index_size.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - pg_size_pretty(sum(c.relpages::bigint*8192)::bigint) AS size -FROM pg_class c -LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) -WHERE NOT n.nspname LIKE ANY($1) -AND c.relkind = 'i' diff --git a/internal/inspect/total_table_sizes/total_table_sizes.go b/internal/inspect/total_table_sizes/total_table_sizes.go index 80b1c89a8..7c7d40cf3 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.go +++ b/internal/inspect/total_table_sizes/total_table_sizes.go @@ -19,9 +19,8 @@ import ( var TotalTableSizesQuery string type Result struct { - Schema string - Name string - Size string + Name string + Size string } func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { @@ -30,7 +29,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } defer conn.Close(context.Background()) - rows, err := conn.Query(ctx, TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)) + rows, err := conn.Query(ctx, TotalTableSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)) if err != nil { return errors.Errorf("failed to query rows: %w", err) } @@ -39,9 +38,9 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "Schema|Table|Size|\n|-|-|-|\n" + table := "|Table|Size|\n|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", r.Schema, r.Name, r.Size) + table += fmt.Sprintf("|`%s`|`%s`|\n", r.Name, r.Size) } return list.RenderTable(table) } diff --git a/internal/inspect/total_table_sizes/total_table_sizes.sql b/internal/inspect/total_table_sizes/total_table_sizes.sql index 471d0655f..b3e7df657 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes.sql +++ b/internal/inspect/total_table_sizes/total_table_sizes.sql @@ -1,6 +1,5 @@ SELECT - n.nspname AS schema, - c.relname AS name, + FORMAT('%I.%I', n.nspname, c.relname) AS name, pg_size_pretty(pg_total_relation_size(c.oid)) AS size FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) diff --git a/internal/inspect/total_table_sizes/total_table_sizes_test.go b/internal/inspect/total_table_sizes/total_table_sizes_test.go index bc548af60..ae16989bd 100644 --- a/internal/inspect/total_table_sizes/total_table_sizes_test.go +++ b/internal/inspect/total_table_sizes/total_table_sizes_test.go @@ -27,11 +27,10 @@ func TestTotalTableSizesCommand(t *testing.T) { // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) - conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.PgSchemas)). + conn.Query(TotalTableSizesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Schema: "public", - Name: "test_table", - Size: "3GB", + Name: "public.test_table", + Size: "3GB", }) // Run test err := Run(context.Background(), dbConfig, fsys, conn.Intercept) diff --git a/internal/inspect/unused_indexes/unused_indexes.go b/internal/inspect/unused_indexes/unused_indexes.go index 2a30a46d7..bfbbf523a 100644 --- a/internal/inspect/unused_indexes/unused_indexes.go +++ b/internal/inspect/unused_indexes/unused_indexes.go @@ -19,7 +19,7 @@ import ( var UnusedIndexesQuery string type Result struct { - Table string + Name string Index string Index_size string Index_scans int64 @@ -42,7 +42,7 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu table := "|Table|Index|Index Size|Index Scans\n|-|-|-|-|\n" for _, r := range result { - table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Table, r.Index, r.Index_size, r.Index_scans) + table += fmt.Sprintf("|`%s`|`%s`|`%s`|`%d`|\n", r.Name, r.Index, r.Index_size, r.Index_scans) } return list.RenderTable(table) } diff --git a/internal/inspect/unused_indexes/unused_indexes.sql b/internal/inspect/unused_indexes/unused_indexes.sql index 6e775967d..aa429b9ed 100644 --- a/internal/inspect/unused_indexes/unused_indexes.sql +++ b/internal/inspect/unused_indexes/unused_indexes.sql @@ -1,5 +1,5 @@ SELECT - schemaname || '.' || relname AS table, + FORMAT('%I.%I', schemaname, relname) AS name, indexrelname AS index, pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, idx_scan as index_scans diff --git a/internal/inspect/unused_indexes/unused_indexes_test.go b/internal/inspect/unused_indexes/unused_indexes_test.go index ee4182094..517f88c61 100644 --- a/internal/inspect/unused_indexes/unused_indexes_test.go +++ b/internal/inspect/unused_indexes/unused_indexes_test.go @@ -29,7 +29,7 @@ func TestUnusedIndexesCommand(t *testing.T) { defer conn.Close(t) conn.Query(UnusedIndexesQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Table: "test_table", + Name: "public.test_table", Index: "test_table_idx", Index_size: "3GB", Index_scans: 2, diff --git a/internal/inspect/vacuum_stats/vacuum_stats.go b/internal/inspect/vacuum_stats/vacuum_stats.go index dc9326d79..b0ce400f8 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.go +++ b/internal/inspect/vacuum_stats/vacuum_stats.go @@ -20,8 +20,7 @@ import ( var VacuumStatsQuery string type Result struct { - Schema string - Table string + Name string Last_vacuum string Last_autovacuum string Rowcount string @@ -45,10 +44,10 @@ func Run(ctx context.Context, config pgconn.Config, fsys afero.Fs, options ...fu return err } - table := "|Schema|Table|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|-|\n" + table := "|Name|Last Vacuum|Last Auto Vacuum|Row count|Dead row count|Expect autovacuum?\n|-|-|-|-|-|-|\n" for _, r := range result { rowcount := strings.Replace(r.Rowcount, "-1", "No stats", 1) - table += fmt.Sprintf("|`%s`|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Schema, r.Table, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) + table += fmt.Sprintf("|`%s`|%s|%s|`%s`|`%s`|`%s`|\n", r.Name, r.Last_vacuum, r.Last_autovacuum, rowcount, r.Dead_rowcount, r.Expect_autovacuum) } return list.RenderTable(table) } diff --git a/internal/inspect/vacuum_stats/vacuum_stats.sql b/internal/inspect/vacuum_stats/vacuum_stats.sql index 707847497..534fa4e64 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats.sql +++ b/internal/inspect/vacuum_stats/vacuum_stats.sql @@ -20,8 +20,7 @@ WITH table_opts AS ( table_opts ) SELECT - vacuum_settings.nspname AS schema, - vacuum_settings.relname AS table, + FORMAT('%I.%I', vacuum_settings.nspname, vacuum_settings.relname) AS name, coalesce(to_char(psut.last_vacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_vacuum, coalesce(to_char(psut.last_autovacuum, 'YYYY-MM-DD HH24:MI'), '') AS last_autovacuum, to_char(pg_class.reltuples, '9G999G999G999') AS rowcount, diff --git a/internal/inspect/vacuum_stats/vacuum_stats_test.go b/internal/inspect/vacuum_stats/vacuum_stats_test.go index 0d3cbec10..7872567b3 100644 --- a/internal/inspect/vacuum_stats/vacuum_stats_test.go +++ b/internal/inspect/vacuum_stats/vacuum_stats_test.go @@ -29,8 +29,7 @@ func TestVacuumCommand(t *testing.T) { defer conn.Close(t) conn.Query(VacuumStatsQuery, reset.LikeEscapeSchema(utils.InternalSchemas)). Reply("SELECT 1", Result{ - Schema: "test_schema", - Table: "test_table", + Name: "test_schema.test_table", Last_vacuum: "2021-01-01 00:00:00", Last_autovacuum: "2021-01-01 00:00:00", Rowcount: "1000", diff --git a/tools/inspect_rules.toml b/tools/inspect_rules.toml new file mode 100644 index 000000000..5f47c6e25 --- /dev/null +++ b/tools/inspect_rules.toml @@ -0,0 +1,43 @@ +# Rules to validate CSV report files + +[[rule]] +query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE age > '00:02:00'" +name = "No old locks" +pass = "✔" +fail = "There is at least one lock older than 2 minutes" + +[[rule]] +query = "SELECT LISTAGG(stmt, ',') AS match FROM `locks.csv` WHERE granted = 'f'" +name = "No ungranted locks" +pass = "✔" +fail = "There is at least one ungranted lock" + +[[rule]] +query = "SELECT LISTAGG(index, ',') AS match FROM `unused_indexes.csv`" +pass = "✔" +fail = "There is at least one unused index" +name = "No unused indexes" + +[[rule]] +name = "Check cache hit is within acceptable bounds" +query = "SELECT LISTAGG(name, ',') AS match FROM `db_stats.csv` WHERE index_hit_rate < 0.94 OR table_hit_rate < 0.94" +pass = "✔" +fail = "There is a cache hit ratio (table or index) below 94%" + +[[rule]] +query = "SELECT LISTAGG(t.name, ',') AS match FROM `table_stats.csv` t WHERE t.seq_scans > t.estimated_row_count * 0.1 AND t.estimated_row_count > 1000;" +name = "No large tables with sequential scans more than 10% of rows" +pass = "✔" +fail = "At least one table is showing sequential scans more than 10% of total row count" + +[[rule]] +query = "SELECT LISTAGG(s.tbl, ',') AS match FROM `vacuum_stats.csv` s WHERE s.expect_autovacuum = 'yes' and s.rowcount > 1000;" +name = "No large tables waiting on autovacuum" +pass = "✔" +fail = "At least one table is waiting on autovacuum" + +[[rule]] +query = "SELECT LISTAGG(s.name, ',') AS match FROM `vacuum_stats.csv` s WHERE s.rowcount > 0 AND (s.last_autovacuum = '' OR s.last_vacuum = '');" +name = "No tables yet to be vacuumed" +pass = "✔" +fail = "At least one table has never had autovacuum or vacuum run on it"