Refine Inspect queries and add CSVQ support for rule parsing

cmd/inspect.go

@@ -1,37 +1,36 @@
 package cmd
 
 import (
+	"database/sql"
 	"fmt"
 	"os"
 	"os/signal"
 	"path/filepath"
+	"time"
 
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/supabase/cli/internal/inspect/bloat"
 	"github.com/supabase/cli/internal/inspect/blocking"
-	"github.com/supabase/cli/internal/inspect/cache"
+	"github.com/supabase/cli/internal/inspect/role_stats"
+
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/internal/utils/flags"
 
+	_ "github.com/mithrandie/csvq-driver"
+	"github.com/pelletier/go-toml/v2"
 	"github.com/supabase/cli/internal/inspect"
 	"github.com/supabase/cli/internal/inspect/calls"
-	"github.com/supabase/cli/internal/inspect/index_sizes"
-	"github.com/supabase/cli/internal/inspect/index_usage"
+	"github.com/supabase/cli/internal/inspect/db_stats"
+	"github.com/supabase/cli/internal/inspect/index_stats"
 	"github.com/supabase/cli/internal/inspect/locks"
 	"github.com/supabase/cli/internal/inspect/long_running_queries"
 	"github.com/supabase/cli/internal/inspect/outliers"
 	"github.com/supabase/cli/internal/inspect/replication_slots"
-	"github.com/supabase/cli/internal/inspect/role_configs"
-	"github.com/supabase/cli/internal/inspect/role_connections"
-	"github.com/supabase/cli/internal/inspect/seq_scans"
-	"github.com/supabase/cli/internal/inspect/table_index_sizes"
-	"github.com/supabase/cli/internal/inspect/table_record_counts"
-	"github.com/supabase/cli/internal/inspect/table_sizes"
-	"github.com/supabase/cli/internal/inspect/total_index_size"
-	"github.com/supabase/cli/internal/inspect/total_table_sizes"
-	"github.com/supabase/cli/internal/inspect/unused_indexes"
+	"github.com/supabase/cli/internal/inspect/table_stats"
+
 	"github.com/supabase/cli/internal/inspect/vacuum_stats"
+	"github.com/supabase/cli/internal/migration/list"
 )
 
 var (
@@ -51,11 +50,11 @@ var (
 		},
 	}
 
-	inspectCacheHitCmd = &cobra.Command{
-		Use:   "cache-hit",
-		Short: "Show cache hit rates for tables and indices",
+	inspectDBStatsCmd = &cobra.Command{
+		Use:   "db-stats",
+		Short: "Show stats such as cache hit rates, total sizes, and WAL size",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return cache.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
@@ -67,14 +66,6 @@ var (
 		},
 	}
 
-	inspectIndexUsageCmd = &cobra.Command{
-		Use:   "index-usage",
-		Short: "Show information about the efficiency of indexes",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			return index_usage.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
-		},
-	}
-
 	inspectLocksCmd = &cobra.Command{
 		Use:   "locks",
 		Short: "Show queries which have taken out an exclusive lock on a relation",
@@ -107,107 +98,117 @@ var (
 		},
 	}
 
-	inspectTotalIndexSizeCmd = &cobra.Command{
-		Use:   "total-index-size",
-		Short: "Show total size of all indexes",
+	inspectIndexStatsCmd = &cobra.Command{
+		Use:   "index-stats",
+		Short: "Show combined index size, usage percent, scan counts, and unused status",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return total_index_size.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectIndexSizesCmd = &cobra.Command{
-		Use:   "index-sizes",
-		Short: "Show index sizes of individual indexes",
+	inspectLongRunningQueriesCmd = &cobra.Command{
+		Use:   "long-running-queries",
+		Short: "Show currently running queries running for longer than 5 minutes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableSizesCmd = &cobra.Command{
-		Use:   "table-sizes",
-		Short: "Show table sizes of individual tables without their index sizes",
+	inspectBloatCmd = &cobra.Command{
+		Use:   "bloat",
+		Short: "Estimates space allocated to a relation that is full of dead tuples",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableIndexSizesCmd = &cobra.Command{
-		Use:   "table-index-sizes",
-		Short: "Show index sizes of individual tables",
+	inspectRoleStatsCmd = &cobra.Command{
+		Use:   "role-stats",
+		Short: "Show information about roles on the database",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTotalTableSizesCmd = &cobra.Command{
-		Use:   "total-table-sizes",
-		Short: "Show total table sizes, including table index sizes",
+	inspectVacuumStatsCmd = &cobra.Command{
+		Use:   "vacuum-stats",
+		Short: "Show statistics related to vacuum operations per table",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return total_table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectUnusedIndexesCmd = &cobra.Command{
-		Use:   "unused-indexes",
-		Short: "Show indexes with low usage",
+	inspectTableStatsCmd = &cobra.Command{
+		Use:   "table-stats",
+		Short: "Show combined table size, index size, and estimated row count",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return unused_indexes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectSeqScansCmd = &cobra.Command{
-		Use:   "seq-scans",
-		Short: "Show number of sequential scans recorded against all tables",
+	// DEPRECATED
+
+	inspectCacheHitCmd = &cobra.Command{
+		Use:   "cache-hit",
+		Short: "DEPRECATED: use db-stats instead. Show cache hit rates for tables and indices",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return seq_scans.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectLongRunningQueriesCmd = &cobra.Command{
-		Use:   "long-running-queries",
-		Short: "Show currently running queries running for longer than 5 minutes",
+	inspectIndexUsageCmd = &cobra.Command{
+		Use:   "index-usage",
+		Short: "DEPRECATED: use index-stats instead. Show information about the efficiency of indexes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableRecordCountsCmd = &cobra.Command{
-		Use:   "table-record-counts",
-		Short: "Show estimated number of rows per table",
+	inspectTotalIndexSizeCmd = &cobra.Command{
+		Use:   "total-index-size",
+		Short: "DEPRECATED: use index-stats instead. Show total size of all indexes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_record_counts.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectBloatCmd = &cobra.Command{
-		Use:   "bloat",
-		Short: "Estimates space allocated to a relation that is full of dead tuples",
+	inspectTableSizesCmd = &cobra.Command{
+		Use:   "table-sizes",
+		Short: "DEPRECATED: use table-stats instead. Show table sizes of individual tables without their index sizes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectVacuumStatsCmd = &cobra.Command{
-		Use:   "vacuum-stats",
-		Short: "Show statistics related to vacuum operations per table",
+	inspectTableIndexSizesCmd = &cobra.Command{
+		Use:   "table-index-sizes",
+		Short: "DEPRECATED: use table-stats instead. Show index sizes of individual tables",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectRoleConfigsCmd = &cobra.Command{
-		Use:   "role-configs",
-		Short: "Show configuration settings for database roles when they have been modified",
+	inspectTotalTableSizesCmd = &cobra.Command{
+		Use:   "total-table-sizes",
+		Short: "DEPRECATED: use table-stats instead. Show total table sizes, including table index sizes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return role_configs.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectRoleConnectionsCmd = &cobra.Command{
-		Use:   "role-connections",
-		Short: "Show number of active connections for all database roles",
+	inspectUnusedIndexesCmd = &cobra.Command{
+		Use:   "unused-indexes",
+		Short: "DEPRECATED: use index-stats instead. Show indexes with low usage",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return role_connections.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectSeqScansCmd = &cobra.Command{
+		Use:   "seq-scans",
+		Short: "DEPRECATED: use index-stats instead. Show number of sequential scans recorded against all tables",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
@@ -218,48 +219,111 @@ var (
 		Short: "Generate a CSV output for all inspect commands",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
-			if len(outputDir) == 0 {
-				defaultPath := filepath.Join(utils.CurrentDirAbs, "report")
-				title := fmt.Sprintf("Enter a directory to save output files (or leave blank to use %s): ", utils.Bold(defaultPath))
-				if dir, err := utils.NewConsole().PromptText(ctx, title); err != nil {
-					return err
-				} else if len(dir) == 0 {
-					outputDir = defaultPath
-				}
+			if err := inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()); err != nil {
+				return err
 			}
-			return inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs())
+			return printReportSummary(outputDir)
 		},
 	}
 )
 
+// Load rules file at runtime (tools/inspect_rules.toml)
+
+// Rule defines a validation rule for a CSV file
+type Rule struct {
+	Query string `toml:"query"`
+	Pass  string `toml:"pass"`
+	Fail  string `toml:"fail"`
+	Name  string `toml:"name"`
+}
+
+// Config holds all rules
+type Config struct {
+	Rules []Rule `toml:"rule"`
+}
+
 func init() {
 	inspectFlags := inspectCmd.PersistentFlags()
 	inspectFlags.String("db-url", "", "Inspect the database specified by the connection string (must be percent-encoded).")
 	inspectFlags.Bool("linked", true, "Inspect the linked project.")
 	inspectFlags.Bool("local", false, "Inspect the local database.")
 	inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local")
-	inspectDBCmd.AddCommand(inspectCacheHitCmd)
 	inspectDBCmd.AddCommand(inspectReplicationSlotsCmd)
-	inspectDBCmd.AddCommand(inspectIndexUsageCmd)
+	inspectDBCmd.AddCommand(inspectIndexStatsCmd)
 	inspectDBCmd.AddCommand(inspectLocksCmd)
 	inspectDBCmd.AddCommand(inspectBlockingCmd)
 	inspectDBCmd.AddCommand(inspectOutliersCmd)
 	inspectDBCmd.AddCommand(inspectCallsCmd)
-	inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd)
-	inspectDBCmd.AddCommand(inspectIndexSizesCmd)
-	inspectDBCmd.AddCommand(inspectTableSizesCmd)
-	inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
-	inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
-	inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
-	inspectDBCmd.AddCommand(inspectSeqScansCmd)
 	inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd)
-	inspectDBCmd.AddCommand(inspectTableRecordCountsCmd)
 	inspectDBCmd.AddCommand(inspectBloatCmd)
 	inspectDBCmd.AddCommand(inspectVacuumStatsCmd)
-	inspectDBCmd.AddCommand(inspectRoleConfigsCmd)
-	inspectDBCmd.AddCommand(inspectRoleConnectionsCmd)
+	inspectDBCmd.AddCommand(inspectTableStatsCmd)
+	inspectDBCmd.AddCommand(inspectRoleStatsCmd)
+	inspectDBCmd.AddCommand(inspectDBStatsCmd)
+	inspectDBCmd.AddCommand(inspectCacheHitCmd)
+	inspectDBCmd.AddCommand(inspectIndexUsageCmd)
+	inspectDBCmd.AddCommand(inspectSeqScansCmd)
+	inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
+	inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
+	inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
+	inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd)
+	inspectDBCmd.AddCommand(inspectTableSizesCmd)
 	inspectCmd.AddCommand(inspectDBCmd)
 	reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in")
 	inspectCmd.AddCommand(reportCmd)
 	rootCmd.AddCommand(inspectCmd)
 }
+
+func printReportSummary(outDir string) error {
+	// point to the date-based subdirectory
+	date := time.Now().Format("2006-01-02")
+	outDir = filepath.Join(outDir, date)
+	// Load rules from tools/inspect_rules.toml
+	data, err := os.ReadFile(filepath.Join(utils.CurrentDirAbs, "tools", "inspect_rules.toml"))
+	if err != nil {
+		return err
+	}
+	var cfg Config
+	if err := toml.Unmarshal(data, &cfg); err != nil {
+		return err
+	}
+	// Open csvq database rooted at the output directory
+	db, err := sql.Open("csvq", outDir)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// Build report summary table
+	table := "RULE|STATUS|MATCHES\n|-|-|-|\n"
+
+	// find matching rule
+	var status string
+	for i := range cfg.Rules {
+		r := &cfg.Rules[i]
+		name := r.Name
+
+		row := db.QueryRow(r.Query)
+		var match sql.NullString
+
+		if err := row.Scan(&match); err != nil {
+			if err == sql.ErrNoRows {
+				status = r.Pass
+			} else {
+				status = err.Error()
+			}
+		} else {
+			if !match.Valid || match.String == "" {
+				status = r.Pass
+			} else {
+				status = r.Fail
+			}
+		}
+		matchStr := "-"
+		if match.Valid {
+			matchStr = match.String
+		}
+		table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, matchStr)
+	}
+	return list.RenderTable(table)
+}