Skip to content

Refine Inspect queries and add CSVQ support for rule parsing #3595

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
+528 −1,044
Open

Refine Inspect queries and add CSVQ support for rule parsing #3595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
258 changes: 161 additions & 97 deletions cmd/inspect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,36 @@
package cmd

import (
"database/sql"
"fmt"
"os"
"os/signal"
"path/filepath"
"time"

"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/supabase/cli/internal/inspect/bloat"
"github.com/supabase/cli/internal/inspect/blocking"
"github.com/supabase/cli/internal/inspect/cache"
"github.com/supabase/cli/internal/inspect/role_stats"

"github.com/supabase/cli/internal/utils"
"github.com/supabase/cli/internal/utils/flags"

_ "github.com/mithrandie/csvq-driver"
"github.com/pelletier/go-toml/v2"
"github.com/supabase/cli/internal/inspect"
"github.com/supabase/cli/internal/inspect/calls"
"github.com/supabase/cli/internal/inspect/index_sizes"
"github.com/supabase/cli/internal/inspect/index_usage"
"github.com/supabase/cli/internal/inspect/db_stats"
"github.com/supabase/cli/internal/inspect/index_stats"
"github.com/supabase/cli/internal/inspect/locks"
"github.com/supabase/cli/internal/inspect/long_running_queries"
"github.com/supabase/cli/internal/inspect/outliers"
"github.com/supabase/cli/internal/inspect/replication_slots"
"github.com/supabase/cli/internal/inspect/role_configs"
"github.com/supabase/cli/internal/inspect/role_connections"
"github.com/supabase/cli/internal/inspect/seq_scans"
"github.com/supabase/cli/internal/inspect/table_index_sizes"
"github.com/supabase/cli/internal/inspect/table_record_counts"
"github.com/supabase/cli/internal/inspect/table_sizes"
"github.com/supabase/cli/internal/inspect/total_index_size"
"github.com/supabase/cli/internal/inspect/total_table_sizes"
"github.com/supabase/cli/internal/inspect/unused_indexes"
"github.com/supabase/cli/internal/inspect/table_stats"

"github.com/supabase/cli/internal/inspect/vacuum_stats"
"github.com/supabase/cli/internal/migration/list"
)

var (
Expand All @@ -51,11 +50,11 @@ var (
},
}

inspectCacheHitCmd = &cobra.Command{
Use: "cache-hit",
Short: "Show cache hit rates for tables and indices",
inspectDBStatsCmd = &cobra.Command{
Use: "db-stats",
Short: "Show stats such as cache hit rates, total sizes, and WAL size",
RunE: func(cmd *cobra.Command, args []string) error {
return cache.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

Expand All @@ -67,14 +66,6 @@ var (
},
}

inspectIndexUsageCmd = &cobra.Command{
Use: "index-usage",
Short: "Show information about the efficiency of indexes",
RunE: func(cmd *cobra.Command, args []string) error {
return index_usage.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectLocksCmd = &cobra.Command{
Use: "locks",
Short: "Show queries which have taken out an exclusive lock on a relation",
Expand Down Expand Up @@ -107,107 +98,117 @@ var (
},
}

inspectTotalIndexSizeCmd = &cobra.Command{
Use: "total-index-size",
Short: "Show total size of all indexes",
inspectIndexStatsCmd = &cobra.Command{
Use: "index-stats",
Short: "Show combined index size, usage percent, scan counts, and unused status",
RunE: func(cmd *cobra.Command, args []string) error {
return total_index_size.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectIndexSizesCmd = &cobra.Command{
Use: "index-sizes",
Short: "Show index sizes of individual indexes",
inspectLongRunningQueriesCmd = &cobra.Command{
Use: "long-running-queries",
Short: "Show currently running queries running for longer than 5 minutes",
RunE: func(cmd *cobra.Command, args []string) error {
return index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectTableSizesCmd = &cobra.Command{
Use: "table-sizes",
Short: "Show table sizes of individual tables without their index sizes",
inspectBloatCmd = &cobra.Command{
Use: "bloat",
Short: "Estimates space allocated to a relation that is full of dead tuples",
RunE: func(cmd *cobra.Command, args []string) error {
return table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectTableIndexSizesCmd = &cobra.Command{
Use: "table-index-sizes",
Short: "Show index sizes of individual tables",
inspectRoleStatsCmd = &cobra.Command{
Use: "role-stats",
Short: "Show information about roles on the database",
RunE: func(cmd *cobra.Command, args []string) error {
return table_index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectTotalTableSizesCmd = &cobra.Command{
Use: "total-table-sizes",
Short: "Show total table sizes, including table index sizes",
inspectVacuumStatsCmd = &cobra.Command{
Use: "vacuum-stats",
Short: "Show statistics related to vacuum operations per table",
RunE: func(cmd *cobra.Command, args []string) error {
return total_table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectUnusedIndexesCmd = &cobra.Command{
Use: "unused-indexes",
Short: "Show indexes with low usage",
inspectTableStatsCmd = &cobra.Command{
Use: "table-stats",
Short: "Show combined table size, index size, and estimated row count",
RunE: func(cmd *cobra.Command, args []string) error {
return unused_indexes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectSeqScansCmd = &cobra.Command{
Use: "seq-scans",
Short: "Show number of sequential scans recorded against all tables",
// DEPRECATED

inspectCacheHitCmd = &cobra.Command{
Use: "cache-hit",
Short: "DEPRECATED: use db-stats instead. Show cache hit rates for tables and indices",
RunE: func(cmd *cobra.Command, args []string) error {
return seq_scans.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectLongRunningQueriesCmd = &cobra.Command{
Use: "long-running-queries",
Short: "Show currently running queries running for longer than 5 minutes",
inspectIndexUsageCmd = &cobra.Command{
Use: "index-usage",
Short: "DEPRECATED: use index-stats instead. Show information about the efficiency of indexes",
RunE: func(cmd *cobra.Command, args []string) error {
return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectTableRecordCountsCmd = &cobra.Command{
Use: "table-record-counts",
Short: "Show estimated number of rows per table",
inspectTotalIndexSizeCmd = &cobra.Command{
Use: "total-index-size",
Short: "DEPRECATED: use index-stats instead. Show total size of all indexes",
RunE: func(cmd *cobra.Command, args []string) error {
return table_record_counts.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectBloatCmd = &cobra.Command{
Use: "bloat",
Short: "Estimates space allocated to a relation that is full of dead tuples",
inspectTableSizesCmd = &cobra.Command{
Use: "table-sizes",
Short: "DEPRECATED: use table-stats instead. Show table sizes of individual tables without their index sizes",
RunE: func(cmd *cobra.Command, args []string) error {
return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectVacuumStatsCmd = &cobra.Command{
Use: "vacuum-stats",
Short: "Show statistics related to vacuum operations per table",
inspectTableIndexSizesCmd = &cobra.Command{
Use: "table-index-sizes",
Short: "DEPRECATED: use table-stats instead. Show index sizes of individual tables",
RunE: func(cmd *cobra.Command, args []string) error {
return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectRoleConfigsCmd = &cobra.Command{
Use: "role-configs",
Short: "Show configuration settings for database roles when they have been modified",
inspectTotalTableSizesCmd = &cobra.Command{
Use: "total-table-sizes",
Short: "DEPRECATED: use table-stats instead. Show total table sizes, including table index sizes",
RunE: func(cmd *cobra.Command, args []string) error {
return role_configs.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectRoleConnectionsCmd = &cobra.Command{
Use: "role-connections",
Short: "Show number of active connections for all database roles",
inspectUnusedIndexesCmd = &cobra.Command{
Use: "unused-indexes",
Short: "DEPRECATED: use index-stats instead. Show indexes with low usage",
RunE: func(cmd *cobra.Command, args []string) error {
return role_connections.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

inspectSeqScansCmd = &cobra.Command{
Use: "seq-scans",
Short: "DEPRECATED: use index-stats instead. Show number of sequential scans recorded against all tables",
RunE: func(cmd *cobra.Command, args []string) error {
return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
},
}

Expand All @@ -218,48 +219,111 @@ var (
Short: "Generate a CSV output for all inspect commands",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if len(outputDir) == 0 {
defaultPath := filepath.Join(utils.CurrentDirAbs, "report")
title := fmt.Sprintf("Enter a directory to save output files (or leave blank to use %s): ", utils.Bold(defaultPath))
if dir, err := utils.NewConsole().PromptText(ctx, title); err != nil {
return err
} else if len(dir) == 0 {
outputDir = defaultPath
}
if err := inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs()); err != nil {
return err
}
return inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs())
return printReportSummary(outputDir)
},
}
)

// Load rules file at runtime (tools/inspect_rules.toml)

// Rule defines a validation rule for a CSV file
type Rule struct {
Query string `toml:"query"`
Pass string `toml:"pass"`
Fail string `toml:"fail"`
Name string `toml:"name"`
}

// Config holds all rules
type Config struct {
Rules []Rule `toml:"rule"`
}

func init() {
inspectFlags := inspectCmd.PersistentFlags()
inspectFlags.String("db-url", "", "Inspect the database specified by the connection string (must be percent-encoded).")
inspectFlags.Bool("linked", true, "Inspect the linked project.")
inspectFlags.Bool("local", false, "Inspect the local database.")
inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local")
inspectDBCmd.AddCommand(inspectCacheHitCmd)
inspectDBCmd.AddCommand(inspectReplicationSlotsCmd)
inspectDBCmd.AddCommand(inspectIndexUsageCmd)
inspectDBCmd.AddCommand(inspectIndexStatsCmd)
inspectDBCmd.AddCommand(inspectLocksCmd)
inspectDBCmd.AddCommand(inspectBlockingCmd)
inspectDBCmd.AddCommand(inspectOutliersCmd)
inspectDBCmd.AddCommand(inspectCallsCmd)
inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd)
inspectDBCmd.AddCommand(inspectIndexSizesCmd)
inspectDBCmd.AddCommand(inspectTableSizesCmd)
inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
inspectDBCmd.AddCommand(inspectSeqScansCmd)
inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd)
inspectDBCmd.AddCommand(inspectTableRecordCountsCmd)
inspectDBCmd.AddCommand(inspectBloatCmd)
inspectDBCmd.AddCommand(inspectVacuumStatsCmd)
inspectDBCmd.AddCommand(inspectRoleConfigsCmd)
inspectDBCmd.AddCommand(inspectRoleConnectionsCmd)
inspectDBCmd.AddCommand(inspectTableStatsCmd)
inspectDBCmd.AddCommand(inspectRoleStatsCmd)
inspectDBCmd.AddCommand(inspectDBStatsCmd)
inspectDBCmd.AddCommand(inspectCacheHitCmd)
inspectDBCmd.AddCommand(inspectIndexUsageCmd)
inspectDBCmd.AddCommand(inspectSeqScansCmd)
inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd)
inspectDBCmd.AddCommand(inspectTableSizesCmd)
inspectCmd.AddCommand(inspectDBCmd)
reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in")
inspectCmd.AddCommand(reportCmd)
rootCmd.AddCommand(inspectCmd)
}

func printReportSummary(outDir string) error {
// point to the date-based subdirectory
date := time.Now().Format("2006-01-02")
outDir = filepath.Join(outDir, date)
// Load rules from tools/inspect_rules.toml
data, err := os.ReadFile(filepath.Join(utils.CurrentDirAbs, "tools", "inspect_rules.toml"))
if err != nil {
return err
}
var cfg Config
if err := toml.Unmarshal(data, &cfg); err != nil {
return err
}
// Open csvq database rooted at the output directory
db, err := sql.Open("csvq", outDir)
if err != nil {
return err
}
defer db.Close()

// Build report summary table
table := "RULE|STATUS|MATCHES\n|-|-|-|\n"

// find matching rule
var status string
for i := range cfg.Rules {
r := &cfg.Rules[i]
name := r.Name

row := db.QueryRow(r.Query)
var match sql.NullString

if err := row.Scan(&match); err != nil {
if err == sql.ErrNoRows {
status = r.Pass
} else {
status = err.Error()
}
} else {
if !match.Valid || match.String == "" {
status = r.Pass
} else {
status = r.Fail
}
}
matchStr := "-"
if match.Valid {
matchStr = match.String
}
table += fmt.Sprintf("|`%s`|`%s`|`%s`|\n", name, status, matchStr)
}
return list.RenderTable(table)
}
Loading