From a0a33657a0f727d580d1f1120f638a518ca3db1a Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Fri, 1 Jul 2022 00:18:19 +1200 Subject: [PATCH 1/4] Avoid dumping sqlite system tables into schema sqlite includes some of its system tables in the .schema output by default, which means it cannot be imported back directly into sqlite (as LoadSchema attempts to do), as you're not allowed to recreate these tables. The -nosys option can be used to prevent this behaviour. Particular issues have been seen with the sqlite_sequence table, which appears if you use an AUTOINCREMENT column as an ID for example. --- dialect_sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dialect_sqlite.go b/dialect_sqlite.go index 35647c3bb..d98b47703 100644 --- a/dialect_sqlite.go +++ b/dialect_sqlite.go @@ -229,7 +229,7 @@ func (m *sqlite) FizzTranslator() fizz.Translator { } func (m *sqlite) DumpSchema(w io.Writer) error { - cmd := exec.Command("sqlite3", m.Details().Database, ".schema") + cmd := exec.Command("sqlite3", m.Details().Database, ".schema --nosys") return genericDumpSchema(m.Details(), cmd, w) } From c21ba468d86370ca7e461847f308c7914161c496 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Fri, 1 Jul 2022 00:51:39 +1200 Subject: [PATCH 2/4] Improve error reporting of sqlite3 LoadSchema --- dialect_sqlite.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dialect_sqlite.go b/dialect_sqlite.go index d98b47703..9a4076f5b 100644 --- a/dialect_sqlite.go +++ b/dialect_sqlite.go @@ -235,9 +235,10 @@ func (m *sqlite) DumpSchema(w io.Writer) error { func (m *sqlite) LoadSchema(r io.Reader) error { cmd := exec.Command("sqlite3", m.ConnectionDetails.Database) + cmd.Stderr = os.Stderr in, err := cmd.StdinPipe() if err != nil { - return err + return fmt.Errorf("could not open stdin to SQLite database %s: %w", m.ConnectionDetails.Database, err) } go func() { defer in.Close() @@ -246,12 +247,12 @@ func (m *sqlite) LoadSchema(r io.Reader) error { log(logging.SQL, strings.Join(cmd.Args, " ")) err = cmd.Start() if err != nil { - return err + return fmt.Errorf("could not load schema to SQLite database %s: %w", m.ConnectionDetails.Database, err) } err = cmd.Wait() if err != nil { - return err + return fmt.Errorf("command failure loading schema to SQLite database %s: %w", m.ConnectionDetails.Database, err) } log(logging.Info, "loaded schema for %s", m.Details().Database) From 89593a778701935bdbd2479a9dfd1b52a4c4d5a4 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Sat, 2 Jul 2022 22:18:56 +1200 Subject: [PATCH 3/4] Add missing logs in the assoc eager creation path --- executors.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/executors.go b/executors.go index 37b03b355..eba3ddeb6 100644 --- a/executors.go +++ b/executors.go @@ -286,6 +286,7 @@ func (c *Connection) Create(model interface{}, excludeColumns ...string) error { } stm := after[index].AfterProcess() if c.TX != nil && !stm.Empty() { + log(logging.SQL, stm.Statement, stm.Args) _, err := c.TX.Exec(c.Dialect.TranslateSQL(stm.Statement), stm.Args...) if err != nil { return err @@ -297,6 +298,7 @@ func (c *Connection) Create(model interface{}, excludeColumns ...string) error { for index := range stms { statements := stms[index].Statements() for _, stm := range statements { + log(logging.SQL, stm.Statement, stm.Args) if c.TX != nil { _, err := c.TX.Exec(c.Dialect.TranslateSQL(stm.Statement), stm.Args...) if err != nil { From 7f904a8d88ca7d6574ccca37413f297275eb6ccf Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Sun, 29 Jan 2023 19:33:02 +1300 Subject: [PATCH 4/4] Add testcase for sqlite dump with -nosys --- dialect_sqlite_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dialect_sqlite_test.go b/dialect_sqlite_test.go index ac5db21ab..6e8f56f6d 100644 --- a/dialect_sqlite_test.go +++ b/dialect_sqlite_test.go @@ -4,6 +4,7 @@ package pop import ( + "bytes" "fmt" "path/filepath" "testing" @@ -196,3 +197,22 @@ func TestSqlite_NewDriver(t *testing.T) { _, err := newSQLiteDriver() require.NoError(t, err) } + +func TestSqlite_NoDumpSysTables(t *testing.T) { + r := require.New(t) + + dir := t.TempDir() + cd := &ConnectionDetails{Dialect: "sqlite", Database: filepath.Join(dir, "testdb.sqlite")} + c, err := NewConnection(cd) + r.NoError(err) + r.NoError(c.Open()) + + r.NoError(c.RawQuery("CREATE TABLE aitest (id integer primary key autoincrement);").Exec()) + + + schema := new(bytes.Buffer) + r.NoError(c.Dialect.DumpSchema(schema)) + + r.Contains(schema.String(), "CREATE TABLE aitest") + r.NotContains(schema.String(), "CREATE TABLE sqlite_sequence") +}