From 342b2b8594c97c7eeb1d6c60be9b207f1bb6bc2a Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 14:23:12 +0100 Subject: [PATCH 01/77] [swarmcd] add `stack_test.go` and first unit test --- swarmcd/stack_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 swarmcd/stack_test.go diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go new file mode 100644 index 0000000..8d347a1 --- /dev/null +++ b/swarmcd/stack_test.go @@ -0,0 +1,49 @@ +package swarmcd + +import ( + "os" + "path" + "testing" +) + +func setupSwarmStackWithValues(t *testing.T, values string) *swarmStack { + t.Helper() + tempDir := t.TempDir() + valuesFilePath := path.Join(tempDir, "values.yaml") + if err := os.WriteFile(valuesFilePath, []byte(values), 0644); err != nil { + t.Fatalf("Failed to write values file: %v", err) + } + + return &swarmStack{ + name: "testStack", + repo: &stackRepo{path: tempDir}, + valuesFile: "values.yaml", + discoverSecrets: false, + } +} + +func TestRenderComposeTemplate(t *testing.T) { + tests := []struct { + name string + values string + template string + expected string + }{ + {"Basic", "key: value", "Service: {{ .Values.key }}", "Service: value"}, + {"Nested", "nested:\n key: nestedValue", "Service: {{ .Values.nested.key }}", "Service: nestedValue"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + swarm := setupSwarmStackWithValues(t, tt.values) + result, err := swarm.renderComposeTemplate([]byte(tt.template)) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if string(result) != tt.expected { + t.Errorf("Expected %q, got %q", tt.expected, string(result)) + } + }) + } +} From c10f913e1b16c19eacd856ff15394b67accfefdb Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 10:04:06 +0100 Subject: [PATCH 02/77] Check git revision while updating the stack. --- swarmcd/stack.go | 5 +++++ swarmcd/swarmcd.go | 14 +++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 2b11584..7eef779 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,6 +49,11 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) + if stackStatus[swarmStack.name].Revision == revision { + logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date", swarmStack.name)) + return revision, err + } + log.Debug("reading stack file...") stackBytes, err := swarmStack.readStack() if err != nil { diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 8163cf6..ff449ba 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -13,7 +13,7 @@ func Run() { logger.Info("starting SwarmCD") for { var waitGroup sync.WaitGroup - logger.Info("updating stacks...") + logger.Info("updating stacks...") for _, swarmStack := range stacks { waitGroup.Add(1) go updateStackThread(swarmStack, &waitGroup) @@ -32,7 +32,13 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { logger.Info(fmt.Sprintf("updating %s stack", swarmStack.name)) revision, err := swarmStack.updateStack() - if err != nil{ + + if stackStatus[swarmStack.name].Revision == revision { + logger.Info(fmt.Sprintf("%s stack is already up-to-date", swarmStack.name)) + return + } + + if err != nil { stackStatus[swarmStack.name].Error = err.Error() logger.Error(err.Error()) return @@ -43,8 +49,6 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { logger.Info(fmt.Sprintf("done updating %s stack", swarmStack.name)) } - - func GetStackStatus() map[string]*StackStatus { return stackStatus -} \ No newline at end of file +} From c6f4a9856d822a71a5e2ac34aadf55eeba266b04 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 11:35:16 +0100 Subject: [PATCH 03/77] add sql_db for persisting git revisions --- README.md | 19 +++++++++++++++++++ go.mod | 9 ++++++++- go.sum | 16 ++++++++++++++++ swarmcd/sql_db.go | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 swarmcd/sql_db.go diff --git a/README.md b/README.md index f648395..a6ddae1 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,17 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro + - swarmcd_data:/data # Mount the volume for SQLite + +# Ensure that swarmdc_data persists across container restarts. +volumes: + swarmcd_data: ``` Run this on a swarm manager node: ```bash +docker volume create swarmcd_data docker stack deploy --compose-file docker-compose.yaml swarm-cd ``` @@ -87,6 +93,7 @@ and set the environment variable SOPS `SOPS_AGE_KEY_FILE` to the path of the key file. See the following docker-compose example ```yaml +# docker-compose.yaml version: '3.7' services: swarm-cd: @@ -104,6 +111,12 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro + - swarmcd_data:/data # Mount the volume for SQLite + +# Ensure that swarmdc_data persists across container restarts. +volumes: + swarmcd_data: + secrets: age: file: age.key @@ -214,9 +227,15 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro + - swarmcd_data:/data # Mount the volume for SQLite secrets: - source: docker-config target: /root/.docker/config.json + +# Ensure that swarmdc_data persists across container restarts. +volumes: + swarmcd_data: + secrets: docker-config: file: docker-config.json diff --git a/go.mod b/go.mod index 4e387ea..b736162 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect @@ -94,8 +95,10 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -119,6 +122,10 @@ require ( google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.61.13 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.8.2 // indirect + modernc.org/sqlite v1.36.0 // indirect ) require ( @@ -202,7 +209,7 @@ require ( golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.30.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index a9005fe..57cb633 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= @@ -425,6 +427,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= @@ -480,6 +484,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -673,6 +679,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= @@ -767,5 +775,13 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8= +modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI= +modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU= +modernc.org/sqlite v1.36.0 h1:EQXNRn4nIS+gfsKeUTymHIz1waxuv5BzU7558dHSfH8= +modernc.org/sqlite v1.36.0/go.mod h1:7MPwH7Z6bREicF9ZVUR78P1IKuxfZ8mRIDHD0iD+8TU= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/swarmcd/sql_db.go b/swarmcd/sql_db.go new file mode 100644 index 0000000..92fe557 --- /dev/null +++ b/swarmcd/sql_db.go @@ -0,0 +1,39 @@ +package swarmcd + +import ( + "database/sql" + _ "modernc.org/sqlite" +) + +const dbFile = "/data/revisions.db" + +func saveRevisionDB(stackName, revision string) error { + db, err := sql.Open("sqlite", dbFile) + if err != nil { + return err + } + defer db.Close() + + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions (stack TEXT PRIMARY KEY, revision TEXT)`) + if err != nil { + return err + } + + _, err = db.Exec(`INSERT INTO revisions (stack, revision) VALUES (?, ?) ON CONFLICT(stack) DO UPDATE SET revision = excluded.revision`, stackName, revision) + return err +} + +func loadRevisionDB(stackName string) string { + db, err := sql.Open("sqlite", dbFile) + if err != nil { + return "" + } + defer db.Close() + + var revision string + err = db.QueryRow(`SELECT revision FROM revisions WHERE stack = ?`, stackName).Scan(&revision) + if err != nil { + return "" + } + return revision +} From 6c00e94fc5bc4e08b9ddd2e72304c882aae31061 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 11:45:23 +0100 Subject: [PATCH 04/77] integrate persisting revision to db --- swarmcd/stack.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 7eef779..d1b10ec 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,15 +49,16 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - if stackStatus[swarmStack.name].Revision == revision { + lastRevision := loadRevisionDB(swarmStack.name) + if lastRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date", swarmStack.name)) - return revision, err + return revision, nil } log.Debug("reading stack file...") stackBytes, err := swarmStack.readStack() if err != nil { - return + return "nil", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } if swarmStack.valuesFile != "" { @@ -65,13 +66,13 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) } if err != nil { - return + return "", fmt.Errorf("failed to render compose template for %s stack: %w", swarmStack.name, err) } log.Debug("parsing stack content...") stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) if err != nil { - return + return "", fmt.Errorf("failed to parse stack content for %s stack: %w", swarmStack.name, err) } log.Debug("decrypting secrets...") @@ -83,17 +84,27 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { log.Debug("rotating configs and secrets...") err = swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { - return + return "", fmt.Errorf("failed to rotate configs and secrets for %s stack: %w", swarmStack.name, err) } log.Debug("writing stack to file...") err = swarmStack.writeStack(stackContents) if err != nil { - return + return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } log.Debug("deploying stack...") err = swarmStack.deployStack() + if err != nil { + return revision, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) + } + + log.Debug("saving current revision to db...") + err = saveRevisionDB(swarmStack.name, revision) + if err != nil { + return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) + } + return } From e734169fb382cccd64f86485fe3ae62140459fdd Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 12:26:20 +0100 Subject: [PATCH 05/77] add error handling in sql_db code --- swarmcd/sql_db.go | 51 ++++++++++++++++++++++++++++++++++++++--------- swarmcd/stack.go | 14 +++++++++++-- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/swarmcd/sql_db.go b/swarmcd/sql_db.go index 92fe557..6619703 100644 --- a/swarmcd/sql_db.go +++ b/swarmcd/sql_db.go @@ -2,38 +2,71 @@ package swarmcd import ( "database/sql" + "errors" + "fmt" _ "modernc.org/sqlite" ) const dbFile = "/data/revisions.db" -func saveRevisionDB(stackName, revision string) error { +// Ensure database and table exist +func initDB() error { db, err := sql.Open("sqlite", dbFile) if err != nil { - return err + return fmt.Errorf("failed to open database: %w", err) } defer db.Close() _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions (stack TEXT PRIMARY KEY, revision TEXT)`) + if err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + return nil +} + +func saveRevisionDB(stackName, revision string) error { + err := initDB() // Ensure DB is initialized if err != nil { return err } + db, err := sql.Open("sqlite", dbFile) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(`INSERT INTO revisions (stack, revision) VALUES (?, ?) ON CONFLICT(stack) DO UPDATE SET revision = excluded.revision`, stackName, revision) - return err + + if err != nil { + return fmt.Errorf("failed to save revision: %w", err) + } + + return nil } -func loadRevisionDB(stackName string) string { +// Load a stack's revision +func loadRevisionDB(stackName string) (revision string, err error) { + err = initDB() // Ensure DB is initialized + if err != nil { + return "", err + } + db, err := sql.Open("sqlite", dbFile) if err != nil { - return "" + return "", fmt.Errorf("failed to open database: %w", err) } defer db.Close() - var revision string err = db.QueryRow(`SELECT revision FROM revisions WHERE stack = ?`, stackName).Scan(&revision) - if err != nil { - return "" + + if errors.Is(err, sql.ErrNoRows) { + // No existing revision found + return "", nil + } else if err != nil { + // Unexpected error + return "", fmt.Errorf("failed to query revision: %w", err) } - return revision + + return revision, nil } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index d1b10ec..2c628c6 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,16 +49,26 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision := loadRevisionDB(swarmStack.name) + lastRevision, err := loadRevisionDB(swarmStack.name) + if err != nil { + return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) + } + + if lastRevision == "" { + logger.Info(fmt.Sprintf("%s no last revision revision found", swarmStack.name)) + } + if lastRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date", swarmStack.name)) return revision, nil } + logger.Info(fmt.Sprintf("%s new revision revision found %s! will update the stack", swarmStack.name, revision)) + log.Debug("reading stack file...") stackBytes, err := swarmStack.readStack() if err != nil { - return "nil", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) + return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } if swarmStack.valuesFile != "" { From e02bed5e24df9c59915265af39248625e723ec69 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 14:03:00 +0100 Subject: [PATCH 06/77] minor fixes --- swarmcd/stack.go | 2 +- swarmcd/swarmcd.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 2c628c6..ac77a39 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -51,7 +51,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { lastRevision, err := loadRevisionDB(swarmStack.name) if err != nil { - return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) + return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } if lastRevision == "" { diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index ff449ba..6771629 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -33,11 +33,6 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { logger.Info(fmt.Sprintf("updating %s stack", swarmStack.name)) revision, err := swarmStack.updateStack() - if stackStatus[swarmStack.name].Revision == revision { - logger.Info(fmt.Sprintf("%s stack is already up-to-date", swarmStack.name)) - return - } - if err != nil { stackStatus[swarmStack.name].Error = err.Error() logger.Error(err.Error()) From cfdbc314a8c2e55d2d7b7d10c0ffcb4dba55a017 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:31:28 +0100 Subject: [PATCH 07/77] ui fixes --- Dockerfile | 4 +++- web/routes.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6493d29..622572e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,8 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates # Copy the built backend binary from the backend build stage COPY --from=backend-build /swarm-cd /app/ # Copy the built frontend from the frontend build stage -COPY --from=frontend-build /ui/dist/ /app/ui/ +COPY --from=frontend-build /ui/dist/ /app/ui/dist/ + +EXPOSE 8080 # Set the entry point for the application CMD ["/app/swarm-cd"] \ No newline at end of file diff --git a/web/routes.go b/web/routes.go index b52b562..d1d8b3e 100644 --- a/web/routes.go +++ b/web/routes.go @@ -19,5 +19,5 @@ func init() { } func RunServer() { - router.Run("localhost:8080") + router.Run(":8080") } From f6e033aa6fa04fee70a39da0c7a9ce8ef4f5d4c5 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:31:51 +0100 Subject: [PATCH 08/77] log the current revision in stack.go --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index ac77a39..2afbc24 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -59,7 +59,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } if lastRevision == revision { - logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date", swarmStack.name)) + logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) return revision, nil } From 11562d4721773e8a0e1ee7829ef992f2b9466607 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:32:11 +0100 Subject: [PATCH 09/77] add repos.yaml/stack.yaml to gitignore --- .gitignore | 3 +++ docker-compose.yaml | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docker-compose.yaml diff --git a/.gitignore b/.gitignore index d4cb05f..fac2a59 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ swarm-cd + +repos.yaml +stacks.yaml \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..4d63cf5 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +# docker-compose.yaml +version: '3.7' +services: + swarm-cd: + image: chris/swarmcd:dev + deploy: + placement: + constraints: + - node.role == manager + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./repos.yaml:/app/repos.yaml:ro + - ./stacks.yaml:/app/stacks.yaml:ro + - swarmcd_data:/data # Mount the volume for SQLite + +# Ensure that swarmdc_data persists across container restarts. +volumes: + swarmcd_data: + driver: local \ No newline at end of file From 3b40eb6bf18e287a68dcf87ab7079448575590de Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:35:50 +0100 Subject: [PATCH 10/77] fix readme regarding volume --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a6ddae1..4e67466 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,12 @@ services: # Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: + driver: local ``` Run this on a swarm manager node: ```bash -docker volume create swarmcd_data docker stack deploy --compose-file docker-compose.yaml swarm-cd ``` @@ -116,6 +116,7 @@ services: # Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: + driver: local secrets: age: @@ -235,6 +236,7 @@ services: # Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: + driver: local secrets: docker-config: From 03afa1f85cbcd0da998d3f3af0c673b3612cb5d1 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:36:15 +0100 Subject: [PATCH 11/77] delete docker compose --- docker-compose.yaml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 4d63cf5..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# docker-compose.yaml -version: '3.7' -services: - swarm-cd: - image: chris/swarmcd:dev - deploy: - placement: - constraints: - - node.role == manager - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./repos.yaml:/app/repos.yaml:ro - - ./stacks.yaml:/app/stacks.yaml:ro - - swarmcd_data:/data # Mount the volume for SQLite - -# Ensure that swarmdc_data persists across container restarts. -volumes: - swarmcd_data: - driver: local \ No newline at end of file From 3d84e0d12449fa01fcb41066985fa42ec7fb4c71 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 28 Feb 2025 15:36:50 +0100 Subject: [PATCH 12/77] ignore docker compose --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fac2a59..fbc3a5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ swarm-cd +docker-compose.yaml repos.yaml stacks.yaml \ No newline at end of file From 0a89d743cc38409e86779b7160134c74c383143b Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:40:25 +0100 Subject: [PATCH 13/77] Rename revision loading/saving Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 2afbc24..1904dc0 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -110,7 +110,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveRevisionDB(swarmStack.name, revision) + err = saveLastDeployedRevision(swarmStack.name, revision) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } From 0e0bd65d07039914631c7e9a92abf08cf679f344 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:40:37 +0100 Subject: [PATCH 14/77] Rename revision loading/saving Co-authored-by: Andrea Ghensi --- swarmcd/sql_db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/sql_db.go b/swarmcd/sql_db.go index 6619703..1aaaf31 100644 --- a/swarmcd/sql_db.go +++ b/swarmcd/sql_db.go @@ -24,7 +24,7 @@ func initDB() error { return nil } -func saveRevisionDB(stackName, revision string) error { +func saveLastDeployedRevision(stackName, revision string) error { err := initDB() // Ensure DB is initialized if err != nil { return err From cf96baeed190c6502266e399883274d79360154b Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:40:46 +0100 Subject: [PATCH 15/77] Rename revision loading/saving Co-authored-by: Andrea Ghensi --- swarmcd/sql_db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/sql_db.go b/swarmcd/sql_db.go index 1aaaf31..f55348e 100644 --- a/swarmcd/sql_db.go +++ b/swarmcd/sql_db.go @@ -46,7 +46,7 @@ func saveLastDeployedRevision(stackName, revision string) error { } // Load a stack's revision -func loadRevisionDB(stackName string) (revision string, err error) { +func loadLastDeployedRevision(stackName string) (revision string, err error) { err = initDB() // Ensure DB is initialized if err != nil { return "", err From d1c4a4da4466867664663019d9fffb8f75aed405 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:40:55 +0100 Subject: [PATCH 16/77] Rename revision loading/saving Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 1904dc0..485092e 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,7 +49,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision, err := loadRevisionDB(swarmStack.name) + lastRevision, err := loadLastDeployedRevision(swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } From a2b73433fc4121f0a8e2fb60152d5496edcc6cd3 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Tue, 4 Mar 2025 09:42:13 +0100 Subject: [PATCH 17/77] rename sql_db to db --- swarmcd/{sql_db.go => database.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename swarmcd/{sql_db.go => database.go} (100%) diff --git a/swarmcd/sql_db.go b/swarmcd/database.go similarity index 100% rename from swarmcd/sql_db.go rename to swarmcd/database.go From 7e7b9193c436e468c7909414c49e37fd4de87f63 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Tue, 4 Mar 2025 09:45:01 +0100 Subject: [PATCH 18/77] read db path from env var --- swarmcd/database.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index f55348e..bed2f86 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -5,9 +5,17 @@ import ( "errors" "fmt" _ "modernc.org/sqlite" + "os" ) -const dbFile = "/data/revisions.db" +var dbFile = getDBFilePath() + +func getDBFilePath() string { + if path := os.Getenv("SWARMCD_DB"); path != "" { + return path + } + return "/data/revisions.db" // Default path +} // Ensure database and table exist func initDB() error { From a91cadfab815676f1807c16338e4011b56ed4c0a Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Tue, 4 Mar 2025 09:49:00 +0100 Subject: [PATCH 19/77] add env var config to Readme --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e67466..6e8e8d8 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,9 @@ services: - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - swarmcd_data:/data # Mount the volume for SQLite + environment: + - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable + # Ensure that swarmdc_data persists across container restarts. volumes: @@ -107,12 +110,14 @@ services: target: /secrets/age.key environment: - SOPS_AGE_KEY_FILE=/secrets/age.key + - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - swarmcd_data:/data # Mount the volume for SQLite - + + # Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: @@ -229,6 +234,9 @@ services: - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - swarmcd_data:/data # Mount the volume for SQLite + environment: + - SOPS_AGE_KEY_FILE=/secrets/age.key + - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable secrets: - source: docker-config target: /root/.docker/config.json From d014319fd6bb7d5aa6dee33d3dce20c8b1685bf2 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:04:18 +0100 Subject: [PATCH 20/77] Fix Typo Co-authored-by: Andrea Ghensi --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e8e8d8..c1340e5 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ services: - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable -# Ensure that swarmdc_data persists across container restarts. +# Ensure that swarmcd_data persists across container restarts. volumes: swarmcd_data: driver: local From ff618091f0e734b84fddce65cca2f019f90aab1a Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Thu, 6 Mar 2025 15:06:59 +0100 Subject: [PATCH 21/77] [Readme] remove unnecessary comments --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c1340e5..e538e68 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - - swarmcd_data:/data # Mount the volume for SQLite + - swarmcd_data:/data environment: - - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable + - SWARMCD_DB=/data/revisions.db # Ensure that swarmcd_data persists across container restarts. @@ -110,12 +110,12 @@ services: target: /secrets/age.key environment: - SOPS_AGE_KEY_FILE=/secrets/age.key - - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable + - SWARMCD_DB=/data/revisions.db volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - - swarmcd_data:/data # Mount the volume for SQLite + - swarmcd_data:/data # Ensure that swarmdc_data persists across container restarts. @@ -233,10 +233,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - - swarmcd_data:/data # Mount the volume for SQLite + - swarmcd_data:/data environment: - SOPS_AGE_KEY_FILE=/secrets/age.key - - SWARMCD_DB=/data/revisions.db # Pass DB file path as env variable + - SWARMCD_DB=/data/revisions.db secrets: - source: docker-config target: /root/.docker/config.json From b60982cba3b64edd354c789e1e31c6fb6d1f7b34 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Thu, 6 Mar 2025 15:16:35 +0100 Subject: [PATCH 22/77] [Readme] add minimal section regarding db config. --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e538e68..7c4b065 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,6 @@ services: environment: - SWARMCD_DB=/data/revisions.db - -# Ensure that swarmcd_data persists across container restarts. volumes: swarmcd_data: driver: local @@ -66,6 +64,12 @@ docker stack deploy --compose-file docker-compose.yaml swarm-cd This will start SwarmCD, it will periodically check the stack repo for new changes, pulling them and updating the stack. +### Configure the database +SwarmCD uses a minimal DB to track the last deployed revision across +container restarts. By default, it stores data in data/revisions.db, +but this can be changed via the `SWARMCD_DB` environment variable as +shown in the above docker-compose file. + ## Manage Encrypted Secrets Using SOPS You can use [sops](https://github.com/getsops/sops) to encrypt secrets in git repos and @@ -117,8 +121,6 @@ services: - ./stacks.yaml:/app/stacks.yaml:ro - swarmcd_data:/data - -# Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: driver: local @@ -241,7 +243,6 @@ services: - source: docker-config target: /root/.docker/config.json -# Ensure that swarmdc_data persists across container restarts. volumes: swarmcd_data: driver: local From b08d085f1c13aa0b12ebc7381dfa886e7343cabf Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 15:14:52 +0100 Subject: [PATCH 23/77] [swarmcd] also persist the hash of the stack content --- swarmcd/database.go | 69 +++++++++++++++++++++------------------- swarmcd/database_test.go | 37 +++++++++++++++++++++ swarmcd/stack.go | 12 +++---- 3 files changed, 80 insertions(+), 38 deletions(-) create mode 100644 swarmcd/database_test.go diff --git a/swarmcd/database.go b/swarmcd/database.go index bed2f86..d02897e 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -1,8 +1,8 @@ package swarmcd import ( + "crypto/sha256" "database/sql" - "errors" "fmt" _ "modernc.org/sqlite" "os" @@ -18,33 +18,40 @@ func getDBFilePath() string { } // Ensure database and table exist -func initDB() error { +func initDB() (*sql.DB, error) { db, err := sql.Open("sqlite", dbFile) if err != nil { - return fmt.Errorf("failed to open database: %w", err) + return nil, fmt.Errorf("failed to open database: %w", err) } - defer db.Close() - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions (stack TEXT PRIMARY KEY, revision TEXT)`) + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions ( + stack TEXT PRIMARY KEY, + revision TEXT, + hash TEXT + )`) if err != nil { - return fmt.Errorf("failed to create table: %w", err) + return nil, fmt.Errorf("failed to create table: %w", err) } - return nil + return db, nil } -func saveLastDeployedRevision(stackName, revision string) error { - err := initDB() // Ensure DB is initialized - if err != nil { - return err - } - - db, err := sql.Open("sqlite", dbFile) +// Save last deployed revision and hash +func saveLastDeployedRevision(stackName, revision string, stackContent []byte) error { + db, err := initDB() if err != nil { return err } defer db.Close() - _, err = db.Exec(`INSERT INTO revisions (stack, revision) VALUES (?, ?) ON CONFLICT(stack) DO UPDATE SET revision = excluded.revision`, stackName, revision) + hash := computeHash(stackContent) + + _, err = db.Exec(` + INSERT INTO revisions (stack, revision, hash) + VALUES (?, ?, ?) + ON CONFLICT(stack) DO UPDATE SET + revision = excluded.revision, + hash = excluded.hash + `, stackName, revision, hash) if err != nil { return fmt.Errorf("failed to save revision: %w", err) @@ -53,28 +60,26 @@ func saveLastDeployedRevision(stackName, revision string) error { return nil } -// Load a stack's revision -func loadLastDeployedRevision(stackName string) (revision string, err error) { - err = initDB() // Ensure DB is initialized +// Load a stack's revision and hash +func loadLastDeployedRevision(stackName string) (revision string, hash string, err error) { + db, err := initDB() if err != nil { - return "", err - } - - db, err := sql.Open("sqlite", dbFile) - if err != nil { - return "", fmt.Errorf("failed to open database: %w", err) + return "", "", err } defer db.Close() - err = db.QueryRow(`SELECT revision FROM revisions WHERE stack = ?`, stackName).Scan(&revision) - - if errors.Is(err, sql.ErrNoRows) { - // No existing revision found - return "", nil + err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) + if err == sql.ErrNoRows { + return "", "", nil } else if err != nil { - // Unexpected error - return "", fmt.Errorf("failed to query revision: %w", err) + return "", "", fmt.Errorf("failed to query revision: %w", err) } - return revision, nil + return revision, hash, nil +} + +// Compute a SHA-256 hash of the stack content +func computeHash(data []byte) string { + hash := sha256.Sum256(data) + return fmt.Sprintf("%x", hash) } diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go new file mode 100644 index 0000000..177b52d --- /dev/null +++ b/swarmcd/database_test.go @@ -0,0 +1,37 @@ +package swarmcd + +import ( + "testing" +) + +import ( + _ "modernc.org/sqlite" +) + +func TestSaveAndLoadLastDeployedRevision(t *testing.T) { + const dbFile = ":memory:" // Use in-memory database for tests + + stackName := "test-stack" + revision := "v1.0.0" + stackContent := []byte("test content") + + err := saveLastDeployedRevision(stackName, revision, stackContent) + if err != nil { + t.Fatalf("Failed to save revision: %v", err) + } + + loadedRevision, loadedHash, err := loadLastDeployedRevision(stackName) + if err != nil { + t.Fatalf("Failed to load revision: %v", err) + } + + expectedHash := computeHash(stackContent) + + if loadedRevision != revision { + t.Errorf("Expected revision %s, got %s", revision, loadedRevision) + } + + if loadedHash != expectedHash { + t.Errorf("Expected hash %s, got %s", expectedHash, loadedHash) + } +} diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 485092e..d68bfd9 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,7 +49,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision, err := loadLastDeployedRevision(swarmStack.name) + lastRevision, _, err := loadLastDeployedRevision(swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -98,7 +98,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("writing stack to file...") - err = swarmStack.writeStack(stackContents) + writenBytes, err := swarmStack.writeStack(stackContents) if err != nil { return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } @@ -110,7 +110,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(swarmStack.name, revision) + err = saveLastDeployedRevision(swarmStack.name, revision, writenBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -232,15 +232,15 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { return nil } -func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error { +func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, error) { composeFileBytes, err := yaml.Marshal(composeMap) if err != nil { - return fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) + return nil, fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) } composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) fileInfo, _ := os.Stat(composeFile) os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode()) - return nil + return composeFileBytes, nil } func (swarmStack *swarmStack) deployStack() error { From 54336f18ea7431116d384772887f55e1f61775d0 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 15:31:22 +0100 Subject: [PATCH 24/77] [swarmcd] add unit tests for persisting revision and hash in to an in memory sqllite db --- swarmcd/database.go | 20 ++++---------------- swarmcd/database_test.go | 8 ++++++-- swarmcd/stack.go | 10 ++++++++-- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index d02897e..9586260 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -18,7 +18,7 @@ func getDBFilePath() string { } // Ensure database and table exist -func initDB() (*sql.DB, error) { +func initDB(dbFile string) (*sql.DB, error) { db, err := sql.Open("sqlite", dbFile) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) @@ -36,16 +36,10 @@ func initDB() (*sql.DB, error) { } // Save last deployed revision and hash -func saveLastDeployedRevision(stackName, revision string, stackContent []byte) error { - db, err := initDB() - if err != nil { - return err - } - defer db.Close() - +func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackContent []byte) error { hash := computeHash(stackContent) - _, err = db.Exec(` + _, err := db.Exec(` INSERT INTO revisions (stack, revision, hash) VALUES (?, ?, ?) ON CONFLICT(stack) DO UPDATE SET @@ -61,13 +55,7 @@ func saveLastDeployedRevision(stackName, revision string, stackContent []byte) e } // Load a stack's revision and hash -func loadLastDeployedRevision(stackName string) (revision string, hash string, err error) { - db, err := initDB() - if err != nil { - return "", "", err - } - defer db.Close() - +func loadLastDeployedRevision(db *sql.DB, stackName string) (revision string, hash string, err error) { err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { return "", "", nil diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 177b52d..ed05760 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -10,17 +10,21 @@ import ( func TestSaveAndLoadLastDeployedRevision(t *testing.T) { const dbFile = ":memory:" // Use in-memory database for tests + db, err := initDB(dbFile) + if err != nil { + t.Fatalf("failed to open database: %v", err) + } stackName := "test-stack" revision := "v1.0.0" stackContent := []byte("test content") - err := saveLastDeployedRevision(stackName, revision, stackContent) + err = saveLastDeployedRevision(db, stackName, revision, stackContent) if err != nil { t.Fatalf("Failed to save revision: %v", err) } - loadedRevision, loadedHash, err := loadLastDeployedRevision(stackName) + loadedRevision, loadedHash, err := loadLastDeployedRevision(db, stackName) if err != nil { t.Fatalf("Failed to load revision: %v", err) } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index d68bfd9..a7681aa 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,7 +49,13 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision, _, err := loadLastDeployedRevision(swarmStack.name) + db, err := initDB(getDBFilePath()) + if err != nil { + return "", err + } + defer db.Close() + + lastRevision, _, err := loadLastDeployedRevision(db, swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -110,7 +116,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(swarmStack.name, revision, writenBytes) + err = saveLastDeployedRevision(db, swarmStack.name, revision, writenBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } From 26ccf8df484e15f8f4071d755ef96745e07e3eb1 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 16:22:59 +0100 Subject: [PATCH 25/77] [swarmcd] create global db --- swarmcd/database.go | 21 +++++++++++++-------- swarmcd/database_test.go | 6 +++--- swarmcd/stack.go | 10 ++-------- swarmcd/swarmcd.go | 9 +++++++++ 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 9586260..bf8ad79 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -8,7 +8,7 @@ import ( "os" ) -var dbFile = getDBFilePath() +var db *sql.DB func getDBFilePath() string { if path := os.Getenv("SWARMCD_DB"); path != "" { @@ -17,11 +17,16 @@ func getDBFilePath() string { return "/data/revisions.db" // Default path } +func closeDB() { + db.Close() +} + // Ensure database and table exist -func initDB(dbFile string) (*sql.DB, error) { - db, err := sql.Open("sqlite", dbFile) +func initDB(dbFile string) error { + var err error + db, err = sql.Open("sqlite", dbFile) if err != nil { - return nil, fmt.Errorf("failed to open database: %w", err) + return fmt.Errorf("failed to open database: %w", err) } _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions ( @@ -30,13 +35,13 @@ func initDB(dbFile string) (*sql.DB, error) { hash TEXT )`) if err != nil { - return nil, fmt.Errorf("failed to create table: %w", err) + return fmt.Errorf("failed to create table: %w", err) } - return db, nil + return nil } // Save last deployed revision and hash -func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackContent []byte) error { +func saveLastDeployedRevision(stackName, revision string, stackContent []byte) error { hash := computeHash(stackContent) _, err := db.Exec(` @@ -55,7 +60,7 @@ func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackConte } // Load a stack's revision and hash -func loadLastDeployedRevision(db *sql.DB, stackName string) (revision string, hash string, err error) { +func loadLastDeployedRevision(stackName string) (revision string, hash string, err error) { err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { return "", "", nil diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index ed05760..1fa2dd8 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -10,7 +10,7 @@ import ( func TestSaveAndLoadLastDeployedRevision(t *testing.T) { const dbFile = ":memory:" // Use in-memory database for tests - db, err := initDB(dbFile) + err := initDB(dbFile) if err != nil { t.Fatalf("failed to open database: %v", err) } @@ -19,12 +19,12 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { revision := "v1.0.0" stackContent := []byte("test content") - err = saveLastDeployedRevision(db, stackName, revision, stackContent) + err = saveLastDeployedRevision(stackName, revision, stackContent) if err != nil { t.Fatalf("Failed to save revision: %v", err) } - loadedRevision, loadedHash, err := loadLastDeployedRevision(db, stackName) + loadedRevision, loadedHash, err := loadLastDeployedRevision(stackName) if err != nil { t.Fatalf("Failed to load revision: %v", err) } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index a7681aa..d68bfd9 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,13 +49,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - db, err := initDB(getDBFilePath()) - if err != nil { - return "", err - } - defer db.Close() - - lastRevision, _, err := loadLastDeployedRevision(db, swarmStack.name) + lastRevision, _, err := loadLastDeployedRevision(swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -116,7 +110,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, writenBytes) + err = saveLastDeployedRevision(swarmStack.name, revision, writenBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 6771629..fbfd80b 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -11,6 +11,15 @@ var stacks []*swarmStack func Run() { logger.Info("starting SwarmCD") + + err := initDB(getDBFilePath()) + defer closeDB() + + if err != nil { + logger.Error(fmt.Sprintf("failed to initialize SwarmCD DB: %s", err)) + return + } + for { var waitGroup sync.WaitGroup logger.Info("updating stacks...") From dc8134c60d84f9a16f32e6fcbf3088f49cc2432c Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 16:40:16 +0100 Subject: [PATCH 26/77] [swarmcd] add global to track if db has been initialized --- swarmcd/database.go | 9 +++++++++ swarmcd/database_test.go | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/swarmcd/database.go b/swarmcd/database.go index bf8ad79..414c51e 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -10,6 +10,9 @@ import ( var db *sql.DB +// Global flag to track if initDB has been called +var initDBCalled = false + func getDBFilePath() string { if path := os.Getenv("SWARMCD_DB"); path != "" { return path @@ -23,6 +26,10 @@ func closeDB() { // Ensure database and table exist func initDB(dbFile string) error { + if initDBCalled { + return nil + } + var err error db, err = sql.Open("sqlite", dbFile) if err != nil { @@ -37,6 +44,8 @@ func initDB(dbFile string) error { if err != nil { return fmt.Errorf("failed to create table: %w", err) } + + initDBCalled = true return nil } diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 1fa2dd8..e63287a 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -15,6 +15,10 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { t.Fatalf("failed to open database: %v", err) } + if !initDBCalled { + t.Fatalf("InitDBCalled should be true") + } + stackName := "test-stack" revision := "v1.0.0" stackContent := []byte("test content") From c18126d06c8844ba64160946ac543090396af3ac Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 16:48:57 +0100 Subject: [PATCH 27/77] [swarmcd] close in memory db ad the end of the tests --- swarmcd/database_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index e63287a..17e92be 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -11,6 +11,7 @@ import ( func TestSaveAndLoadLastDeployedRevision(t *testing.T) { const dbFile = ":memory:" // Use in-memory database for tests err := initDB(dbFile) + defer closeDB() if err != nil { t.Fatalf("failed to open database: %v", err) } @@ -43,3 +44,21 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { t.Errorf("Expected hash %s, got %s", expectedHash, loadedHash) } } + +func TestInitDbTwiceShouldWork(t *testing.T) { + const dbFile = ":memory:" // Use in-memory database for tests + err := initDB(dbFile) + defer closeDB() + if err != nil { + t.Fatalf("failed to open database: %v", err) + } + + if !initDBCalled { + t.Fatalf("InitDBCalled should be true") + } + + err = initDB(dbFile) + if err != nil { + t.Fatalf("failed to open database: %v", err) + } +} From c664360df35f4363170d8647043baeed187124e1 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 16:55:13 +0100 Subject: [PATCH 28/77] [swarmcd] compare hashes of deployed filed --- swarmcd/stack.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index d68bfd9..ed6a21b 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,7 +49,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision, _, err := loadLastDeployedRevision(swarmStack.name) + lastRevision, hash, err := loadLastDeployedRevision(swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -71,6 +71,17 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } + if computeHash(stackBytes) == hash { + logger.Info(fmt.Sprintf("%s stack hash unchanged, will skip deployment: %s", swarmStack.name, revision)) + log.Debug("saving current revision to db...") + err = saveLastDeployedRevision(swarmStack.name, revision, stackBytes) + if err != nil { + return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) + } + + return revision, nil + } + if swarmStack.valuesFile != "" { log.Debug("rendering template...") stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) From 3b115d22421bb419955b6c325b351473f6455697 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 17:11:45 +0100 Subject: [PATCH 29/77] [gitignore] add config.yaml to identity --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fbc3a5a..ca9902c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ swarm-cd docker-compose.yaml repos.yaml -stacks.yaml \ No newline at end of file +stacks.yaml +config.yaml \ No newline at end of file From 90c4c3dd4a387a8e9c4f0bd4c78be96f748e5d34 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Wed, 19 Mar 2025 17:55:17 +0100 Subject: [PATCH 30/77] [gitignore] don't use global db references --- swarmcd/database.go | 29 +++++++---------------------- swarmcd/database_test.go | 30 ++++-------------------------- swarmcd/stack.go | 13 ++++++++++--- swarmcd/swarmcd.go | 8 -------- 4 files changed, 21 insertions(+), 59 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 414c51e..07579f6 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -8,11 +8,6 @@ import ( "os" ) -var db *sql.DB - -// Global flag to track if initDB has been called -var initDBCalled = false - func getDBFilePath() string { if path := os.Getenv("SWARMCD_DB"); path != "" { return path @@ -20,20 +15,11 @@ func getDBFilePath() string { return "/data/revisions.db" // Default path } -func closeDB() { - db.Close() -} - // Ensure database and table exist -func initDB(dbFile string) error { - if initDBCalled { - return nil - } - - var err error - db, err = sql.Open("sqlite", dbFile) +func initDB(dbFile string) (*sql.DB, error) { + db, err := sql.Open("sqlite", dbFile) if err != nil { - return fmt.Errorf("failed to open database: %w", err) + return nil, fmt.Errorf("failed to open database: %w", err) } _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions ( @@ -42,15 +28,14 @@ func initDB(dbFile string) error { hash TEXT )`) if err != nil { - return fmt.Errorf("failed to create table: %w", err) + return nil, fmt.Errorf("failed to create table: %w", err) } - initDBCalled = true - return nil + return db, nil } // Save last deployed revision and hash -func saveLastDeployedRevision(stackName, revision string, stackContent []byte) error { +func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackContent []byte) error { hash := computeHash(stackContent) _, err := db.Exec(` @@ -69,7 +54,7 @@ func saveLastDeployedRevision(stackName, revision string, stackContent []byte) e } // Load a stack's revision and hash -func loadLastDeployedRevision(stackName string) (revision string, hash string, err error) { +func loadLastDeployedRevision(db *sql.DB, stackName string) (revision string, hash string, err error) { err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { return "", "", nil diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 17e92be..b1e3b6d 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -10,26 +10,22 @@ import ( func TestSaveAndLoadLastDeployedRevision(t *testing.T) { const dbFile = ":memory:" // Use in-memory database for tests - err := initDB(dbFile) - defer closeDB() + db, err := initDB(dbFile) if err != nil { t.Fatalf("failed to open database: %v", err) } - - if !initDBCalled { - t.Fatalf("InitDBCalled should be true") - } + defer db.Close() stackName := "test-stack" revision := "v1.0.0" stackContent := []byte("test content") - err = saveLastDeployedRevision(stackName, revision, stackContent) + err = saveLastDeployedRevision(db, stackName, revision, stackContent) if err != nil { t.Fatalf("Failed to save revision: %v", err) } - loadedRevision, loadedHash, err := loadLastDeployedRevision(stackName) + loadedRevision, loadedHash, err := loadLastDeployedRevision(db, stackName) if err != nil { t.Fatalf("Failed to load revision: %v", err) } @@ -44,21 +40,3 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { t.Errorf("Expected hash %s, got %s", expectedHash, loadedHash) } } - -func TestInitDbTwiceShouldWork(t *testing.T) { - const dbFile = ":memory:" // Use in-memory database for tests - err := initDB(dbFile) - defer closeDB() - if err != nil { - t.Fatalf("failed to open database: %v", err) - } - - if !initDBCalled { - t.Fatalf("InitDBCalled should be true") - } - - err = initDB(dbFile) - if err != nil { - t.Fatalf("failed to open database: %v", err) - } -} diff --git a/swarmcd/stack.go b/swarmcd/stack.go index ed6a21b..3107889 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,7 +49,14 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - lastRevision, hash, err := loadLastDeployedRevision(swarmStack.name) + db, err := initDB(getDBFilePath()) + if err != nil { + logger.Error(fmt.Sprintf("failed to open database: %s", err)) + return + } + defer db.Close() + + lastRevision, hash, err := loadLastDeployedRevision(db, swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -74,7 +81,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { if computeHash(stackBytes) == hash { logger.Info(fmt.Sprintf("%s stack hash unchanged, will skip deployment: %s", swarmStack.name, revision)) log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(swarmStack.name, revision, stackBytes) + err = saveLastDeployedRevision(db, swarmStack.name, revision, stackBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -121,7 +128,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(swarmStack.name, revision, writenBytes) + err = saveLastDeployedRevision(db, swarmStack.name, revision, writenBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index fbfd80b..ef0209b 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -12,14 +12,6 @@ var stacks []*swarmStack func Run() { logger.Info("starting SwarmCD") - err := initDB(getDBFilePath()) - defer closeDB() - - if err != nil { - logger.Error(fmt.Sprintf("failed to initialize SwarmCD DB: %s", err)) - return - } - for { var waitGroup sync.WaitGroup logger.Info("updating stacks...") From 23cac4dcb8e2782b5fc744a1fee606d5a51d4696 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 08:57:33 +0100 Subject: [PATCH 31/77] [stack] add unit tests for rotateObjects --- swarmcd/stack_test.go | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 8d347a1..f147df9 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -1,6 +1,8 @@ package swarmcd import ( + "crypto/md5" + "fmt" "os" "path" "testing" @@ -47,3 +49,73 @@ func TestRenderComposeTemplate(t *testing.T) { }) } } + +func TestRotateObjects(t *testing.T) { + fileContent, swarm := setupTestStack(t) + + objects := map[string]any{ + "service1": map[string]any{"file": "testfile.txt"}, + } + + err := swarm.rotateObjects(objects) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + expectedHash := fmt.Sprintf("%x", md5.Sum(fileContent))[:8] + expectedName := "test-stack-service1-" + expectedHash + if objects["service1"].(map[string]any)["name"] != expectedName { + t.Errorf("Expected name %s, got %s", expectedName, objects["service1"].(map[string]any)["name"]) + } +} + +func TestRotateObjectsInvalidMap(t *testing.T) { + _, swarm := setupTestStack(t) + + objects := map[string]any{"service1": "invalid"} + + err := swarm.rotateObjects(objects) + if err == nil { + t.Fatalf("Expected an error but got none") + } + expectedErr := "invalid compose file: service1 object must be a map" + if err.Error() != expectedErr { + t.Errorf("Expected error %q, got %q", expectedErr, err.Error()) + } +} + +func TestRotateObjectsMissingFileField(t *testing.T) { + _, swarm := setupTestStack(t) + + objects := map[string]any{"service1": map[string]any{}} + + err := swarm.rotateObjects(objects) + if err == nil { + t.Fatalf("Expected an error but got none") + } + expectedErr := "invalid compose file: service1 file field must be a string" + if err.Error() != expectedErr { + t.Errorf("Expected error %q, got %q", expectedErr, err.Error()) + } +} + +func TestRotateObjectsFileNotFound(t *testing.T) { + swarm := &swarmStack{name: "test-stack", repo: &stackRepo{path: "nonexistent"}, composePath: "docker-compose.yml"} + objects := map[string]any{"service1": map[string]any{"file": "missing.txt"}} + + err := swarm.rotateObjects(objects) + if err == nil { + t.Fatalf("Expected an error but got none") + } +} + +func setupTestStack(t *testing.T) ([]byte, *swarmStack) { + tempDir := t.TempDir() + filePath := path.Join(tempDir, "testfile.txt") + fileContent := []byte("test content") + os.WriteFile(filePath, fileContent, 0644) + + repo := &stackRepo{path: tempDir} + swarm := &swarmStack{name: "test-stack", repo: repo, composePath: "docker-compose.yml"} + return fileContent, swarm +} From 408603fe9910d5e424e36745e7ea9e8954141743 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 09:14:53 +0100 Subject: [PATCH 32/77] [stack] rotating configs works now with external: true instead of file. --- swarmcd/stack.go | 4 ++++ swarmcd/stack_test.go | 32 +++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 3107889..2ab6f8c 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -235,6 +235,10 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { if !ok { return fmt.Errorf("invalid compose file: %s object must be a map", objectName) } + // If "external" field exists and is true, skip processing + if external, exists := objectMap["external"].(bool); exists && external { + continue + } objectFile, ok := objectMap["file"].(string) if !ok { return fmt.Errorf("invalid compose file: %s file field must be a string", objectName) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index f147df9..65c0a01 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -51,10 +51,10 @@ func TestRenderComposeTemplate(t *testing.T) { } func TestRotateObjects(t *testing.T) { - fileContent, swarm := setupTestStack(t) + fileName, fileContent, swarm := setupTestStack(t) objects := map[string]any{ - "service1": map[string]any{"file": "testfile.txt"}, + "service1": map[string]any{"file": fileName}, } err := swarm.rotateObjects(objects) @@ -69,8 +69,25 @@ func TestRotateObjects(t *testing.T) { } } +func TestRotateObjectsHandlesExternalTrue(t *testing.T) { + configFile, _, swarm := setupTestStack(t) + + objects := map[string]any{ + "config1": map[string]any{"external": true}, + "config2": map[string]any{"file": configFile}, + } + + err := swarm.rotateObjects(objects) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if _, exists := objects["config1"].(map[string]any)["name"]; exists { + t.Errorf("Expected config1 to be unmodified, but 'name' was set") + } +} + func TestRotateObjectsInvalidMap(t *testing.T) { - _, swarm := setupTestStack(t) + _, _, swarm := setupTestStack(t) objects := map[string]any{"service1": "invalid"} @@ -85,7 +102,7 @@ func TestRotateObjectsInvalidMap(t *testing.T) { } func TestRotateObjectsMissingFileField(t *testing.T) { - _, swarm := setupTestStack(t) + _, _, swarm := setupTestStack(t) objects := map[string]any{"service1": map[string]any{}} @@ -109,13 +126,14 @@ func TestRotateObjectsFileNotFound(t *testing.T) { } } -func setupTestStack(t *testing.T) ([]byte, *swarmStack) { +func setupTestStack(t *testing.T) (string, []byte, *swarmStack) { tempDir := t.TempDir() - filePath := path.Join(tempDir, "testfile.txt") + fileName := "testfile.txt" + filePath := path.Join(tempDir, fileName) fileContent := []byte("test content") os.WriteFile(filePath, fileContent, 0644) repo := &stackRepo{path: tempDir} swarm := &swarmStack{name: "test-stack", repo: repo, composePath: "docker-compose.yml"} - return fileContent, swarm + return fileName, fileContent, swarm } From 9440ea238305153a480f9afa45538adfeb3634a4 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 10:04:59 +0100 Subject: [PATCH 33/77] [stack] skip writing to db with the new revision --- swarmcd/stack.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 2ab6f8c..28fd4c0 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -56,7 +56,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } defer db.Close() - lastRevision, hash, err := loadLastDeployedRevision(db, swarmStack.name) + lastRevision, deployedStackHash, err := loadLastDeployedRevision(db, swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -78,15 +78,12 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } - if computeHash(stackBytes) == hash { - logger.Info(fmt.Sprintf("%s stack hash unchanged, will skip deployment: %s", swarmStack.name, revision)) - log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, stackBytes) - if err != nil { - return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) - } - + newStackHash := computeHash(stackBytes) + if newStackHash == deployedStackHash { + logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s, will skip deployment of revision: %s", swarmStack.name, deployedStackHash, revision)) return revision, nil + } else { + logger.Info(fmt.Sprintf("%s new stack file with hash found: %s. Will continue with deployment of revision: %s", swarmStack.name, newStackHash, revision)) } if swarmStack.valuesFile != "" { From 444a0668e0a442d93b947da95487f5e7623b6cf8 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 10:23:55 +0100 Subject: [PATCH 34/77] [stack] hash the correct bytes to prevent a mismatch of hashes --- swarmcd/stack.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 28fd4c0..0ee785f 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -79,11 +79,13 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } newStackHash := computeHash(stackBytes) + logger.Info(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) + logger.Info(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s, will skip deployment of revision: %s", swarmStack.name, deployedStackHash, revision)) + logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s, will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) return revision, nil } else { - logger.Info(fmt.Sprintf("%s new stack file with hash found: %s. Will continue with deployment of revision: %s", swarmStack.name, newStackHash, revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash found: %s. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) } if swarmStack.valuesFile != "" { @@ -113,7 +115,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("writing stack to file...") - writenBytes, err := swarmStack.writeStack(stackContents) + err = swarmStack.writeStack(stackContents) if err != nil { return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } @@ -125,7 +127,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, writenBytes) + err = saveLastDeployedRevision(db, swarmStack.name, revision, stackBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -251,15 +253,15 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { return nil } -func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, error) { +func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error { composeFileBytes, err := yaml.Marshal(composeMap) if err != nil { - return nil, fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) + return fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) } composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) fileInfo, _ := os.Stat(composeFile) os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode()) - return composeFileBytes, nil + return nil } func (swarmStack *swarmStack) deployStack() error { From 8302c416ff6348be3ae8b8affd5f0be00df6a19f Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 10:37:10 +0100 Subject: [PATCH 35/77] [stack] better logs --- swarmcd/stack.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 0ee785f..287c6b7 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -79,13 +79,13 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } newStackHash := computeHash(stackBytes) - logger.Info(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) - logger.Info(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s, will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) return revision, nil } else { - logger.Info(fmt.Sprintf("%s new stack file with hash found: %s. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash= %s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) } if swarmStack.valuesFile != "" { From e582c582bc837d68a76b2b76f5867909d727deb9 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 10:40:13 +0100 Subject: [PATCH 36/77] [stack] better logs --- swarmcd/stack.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 287c6b7..5a49593 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -82,10 +82,11 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file deployedStackHash unchanged %s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) return revision, nil } else { - logger.Info(fmt.Sprintf("%s new stack file with hash= %s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, revision)) } if swarmStack.valuesFile != "" { From b7dd7fff97c9828a384d48ca60d848fb7fa59ddf Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 11:01:26 +0100 Subject: [PATCH 37/77] [stack] better test setup --- swarmcd/stack_test.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 65c0a01..b5e0288 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -8,22 +8,6 @@ import ( "testing" ) -func setupSwarmStackWithValues(t *testing.T, values string) *swarmStack { - t.Helper() - tempDir := t.TempDir() - valuesFilePath := path.Join(tempDir, "values.yaml") - if err := os.WriteFile(valuesFilePath, []byte(values), 0644); err != nil { - t.Fatalf("Failed to write values file: %v", err) - } - - return &swarmStack{ - name: "testStack", - repo: &stackRepo{path: tempDir}, - valuesFile: "values.yaml", - discoverSecrets: false, - } -} - func TestRenderComposeTemplate(t *testing.T) { tests := []struct { name string @@ -126,6 +110,22 @@ func TestRotateObjectsFileNotFound(t *testing.T) { } } +func setupSwarmStackWithValues(t *testing.T, values string) *swarmStack { + t.Helper() + tempDir := t.TempDir() + valuesFilePath := path.Join(tempDir, "values.yaml") + if err := os.WriteFile(valuesFilePath, []byte(values), 0644); err != nil { + t.Fatalf("Failed to write values file: %v", err) + } + + return &swarmStack{ + name: "testStack", + repo: &stackRepo{path: tempDir}, + valuesFile: "values.yaml", + discoverSecrets: false, + } +} + func setupTestStack(t *testing.T) (string, []byte, *swarmStack) { tempDir := t.TempDir() fileName := "testfile.txt" From cd806e04c3cd0d2609ca4078da47ec05db4d265e Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 11:47:15 +0100 Subject: [PATCH 38/77] [stack] fix readme review comments --- README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7c4b065..e1be86f 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,6 @@ services: - ./repos.yaml:/app/repos.yaml:ro - ./stacks.yaml:/app/stacks.yaml:ro - swarmcd_data:/data - environment: - - SWARMCD_DB=/data/revisions.db volumes: swarmcd_data: @@ -68,7 +66,30 @@ for new changes, pulling them and updating the stack. SwarmCD uses a minimal DB to track the last deployed revision across container restarts. By default, it stores data in data/revisions.db, but this can be changed via the `SWARMCD_DB` environment variable as -shown in the above docker-compose file. +shown in the below docker-compose file. + +```yaml +# docker-compose.yaml +version: '3.7' +services: + swarm-cd: + image: ghcr.io/m-adawi/swarm-cd:latest + deploy: + placement: + constraints: + - node.role == manager + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./repos.yaml:/app/repos.yaml:ro + - ./stacks.yaml:/app/stacks.yaml:ro + - swarmcd_data:/data + environment: + - SWARMCD_DB=/data/revisions.db + +volumes: + swarmcd_data: + driver: local +``` ## Manage Encrypted Secrets Using SOPS @@ -114,7 +135,6 @@ services: target: /secrets/age.key environment: - SOPS_AGE_KEY_FILE=/secrets/age.key - - SWARMCD_DB=/data/revisions.db volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./repos.yaml:/app/repos.yaml:ro @@ -238,7 +258,6 @@ services: - swarmcd_data:/data environment: - SOPS_AGE_KEY_FILE=/secrets/age.key - - SWARMCD_DB=/data/revisions.db secrets: - source: docker-config target: /root/.docker/config.json From d083340e5cb1701ab5baeb81c29b996d8160561f Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 12:02:11 +0100 Subject: [PATCH 39/77] [database] use git hash as revision to prevent confusion --- swarmcd/database_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index b1e3b6d..2ad8427 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -17,7 +17,7 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { defer db.Close() stackName := "test-stack" - revision := "v1.0.0" + revision := "abcdefgh" stackContent := []byte("test content") err = saveLastDeployedRevision(db, stackName, revision, stackContent) From 969f74d9600944257318879fe6ec171426201820 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 13:23:47 +0100 Subject: [PATCH 40/77] [stack] fix log about remaining at current revision --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 5a49593..f33774b 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -83,10 +83,10 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) if newStackHash == deployedStackHash { logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) return revision, nil } else { logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) - logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, revision)) } if swarmStack.valuesFile != "" { From bd094715685e9b73de49dbdd0622c5856ff87392 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 15:27:41 +0100 Subject: [PATCH 41/77] [stack] include all the read data into the hash --- swarmcd/stack.go | 58 ++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index f33774b..904eb06 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -78,17 +78,6 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } - newStackHash := computeHash(stackBytes) - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) - if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) - logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) - return revision, nil - } else { - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) - } - if swarmStack.valuesFile != "" { log.Debug("rendering template...") stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) @@ -110,11 +99,26 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("rotating configs and secrets...") - err = swarmStack.rotateConfigsAndSecrets(stackContents) + // This contains all the new data of the config and secrets. + // We use this to determine if the stack has changed, and we + // need to redeploy. + dataBytes, err := swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { return "", fmt.Errorf("failed to rotate configs and secrets for %s stack: %w", swarmStack.name, err) } + dataToHash := append(stackBytes, dataBytes...) + newStackHash := computeHash(dataToHash) + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) + if newStackHash == deployedStackHash { + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) + return revision, nil + } else { + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) + } + log.Debug("writing stack to file...") err = swarmStack.writeStack(stackContents) if err != nil { @@ -128,7 +132,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, stackBytes) + err = saveLastDeployedRevision(db, swarmStack.name, revision, dataToHash) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -212,28 +216,33 @@ func discoverSecrets(composeMap map[string]any, composePath string) ([]string, e return sopsFiles, nil } -func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) error { +func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) ([]byte, error) { + var dataBytes []byte if configs, ok := composeMap["configs"].(map[string]any); ok { - err := swarmStack.rotateObjects(configs) + configBytes, err := swarmStack.rotateObjects(configs) + dataBytes = append(dataBytes, configBytes...) + if err != nil { - return fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) + return nil, fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) } } if secrets, ok := composeMap["secrets"].(map[string]any); ok { - err := swarmStack.rotateObjects(secrets) + secretsByte, err := swarmStack.rotateObjects(secrets) + dataBytes = append(dataBytes, secretsByte...) if err != nil { - return fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err) + return nil, fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err) } } - return nil + return dataBytes, nil } -func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { +func (swarmStack *swarmStack) rotateObjects(objects map[string]any) ([]byte, error) { objectsDir := path.Dir(path.Join(swarmStack.repo.path, swarmStack.composePath)) + var configBytes []byte for objectName, object := range objects { objectMap, ok := object.(map[string]any) if !ok { - return fmt.Errorf("invalid compose file: %s object must be a map", objectName) + return nil, fmt.Errorf("invalid compose file: %s object must be a map", objectName) } // If "external" field exists and is true, skip processing if external, exists := objectMap["external"].(bool); exists && external { @@ -241,17 +250,18 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error { } objectFile, ok := objectMap["file"].(string) if !ok { - return fmt.Errorf("invalid compose file: %s file field must be a string", objectName) + return nil, fmt.Errorf("invalid compose file: %s file field must be a string", objectName) } objectFilePath := path.Join(objectsDir, objectFile) configFileBytes, err := os.ReadFile(objectFilePath) + configBytes = append(configBytes, configFileBytes...) if err != nil { - return fmt.Errorf("could not read file %s for rotation: %w", objectFilePath, err) + return nil, fmt.Errorf("could not read file %s for rotation: %w", objectFilePath, err) } hash := fmt.Sprintf("%x", md5.Sum(configFileBytes))[:8] objectMap["name"] = swarmStack.name + "-" + objectName + "-" + hash } - return nil + return configBytes, nil } func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error { From 71947391d7d045cc835c152416e26d9472fa77d0 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 16:13:59 +0100 Subject: [PATCH 42/77] [stack] fix hash display for empty db --- swarmcd/stack.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 904eb06..c4609d3 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -109,14 +109,14 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { dataToHash := append(stackBytes, dataBytes...) newStackHash := computeHash(dataToHash) - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedStackHash[:8])) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newStackHash[:8])) + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedStackHash))) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newStackHash))) if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedStackHash), revision)) logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) return revision, nil } else { - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newStackHash[:8], revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) } log.Debug("writing stack to file...") @@ -292,3 +292,14 @@ func (swarmStack *swarmStack) deployStack() error { } return nil } + +func fmtHash(hash string) string { + var shortHash string + if len(hash) >= 8 { + shortHash = hash[:8] + } else { + shortHash = hash // Use as-is (empty or shorter than 8) + } + + return shortHash +} From bb7f76807dfa3910d925fbfa43785be957e0d725 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 16:16:54 +0100 Subject: [PATCH 43/77] fix tests --- swarmcd/stack_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index b5e0288..6d00411 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -41,7 +41,7 @@ func TestRotateObjects(t *testing.T) { "service1": map[string]any{"file": fileName}, } - err := swarm.rotateObjects(objects) + _, err := swarm.rotateObjects(objects) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -61,7 +61,7 @@ func TestRotateObjectsHandlesExternalTrue(t *testing.T) { "config2": map[string]any{"file": configFile}, } - err := swarm.rotateObjects(objects) + _, err := swarm.rotateObjects(objects) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -75,7 +75,7 @@ func TestRotateObjectsInvalidMap(t *testing.T) { objects := map[string]any{"service1": "invalid"} - err := swarm.rotateObjects(objects) + _, err := swarm.rotateObjects(objects) if err == nil { t.Fatalf("Expected an error but got none") } @@ -90,7 +90,7 @@ func TestRotateObjectsMissingFileField(t *testing.T) { objects := map[string]any{"service1": map[string]any{}} - err := swarm.rotateObjects(objects) + _, err := swarm.rotateObjects(objects) if err == nil { t.Fatalf("Expected an error but got none") } @@ -104,7 +104,7 @@ func TestRotateObjectsFileNotFound(t *testing.T) { swarm := &swarmStack{name: "test-stack", repo: &stackRepo{path: "nonexistent"}, composePath: "docker-compose.yml"} objects := map[string]any{"service1": map[string]any{"file": "missing.txt"}} - err := swarm.rotateObjects(objects) + _, err := swarm.rotateObjects(objects) if err == nil { t.Fatalf("Expected an error but got none") } From 986cb7bdd73f2001938f271816c42c4f1697719e Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Fri, 21 Mar 2025 16:30:23 +0100 Subject: [PATCH 44/77] be more explicit with the empty hash --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index c4609d3..45a57d8 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -298,7 +298,7 @@ func fmtHash(hash string) string { if len(hash) >= 8 { shortHash = hash[:8] } else { - shortHash = hash // Use as-is (empty or shorter than 8) + shortHash = "" } return shortHash From db586796d8d6b574bdb553694bb73e5cb3dcc7b2 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Sat, 22 Mar 2025 06:57:58 +0100 Subject: [PATCH 45/77] Update swarmcd/database.go Co-authored-by: Andrea Ghensi --- swarmcd/database.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 07579f6..6f0ed05 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -58,7 +58,8 @@ func loadLastDeployedRevision(db *sql.DB, stackName string) (revision string, ha err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { return "", "", nil - } else if err != nil { + } + if err != nil { return "", "", fmt.Errorf("failed to query revision: %w", err) } From b31247a7e4dcd1b2ab6d30f88652dc722829806a Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Sat, 22 Mar 2025 06:58:12 +0100 Subject: [PATCH 46/77] Update swarmcd/stack.go Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 45a57d8..8b8981e 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -62,7 +62,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } if lastRevision == "" { - logger.Info(fmt.Sprintf("%s no last revision revision found", swarmStack.name)) + logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } if lastRevision == revision { From b63120eef6b348e5a6d4eed5349f5c5a815b4dd0 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Sat, 22 Mar 2025 06:58:25 +0100 Subject: [PATCH 47/77] Update swarmcd/stack.go Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 8b8981e..fafc58c 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -296,10 +296,7 @@ func (swarmStack *swarmStack) deployStack() error { func fmtHash(hash string) string { var shortHash string if len(hash) >= 8 { - shortHash = hash[:8] - } else { - shortHash = "" + return hash[:8] } - - return shortHash + return "" } From a120d6fa841bf0e5b35bf780f5f8a1d7ee7604f9 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 07:38:51 +0100 Subject: [PATCH 48/77] fix merge error --- swarmcd/stack.go | 1 - 1 file changed, 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index fd6b659..e94dba6 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -316,7 +316,6 @@ func (swarmStack *swarmStack) deployStack() error { } func fmtHash(hash string) string { - var shortHash string if len(hash) >= 8 { return hash[:8] } From 20e2c8032c26c4e7bcabf75753dee1494082383f Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 07:53:39 +0100 Subject: [PATCH 49/77] better approach to calculate the stack hash --- swarmcd/stack.go | 55 ++++++++++++++++++------------------------- swarmcd/stack_test.go | 12 ++++------ 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index e94dba6..30772cd 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -99,16 +99,18 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("rotating configs and secrets...") - // This contains all the new data of the config and secrets. - // We use this to determine if the stack has changed, and we - // need to redeploy. - dataBytes, err := swarmStack.rotateConfigsAndSecrets(stackContents) + err = swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { return "", fmt.Errorf("failed to rotate configs and secrets for %s stack: %w", swarmStack.name, err) } - dataToHash := append(stackBytes, dataBytes...) - newStackHash := computeHash(dataToHash) + log.Debug("writing stack to file...") + writtenBytes, err := swarmStack.writeStack(stackContents) + if err != nil { + return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) + } + + newStackHash := computeHash(writtenBytes) logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedStackHash))) logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newStackHash))) if newStackHash == deployedStackHash { @@ -119,12 +121,6 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) } - log.Debug("writing stack to file...") - err = swarmStack.writeStack(stackContents) - if err != nil { - return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) - } - log.Debug("deploying stack...") err = swarmStack.deployStack() if err != nil { @@ -132,7 +128,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, dataToHash) + err = saveLastDeployedRevision(db, swarmStack.name, revision, writtenBytes) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -225,29 +221,25 @@ func discoverSecrets(composeMap map[string]any, composePath string) ([]string, e return sopsFiles, nil } -func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) ([]byte, error) { - var dataBytes []byte +func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) error { if configs, ok := composeMap["configs"].(map[string]any); ok { - configBytes, err := swarmStack.rotateObjects(configs, "configs") - dataBytes = append(dataBytes, configBytes...) + err := swarmStack.rotateObjects(configs, "configs") if err != nil { - return nil, fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) + return fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) } } if secrets, ok := composeMap["secrets"].(map[string]any); ok { - secretsByte, err := swarmStack.rotateObjects(secrets, "secrets") - dataBytes = append(dataBytes, secretsByte...) + err := swarmStack.rotateObjects(secrets, "secrets") if err != nil { - return nil, fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err) + return fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err) } } - return dataBytes, nil + return nil } -func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType string) ([]byte, error) { +func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType string) error { objectsDir := path.Dir(path.Join(swarmStack.repo.path, swarmStack.composePath)) - var configBytes []byte for objectName, object := range objects { log := logger.With( slog.String("stack", swarmStack.name), @@ -256,7 +248,7 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType s ) objectMap, ok := object.(map[string]any) if !ok { - return nil, fmt.Errorf("invalid compose file: %s object must be a map", objectName) + return fmt.Errorf("invalid compose file: %s object must be a map", objectName) } // If "external" field exists and is true, skip processing if external, exists := objectMap["external"].(bool); exists && external { @@ -268,14 +260,13 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType s } objectFile, ok := objectMap["file"].(string) if !ok { - return nil, fmt.Errorf("invalid compose file: %s file field must be a string", objectName) + return fmt.Errorf("invalid compose file: %s file field must be a string", objectName) } log.Debug("reading...", "file", objectFile) objectFilePath := path.Join(objectsDir, objectFile) configFileBytes, err := os.ReadFile(objectFilePath) - configBytes = append(configBytes, configFileBytes...) if err != nil { - return nil, fmt.Errorf("could not read file %s for rotation: %w", objectFilePath, err) + return fmt.Errorf("could not read file %s for rotation: %w", objectFilePath, err) } log.Debug("computing hash...", "file", objectFile) hash := fmt.Sprintf("%x", md5.Sum(configFileBytes))[:8] @@ -283,18 +274,18 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType s log.Debug("renaming...", "new_name", newObjectName) objectMap["name"] = newObjectName } - return configBytes, nil + return nil } -func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error { +func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, error) { composeFileBytes, err := yaml.Marshal(composeMap) if err != nil { - return fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) + return nil, fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name) } composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath) fileInfo, _ := os.Stat(composeFile) os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode()) - return nil + return composeFileBytes, nil } func (swarmStack *swarmStack) deployStack() error { diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 2b3eeb1..91561fb 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -1,8 +1,6 @@ package swarmcd import ( - "crypto/md5" - "fmt" "os" "path" "sync" @@ -42,7 +40,7 @@ func TestRotateExternalObjects(t *testing.T) { objects := map[string]any{ "my-secret": map[string]any{"external": true}, } - _, err := stack.rotateObjects(objects, "secrets") + err := stack.rotateObjects(objects, "secrets") if err != nil { t.Errorf("unexpected error: %s", err) } @@ -56,7 +54,7 @@ func TestRotateObjectsHandlesExternalTrue(t *testing.T) { "config2": map[string]any{"file": configFile}, } - _, err := swarm.rotateObjects(objects, "secrets") + err := swarm.rotateObjects(objects, "secrets") if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -70,7 +68,7 @@ func TestRotateObjectsInvalidMap(t *testing.T) { objects := map[string]any{"service1": "invalid"} - _, err := swarm.rotateObjects(objects, "secrets") + err := swarm.rotateObjects(objects, "secrets") if err == nil { t.Fatalf("Expected an error but got none") } @@ -85,7 +83,7 @@ func TestRotateObjectsMissingFileField(t *testing.T) { objects := map[string]any{"service1": map[string]any{}} - _, err := swarm.rotateObjects(objects, "secrets") + err := swarm.rotateObjects(objects, "secrets") if err == nil { t.Fatalf("Expected an error but got none") } @@ -99,7 +97,7 @@ func TestRotateObjectsFileNotFound(t *testing.T) { swarm := &swarmStack{name: "test-stack", repo: &stackRepo{path: "nonexistent"}, composePath: "docker-compose.yml"} objects := map[string]any{"service1": map[string]any{"file": "missing.txt"}} - _, err := swarm.rotateObjects(objects, "secrets") + err := swarm.rotateObjects(objects, "secrets") if err == nil { t.Fatalf("Expected an error but got none") } From 80389c0c35f93f8f078be52b408729e7f8e079e8 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:01:09 +0100 Subject: [PATCH 50/77] earlier return --- swarmcd/stack.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 30772cd..5a80780 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -61,13 +61,11 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } - if lastRevision == "" { - logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) - } - if lastRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) return revision, nil + } else if lastRevision == "" { + logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } logger.Info(fmt.Sprintf("%s new revision revision found %s! will update the stack", swarmStack.name, revision)) From d915ff9f0e22da7992dcf496088aa6f409547a1c Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:03:01 +0100 Subject: [PATCH 51/77] fix unnecessary diff --- swarmcd/swarmcd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 61b798f..0e6da95 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -11,7 +11,6 @@ var stacks []*swarmStack func Run() { logger.Info("starting SwarmCD") - for { var waitGroup sync.WaitGroup logger.Info("updating stacks...") From 07b0fe9ebf563e328a5ad71383285de6f8fbd5ff Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:06:35 +0100 Subject: [PATCH 52/77] better organize test file --- swarmcd/stack_test.go | 68 ++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 91561fb..aec9fc2 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -34,18 +34,6 @@ func TestRenderComposeTemplate(t *testing.T) { } // External objects are ignored by the rotation -func TestRotateExternalObjects(t *testing.T) { - repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} - stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false) - objects := map[string]any{ - "my-secret": map[string]any{"external": true}, - } - err := stack.rotateObjects(objects, "secrets") - if err != nil { - t.Errorf("unexpected error: %s", err) - } -} - func TestRotateObjectsHandlesExternalTrue(t *testing.T) { configFile, _, swarm := setupTestStack(t) @@ -103,34 +91,6 @@ func TestRotateObjectsFileNotFound(t *testing.T) { } } -func setupSwarmStackWithValues(t *testing.T, values string) *swarmStack { - t.Helper() - tempDir := t.TempDir() - valuesFilePath := path.Join(tempDir, "values.yaml") - if err := os.WriteFile(valuesFilePath, []byte(values), 0644); err != nil { - t.Fatalf("Failed to write values file: %v", err) - } - - return &swarmStack{ - name: "testStack", - repo: &stackRepo{path: tempDir}, - valuesFile: "values.yaml", - discoverSecrets: false, - } -} - -func setupTestStack(t *testing.T) (string, []byte, *swarmStack) { - tempDir := t.TempDir() - fileName := "testfile.txt" - filePath := path.Join(tempDir, fileName) - fileContent := []byte("test content") - os.WriteFile(filePath, fileContent, 0644) - - repo := &stackRepo{path: tempDir} - swarm := &swarmStack{name: "test-stack", repo: repo, composePath: "docker-compose.yml"} - return fileName, fileContent, swarm -} - // Secrets are discovered, external secrets are ignored func TestSecretDiscovery(t *testing.T) { repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} @@ -161,3 +121,31 @@ secrets: t.Errorf("unexpected sops file: %s", sopsFiles[0]) } } + +func setupSwarmStackWithValues(t *testing.T, values string) *swarmStack { + t.Helper() + tempDir := t.TempDir() + valuesFilePath := path.Join(tempDir, "values.yaml") + if err := os.WriteFile(valuesFilePath, []byte(values), 0644); err != nil { + t.Fatalf("Failed to write values file: %v", err) + } + + return &swarmStack{ + name: "testStack", + repo: &stackRepo{path: tempDir}, + valuesFile: "values.yaml", + discoverSecrets: false, + } +} + +func setupTestStack(t *testing.T) (string, []byte, *swarmStack) { + tempDir := t.TempDir() + fileName := "testfile.txt" + filePath := path.Join(tempDir, fileName) + fileContent := []byte("test content") + os.WriteFile(filePath, fileContent, 0644) + + repo := &stackRepo{path: tempDir} + swarm := &swarmStack{name: "test-stack", repo: repo, composePath: "docker-compose.yml"} + return fileName, fileContent, swarm +} From 400f9be4ccdd5095a09e8cc4f393a30c09004c73 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:07:54 +0100 Subject: [PATCH 53/77] fix merge errors --- swarmcd/stack.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 5a80780..29f8afb 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -222,7 +222,6 @@ func discoverSecrets(composeMap map[string]any, composePath string) ([]string, e func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) error { if configs, ok := composeMap["configs"].(map[string]any); ok { err := swarmStack.rotateObjects(configs, "configs") - if err != nil { return fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err) } @@ -248,10 +247,6 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any, objectType s if !ok { return fmt.Errorf("invalid compose file: %s object must be a map", objectName) } - // If "external" field exists and is true, skip processing - if external, exists := objectMap["external"].(bool); exists && external { - continue - } isExternal, ok := objectMap["external"].(bool) if ok && isExternal { continue From 66fe6d9d06e60f12034f8a65b58ad9e77114593e Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:13:30 +0100 Subject: [PATCH 54/77] better naming --- swarmcd/stack_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index aec9fc2..e5bac26 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -33,8 +33,7 @@ func TestRenderComposeTemplate(t *testing.T) { } } -// External objects are ignored by the rotation -func TestRotateObjectsHandlesExternalTrue(t *testing.T) { +func TestRotateObjects(t *testing.T) { configFile, _, swarm := setupTestStack(t) objects := map[string]any{ @@ -46,6 +45,8 @@ func TestRotateObjectsHandlesExternalTrue(t *testing.T) { if err != nil { t.Fatalf("Unexpected error: %v", err) } + + // External objects are ignored by the rotation if _, exists := objects["config1"].(map[string]any)["name"]; exists { t.Errorf("Expected config1 to be unmodified, but 'name' was set") } From f9ebdd66ec62006a5beed5e71825a48090163337 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:14:12 +0100 Subject: [PATCH 55/77] add line break in gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ca9902c..2feb33b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ swarm-cd docker-compose.yaml repos.yaml stacks.yaml -config.yaml \ No newline at end of file +config.yaml From 78d17539455765c9b57b100f88dd0a16c35dc9d7 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:19:15 +0100 Subject: [PATCH 56/77] extract should deploy method --- swarmcd/stack.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 29f8afb..d85d655 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -108,15 +108,8 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } - newStackHash := computeHash(writtenBytes) - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedStackHash))) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newStackHash))) - if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedStackHash), revision)) - logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) + if !swarmStack.shouldDeploy(writtenBytes, deployedStackHash, revision, lastRevision) { return revision, nil - } else { - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) } log.Debug("deploying stack...") @@ -281,6 +274,20 @@ func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, err return composeFileBytes, nil } +func (swarmStack *swarmStack) shouldDeploy(writtenBytes []byte, deployedStackHash string, revision string, lastRevision string) bool { + newStackHash := computeHash(writtenBytes) + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedStackHash))) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newStackHash))) + if newStackHash == deployedStackHash { + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedStackHash), revision)) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) + return false + } else { + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) + return true + } +} + func (swarmStack *swarmStack) deployStack() error { cmd := stack.NewStackCommand(dockerCli) cmd.SetArgs([]string{ From dd23badcfdcb26cc5f39a4a7acad7526f40a1bf6 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:30:37 +0100 Subject: [PATCH 57/77] return last revision if deployment was skipped --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index d85d655..2e53161 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -109,7 +109,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } if !swarmStack.shouldDeploy(writtenBytes, deployedStackHash, revision, lastRevision) { - return revision, nil + return lastRevision, nil } log.Debug("deploying stack...") From 638e81997f47b5ecbabd8a51eb5742c32604e57c Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 08:30:46 +0100 Subject: [PATCH 58/77] uniform log format --- swarmcd/swarmcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 0e6da95..3a7ab6c 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -30,7 +30,7 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { defer repoLock.Unlock() defer waitGroup.Done() - logger.Info(fmt.Sprintf("updating %s stack", swarmStack.name)) + logger.Info(fmt.Sprintf("%s updating stack", swarmStack.name)) revision, err := swarmStack.updateStack() if err != nil { stackStatus[swarmStack.name].Error = err.Error() @@ -40,7 +40,7 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { stackStatus[swarmStack.name].Error = "" stackStatus[swarmStack.name].Revision = revision - logger.Info(fmt.Sprintf("done updating %s stack", swarmStack.name)) + logger.Info(fmt.Sprintf("%s done updating stack", swarmStack.name)) } func GetStackStatus() map[string]*StackStatus { From 291b2c99cdfd73b563c00b74cf1cd1b0df422022 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 09:00:18 +0100 Subject: [PATCH 59/77] remove that are no longer necessary after merging master --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f4c0ee..289f7d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,10 +27,8 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates # Copy the built backend binary from the backend build stage COPY --from=backend-build /swarm-cd /app/ # Copy the built frontend from the frontend build stage -COPY --from=frontend-build /ui/dist/ /app/ui/dist/ +COPY --from=frontend-build /ui/dist/ /app/ui/ # Sets the web server mode to release ENV GIN_MODE=release - -EXPOSE 8080 # Set the entry point for the application CMD ["/app/swarm-cd"] \ No newline at end of file From 399130390b2600913c8814c261feb74f2a1f8be0 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 09:02:44 +0100 Subject: [PATCH 60/77] return error instead of nothing after opening database --- swarmcd/stack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 2e53161..aadcdf8 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -51,8 +51,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { db, err := initDB(getDBFilePath()) if err != nil { - logger.Error(fmt.Sprintf("failed to open database: %s", err)) - return + return "", fmt.Errorf("failed to open database: %s", err) } defer db.Close() From 621e2a55b1172f47c5a8795e0e71883d81b9dfd6 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Sat, 22 Mar 2025 10:20:43 +0100 Subject: [PATCH 61/77] Update swarmcd/stack.go Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index aadcdf8..60d759f 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -281,10 +281,9 @@ func (swarmStack *swarmStack) shouldDeploy(writtenBytes []byte, deployedStackHas logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedStackHash), revision)) logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) return false - } else { - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) - return true } + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) + return true } func (swarmStack *swarmStack) deployStack() error { From a5d89991a9fdf74ac0ce24a0f81236307d64eac1 Mon Sep 17 00:00:00 2001 From: clangenb <37865735+clangenb@users.noreply.github.com> Date: Sat, 22 Mar 2025 10:20:52 +0100 Subject: [PATCH 62/77] Update swarmcd/stack.go Co-authored-by: Andrea Ghensi --- swarmcd/stack.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 60d759f..decbc16 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -63,7 +63,8 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { if lastRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) return revision, nil - } else if lastRevision == "" { + } + if lastRevision == "" { logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } From 98b1b0664fdb20e72222201017bcd8b3a267b6d0 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 11:00:18 +0100 Subject: [PATCH 63/77] introduce a struct for nicer interface --- swarmcd/database.go | 35 +++++++++++++++++++++++++++-------- swarmcd/database_test.go | 13 +++++++------ swarmcd/stack.go | 31 ++++++++++++++++--------------- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 6f0ed05..a49c05b 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -8,6 +8,25 @@ import ( "os" ) +type version struct { + revision string + hash string +} + +func newVersion(revision string, hash string) *version { + return &version{ + revision: revision, + hash: hash, + } +} + +func newVersionFromData(revision string, data []byte) *version { + return &version{ + revision: revision, + hash: computeHash(data), + } +} + func getDBFilePath() string { if path := os.Getenv("SWARMCD_DB"); path != "" { return path @@ -35,8 +54,7 @@ func initDB(dbFile string) (*sql.DB, error) { } // Save last deployed revision and hash -func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackContent []byte) error { - hash := computeHash(stackContent) +func saveLastDeployedRevision(db *sql.DB, stackName string, version *version) error { _, err := db.Exec(` INSERT INTO revisions (stack, revision, hash) @@ -44,7 +62,7 @@ func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackConte ON CONFLICT(stack) DO UPDATE SET revision = excluded.revision, hash = excluded.hash - `, stackName, revision, hash) + `, stackName, version.revision, version.hash) if err != nil { return fmt.Errorf("failed to save revision: %w", err) @@ -54,16 +72,17 @@ func saveLastDeployedRevision(db *sql.DB, stackName, revision string, stackConte } // Load a stack's revision and hash -func loadLastDeployedRevision(db *sql.DB, stackName string) (revision string, hash string, err error) { +func loadLastDeployedRevision(db *sql.DB, stackName string) (version *version, err error) { + var revision, hash string err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { - return "", "", nil - } + return nil, nil + } if err != nil { - return "", "", fmt.Errorf("failed to query revision: %w", err) + return nil, fmt.Errorf("failed to query revision: %w", err) } - return revision, hash, nil + return newVersion(revision, hash), nil } // Compute a SHA-256 hash of the stack content diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 2ad8427..f4359da 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -20,23 +20,24 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { revision := "abcdefgh" stackContent := []byte("test content") - err = saveLastDeployedRevision(db, stackName, revision, stackContent) + version := newVersionFromData(revision, stackContent) + err = saveLastDeployedRevision(db, stackName, version) if err != nil { t.Fatalf("Failed to save revision: %v", err) } - loadedRevision, loadedHash, err := loadLastDeployedRevision(db, stackName) + loadedVersion, err := loadLastDeployedRevision(db, stackName) if err != nil { t.Fatalf("Failed to load revision: %v", err) } expectedHash := computeHash(stackContent) - if loadedRevision != revision { - t.Errorf("Expected revision %s, got %s", revision, loadedRevision) + if loadedVersion.revision != revision { + t.Errorf("Expected revision %s, got %s", revision, loadedVersion.revision) } - if loadedHash != expectedHash { - t.Errorf("Expected hash %s, got %s", expectedHash, loadedHash) + if loadedVersion.hash != expectedHash { + t.Errorf("Expected hash %s, got %s", expectedHash, loadedVersion.hash) } } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index decbc16..036cb81 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -55,16 +55,16 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } defer db.Close() - lastRevision, deployedStackHash, err := loadLastDeployedRevision(db, swarmStack.name) + deployedVersion, err := loadLastDeployedRevision(db, swarmStack.name) if err != nil { return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } - if lastRevision == revision { + if deployedVersion.revision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) return revision, nil - } - if lastRevision == "" { + } + if deployedVersion.revision == "" { logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } @@ -108,8 +108,10 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } - if !swarmStack.shouldDeploy(writtenBytes, deployedStackHash, revision, lastRevision) { - return lastRevision, nil + updatedVersion := newVersionFromData(revision, writtenBytes) + + if !swarmStack.shouldDeploy(updatedVersion, deployedVersion) { + return deployedVersion.revision, nil } log.Debug("deploying stack...") @@ -119,7 +121,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, revision, writtenBytes) + err = saveLastDeployedRevision(db, swarmStack.name, updatedVersion) if err != nil { return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } @@ -274,16 +276,15 @@ func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, err return composeFileBytes, nil } -func (swarmStack *swarmStack) shouldDeploy(writtenBytes []byte, deployedStackHash string, revision string, lastRevision string) bool { - newStackHash := computeHash(writtenBytes) - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedStackHash))) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newStackHash))) - if newStackHash == deployedStackHash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedStackHash), revision)) - logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, lastRevision)) +func (swarmStack *swarmStack) shouldDeploy(newVersion *version, deployedVersion *version) bool { + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedVersion.hash))) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newVersion.hash))) + if newVersion.hash == deployedVersion.hash { + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedVersion.hash), fmtHash(newVersion.hash))) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, deployedVersion.hash)) return false } - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newStackHash), revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newVersion.hash), newVersion.hash)) return true } From 965961d8b48a733e5cc920b55cbc2c6d89a67838 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 11:06:04 +0100 Subject: [PATCH 64/77] add fmtHash method to struct --- swarmcd/database.go | 11 +++++++++++ swarmcd/stack.go | 15 ++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index a49c05b..057a25f 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -27,6 +27,10 @@ func newVersionFromData(revision string, data []byte) *version { } } +func (version *version) fmtHash() string { + return fmtHash(version.hash) +} + func getDBFilePath() string { if path := os.Getenv("SWARMCD_DB"); path != "" { return path @@ -90,3 +94,10 @@ func computeHash(data []byte) string { hash := sha256.Sum256(data) return fmt.Sprintf("%x", hash) } + +func fmtHash(hash string) string { + if len(hash) >= 8 { + return hash[:8] + } + return "" +} diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 036cb81..6e114ee 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -277,14 +277,14 @@ func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, err } func (swarmStack *swarmStack) shouldDeploy(newVersion *version, deployedVersion *version) bool { - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, fmtHash(deployedVersion.hash))) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, fmtHash(newVersion.hash))) + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedVersion.fmtHash())) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newVersion.fmtHash())) if newVersion.hash == deployedVersion.hash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, fmtHash(deployedVersion.hash), fmtHash(newVersion.hash))) + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedVersion.fmtHash()), newVersion.revision) logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, deployedVersion.hash)) return false } - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, fmtHash(newVersion.hash), newVersion.hash)) + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newVersion.fmtHash(), newVersion.revision)) return true } @@ -305,10 +305,3 @@ func (swarmStack *swarmStack) deployStack() error { } return nil } - -func fmtHash(hash string) string { - if len(hash) >= 8 { - return hash[:8] - } - return "" -} From 47837413055684fe14d488c3b75da7ebbfc30058 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 11:07:56 +0100 Subject: [PATCH 65/77] fix sprintf statement --- swarmcd/stack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 6e114ee..9708838 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -280,7 +280,7 @@ func (swarmStack *swarmStack) shouldDeploy(newVersion *version, deployedVersion logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedVersion.fmtHash())) logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newVersion.fmtHash())) if newVersion.hash == deployedVersion.hash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedVersion.fmtHash()), newVersion.revision) + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedVersion.fmtHash(), newVersion.revision)) logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, deployedVersion.hash)) return false } From c40c9e5757e7ef8a09720c546085269824ac093b Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 11:10:40 +0100 Subject: [PATCH 66/77] fix null-pointer upon first startup --- swarmcd/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 057a25f..0a92f1f 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -80,7 +80,7 @@ func loadLastDeployedRevision(db *sql.DB, stackName string) (version *version, e var revision, hash string err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) if err == sql.ErrNoRows { - return nil, nil + return newVersion("", ""), nil } if err != nil { return nil, fmt.Errorf("failed to query revision: %w", err) From 793f0c44680fef084357703a6d205054800c5561 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 11:17:33 +0100 Subject: [PATCH 67/77] persist current revision to DB even if we don't deploy --- swarmcd/stack.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 9708838..1a188dd 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -110,14 +110,12 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { updatedVersion := newVersionFromData(revision, writtenBytes) - if !swarmStack.shouldDeploy(updatedVersion, deployedVersion) { - return deployedVersion.revision, nil - } - - log.Debug("deploying stack...") - err = swarmStack.deployStack() - if err != nil { - return revision, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) + if swarmStack.shouldDeploy(updatedVersion, deployedVersion) { + log.Debug("deploying stack...") + err = swarmStack.deployStack() + if err != nil { + return revision, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) + } } log.Debug("saving current revision to db...") From 85d20e75a9b6ac7880d2d0117a8ffc007894fd43 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:01:07 +0100 Subject: [PATCH 68/77] save deployment date --- swarmcd/database.go | 77 +++++++++++++++++++++++++--------------- swarmcd/database_test.go | 34 ++++++++++++++---- swarmcd/init.go | 17 ++++----- swarmcd/stack.go | 55 +++++++++++++++------------- swarmcd/swarmcd.go | 6 ++-- web/controllers.go | 12 ++++--- 6 files changed, 127 insertions(+), 74 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 0a92f1f..293ea5a 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -6,28 +6,35 @@ import ( "fmt" _ "modernc.org/sqlite" "os" + "time" ) -type version struct { - revision string - hash string +type stackMetadata struct { + repoRevision string + deployedStackRevision string + deployedAt time.Time + hash string } -func newVersion(revision string, hash string) *version { - return &version{ - revision: revision, - hash: hash, +func newVersion(repoRevision string, stackRevision string, hash string, time time.Time) *stackMetadata { + return &stackMetadata{ + repoRevision: repoRevision, + deployedStackRevision: stackRevision, + hash: hash, + deployedAt: time, } } -func newVersionFromData(revision string, data []byte) *version { - return &version{ - revision: revision, - hash: computeHash(data), +func newVersionFromData(repoRevision string, stackRevision string, data []byte) *stackMetadata { + return &stackMetadata{ + repoRevision: repoRevision, + deployedStackRevision: stackRevision, + hash: computeHash(data), + deployedAt: time.Now(), } } -func (version *version) fmtHash() string { +func (version *stackMetadata) fmtHash() string { return fmtHash(version.hash) } @@ -47,8 +54,10 @@ func initDB(dbFile string) (*sql.DB, error) { _, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions ( stack TEXT PRIMARY KEY, - revision TEXT, - hash TEXT + repo_revision TEXT, + deployed_stack_revision TEXT, + hash TEXT, + deployed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )`) if err != nil { return nil, fmt.Errorf("failed to create table: %w", err) @@ -57,16 +66,17 @@ func initDB(dbFile string) (*sql.DB, error) { return db, nil } -// Save last deployed revision and hash -func saveLastDeployedRevision(db *sql.DB, stackName string, version *version) error { - +// Save last deployed stackMetadata +func saveLastDeployedRevision(db *sql.DB, stackName string, stackMetadata *stackMetadata) error { _, err := db.Exec(` - INSERT INTO revisions (stack, revision, hash) - VALUES (?, ?, ?) + INSERT INTO revisions (stack, repo_revision, deployed_stack_revision, hash, deployed_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(stack) DO UPDATE SET - revision = excluded.revision, - hash = excluded.hash - `, stackName, version.revision, version.hash) + repo_revision = excluded.repo_revision, + deployed_stack_revision = excluded.deployed_stack_revision, + hash = excluded.hash, + deployed_at = excluded.deployed_at + `, stackName, stackMetadata.repoRevision, stackMetadata.deployedStackRevision, stackMetadata.hash, stackMetadata.deployedAt) if err != nil { return fmt.Errorf("failed to save revision: %w", err) @@ -75,18 +85,29 @@ func saveLastDeployedRevision(db *sql.DB, stackName string, version *version) er return nil } -// Load a stack's revision and hash -func loadLastDeployedRevision(db *sql.DB, stackName string) (version *version, err error) { - var revision, hash string - err = db.QueryRow(`SELECT revision, hash FROM revisions WHERE stack = ?`, stackName).Scan(&revision, &hash) +// Load a stack's stackMetadata +func loadLastDeployedRevision(db *sql.DB, stackName string) (*stackMetadata, error) { + var repoRevision, deployedStackRevision, hash string + var deployedAt time.Time + + err := db.QueryRow(` + SELECT repo_revision, deployed_stack_revision, hash, deployed_at + FROM revisions + WHERE stack = ?`, stackName).Scan(&repoRevision, &deployedStackRevision, &hash, &deployedAt) + if err == sql.ErrNoRows { - return newVersion("", ""), nil + return newVersion("", "", "", time.Now()), nil } if err != nil { return nil, fmt.Errorf("failed to query revision: %w", err) } - return newVersion(revision, hash), nil + return &stackMetadata{ + repoRevision: repoRevision, + deployedStackRevision: deployedStackRevision, + hash: hash, + deployedAt: deployedAt, + }, nil } // Compute a SHA-256 hash of the stack content diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index f4359da..483d971 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -2,6 +2,7 @@ package swarmcd import ( "testing" + "time" ) import ( @@ -17,27 +18,48 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { defer db.Close() stackName := "test-stack" - revision := "abcdefgh" + repoRevision := "abcdefgh" + stackRevision := "12345678" stackContent := []byte("test content") - version := newVersionFromData(revision, stackContent) + version := newVersionFromData(repoRevision, stackRevision, stackContent) + now := time.Now() + version.deployedAt = now + err = saveLastDeployedRevision(db, stackName, version) if err != nil { - t.Fatalf("Failed to save revision: %v", err) + t.Fatalf("Failed to save repoRevision: %v", err) } loadedVersion, err := loadLastDeployedRevision(db, stackName) if err != nil { - t.Fatalf("Failed to load revision: %v", err) + t.Fatalf("Failed to load repoRevision: %v", err) } expectedHash := computeHash(stackContent) - if loadedVersion.revision != revision { - t.Errorf("Expected revision %s, got %s", revision, loadedVersion.revision) + if loadedVersion.repoRevision != repoRevision { + t.Errorf("Expected repoRevision %s, got %s", repoRevision, loadedVersion.repoRevision) + } + + if loadedVersion.deployedStackRevision != stackRevision { + t.Errorf("Expected repoRevision %s, got %s", repoRevision, loadedVersion.deployedStackRevision) + } + + if !isRoughlyEqual(loadedVersion.deployedAt, now, 1*time.Microsecond) { + t.Errorf("Expected time %s, got %s", now, loadedVersion.deployedAt) } if loadedVersion.hash != expectedHash { t.Errorf("Expected hash %s, got %s", expectedHash, loadedVersion.hash) } } + +func isRoughlyEqual(t1, t2 time.Time, tolerance time.Duration) bool { + diff := t2.Sub(t1) + // Check if the difference is within the tolerance + if diff < 0 { + diff = -diff // Handle negative difference + } + return diff <= tolerance +} diff --git a/swarmcd/init.go b/swarmcd/init.go index e17940d..cde495c 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -2,21 +2,22 @@ package swarmcd import ( "fmt" - "log/slog" - "os" - "path" - "strings" - "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/flags" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/m-adawi/swarm-cd/util" + "log/slog" + "os" + "path" + "strings" ) type StackStatus struct { - Error string - Revision string - RepoURL string + Error string + Revision string + DeployedStackRevision string + DeployedAt string + RepoURL string } var config *util.Config = &util.Configs diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 1a188dd..07ace83 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -36,14 +36,14 @@ func newSwarmStack(name string, repo *stackRepo, branch string, composePath stri } } -func (swarmStack *swarmStack) updateStack() (revision string, err error) { +func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err error) { log := logger.With( slog.String("stack", swarmStack.name), slog.String("branch", swarmStack.branch), ) log.Debug("pulling changes...") - revision, err = swarmStack.repo.pullChanges(swarmStack.branch) + revision, err := swarmStack.repo.pullChanges(swarmStack.branch) if err != nil { return } @@ -51,20 +51,20 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { db, err := initDB(getDBFilePath()) if err != nil { - return "", fmt.Errorf("failed to open database: %s", err) + return nil, fmt.Errorf("failed to open database: %s", err) } defer db.Close() deployedVersion, err := loadLastDeployedRevision(db, swarmStack.name) if err != nil { - return "", fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } - if deployedVersion.revision == revision { + if deployedVersion.repoRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) - return revision, nil + return deployedVersion, nil } - if deployedVersion.revision == "" { + if deployedVersion.repoRevision == "" { logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } @@ -73,7 +73,7 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { log.Debug("reading stack file...") stackBytes, err := swarmStack.readStack() if err != nil { - return "", fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to read stack for %s stack: %w", swarmStack.name, err) } if swarmStack.valuesFile != "" { @@ -81,47 +81,52 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) } if err != nil { - return "", fmt.Errorf("failed to render compose template for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to render compose template for %s stack: %w", swarmStack.name, err) } log.Debug("parsing stack content...") stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) if err != nil { - return "", fmt.Errorf("failed to parse stack content for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to parse stack content for %s stack: %w", swarmStack.name, err) } log.Debug("decrypting secrets...") err = swarmStack.decryptSopsFiles(stackContents) if err != nil { - return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err) } log.Debug("rotating configs and secrets...") err = swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { - return "", fmt.Errorf("failed to rotate configs and secrets for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to rotate configs and secrets for %s stack: %w", swarmStack.name, err) } log.Debug("writing stack to file...") writtenBytes, err := swarmStack.writeStack(stackContents) if err != nil { - return "", fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } - updatedVersion := newVersionFromData(revision, writtenBytes) + updatedVersion := newVersionFromData(revision, revision, writtenBytes) if swarmStack.shouldDeploy(updatedVersion, deployedVersion) { log.Debug("deploying stack...") err = swarmStack.deployStack() if err != nil { - return revision, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) } + + log.Debug("saving current stack's metadata to db...") + err = saveLastDeployedRevision(db, swarmStack.name, updatedVersion) + } else { + log.Debug("updating stacks.repoRevision in db...") + deployedVersion.repoRevision = revision + err = saveLastDeployedRevision(db, swarmStack.name, deployedVersion) } - log.Debug("saving current revision to db...") - err = saveLastDeployedRevision(db, swarmStack.name, updatedVersion) if err != nil { - return revision, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) } return @@ -274,15 +279,15 @@ func (swarmStack *swarmStack) writeStack(composeMap map[string]any) ([]byte, err return composeFileBytes, nil } -func (swarmStack *swarmStack) shouldDeploy(newVersion *version, deployedVersion *version) bool { - logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedVersion.fmtHash())) - logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newVersion.fmtHash())) - if newVersion.hash == deployedVersion.hash { - logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedVersion.fmtHash(), newVersion.revision)) - logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, deployedVersion.hash)) +func (swarmStack *swarmStack) shouldDeploy(newMetadata *stackMetadata, deployedMetadata *stackMetadata) bool { + logger.Debug(fmt.Sprintf("%s Old Stack hash: %s", swarmStack.name, deployedMetadata.fmtHash())) + logger.Debug(fmt.Sprintf("%s New Stack hash: %s", swarmStack.name, newMetadata.fmtHash())) + if newMetadata.hash == deployedMetadata.hash { + logger.Info(fmt.Sprintf("%s stack file hash unchanged, hash=%s. Will skip deployment of revision: %s", swarmStack.name, deployedMetadata.fmtHash(), newMetadata.repoRevision)) + logger.Info(fmt.Sprintf("%s stack remains at revision: %s", swarmStack.name, deployedMetadata.deployedStackRevision)) return false } - logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newVersion.fmtHash(), newVersion.revision)) + logger.Info(fmt.Sprintf("%s new stack file with hash=%s found. Will continue with deployment of revision: %s", swarmStack.name, newMetadata.fmtHash(), newMetadata.repoRevision)) return true } diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 3a7ab6c..2d79125 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -31,7 +31,7 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { defer waitGroup.Done() logger.Info(fmt.Sprintf("%s updating stack", swarmStack.name)) - revision, err := swarmStack.updateStack() + stackMetadata, err := swarmStack.updateStack() if err != nil { stackStatus[swarmStack.name].Error = err.Error() logger.Error(err.Error()) @@ -39,7 +39,9 @@ func updateStackThread(swarmStack *swarmStack, waitGroup *sync.WaitGroup) { } stackStatus[swarmStack.name].Error = "" - stackStatus[swarmStack.name].Revision = revision + stackStatus[swarmStack.name].Revision = stackMetadata.repoRevision + stackStatus[swarmStack.name].DeployedStackRevision = stackMetadata.deployedStackRevision + stackStatus[swarmStack.name].DeployedAt = stackMetadata.deployedAt.Format(time.RFC3339) logger.Info(fmt.Sprintf("%s done updating stack", swarmStack.name)) } diff --git a/web/controllers.go b/web/controllers.go index 14a4721..0da7ede 100644 --- a/web/controllers.go +++ b/web/controllers.go @@ -13,14 +13,16 @@ func getStacks(ctx *gin.Context) { var stacks []map[string]string for k, v := range stacksStatus { stacks = append(stacks, map[string]string{ - "Name": k, - "Error": v.Error, - "RepoURL": v.RepoURL, - "Revision": v.Revision, + "Name": k, + "Error": v.Error, + "RepoURL": v.RepoURL, + "Revision": v.Revision, + "DeployedStackRevision": v.DeployedStackRevision, + "DeployedAt": v.DeployedAt, }) } sort.Slice(stacks, func(i, j int) bool { return stacks[i]["Name"] < stacks[j]["Name"] }) ctx.JSON(http.StatusOK, stacks) -} \ No newline at end of file +} From 23b3dff5dc974a24e7de7ef20cf92f4bcdcbabe8 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:07:27 +0100 Subject: [PATCH 69/77] fix null pointer error --- swarmcd/stack.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 07ace83..f29c50e 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -45,7 +45,7 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e log.Debug("pulling changes...") revision, err := swarmStack.repo.pullChanges(swarmStack.branch) if err != nil { - return + return nil, err } log.Debug("changes pulled", "revision", revision) @@ -119,17 +119,21 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e log.Debug("saving current stack's metadata to db...") err = saveLastDeployedRevision(db, swarmStack.name, updatedVersion) + if err != nil { + return nil, fmt.Errorf("failed to save new stackMetadata to db for %s stack: %w", swarmStack.name, err) + } + + return updatedVersion, nil } else { - log.Debug("updating stacks.repoRevision in db...") + log.Debug("updating deployedVersion.repoRevision in db...") deployedVersion.repoRevision = revision err = saveLastDeployedRevision(db, swarmStack.name, deployedVersion) - } + if err != nil { + return nil, fmt.Errorf("failed to update stackMetadata in db for %s stack: %w", swarmStack.name, err) + } - if err != nil { - return nil, fmt.Errorf("failed to save revision to db for %s stack: %w", swarmStack.name, err) + return deployedVersion, nil } - - return } func (swarmStack *swarmStack) readStack() ([]byte, error) { From ea1319708242711bf447bdfd68e11efb97722e5a Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:08:43 +0100 Subject: [PATCH 70/77] some renaming --- swarmcd/database.go | 12 ++++++------ swarmcd/database_test.go | 2 +- swarmcd/stack.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 293ea5a..4f33605 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -16,7 +16,7 @@ type stackMetadata struct { hash string } -func newVersion(repoRevision string, stackRevision string, hash string, time time.Time) *stackMetadata { +func newStackMetadata(repoRevision string, stackRevision string, hash string, time time.Time) *stackMetadata { return &stackMetadata{ repoRevision: repoRevision, deployedStackRevision: stackRevision, @@ -25,17 +25,17 @@ func newVersion(repoRevision string, stackRevision string, hash string, time tim } } -func newVersionFromData(repoRevision string, stackRevision string, data []byte) *stackMetadata { +func newStackMetadataFromStackData(repoRevision string, stackRevision string, stackData []byte) *stackMetadata { return &stackMetadata{ repoRevision: repoRevision, deployedStackRevision: stackRevision, - hash: computeHash(data), + hash: computeHash(stackData), deployedAt: time.Now(), } } -func (version *stackMetadata) fmtHash() string { - return fmtHash(version.hash) +func (stackMetadata *stackMetadata) fmtHash() string { + return fmtHash(stackMetadata.hash) } func getDBFilePath() string { @@ -96,7 +96,7 @@ func loadLastDeployedRevision(db *sql.DB, stackName string) (*stackMetadata, err WHERE stack = ?`, stackName).Scan(&repoRevision, &deployedStackRevision, &hash, &deployedAt) if err == sql.ErrNoRows { - return newVersion("", "", "", time.Now()), nil + return newStackMetadata("", "", "", time.Now()), nil } if err != nil { return nil, fmt.Errorf("failed to query revision: %w", err) diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 483d971..d34e9da 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -22,7 +22,7 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { stackRevision := "12345678" stackContent := []byte("test content") - version := newVersionFromData(repoRevision, stackRevision, stackContent) + version := newStackMetadataFromStackData(repoRevision, stackRevision, stackContent) now := time.Now() version.deployedAt = now diff --git a/swarmcd/stack.go b/swarmcd/stack.go index f29c50e..425eb69 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -108,7 +108,7 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e return nil, fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } - updatedVersion := newVersionFromData(revision, revision, writtenBytes) + updatedVersion := newStackMetadataFromStackData(revision, revision, writtenBytes) if swarmStack.shouldDeploy(updatedVersion, deployedVersion) { log.Debug("deploying stack...") From 7c77900ff2954837fd393d57bb3c8966c00bd0b5 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:10:10 +0100 Subject: [PATCH 71/77] more renaming --- swarmcd/database.go | 4 ++-- swarmcd/database_test.go | 4 ++-- swarmcd/stack.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index 4f33605..da44138 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -67,7 +67,7 @@ func initDB(dbFile string) (*sql.DB, error) { } // Save last deployed stackMetadata -func saveLastDeployedRevision(db *sql.DB, stackName string, stackMetadata *stackMetadata) error { +func saveLastDeployedMetadata(db *sql.DB, stackName string, stackMetadata *stackMetadata) error { _, err := db.Exec(` INSERT INTO revisions (stack, repo_revision, deployed_stack_revision, hash, deployed_at) VALUES (?, ?, ?, ?, ?) @@ -86,7 +86,7 @@ func saveLastDeployedRevision(db *sql.DB, stackName string, stackMetadata *stack } // Load a stack's stackMetadata -func loadLastDeployedRevision(db *sql.DB, stackName string) (*stackMetadata, error) { +func loadLastDeployedMetadata(db *sql.DB, stackName string) (*stackMetadata, error) { var repoRevision, deployedStackRevision, hash string var deployedAt time.Time diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index d34e9da..3980aa5 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -26,12 +26,12 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { now := time.Now() version.deployedAt = now - err = saveLastDeployedRevision(db, stackName, version) + err = saveLastDeployedMetadata(db, stackName, version) if err != nil { t.Fatalf("Failed to save repoRevision: %v", err) } - loadedVersion, err := loadLastDeployedRevision(db, stackName) + loadedVersion, err := loadLastDeployedMetadata(db, stackName) if err != nil { t.Fatalf("Failed to load repoRevision: %v", err) } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 425eb69..034aabf 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -55,7 +55,7 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } defer db.Close() - deployedVersion, err := loadLastDeployedRevision(db, swarmStack.name) + deployedVersion, err := loadLastDeployedMetadata(db, swarmStack.name) if err != nil { return nil, fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) } @@ -118,7 +118,7 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } log.Debug("saving current stack's metadata to db...") - err = saveLastDeployedRevision(db, swarmStack.name, updatedVersion) + err = saveLastDeployedMetadata(db, swarmStack.name, updatedVersion) if err != nil { return nil, fmt.Errorf("failed to save new stackMetadata to db for %s stack: %w", swarmStack.name, err) } @@ -127,7 +127,7 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } else { log.Debug("updating deployedVersion.repoRevision in db...") deployedVersion.repoRevision = revision - err = saveLastDeployedRevision(db, swarmStack.name, deployedVersion) + err = saveLastDeployedMetadata(db, swarmStack.name, deployedVersion) if err != nil { return nil, fmt.Errorf("failed to update stackMetadata in db for %s stack: %w", swarmStack.name, err) } From c5e9f1c4cb9d4dc9db0045fd6184d103844a689b Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:23:00 +0100 Subject: [PATCH 72/77] introduce stackDb abstraction --- swarmcd/database.go | 29 ++++++++++++++++++++++++++++- swarmcd/database_test.go | 9 ++++----- swarmcd/stack.go | 20 ++++++++++---------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/swarmcd/database.go b/swarmcd/database.go index da44138..47a75bc 100644 --- a/swarmcd/database.go +++ b/swarmcd/database.go @@ -45,8 +45,35 @@ func getDBFilePath() string { return "/data/revisions.db" // Default path } +type stackDB struct { + db *sql.DB + stackName string +} + +// Ensure database and table exist +func initStackDB(dbFile string, stackName string) (*stackDB, error) { + db, err := initSqlDB(dbFile) + if err != nil { + return nil, err + } + + return &stackDB{db: db, stackName: stackName}, nil +} + +func (stackDb *stackDB) saveLastDeployedMetadata(stackMetadata *stackMetadata) error { + return saveLastDeployedMetadata(stackDb.db, stackDb.stackName, stackMetadata) +} + +func (stackDb *stackDB) loadLastDeployedMetadata() (*stackMetadata, error) { + return loadLastDeployedMetadata(stackDb.db, stackDb.stackName) +} + +func (stackDb *stackDB) close() error { + return stackDb.db.Close() +} + // Ensure database and table exist -func initDB(dbFile string) (*sql.DB, error) { +func initSqlDB(dbFile string) (*sql.DB, error) { db, err := sql.Open("sqlite", dbFile) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) diff --git a/swarmcd/database_test.go b/swarmcd/database_test.go index 3980aa5..1849a67 100644 --- a/swarmcd/database_test.go +++ b/swarmcd/database_test.go @@ -11,13 +11,12 @@ import ( func TestSaveAndLoadLastDeployedRevision(t *testing.T) { const dbFile = ":memory:" // Use in-memory database for tests - db, err := initDB(dbFile) + db, err := initStackDB(dbFile, "test-stack") if err != nil { t.Fatalf("failed to open database: %v", err) } - defer db.Close() + defer db.close() - stackName := "test-stack" repoRevision := "abcdefgh" stackRevision := "12345678" stackContent := []byte("test content") @@ -26,12 +25,12 @@ func TestSaveAndLoadLastDeployedRevision(t *testing.T) { now := time.Now() version.deployedAt = now - err = saveLastDeployedMetadata(db, stackName, version) + err = db.saveLastDeployedMetadata(version) if err != nil { t.Fatalf("Failed to save repoRevision: %v", err) } - loadedVersion, err := loadLastDeployedMetadata(db, stackName) + loadedVersion, err := db.loadLastDeployedMetadata() if err != nil { t.Fatalf("Failed to load repoRevision: %v", err) } diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 034aabf..1f6bcb0 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -49,15 +49,15 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } log.Debug("changes pulled", "revision", revision) - db, err := initDB(getDBFilePath()) + stackDb, err := initStackDB(getDBFilePath(), swarmStack.name) if err != nil { return nil, fmt.Errorf("failed to open database: %s", err) } - defer db.Close() + defer stackDb.close() - deployedVersion, err := loadLastDeployedMetadata(db, swarmStack.name) + deployedVersion, err := stackDb.loadLastDeployedMetadata() if err != nil { - return nil, fmt.Errorf("failed to read revision from db for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to read revision from stackDb for %s stack: %w", swarmStack.name, err) } if deployedVersion.repoRevision == revision { @@ -117,19 +117,19 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e return nil, fmt.Errorf("failed to deploy stack for %s stack: %w", swarmStack.name, err) } - log.Debug("saving current stack's metadata to db...") - err = saveLastDeployedMetadata(db, swarmStack.name, updatedVersion) + log.Debug("saving current stack's metadata to stackDb...") + err = stackDb.saveLastDeployedMetadata(updatedVersion) if err != nil { - return nil, fmt.Errorf("failed to save new stackMetadata to db for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to save new stackMetadata to stackDb for %s stack: %w", swarmStack.name, err) } return updatedVersion, nil } else { - log.Debug("updating deployedVersion.repoRevision in db...") + log.Debug("updating deployedVersion.repoRevision in stackDb...") deployedVersion.repoRevision = revision - err = saveLastDeployedMetadata(db, swarmStack.name, deployedVersion) + err = stackDb.saveLastDeployedMetadata(deployedVersion) if err != nil { - return nil, fmt.Errorf("failed to update stackMetadata in db for %s stack: %w", swarmStack.name, err) + return nil, fmt.Errorf("failed to update stackMetadata in stackDb for %s stack: %w", swarmStack.name, err) } return deployedVersion, nil From 4b800f9bf3af4272ebbaf723cb195cc5d31e4c2c Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 12:26:24 +0100 Subject: [PATCH 73/77] some renaming --- swarmcd/stack.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 1f6bcb0..79f6f4e 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -55,16 +55,16 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } defer stackDb.close() - deployedVersion, err := stackDb.loadLastDeployedMetadata() + deployedMetadata, err := stackDb.loadLastDeployedMetadata() if err != nil { return nil, fmt.Errorf("failed to read revision from stackDb for %s stack: %w", swarmStack.name, err) } - if deployedVersion.repoRevision == revision { + if deployedMetadata.repoRevision == revision { logger.Info(fmt.Sprintf("%s revision unchanged: stack up-to-date on rev: %s", swarmStack.name, revision)) - return deployedVersion, nil + return deployedMetadata, nil } - if deployedVersion.repoRevision == "" { + if deployedMetadata.repoRevision == "" { logger.Info(fmt.Sprintf("%s no last revision found", swarmStack.name)) } @@ -108,9 +108,9 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e return nil, fmt.Errorf("failed to write stack to file for %s stack: %w", swarmStack.name, err) } - updatedVersion := newStackMetadataFromStackData(revision, revision, writtenBytes) + updatedMetadata := newStackMetadataFromStackData(revision, revision, writtenBytes) - if swarmStack.shouldDeploy(updatedVersion, deployedVersion) { + if swarmStack.shouldDeploy(updatedMetadata, deployedMetadata) { log.Debug("deploying stack...") err = swarmStack.deployStack() if err != nil { @@ -118,21 +118,21 @@ func (swarmStack *swarmStack) updateStack() (stackMetadata *stackMetadata, err e } log.Debug("saving current stack's metadata to stackDb...") - err = stackDb.saveLastDeployedMetadata(updatedVersion) + err = stackDb.saveLastDeployedMetadata(updatedMetadata) if err != nil { return nil, fmt.Errorf("failed to save new stackMetadata to stackDb for %s stack: %w", swarmStack.name, err) } - return updatedVersion, nil + return updatedMetadata, nil } else { - log.Debug("updating deployedVersion.repoRevision in stackDb...") - deployedVersion.repoRevision = revision - err = stackDb.saveLastDeployedMetadata(deployedVersion) + log.Debug("updating deployedMetadata.repoRevision in stackDb...") + deployedMetadata.repoRevision = revision + err = stackDb.saveLastDeployedMetadata(deployedMetadata) if err != nil { return nil, fmt.Errorf("failed to update stackMetadata in stackDb for %s stack: %w", swarmStack.name, err) } - return deployedVersion, nil + return deployedMetadata, nil } } From 38075e3bff948bd2d7d831e7425d141ae4c3b194 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 10:13:46 +0100 Subject: [PATCH 74/77] dynamically add new stacks --- swarmcd/init.go | 44 ++++++++++++++++++++++++++++++++++++++++---- swarmcd/swarmcd.go | 27 +++++++++++++++++++++++++-- util/config.go | 18 ++++++++---------- 3 files changed, 73 insertions(+), 16 deletions(-) diff --git a/swarmcd/init.go b/swarmcd/init.go index cde495c..6b6c2c4 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -45,17 +45,32 @@ func Init() (err error) { } func initRepos() error { + var newRepos = map[string]*stackRepo{} + for repoName, repoConfig := range config.RepoConfigs { + if repo, exists := repos[repoName]; exists { + newRepos[repoName] = repo + delete(repos, repoName) + continue + } + repoPath := path.Join(config.ReposPath, repoName) auth, err := createHTTPBasicAuth(repoName) if err != nil { return err } - repos[repoName], err = newStackRepo(repoName, repoPath, repoConfig.Url, auth) + newRepos[repoName], err = newStackRepo(repoName, repoPath, repoConfig.Url, auth) if err != nil { return err } } + + if len(repos) != 0 { + logger.Info("Some repos were removed from the stack: %v", repos) + } + + repos = newRepos + return nil } @@ -93,17 +108,38 @@ func createHTTPBasicAuth(repoName string) (*http.BasicAuth, error) { } func initStacks() error { + var newStacks = map[string]*swarmStack{} + var newStackStatus = map[string]*StackStatus{} + for stack, stackConfig := range config.StackConfigs { stackRepo, ok := repos[stackConfig.Repo] if !ok { return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) } + discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets) - stacks = append(stacks, swarmStack) - stackStatus[stack] = &StackStatus{} - stackStatus[stack].RepoURL = stackRepo.url + + newStacks[stack] = swarmStack + if _, exists := stacks[stack]; exists { + delete(stacks, stack) + } + + newStackStatus[stack] = &StackStatus{} + newStackStatus[stack].RepoURL = stackRepo.url + if _, exists := stackStatus[stack]; exists { + delete(stacks, stack) + } } + + if len(stacks) != 0 { + logger.Info("Some stacks were removed: %v", stacks) + // Todo: do we need to do something for this. + } + + stacks = newStacks + stackStatus = newStackStatus + return nil } diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 2d79125..90f775c 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -2,12 +2,13 @@ package swarmcd import ( "fmt" + "github.com/m-adawi/swarm-cd/util" "sync" "time" ) -var stackStatus map[string]*StackStatus = map[string]*StackStatus{} -var stacks []*swarmStack +var stackStatus = map[string]*StackStatus{} +var stacks = map[string]*swarmStack{} func Run() { logger.Info("starting SwarmCD") @@ -15,12 +16,34 @@ func Run() { var waitGroup sync.WaitGroup logger.Info("updating stacks...") for _, swarmStack := range stacks { + logger.Info("Starting go routine for %v", swarmStack.name) + waitGroup.Add(1) go updateStackThread(swarmStack, &waitGroup) } waitGroup.Wait() logger.Info("waiting for the update interval") time.Sleep(time.Duration(config.UpdateInterval) * time.Second) + + logger.Info("check if new repos or new stacks are available") + updateStackConfigs() + } +} + +func updateStackConfigs() { + err := util.LoadConfigs() + if err != nil { + logger.Info("Error calling loadConfig again: %v", err) + } + + err = initRepos() + if err != nil { + logger.Info("Error calling initRepos again: %v", err) + } + + err = initStacks() + if err != nil { + logger.Info("Error calling initStacks again: %v", err) } } diff --git a/util/config.go b/util/config.go index 174ced2..11ef04d 100644 --- a/util/config.go +++ b/util/config.go @@ -40,17 +40,15 @@ func LoadConfigs() (err error) { if err != nil { return fmt.Errorf("could not read configuration file: %w", err) } - if Configs.RepoConfigs == nil { - err = readRepoConfigs() - if err != nil { - return fmt.Errorf("could not read repos file: %w", err) - } + + err = readRepoConfigs() + if err != nil { + return fmt.Errorf("could not read repos file: %w", err) } - if Configs.StackConfigs == nil { - err = readStackConfigs() - if err != nil { - return fmt.Errorf("could not load stacks file: %w", err) - } + + err = readStackConfigs() + if err != nil { + return fmt.Errorf("could not load stacks file: %w", err) } return } From d7161f667334ba0dc41ccf54576a5503021493b7 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sat, 22 Mar 2025 10:41:20 +0100 Subject: [PATCH 75/77] dynamically add and remove new stacks --- swarmcd/init.go | 2 ++ swarmcd/swarmcd.go | 4 ++-- util/config.go | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/swarmcd/init.go b/swarmcd/init.go index 6b6c2c4..32b52a2 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -112,6 +112,8 @@ func initStacks() error { var newStackStatus = map[string]*StackStatus{} for stack, stackConfig := range config.StackConfigs { + logger.Info(fmt.Sprintf("Initializing Stack %v", stack)) + stackRepo, ok := repos[stackConfig.Repo] if !ok { return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) diff --git a/swarmcd/swarmcd.go b/swarmcd/swarmcd.go index 90f775c..d5e6443 100644 --- a/swarmcd/swarmcd.go +++ b/swarmcd/swarmcd.go @@ -16,8 +16,7 @@ func Run() { var waitGroup sync.WaitGroup logger.Info("updating stacks...") for _, swarmStack := range stacks { - logger.Info("Starting go routine for %v", swarmStack.name) - + logger.Debug(fmt.Sprintf("Starting go routine for %v", swarmStack.name)) waitGroup.Add(1) go updateStackThread(swarmStack, &waitGroup) } @@ -34,6 +33,7 @@ func updateStackConfigs() { err := util.LoadConfigs() if err != nil { logger.Info("Error calling loadConfig again: %v", err) + return } err = initRepos() diff --git a/util/config.go b/util/config.go index 11ef04d..86478c7 100644 --- a/util/config.go +++ b/util/config.go @@ -77,6 +77,10 @@ func readRepoConfigs() (err error) { if err != nil { return } + + // **Reset maps before unmarshaling to remove old keys** + Configs.RepoConfigs = make(map[string]*RepoConfig) + return reposViper.Unmarshal(&Configs.RepoConfigs) } @@ -88,5 +92,9 @@ func readStackConfigs() (err error) { if err != nil { return } + + // **Reset maps before unmarshaling to remove old keys** + Configs.StackConfigs = make(map[string]*StackConfig) + return stacksViper.Unmarshal(&Configs.StackConfigs) } From cc85a63385f759704b8dea2519e06e777dd37086 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Sun, 23 Mar 2025 13:00:49 +0100 Subject: [PATCH 76/77] fix logger --- swarmcd/init.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarmcd/init.go b/swarmcd/init.go index 32b52a2..1bdbc97 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -66,7 +66,7 @@ func initRepos() error { } if len(repos) != 0 { - logger.Info("Some repos were removed from the stack: %v", repos) + logger.Info(fmt.Sprintf("Some repos were removed from the stack: %v", repos)) } repos = newRepos @@ -135,7 +135,7 @@ func initStacks() error { } if len(stacks) != 0 { - logger.Info("Some stacks were removed: %v", stacks) + logger.Info(fmt.Sprintf("Some stacks were removed: %v", stacks)) // Todo: do we need to do something for this. } From d91feb9ca1a46174752d1ba10cb393f0a4446c26 Mon Sep 17 00:00:00 2001 From: Christian Langenbacher Date: Mon, 31 Mar 2025 22:28:56 +0200 Subject: [PATCH 77/77] better log messages --- swarmcd/init.go | 4 ++-- util/config.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/swarmcd/init.go b/swarmcd/init.go index 1bdbc97..239f816 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -112,11 +112,11 @@ func initStacks() error { var newStackStatus = map[string]*StackStatus{} for stack, stackConfig := range config.StackConfigs { - logger.Info(fmt.Sprintf("Initializing Stack %v", stack)) + logger.Info(fmt.Sprintf("reading stackConfig for stack: %v", stack)) stackRepo, ok := repos[stackConfig.Repo] if !ok { - return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) + return fmt.Errorf("error reading %s stack, no such repo: %s", stack, stackConfig.Repo) } discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery diff --git a/util/config.go b/util/config.go index 86478c7..755344a 100644 --- a/util/config.go +++ b/util/config.go @@ -78,7 +78,7 @@ func readRepoConfigs() (err error) { return } - // **Reset maps before unmarshaling to remove old keys** + // Reset maps before unmarshalling to remove old keys** Configs.RepoConfigs = make(map[string]*RepoConfig) return reposViper.Unmarshal(&Configs.RepoConfigs) @@ -93,7 +93,7 @@ func readStackConfigs() (err error) { return } - // **Reset maps before unmarshaling to remove old keys** + // Reset maps before unmarshalling to remove old keys** Configs.StackConfigs = make(map[string]*StackConfig) return stacksViper.Unmarshal(&Configs.StackConfigs)