Detect new stacks on the fly

.gitignore

@@ -1 +1,6 @@
 swarm-cd
+
+docker-compose.yaml
+repos.yaml
+stacks.yaml
+config.yaml

README.md

@@ -46,6 +46,11 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./repos.yaml:/app/repos.yaml:ro
       - ./stacks.yaml:/app/stacks.yaml:ro
+      - swarmcd_data:/data
+
+volumes:
+  swarmcd_data:
+    driver: local
 ```
 
 Run this on a swarm manager node:
@@ -57,6 +62,35 @@ docker stack deploy --compose-file docker-compose.yaml swarm-cd
 This will start SwarmCD, it will periodically check the stack repo
 for new changes, pulling them and updating the stack.
 
+### Configure the database
+SwarmCD uses a minimal DB to track the last deployed revision across 
+container restarts. By default, it stores data in data/revisions.db,
+but this can be changed via the `SWARMCD_DB` environment variable as 
+shown in the below docker-compose file.
+
+```yaml
+# docker-compose.yaml
+version: '3.7'
+services:
+  swarm-cd:
+    image: ghcr.io/m-adawi/swarm-cd:latest
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./repos.yaml:/app/repos.yaml:ro
+      - ./stacks.yaml:/app/stacks.yaml:ro
+      - swarmcd_data:/data
+    environment:
+      - SWARMCD_DB=/data/revisions.db
+
+volumes:
+  swarmcd_data:
+    driver: local
+```
+
 ## Manage Encrypted Secrets Using SOPS
 
 You can use [sops](https://github.com/getsops/sops) to encrypt secrets in git repos and
@@ -87,6 +121,7 @@ and set the environment variable SOPS `SOPS_AGE_KEY_FILE`
 to the path of the key file. See the following docker-compose example
 
 ```yaml
+# docker-compose.yaml
 version: '3.7'
 services:
   swarm-cd:
@@ -104,6 +139,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./repos.yaml:/app/repos.yaml:ro
       - ./stacks.yaml:/app/stacks.yaml:ro
+      - swarmcd_data:/data
+
+volumes:
+  swarmcd_data:
+    driver: local
+
 secrets:
   age:
     file: age.key
@@ -214,9 +255,17 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./repos.yaml:/app/repos.yaml:ro
       - ./stacks.yaml:/app/stacks.yaml:ro
+      - swarmcd_data:/data
+    environment:
+      - SOPS_AGE_KEY_FILE=/secrets/age.key
     secrets:
       - source: docker-config
         target: /root/.docker/config.json
+
+volumes:
+  swarmcd_data:
+    driver: local
+
 secrets:
   docker-config:
     file: docker-config.json

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@@ -94,8 +95,10 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -119,6 +122,10 @@ require (
 	google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.61.13 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.8.2 // indirect
+	modernc.org/sqlite v1.36.0 // indirect
 )
 
 require (
@@ -202,7 +209,7 @@ require (
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/term v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect

go.sum

@@ -177,6 +177,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
@@ -425,6 +427,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
@@ -480,6 +484,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -673,6 +679,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
@@ -767,5 +775,13 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=
 k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8=
+modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI=
+modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU=
+modernc.org/sqlite v1.36.0 h1:EQXNRn4nIS+gfsKeUTymHIz1waxuv5BzU7558dHSfH8=
+modernc.org/sqlite v1.36.0/go.mod h1:7MPwH7Z6bREicF9ZVUR78P1IKuxfZ8mRIDHD0iD+8TU=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

swarmcd/database.go

@@ -0,0 +1,151 @@
+package swarmcd
+
+import (
+	"crypto/sha256"
+	"database/sql"
+	"fmt"
+	_ "modernc.org/sqlite"
+	"os"
+	"time"
+)
+
+type stackMetadata struct {
+	repoRevision          string
+	deployedStackRevision string
+	deployedAt            time.Time
+	hash                  string
+}
+
+func newStackMetadata(repoRevision string, stackRevision string, hash string, time time.Time) *stackMetadata {
+	return &stackMetadata{
+		repoRevision:          repoRevision,
+		deployedStackRevision: stackRevision,
+		hash:                  hash,
+		deployedAt:            time,
+	}
+}
+
+func newStackMetadataFromStackData(repoRevision string, stackRevision string, stackData []byte) *stackMetadata {
+	return &stackMetadata{
+		repoRevision:          repoRevision,
+		deployedStackRevision: stackRevision,
+		hash:                  computeHash(stackData),
+		deployedAt:            time.Now(),
+	}
+}
+
+func (stackMetadata *stackMetadata) fmtHash() string {
+	return fmtHash(stackMetadata.hash)
+}
+
+func getDBFilePath() string {
+	if path := os.Getenv("SWARMCD_DB"); path != "" {
+		return path
+	}
+	return "/data/revisions.db" // Default path
+}
+
+type stackDB struct {
+	db        *sql.DB
+	stackName string
+}
+
+// Ensure database and table exist
+func initStackDB(dbFile string, stackName string) (*stackDB, error) {
+	db, err := initSqlDB(dbFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &stackDB{db: db, stackName: stackName}, nil
+}
+
+func (stackDb *stackDB) saveLastDeployedMetadata(stackMetadata *stackMetadata) error {
+	return saveLastDeployedMetadata(stackDb.db, stackDb.stackName, stackMetadata)
+}
+
+func (stackDb *stackDB) loadLastDeployedMetadata() (*stackMetadata, error) {
+	return loadLastDeployedMetadata(stackDb.db, stackDb.stackName)
+}
+
+func (stackDb *stackDB) close() error {
+	return stackDb.db.Close()
+}
+
+// Ensure database and table exist
+func initSqlDB(dbFile string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", dbFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open database: %w", err)
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS revisions (
+		stack TEXT PRIMARY KEY, 
+		repo_revision TEXT, 
+		deployed_stack_revision TEXT, 
+		hash TEXT, 
+		deployed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+	)`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create table: %w", err)
+	}
+
+	return db, nil
+}
+
+// Save last deployed stackMetadata
+func saveLastDeployedMetadata(db *sql.DB, stackName string, stackMetadata *stackMetadata) error {
+	_, err := db.Exec(`
+		INSERT INTO revisions (stack, repo_revision, deployed_stack_revision, hash, deployed_at) 
+		VALUES (?, ?, ?, ?, ?) 
+		ON CONFLICT(stack) DO UPDATE SET 
+			repo_revision = excluded.repo_revision, 
+			deployed_stack_revision = excluded.deployed_stack_revision, 
+			hash = excluded.hash,
+			deployed_at = excluded.deployed_at
+	`, stackName, stackMetadata.repoRevision, stackMetadata.deployedStackRevision, stackMetadata.hash, stackMetadata.deployedAt)
+
+	if err != nil {
+		return fmt.Errorf("failed to save revision: %w", err)
+	}
+
+	return nil
+}
+
+// Load a stack's stackMetadata
+func loadLastDeployedMetadata(db *sql.DB, stackName string) (*stackMetadata, error) {
+	var repoRevision, deployedStackRevision, hash string
+	var deployedAt time.Time
+
+	err := db.QueryRow(`
+		SELECT repo_revision, deployed_stack_revision, hash, deployed_at 
+		FROM revisions 
+		WHERE stack = ?`, stackName).Scan(&repoRevision, &deployedStackRevision, &hash, &deployedAt)
+
+	if err == sql.ErrNoRows {
+		return newStackMetadata("", "", "", time.Now()), nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to query revision: %w", err)
+	}
+
+	return &stackMetadata{
+		repoRevision:          repoRevision,
+		deployedStackRevision: deployedStackRevision,
+		hash:                  hash,
+		deployedAt:            deployedAt,
+	}, nil
+}
+
+// Compute a SHA-256 hash of the stack content
+func computeHash(data []byte) string {
+	hash := sha256.Sum256(data)
+	return fmt.Sprintf("%x", hash)
+}
+
+func fmtHash(hash string) string {
+	if len(hash) >= 8 {
+		return hash[:8]
+	}
+	return "<empty-hash>"
+}

swarmcd/database_test.go

@@ -0,0 +1,64 @@
+package swarmcd
+
+import (
+	"testing"
+	"time"
+)
+
+import (
+	_ "modernc.org/sqlite"
+)
+
+func TestSaveAndLoadLastDeployedRevision(t *testing.T) {
+	const dbFile = ":memory:" // Use in-memory database for tests
+	db, err := initStackDB(dbFile, "test-stack")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.close()
+
+	repoRevision := "abcdefgh"
+	stackRevision := "12345678"
+	stackContent := []byte("test content")
+
+	version := newStackMetadataFromStackData(repoRevision, stackRevision, stackContent)
+	now := time.Now()
+	version.deployedAt = now
+
+	err = db.saveLastDeployedMetadata(version)
+	if err != nil {
+		t.Fatalf("Failed to save repoRevision: %v", err)
+	}
+
+	loadedVersion, err := db.loadLastDeployedMetadata()
+	if err != nil {
+		t.Fatalf("Failed to load repoRevision: %v", err)
+	}
+
+	expectedHash := computeHash(stackContent)
+
+	if loadedVersion.repoRevision != repoRevision {
+		t.Errorf("Expected repoRevision %s, got %s", repoRevision, loadedVersion.repoRevision)
+	}
+
+	if loadedVersion.deployedStackRevision != stackRevision {
+		t.Errorf("Expected repoRevision %s, got %s", repoRevision, loadedVersion.deployedStackRevision)
+	}
+
+	if !isRoughlyEqual(loadedVersion.deployedAt, now, 1*time.Microsecond) {
+		t.Errorf("Expected time %s, got %s", now, loadedVersion.deployedAt)
+	}
+
+	if loadedVersion.hash != expectedHash {
+		t.Errorf("Expected hash %s, got %s", expectedHash, loadedVersion.hash)
+	}
+}
+
+func isRoughlyEqual(t1, t2 time.Time, tolerance time.Duration) bool {
+	diff := t2.Sub(t1)
+	// Check if the difference is within the tolerance
+	if diff < 0 {
+		diff = -diff // Handle negative difference
+	}
+	return diff <= tolerance
+}