feat: templatize migrations

.github/workflows/push.yaml

@@ -48,3 +48,42 @@ jobs:
         run: make build-all
       - name: Build and push docker image
         run: make build-and-push-signoz-collector
+  image-build-and-push-schema-migrator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: benjlevesque/[email protected]
+        id: short-sha
+      - name: Get branch name
+        id: branch-name
+        uses: tj-actions/[email protected]
+      - name: Set docker tag environment
+        run: |
+          if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then
+            tag="${{ steps.branch-name.outputs.tag }}"
+            tag="${tag:1}"
+            echo "DOCKER_TAG=$tag" >> $GITHUB_ENV
+          else
+            echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV
+          fi
+      - name: Install cross-compilation tools
+        run: |
+          set -ex
+          sudo apt-get update
+          sudo apt-get install -y gcc-aarch64-linux-gnu musl-tools
+      - name: Build artifacts for arm64/amd64
+        run: make build-all
+      - name: Build and push docker image
+        run: make build-and-push-signoz-schema-migrator

.github/workflows/release.yaml

@@ -25,9 +25,13 @@ jobs:
           cd .build/linux-amd64
           sha256sum signoz-collector > signoz-collector-linux-amd64.sha256
           tar -czvf signoz-collector-linux-amd64.tar.gz signoz-collector
+          sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-amd64.sha256
+          tar -czvf signoz-schema-migrator-linux-amd64.tar.gz signoz-schema-migrator
           cd ../linux-arm64
           sha256sum signoz-collector > signoz-collector-linux-arm64.sha256
           tar -czvf signoz-collector-linux-arm64.tar.gz signoz-collector
+          sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-arm64.sha256
+          tar -czvf signoz-schema-migrator-linux-arm64.tar.gz signoz-schema-migrator
       - name: Create release and upload assets
         uses: softprops/action-gh-release@v1
         with:
@@ -36,4 +40,8 @@ jobs:
             .build/linux-amd64/signoz-collector-linux-amd64.tar.gz
             .build/linux-arm64/signoz-collector-linux-arm64.sha256
             .build/linux-arm64/signoz-collector-linux-arm64.tar.gz
+            .build/linux-amd64/signoz-schema-migrator-linux-amd64.sha256
+            .build/linux-amd64/signoz-schema-migrator-linux-amd64.tar.gz
+            .build/linux-arm64/signoz-schema-migrator-linux-arm64.sha256
+            .build/linux-arm64/signoz-schema-migrator-linux-arm64.tar.gz
           generate_release_notes: true

Makefile

@@ -1,6 +1,7 @@
 COMMIT_SHA ?= $(shell git rev-parse HEAD)
 REPONAME ?= signoz
 IMAGE_NAME ?= signoz-otel-collector
+MIGRATOR_IMAGE_NAME ?= signoz-schema-migrator
 CONFIG_FILE ?= ./config/default-config.yaml
 DOCKER_TAG ?= latest
 
@@ -34,6 +35,7 @@ test:
 .PHONY: build
 build:
 	CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-collector -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozcollector
+	CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-schema-migrator -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozschemamigrator
 
 .PHONY: amd64
 amd64:
@@ -73,6 +75,24 @@ build-signoz-collector:
 		--no-cache -f cmd/signozcollector/Dockerfile --progress plain \
 		--tag $(REPONAME)/$(IMAGE_NAME):$(DOCKER_TAG) .
 
+.PHONY: build-signoz-schema-migrator
+build-signoz-schema-migrator:
+	@echo "------------------"
+	@echo  "--> Build schema migrator docker image"
+	@echo "------------------"
+	docker build --build-arg TARGETPLATFORM="linux/amd64" \
+		--no-cache -f cmd/signozschemamigrator/Dockerfile --progress plain \
+		--tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) .
+
+.PHONY: build-and-push-signoz-schema-migrator
+build-and-push-signoz-schema-migrator:
+	@echo "------------------"
+	@echo  "--> Build and push schema migrator docker image"
+	@echo "------------------"
+	docker buildx build --platform linux/amd64,linux/arm64 --progress plain \
+		--no-cache --push -f cmd/signozschemamigrator/Dockerfile \
+		--tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) .
+
 .PHONY: lint
 lint:
 	@echo "Running linters..."

cmd/signozcollector/Dockerfile

@@ -14,12 +14,8 @@ USER ${USER_UID}
 # copy the binaries from the multi-stage build
 COPY .build/${TARGETOS}-${TARGETARCH}/signoz-collector /signoz-collector
 
-# copy the config and migration files
+# copy the config file
 COPY config/default-config.yaml /etc/otel/config.yaml
-COPY exporter/clickhousetracesexporter/migrations /migrations
-COPY exporter/clickhouselogsexporter/migrations /logsmigrations
-
-ENV LOG_MIGRATIONS_FOLDER="/logsmigrations"
 
 # expose OTLP ports for the collector
 EXPOSE 4317 4318

cmd/signozschemamigrator/Dockerfile

@@ -0,0 +1,17 @@
+# use a minimal alpine image
+FROM alpine:3.17
+
+# define arguments and default values
+ARG TARGETOS TARGETARCH
+ARG USER_UID=10001
+
+# create a non-root user for running the migrator
+USER ${USER_UID}
+
+# copy the binaries from the multi-stage build
+COPY .build/${TARGETOS}-${TARGETARCH}/signoz-schema-migrator /signoz-schema-migrator
+COPY migrationmanager ./migrationmanager
+
+# run the binary as the entrypoint and pass the default dsn as a flag
+ENTRYPOINT [ "/signoz-schema-migrator" ]
+CMD ["--dsn", "tcp://localhost:9000"]

cmd/signozschemamigrator/migrate.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/SigNoz/signoz-otel-collector/migrationmanager"
+	"github.com/spf13/pflag"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func init() {
+	// init zap logger
+	config := zap.NewProductionConfig()
+	config.EncoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder
+	config.EncoderConfig.TimeKey = "timestamp"
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+	logger, err := config.Build()
+	if err != nil {
+		log.Fatalf("Failed to initialize zap logger %v", err)
+	}
+	// replace global logger
+	// TODO(dhawal1248): move away from global logger
+	zap.ReplaceGlobals(logger)
+}
+
+func main() {
+	logger := zap.L().With(zap.String("component", "migrate cli"))
+	f := pflag.NewFlagSet("Schema Migrator CLI Options", pflag.ExitOnError)
+
+	f.Usage = func() {
+		fmt.Println(f.FlagUsages())
+		os.Exit(1)
+	}
+
+	f.String("dsn", "", "Clickhouse DSN")
+	f.String("cluster-name", "cluster", "Cluster name to use while running migrations")
+	f.Bool("disable-duration-sort-feature", false, "Flag to disable the duration sort feature. Defaults to false.")
+	f.Bool("disable-timestamp-sort-feature", false, "Flag to disable the timestamp sort feature. Defaults to false.")
+
+	err := f.Parse(os.Args[1:])
+	if err != nil {
+		logger.Fatal("Failed to parse args", zap.Error(err))
+	}
+
+	dsn, err := f.GetString("dsn")
+	if err != nil {
+		logger.Fatal("Failed to get dsn from args", zap.Error(err))
+	}
+
+	clusterName, err := f.GetString("cluster-name")
+	if err != nil {
+		logger.Fatal("Failed to get cluster name from args", zap.Error(err))
+	}
+
+	disableDurationSortFeature, err := f.GetBool("disable-duration-sort-feature")
+	if err != nil {
+		logger.Fatal("Failed to get disable duration sort feature flag from args", zap.Error(err))
+	}
+
+	disableTimestampSortFeature, err := f.GetBool("disable-timestamp-sort-feature")
+	if err != nil {
+		logger.Fatal("Failed to get disable timestamp sort feature flag from args", zap.Error(err))
+	}
+
+	if dsn == "" {
+		logger.Fatal("dsn is a required field")
+	}
+
+	// set cluster env so that golang-migrate can use it
+	// the value of this env would replace all occurences of {{.SIGNOZ_CLUSTER}} in the migration files
+	os.Setenv("SIGNOZ_CLUSTER", clusterName)
+
+	manager, err := migrationmanager.New(dsn, clusterName, disableDurationSortFeature, disableTimestampSortFeature)
+	if err != nil {
+		logger.Fatal("Failed to create migration manager", zap.Error(err))
+	}
+	defer manager.Close()
+
+	err = manager.Migrate(context.Background())
+	if err != nil {
+		logger.Fatal("Failed to run migrations", zap.Error(err))
+	}
+}

example/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.8"
+
+services:
+  schema-migrator:
+    image: signoz/signoz-schema-migrator:templatize-migrations
+    extra_hosts:
+      - signoz-host:host-gateway
+    command:
+      - "--dsn=tcp://signoz-host:9000"
+
+  otel-collector:
+    image: signoz/signoz-otel-collector:templatize-migrations
+    deploy:
+      replicas: 3
+    user: root # required for reading docker container logs
+    extra_hosts:
+      - signoz-host:host-gateway
+    command:
+      - "--config=/etc/otel-collector-config.yml"
+      # - "--set=service.telemetry.logs.level=DEBUG"
+    volumes:
+      - ./example-config.yaml:/etc/otel-collector-config.yml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+    # required to read logs from /var/lib/docker/containers/
+    ports:
+      - "1888" # pprof extension
+      - "8888" # Prometheus metrics exposed by the collector
+      - "13133" # health_check extension
+      - "55679" # zpages extension
+    depends_on:
+      schema-migrator:
+        condition: service_completed_successfully