Remove hardcoded cluster name, dead code and unused config options (#438

)

Makefile

@@ -103,12 +103,3 @@ install-ci: install-tools
 
 .PHONY: test-ci
 test-ci: lint
-
-
-.PHONY: migrate-logs
-migrate-logs: 
-	migrate -verbose  -path "./exporter/clickhouselogsexporter/migrations/" -database "clickhouse://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}?database=signoz_logs&x-multi-statement=true" up
-
-.PHONY: migrate-logs-down
-migrate-logs-down: 
-	migrate -verbose  -path "./exporter/clickhouselogsexporter/migrations/" -database "clickhouse://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}?database=signoz_logs&x-multi-statement=true" down

cmd/signozschemamigrator/Dockerfile

@@ -10,7 +10,6 @@ USER ${USER_UID}
 
 # copy the binaries from the multi-stage build
 COPY .build/${TARGETOS}-${TARGETARCH}/signoz-schema-migrator /signoz-schema-migrator
-COPY migrationmanager ./migrationmanager
 
 # run the binary as the entrypoint and pass the default dsn as a flag
 ENTRYPOINT [ "/signoz-schema-migrator" ]

cmd/signozschemamigrator/schema_migrator/manager.go

@@ -86,7 +86,6 @@ func NewMigrationManager(opts ...Option) (*MigrationManager, error) {
 		// the default backoff is good enough for our use case
 		// no mutation should be running for more than 15 minutes, if it is, we should fail fast
 		backoff:            backoff.NewExponentialBackOff(),
-		clusterName:        "cluster",
 		replicationEnabled: false,
 		conns:              make(map[string]clickhouse.Conn),
 	}

cmd/signozschemamigrator/schema_migrator/metrics_migrations.go

@@ -362,7 +362,6 @@ var MetricsMigrations = []SchemaMigrationRecord{
 					{Name: "last_reported_unix_milli", Type: SimpleAggregateFunction{FunctionName: "max", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"},
 				},
 				Engine: Distributed{
-					Cluster:     "cluster",
 					Database:    "signoz_metrics",
 					Table:       "metadata",
 					ShardingKey: "rand()",

config/default-config.yaml

@@ -53,7 +53,6 @@ extensions:
 exporters:
   clickhousetraces:
     datasource: tcp://localhost:9000/signoz_traces
-    migrations: exporter/clickhousetracesexporter/migrations
     retry_on_failure:
       enabled: true
       initial_interval: 5s

exporter/clickhouselogsexporter/config.go

@@ -31,10 +31,8 @@ type Config struct {
 	// DSN is the ClickHouse server Data Source Name.
 	// For tcp protocol reference: [ClickHouse/clickhouse-go#dsn](https://github.com/ClickHouse/clickhouse-go#dsn).
 	// For http protocol reference: [mailru/go-clickhouse/#dsn](https://github.com/mailru/go-clickhouse/#dsn).
-	DSN string `mapstructure:"dsn"`
-	// Docker Multi Node Cluster is a flag to enable the docker multi node cluster. Default is false.
-	DockerMultiNodeCluster bool `mapstructure:"docker_multi_node_cluster" default:"false"`
-	UseNewSchema           bool `mapstructure:"use_new_schema" default:"false"`
+	DSN          string `mapstructure:"dsn"`
+	UseNewSchema bool   `mapstructure:"use_new_schema" default:"false"`
 }
 
 var (

exporter/clickhouselogsexporter/exporter.go

@@ -88,9 +88,6 @@ func newExporter(logger *zap.Logger, cfg *Config) (*clickhouseLogsExporter, erro
 		"signoz_logs",
 		UsageExporter,
 	)
-	if err != nil {
-		return nil, fmt.Errorf("error creating usage collector for logs : %v", err)
-	}
 
 	collector.Start()
 
@@ -778,7 +775,7 @@ const (
 )
 
 // newClickhouseClient create a clickhouse client.
-func newClickhouseClient(logger *zap.Logger, cfg *Config) (clickhouse.Conn, error) {
+func newClickhouseClient(_ *zap.Logger, cfg *Config) (clickhouse.Conn, error) {
 	// use empty database to create database
 	ctx := context.Background()
 	options, err := clickhouse.ParseDSN(cfg.DSN)
@@ -804,10 +801,10 @@ func newClickhouseClient(logger *zap.Logger, cfg *Config) (clickhouse.Conn, erro
 	return db, nil
 }
 
-func renderInsertLogsSQL(cfg *Config) string {
+func renderInsertLogsSQL(_ *Config) string {
 	return fmt.Sprintf(insertLogsSQLTemplate, databaseName, DISTRIBUTED_LOGS_TABLE)
 }
 
-func renderInsertLogsSQLV2(cfg *Config) string {
+func renderInsertLogsSQLV2(_ *Config) string {
 	return fmt.Sprintf(insertLogsSQLTemplateV2, databaseName, DISTRIBUTED_LOGS_TABLE_V2)
 }

exporter/clickhouselogsexporter/factory.go

@@ -34,7 +34,6 @@ const (
 	archiveNamespace = "clickhouselogs-archive"
 	databaseName     = "signoz_logs"
 	tableName        = "logs"
-	migrationsFolder = "./migrations"
 )
 
 var (

exporter/clickhousemetricsexporter/clickhouse.go

@@ -158,7 +158,7 @@ func (ch *clickHouse) shardCountWatcher(ctx context.Context) {
 	ticker := time.NewTicker(ch.watcherInterval)
 	defer ticker.Stop()
 
-	q := `SELECT count() FROM system.clusters WHERE cluster='cluster'`
+	q := `SELECT count() FROM system.clusters`
 	for {
 
 		err := func() error {

exporter/clickhousetracesexporter/clickhouse_exporter.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/url"
 	"strconv"
 	"strings"
@@ -72,9 +71,6 @@ func newExporter(cfg component.Config, logger *zap.Logger) (*storage, error) {
 		"signoz_traces",
 		UsageExporter,
 	)
-	if err != nil {
-		log.Fatalf("Error creating usage collector for traces: %v", err)
-	}
 	collector.Start()
 
 	if err := view.Register(SpansCountView, SpansCountBytesView); err != nil {

exporter/clickhousetracesexporter/clickhouse_factory.go

@@ -16,12 +16,10 @@ package clickhousetracesexporter
 
 import (
 	"context"
-	"flag"
 	"fmt"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/google/uuid"
-	"github.com/spf13/viper"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
@@ -34,7 +32,6 @@ type Factory struct {
 	Options    *Options
 	db         clickhouse.Conn
 	archive    clickhouse.Conn
-	datasource string
 	makeWriter writerMaker
 }
 
@@ -106,16 +103,6 @@ func (f *Factory) connect(cfg *namespaceConfig) (clickhouse.Conn, error) {
 	return cfg.Connector(cfg)
 }
 
-// AddFlags implements plugin.Configurable
-func (f *Factory) AddFlags(flagSet *flag.FlagSet) {
-	f.Options.AddFlags(flagSet)
-}
-
-// InitFromViper implements plugin.Configurable
-func (f *Factory) InitFromViper(v *viper.Viper) {
-	f.Options.InitFromViper(v)
-}
-
 // CreateSpanWriter implements storage.Factory
 func (f *Factory) CreateSpanWriter() (Writer, error) {
 	cfg := f.Options.getPrimary()

exporter/clickhousetracesexporter/config.go

@@ -26,15 +26,11 @@ import (
 type Config struct {
 	Options    `mapstructure:",squash"`
 	Datasource string `mapstructure:"datasource"`
-	Migrations string `mapstructure:"migrations"`
-	// Docker Multi Node Cluster is a flag to enable the docker multi node cluster. Default is false.
-	DockerMultiNodeCluster bool `mapstructure:"docker_multi_node_cluster"`
 	// LowCardinalExceptionGrouping is a flag to enable exception grouping by serviceName + exceptionType. Default is false.
 	LowCardinalExceptionGrouping bool `mapstructure:"low_cardinal_exception_grouping"`
 	exporterhelper.TimeoutConfig `mapstructure:",squash"`
 	configretry.BackOffConfig    `mapstructure:"retry_on_failure"`
 	exporterhelper.QueueConfig   `mapstructure:"sending_queue"`
-	ClusterName string `mapstructure:"cluster_name" default:"cluster"`
 }
 
 var _ component.Config = (*Config)(nil)

exporter/clickhousetracesexporter/options.go

@@ -16,18 +16,14 @@ package clickhousetracesexporter
 
 import (
 	"context"
-	"flag"
-	"fmt"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/google/uuid"
-	"github.com/spf13/viper"
 )
 
 const (
 	defaultDatasource               string   = "tcp://127.0.0.1:9000/?database=signoz_traces"
 	DefaultTraceDatabase            string   = "signoz_traces"
-	defaultMigrations               string   = "/migrations"
 	defaultOperationsTable          string   = "distributed_signoz_operations"
 	DefaultIndexTable               string   = "distributed_signoz_index_v2"
 	LocalIndexTable                 string   = "signoz_index_v2"
@@ -45,23 +41,11 @@ const (
 	defaultEncoding                 Encoding = EncodingJSON
 )
 
-const (
-	suffixEnabled         = ".enabled"
-	suffixDatasource      = ".datasource"
-	suffixTraceDatabase   = ".trace-database"
-	suffixMigrations      = ".migrations"
-	suffixOperationsTable = ".operations-table"
-	suffixIndexTable      = ".index-table"
-	suffixSpansTable      = ".spans-table"
-	suffixEncoding        = ".encoding"
-)
-
 // NamespaceConfig is Clickhouse's internal configuration data
 type namespaceConfig struct {
 	namespace                  string
 	Enabled                    bool
 	Datasource                 string
-	Migrations                 string
 	TraceDatabase              string
 	OperationsTable            string
 	IndexTable                 string
@@ -70,14 +54,12 @@ type namespaceConfig struct {
 	ErrorTable                 string
 	AttributeTable             string
 	AttributeKeyTable          string
-	Cluster                    string
 	DurationSortTable          string
 	DurationSortMVTable        string
 	DependencyGraphServiceMV   string
 	DependencyGraphDbMV        string
 	DependencyGraphMessagingMV string
 	DependencyGraphTable       string
-	DockerMultiNodeCluster     bool
 	NumConsumers               int
 	Encoding                   Encoding
 	Connector                  Connector
@@ -112,10 +94,6 @@ func defaultConnector(cfg *namespaceConfig) (clickhouse.Conn, error) {
 		return nil, err
 	}
 
-	query := fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s ON CLUSTER %s`, options.Auth.Database, cfg.Cluster)
-	if err := db.Exec(ctx, query); err != nil {
-		return nil, err
-	}
 	return db, nil
 }
 
@@ -133,17 +111,12 @@ func NewOptions(exporterId uuid.UUID, config Config, primaryNamespace string, ot
 	if datasource == "" {
 		datasource = defaultDatasource
 	}
-	migrations := config.Migrations
-	if migrations == "" {
-		migrations = defaultMigrations
-	}
 
 	options := &Options{
 		primary: &namespaceConfig{
 			namespace:                  primaryNamespace,
 			Enabled:                    true,
 			Datasource:                 datasource,
-			Migrations:                 migrations,
 			TraceDatabase:              DefaultTraceDatabase,
 			OperationsTable:            defaultOperationsTable,
 			IndexTable:                 DefaultIndexTable,
@@ -154,12 +127,10 @@ func NewOptions(exporterId uuid.UUID, config Config, primaryNamespace string, ot
 			AttributeKeyTable:          defaultAttributeKeyTable,
 			DurationSortTable:          DefaultDurationSortTable,
 			DurationSortMVTable:        DefaultDurationSortMVTable,
-			Cluster:                    config.ClusterName,
 			DependencyGraphTable:       defaultDependencyGraphTable,
 			DependencyGraphServiceMV:   defaultDependencyGraphServiceMV,
 			DependencyGraphDbMV:        defaultDependencyGraphDbMV,
 			DependencyGraphMessagingMV: DependencyGraphMessagingMV,
-			DockerMultiNodeCluster:     config.DockerMultiNodeCluster,
 			NumConsumers:               config.QueueConfig.NumConsumers,
 			Encoding:                   defaultEncoding,
 			Connector:                  defaultConnector,
@@ -173,7 +144,6 @@ func NewOptions(exporterId uuid.UUID, config Config, primaryNamespace string, ot
 			options.others[namespace] = &namespaceConfig{
 				namespace:       namespace,
 				Datasource:      datasource,
-				Migrations:      migrations,
 				OperationsTable: "",
 				IndexTable:      "",
 				SpansTable:      defaultArchiveSpansTable,
@@ -189,73 +159,6 @@ func NewOptions(exporterId uuid.UUID, config Config, primaryNamespace string, ot
 	return options
 }
 
-// AddFlags adds flags for Options
-func (opt *Options) AddFlags(flagSet *flag.FlagSet) {
-	addFlags(flagSet, opt.primary)
-	for _, cfg := range opt.others {
-		addFlags(flagSet, cfg)
-	}
-}
-
-func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) {
-	if nsConfig.namespace == archiveNamespace {
-		flagSet.Bool(
-			nsConfig.namespace+suffixEnabled,
-			nsConfig.Enabled,
-			"Enable archive storage")
-	}
-
-	flagSet.String(
-		nsConfig.namespace+suffixDatasource,
-		nsConfig.Datasource,
-		"Clickhouse datasource string.",
-	)
-
-	if nsConfig.namespace != archiveNamespace {
-		flagSet.String(
-			nsConfig.namespace+suffixOperationsTable,
-			nsConfig.OperationsTable,
-			"Clickhouse operations table name.",
-		)
-
-		flagSet.String(
-			nsConfig.namespace+suffixIndexTable,
-			nsConfig.IndexTable,
-			"Clickhouse index table name.",
-		)
-	}
-
-	flagSet.String(
-		nsConfig.namespace+suffixSpansTable,
-		nsConfig.SpansTable,
-		"Clickhouse spans table name.",
-	)
-
-	flagSet.String(
-		nsConfig.namespace+suffixEncoding,
-		string(nsConfig.Encoding),
-		"Encoding to store spans (json allows out of band queries, protobuf is more compact)",
-	)
-}
-
-// InitFromViper initializes Options with properties from viper
-func (opt *Options) InitFromViper(v *viper.Viper) {
-	initFromViper(opt.primary, v)
-	for _, cfg := range opt.others {
-		initFromViper(cfg, v)
-	}
-}
-
-func initFromViper(cfg *namespaceConfig, v *viper.Viper) {
-	cfg.Enabled = v.GetBool(cfg.namespace + suffixEnabled)
-	cfg.Datasource = v.GetString(cfg.namespace + suffixDatasource)
-	cfg.TraceDatabase = v.GetString(cfg.namespace + suffixTraceDatabase)
-	cfg.IndexTable = v.GetString(cfg.namespace + suffixIndexTable)
-	cfg.SpansTable = v.GetString(cfg.namespace + suffixSpansTable)
-	cfg.OperationsTable = v.GetString(cfg.namespace + suffixOperationsTable)
-	cfg.Encoding = Encoding(v.GetString(cfg.namespace + suffixEncoding))
-}
-
 // getPrimary returns the primary namespace configuration
 func (opt *Options) getPrimary() *namespaceConfig {
 	return opt.primary