diff --git a/cmd/signozschemamigrator/schema_migrator/manager.go b/cmd/signozschemamigrator/schema_migrator/manager.go index 739c21727..a51056526 100644 --- a/cmd/signozschemamigrator/schema_migrator/manager.go +++ b/cmd/signozschemamigrator/schema_migrator/manager.go @@ -61,6 +61,7 @@ type SchemaMigrationRecord struct { MigrationID uint64 UpItems []Operation DownItems []Operation + IsNecessary bool } // MigrationManager is the manager for the schema migrations. @@ -637,7 +638,7 @@ func (m *MigrationManager) MigrateUpSync(ctx context.Context, upVersions []uint6 continue } for _, item := range migration.UpItems { - if !item.IsMutation() && item.IsIdempotent() && item.IsLightweight() { + if (!item.IsMutation() && item.IsIdempotent() && item.IsLightweight()) || migration.IsNecessary { if err := m.RunOperation(ctx, item, migration.MigrationID, signozMetricsDB, false); err != nil { return err } diff --git a/cmd/signozschemamigrator/schema_migrator/metrics_migrations.go b/cmd/signozschemamigrator/schema_migrator/metrics_migrations.go index e6f03b3c2..be5795c1c 100644 --- a/cmd/signozschemamigrator/schema_migrator/metrics_migrations.go +++ b/cmd/signozschemamigrator/schema_migrator/metrics_migrations.go @@ -883,4 +883,404 @@ var MetricsMigrations = []SchemaMigrationRecord{ }, }, }, + { + MigrationID: 1006, + UpItems: []Operation{ + // Drop existing tables and materialized views + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_6hrs", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1day", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1week", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs_mv", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day_mv", + }, + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week_mv", + }, + // Create new tables with AggregatingMergeTree + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeString}}, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "true", Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeString}}, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "true", Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeString}}, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "true", Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{LowCardinalityColumnType{ColumnTypeString}}}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeString}}, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{MapColumnType{KeyType: ColumnTypeString, ValueType: ColumnTypeString}}}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeBool}}, Default: "true", Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_6hrs_mv", + DestTable: "time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 21600000) * 21600000 AS unix_milli, + labels, + attrs, + scope_attrs, + resource_attrs, + __normalized + FROM signoz_metrics.time_series_v4`, + }, + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_1day_mv", + DestTable: "time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 86400000) * 86400000 AS unix_milli, + labels, + attrs, + scope_attrs, + resource_attrs, + __normalized + FROM signoz_metrics.time_series_v4`, + }, + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_1week_mv", + DestTable: "time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Default: "0", Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Default: "''", Codec: "ZSTD(5)"}, + {Name: "attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resource_attrs", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "__normalized", Type: ColumnTypeBool, Default: "true", Codec: "ZSTD(1)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 604800000) * 604800000 AS unix_milli, + labels, + attrs, + scope_attrs, + resource_attrs, + __normalized + FROM signoz_metrics.time_series_v4_1day`, + }, + }, + //added is necessary + IsNecessary: true, + }, } diff --git a/exporter/clickhousemetricsexporter/clickhouse.go b/exporter/clickhousemetricsexporter/clickhouse.go index b4cb0781c..3791aecf4 100644 --- a/exporter/clickhousemetricsexporter/clickhouse.go +++ b/exporter/clickhousemetricsexporter/clickhouse.go @@ -99,6 +99,7 @@ type ClickHouseParams struct { WatcherInterval time.Duration WriteTSToV4 bool DisableV2 bool + DisableTTLCache bool ExporterId uuid.UUID Settings exporter.Settings } @@ -129,11 +130,14 @@ func NewClickHouse(params *ClickHouseParams) (base.Storage, error) { return nil, fmt.Errorf("could not connect to clickhouse: %s", err) } - cache := ttlcache.New[string, bool]( - ttlcache.WithTTL[string, bool](45*time.Minute), - ttlcache.WithDisableTouchOnHit[string, bool](), - ) - go cache.Start() + var cache *ttlcache.Cache[string, bool] + if !params.DisableTTLCache { + cache = ttlcache.New[string, bool]( + ttlcache.WithTTL[string, bool](45*time.Minute), + ttlcache.WithDisableTouchOnHit[string, bool](), + ) + go cache.Start() + } durationHistogram, err := meter.Float64Histogram( "exporter_db_write_latency", @@ -467,9 +471,11 @@ func (ch *clickHouse) Write(ctx context.Context, data *prompb.WriteRequest, metr for fingerprint, labels := range timeSeries { key := fmt.Sprintf("%d:%d", fingerprint, unixMilli) - if item := ch.cache.Get(key); item != nil { - if value := item.Value(); value { - continue + if ch.cache != nil { + if item := ch.cache.Get(key); item != nil { + if value := item.Value(); value { + continue + } } } encodedLabels := string(marshalLabels(labels, make([]byte, 0, 128))) @@ -489,7 +495,9 @@ func (ch *clickHouse) Write(ctx context.Context, data *prompb.WriteRequest, metr if err != nil { return err } - ch.cache.Set(key, true, ttlcache.DefaultTTL) + if ch.cache != nil { + ch.cache.Set(key, true, ttlcache.DefaultTTL) + } } start := time.Now() diff --git a/exporter/clickhousemetricsexporter/config.go b/exporter/clickhousemetricsexporter/config.go index 092490778..5caff1c91 100644 --- a/exporter/clickhousemetricsexporter/config.go +++ b/exporter/clickhousemetricsexporter/config.go @@ -48,9 +48,10 @@ type Config struct { WatcherInterval time.Duration `mapstructure:"watcher_interval"` - WriteTSToV4 bool `mapstructure:"write_ts_to_v4"` - DisableV2 bool `mapstructure:"disable_v2"` - EnableExpHist bool `mapstructure:"enable_exp_hist"` + WriteTSToV4 bool `mapstructure:"write_ts_to_v4"` + DisableV2 bool `mapstructure:"disable_v2"` + EnableExpHist bool `mapstructure:"enable_exp_hist"` + DisableTTLCache bool `mapstructure:"disable_ttl_cache"` } // RemoteWriteQueue allows to configure the remote write queue. diff --git a/exporter/clickhousemetricsexporter/exporter.go b/exporter/clickhousemetricsexporter/exporter.go index 9599d0703..081eb7b13 100644 --- a/exporter/clickhousemetricsexporter/exporter.go +++ b/exporter/clickhousemetricsexporter/exporter.go @@ -95,6 +95,7 @@ func NewPrwExporter(cfg *Config, set exporter.Settings) (*PrwExporter, error) { WatcherInterval: cfg.WatcherInterval, WriteTSToV4: cfg.WriteTSToV4, DisableV2: cfg.DisableV2, + DisableTTLCache: cfg.DisableTTLCache, ExporterId: id, Settings: set, } diff --git a/exporter/clickhousemetricsexporter/factory.go b/exporter/clickhousemetricsexporter/factory.go index cc3bd7b6b..710dde1f1 100644 --- a/exporter/clickhousemetricsexporter/factory.go +++ b/exporter/clickhousemetricsexporter/factory.go @@ -106,5 +106,6 @@ func createDefaultConfig() component.Config { WriteTSToV4: true, DisableV2: false, EnableExpHist: false, + DisableTTLCache: false, } } diff --git a/exporter/signozclickhousemetrics/config.go b/exporter/signozclickhousemetrics/config.go index 7538a1129..e8fa84fa1 100644 --- a/exporter/signozclickhousemetrics/config.go +++ b/exporter/signozclickhousemetrics/config.go @@ -23,6 +23,7 @@ type Config struct { TimeSeriesTable string `mapstructure:"time_series_table"` ExpHistTable string `mapstructure:"exp_hist_table"` MetadataTable string `mapstructure:"metadata_table"` + DisableTtlCache bool `mapstructure:"disable_ttl_cache"` } var _ component.Config = (*Config)(nil) diff --git a/exporter/signozclickhousemetrics/exporter.go b/exporter/signozclickhousemetrics/exporter.go index 6e8a20d03..8d6ec6190 100644 --- a/exporter/signozclickhousemetrics/exporter.go +++ b/exporter/signozclickhousemetrics/exporter.go @@ -61,6 +61,7 @@ type clickhouseMetricsExporter struct { processMetricsDuration metricapi.Float64Histogram exportMetricsDuration metricapi.Float64Histogram settings exporter.Settings + disableTtlCache bool } // sample represents a single metric sample @@ -147,9 +148,14 @@ func WithEnableExpHist(enableExpHist bool) ExporterOption { } } -func WithCache(cache *ttlcache.Cache[string, bool]) ExporterOption { +func WithCache(cache *ttlcache.Cache[string, bool], enabled bool) ExporterOption { return func(e *clickhouseMetricsExporter) error { - e.cache = cache + e.disableTtlCache = enabled + if e.disableTtlCache { + e.cache = nil + } else { + e.cache = cache + } return nil } } @@ -176,13 +182,8 @@ func WithSettings(settings exporter.Settings) ExporterOption { } func defaultOptions() []ExporterOption { - cache := ttlcache.New( - ttlcache.WithTTL[string, bool](45*time.Minute), - ttlcache.WithDisableTouchOnHit[string, bool](), - ) return []ExporterOption{ - WithCache(cache), WithLogger(zap.NewNop()), WithEnableExpHist(false), WithMeter(noop.NewMeterProvider().Meter(internalmetadata.ScopeName)), @@ -228,8 +229,10 @@ func NewClickHouseExporter(opts ...ExporterOption) (*clickhouseMetricsExporter, } func (c *clickhouseMetricsExporter) Start(ctx context.Context, host component.Host) error { - go c.cache.Start() - c.cacheRunning = true + if c.cache != nil { + go c.cache.Start() + c.cacheRunning = true + } return nil } @@ -949,9 +952,11 @@ func (c *clickhouseMetricsExporter) writeBatch(ctx context.Context, batch *batch for _, ts := range timeSeries { roundedUnixMilli := ts.unixMilli / 3600000 * 3600000 cacheKey := makeCacheKey(ts.fingerprint, uint64(roundedUnixMilli)) - if item := c.cache.Get(cacheKey); item != nil { - if value := item.Value(); value { - continue + if c.cache != nil { + if item := c.cache.Get(cacheKey); item != nil { + if value := item.Value(); value { + continue + } } } err = statement.Append( @@ -973,7 +978,9 @@ func (c *clickhouseMetricsExporter) writeBatch(ctx context.Context, batch *batch if err != nil { return err } - c.cache.Set(cacheKey, true, ttlcache.DefaultTTL) + if c.cache != nil { + c.cache.Set(cacheKey, true, ttlcache.DefaultTTL) + } } return statement.Send() } diff --git a/exporter/signozclickhousemetrics/factory.go b/exporter/signozclickhousemetrics/factory.go index 92aa7e365..2eef696b8 100644 --- a/exporter/signozclickhousemetrics/factory.go +++ b/exporter/signozclickhousemetrics/factory.go @@ -3,6 +3,8 @@ package signozclickhousemetrics import ( "context" "errors" + "github.com/jellydator/ttlcache/v3" + "time" "github.com/ClickHouse/clickhouse-go/v2" "go.opentelemetry.io/collector/component" @@ -39,6 +41,11 @@ func createMetricsExporter(ctx context.Context, set exporter.Settings, return nil, err } + cache := ttlcache.New[string, bool]( + ttlcache.WithTTL[string, bool](45*time.Minute), + ttlcache.WithDisableTouchOnHit[string, bool](), + ) + chExporter, err := NewClickHouseExporter( WithConfig(chCfg), WithConn(conn), @@ -46,6 +53,7 @@ func createMetricsExporter(ctx context.Context, set exporter.Settings, WithMeter(set.MeterProvider.Meter(internalmetadata.ScopeName)), WithEnableExpHist(chCfg.EnableExpHist), WithSettings(set), + WithCache(cache, chCfg.DisableTtlCache), ) if err != nil { return nil, err @@ -82,5 +90,6 @@ func createDefaultConfig() component.Config { TimeSeriesTable: "distributed_time_series_v4", ExpHistTable: "distributed_exp_hist", MetadataTable: "distributed_metadata", + DisableTtlCache: false, } }