Skip to content

Commit

Permalink
fix WITH AS corner case for restore_schema_on_cluster option, fix #…
Browse files Browse the repository at this point in the history 
…1075

Signed-off-by: Slach <[email protected]>
  • Loading branch information
@Slach
Slach committed Jan 17, 2025
1 parent 7562e26 commit 3b7dc7c
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 11 deletions.
2 changes: 2 additions & 0 deletions ChangeLog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ BUG FIXES
fix [877](https://github.com/Altinity/clickhouse-backup/issues/877#issuecomment-2589164718),
fix [505](https://github.com/Altinity/clickhouse-backup/issues/505#issuecomment-2589163706)
- add test for COS, fix [1053](https://github.com/Altinity/clickhouse-backup/issues/1053)
- fix `AS WITH x AS` corner case for `restore_schema_on_cluster` option,
fix [1075](https://github.com/Altinity/clickhouse-backup/issues/1075)

# v2.6.5

Expand Down
29 changes: 18 additions & 11 deletions pkg/clickhouse/clickhouse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -932,9 +932,11 @@ func (ch *ClickHouse) DropTable(table Table, query string, onCluster string, ign
}

var createViewToClauseRe = regexp.MustCompile(`(?im)^(CREATE[\s\w]+VIEW[^(]+)(\s+TO\s+.+)`)
var createViewSelectRe = regexp.MustCompile(`(?im)^(CREATE[\s\w]+VIEW[^(]+)(\s+AS\s+SELECT.+)`)
var createViewAsWithRe = regexp.MustCompile(`(?im)^(CREATE[\s\w]+VIEW[^(]+)(\s+AS\s+WITH\s+.+)`)
var createViewRe = regexp.MustCompile(`(?im)^(CREATE[\s\w]+VIEW[^(]+)(\s+AS\s+.+)`)
var attachViewToClauseRe = regexp.MustCompile(`(?im)^(ATTACH[\s\w]+VIEW[^(]+)(\s+TO\s+.+)`)
var attachViewSelectRe = regexp.MustCompile(`(?im)^(ATTACH[\s\w]+VIEW[^(]+)(\s+AS\s+SELECT.+)`)
var attachViewAsWithRe = regexp.MustCompile(`(?im)^(ATTACH[\s\w]+VIEW[^(]+)(\s+AS\s+WITH\s+.+)`)
var attachViewRe = regexp.MustCompile(`(?im)^(ATTACH[\s\w]+VIEW[^(]+)(\s+AS\s+.+)`)
var createObjRe = regexp.MustCompile(`(?is)^(CREATE [^(]+)(\(.+)`)
var onClusterRe = regexp.MustCompile(`(?im)\s+ON\s+CLUSTER\s+`)
var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^)]+)\)`)
Expand All @@ -948,15 +950,7 @@ func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDe
}
}

if version > 19000000 && onCluster != "" && !onClusterRe.MatchString(query) {
tryMatchReList := []*regexp.Regexp{attachViewToClauseRe, attachViewSelectRe, createViewToClauseRe, createViewSelectRe, createObjRe}
for _, tryMatchRe := range tryMatchReList {
if tryMatchRe.MatchString(query) {
query = tryMatchRe.ReplaceAllString(query, "$1 ON CLUSTER '"+onCluster+"' $2")
break
}
}
}
query = ch.enrichQueryWithOnCluster(query, onCluster, version)

if !strings.Contains(query, table.Name) {
return errors.New(fmt.Sprintf("schema query ```%s``` doesn't contains table name `%s`", query, table.Name))
Expand Down Expand Up @@ -1010,6 +1004,19 @@ func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDe
return nil
}

func (ch *ClickHouse) enrichQueryWithOnCluster(query string, onCluster string, version int) string {
if version > 19000000 && onCluster != "" && !onClusterRe.MatchString(query) {
tryMatchReList := []*regexp.Regexp{attachViewToClauseRe, attachViewAsWithRe, attachViewRe, createViewToClauseRe, createViewAsWithRe, createViewRe, createObjRe}
for _, tryMatchRe := range tryMatchReList {
if tryMatchRe.MatchString(query) {
query = tryMatchRe.ReplaceAllString(query, "$1 ON CLUSTER '"+onCluster+"' $2")
break
}
}
}
return query
}

func (ch *ClickHouse) TurnAnalyzerOnIfNecessary(version int, query string, allowExperimentalAnalyzer string) error {
if version > 24003000 && (strings.HasPrefix(query, "CREATE LIVE VIEW") || strings.HasPrefix(query, "ATTACH LIVE VIEW") || strings.HasPrefix(query, "CREATE WINDOW VIEW") || strings.HasPrefix(query, "ATTACH WINDOW VIEW")) && allowExperimentalAnalyzer == "1" {
if err := ch.Query("SET allow_experimental_analyzer=1"); err != nil {
Expand Down
76 changes: 76 additions & 0 deletions pkg/clickhouse/clickhouse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,79 @@ func TestExtractStoragePolicy(t *testing.T) {
assert.Equal(t, policy, ch.ExtractStoragePolicy(query))
}
}

func TestEnrichQueryWithOnCluster(t *testing.T) {
ch := ClickHouse{}

testCases := []struct {
Name string
Query string
OnCluster string
Version int
ExpectedQuery string
}{
{
Name: "No OnCluster provided, version < 19000000",
Query: "CREATE TABLE test (id UInt64) ENGINE = MergeTree ORDER BY id",
OnCluster: "my_cluster",
Version: 19000000,
ExpectedQuery: "CREATE TABLE test (id UInt64) ENGINE = MergeTree ORDER BY id",
},
{
Name: "OnCluster provided, version >= 19000000, CREATE TABLE",
Query: "CREATE TABLE test (id UInt64) ENGINE = MergeTree ORDER BY id",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "CREATE TABLE test ON CLUSTER 'my_cluster' (id UInt64) ENGINE = MergeTree ORDER BY id",
},
{
Name: "OnCluster provided, version >= 19000000, CREATE VIEW with TO clause",
Query: "CREATE VIEW test_view TO test_table AS SELECT * FROM test_table",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "CREATE VIEW test_view ON CLUSTER 'my_cluster' TO test_table AS SELECT * FROM test_table",
},
{
Name: "OnCluster provided, version >= 19000000, CREATE VIEW with AS SELECT",
Query: "CREATE VIEW test_view AS SELECT * FROM test_table",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "CREATE VIEW test_view ON CLUSTER 'my_cluster' AS SELECT * FROM test_table",
},
{
Name: "OnCluster provided, version >= 19000000, ATTACH VIEW with TO clause",
Query: "ATTACH VIEW test_view TO test_table AS SELECT * FROM test_table",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "ATTACH VIEW test_view ON CLUSTER 'my_cluster' TO test_table AS SELECT * FROM test_table",
},
{
Name: "OnCluster provided, version >= 19000000, ATTACH VIEW with AS SELECT",
Query: "ATTACH VIEW test_view AS SELECT * FROM test_table",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "ATTACH VIEW test_view ON CLUSTER 'my_cluster' AS SELECT * FROM test_table",
},
{
Name: "OnCluster provided, version >= 19000000, CREATE DICTIONARY",
Query: "CREATE DICTIONARY test_dict (id UInt64) PRIMARY KEY id SOURCE(CLICKHOUSE(HOST 'localhost')) LAYOUT(HASHED())",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "CREATE DICTIONARY test_dict ON CLUSTER 'my_cluster' (id UInt64) PRIMARY KEY id SOURCE(CLICKHOUSE(HOST 'localhost')) LAYOUT(HASHED())",
},
{
Name: "OnCluster provided, version >= 19000000, CREATE VIEW ...AS WITH AS..., https://github.com/Altinity/clickhouse-backup/issues/1075",
Query: "CREATE VIEW feature_data.insert_price_history_it_az UUID '12a40994-ff52-4a14-8f0b-bccdf1a3a4ba' AS WITH a AS (SELECT {input_timestamp:DateTime} AS entity_timestamp, concat(it.provider, '_', z.name, '_', it.instance_type) AS entity_id_cloud_az_it, it.provider AS cloud, z.name AS availability_zone, it.instance_type AS instance_type, ph.is_spot AS is_spot, CAST(ph.price, 'float') AS price, now() AS created_timestamp, row_number() OVER (PARTITION BY entity_id_cloud_az_it ORDER BY ph.effective_from DESC) AS rn FROM raw_data.price_history AS ph INNER JOIN raw_data.zones AS z ON z.id = ph.availability_zone INNER JOIN raw_data.instance_types AS it ON it.id = ph.instance_type WHERE (ph.effective_to < {input_timestamp:DateTime}) AND z.latest_value AND it.latest_value) SELECT entity_timestamp, entity_id_cloud_az_it, cloud, availability_zone, instance_type, is_spot, price, created_timestamp FROM a WHERE rn = 1 SETTINGS final = 1",
OnCluster: "my_cluster",
Version: 19000001,
ExpectedQuery: "CREATE VIEW feature_data.insert_price_history_it_az UUID '12a40994-ff52-4a14-8f0b-bccdf1a3a4ba' ON CLUSTER 'my_cluster' AS WITH a AS (SELECT {input_timestamp:DateTime} AS entity_timestamp, concat(it.provider, '_', z.name, '_', it.instance_type) AS entity_id_cloud_az_it, it.provider AS cloud, z.name AS availability_zone, it.instance_type AS instance_type, ph.is_spot AS is_spot, CAST(ph.price, 'float') AS price, now() AS created_timestamp, row_number() OVER (PARTITION BY entity_id_cloud_az_it ORDER BY ph.effective_from DESC) AS rn FROM raw_data.price_history AS ph INNER JOIN raw_data.zones AS z ON z.id = ph.availability_zone INNER JOIN raw_data.instance_types AS it ON it.id = ph.instance_type WHERE (ph.effective_to < {input_timestamp:DateTime}) AND z.latest_value AND it.latest_value) SELECT entity_timestamp, entity_id_cloud_az_it, cloud, availability_zone, instance_type, is_spot, price, created_timestamp FROM a WHERE rn = 1 SETTINGS final = 1",
},
}

for _, tc := range testCases {
t.Run(tc.Name, func(t *testing.T) {
result := ch.enrichQueryWithOnCluster(tc.Query, tc.OnCluster, tc.Version)
assert.Equal(t, tc.ExpectedQuery, result)
})
}
}

0 comments on commit 3b7dc7c

Please sign in to comment.