From 7893b8a70da27e82ef31969a950b2e2d757bcbc7 Mon Sep 17 00:00:00 2001 From: Slach Date: Tue, 14 Jan 2025 21:13:18 +0500 Subject: [PATCH] add test for COS, fix https://github.com/Altinity/clickhouse-backup/issues/1053 Signed-off-by: Slach --- ChangeLog.md | 1 + test/integration/config-cos.yml | 33 ++++++++++++++++++++ test/integration/docker-compose.yml | 7 +++++ test/integration/docker-compose_advanced.yml | 10 +++++- test/integration/integration_test.go | 12 +++++++ 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 test/integration/config-cos.yml diff --git a/ChangeLog.md b/ChangeLog.md index 5b41a5ff..f688dd95 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,6 +7,7 @@ BUG FIXES - add alibaba/oss/aliyuncs worked config, fix [877](https://github.com/Altinity/clickhouse-backup/issues/877#issuecomment-2589164718), fix [505](https://github.com/Altinity/clickhouse-backup/issues/505#issuecomment-2589163706) +- add test for COS, fix [1053](https://github.com/Altinity/clickhouse-backup/issues/1053) # v2.6.5 diff --git a/test/integration/config-cos.yml b/test/integration/config-cos.yml new file mode 100644 index 00000000..86ce9706 --- /dev/null +++ b/test/integration/config-cos.yml @@ -0,0 +1,33 @@ +general: + remote_storage: cos + upload_concurrency: 4 + download_concurrency: 4 + restore_schema_on_cluster: "{cluster}" + allow_object_disk_streaming: true + object_disk_server_side_copy_concurrency: 2 +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN" + # restart_command: bash -c 'echo "FAKE RESTART"' + backup_mutations: true + timeout: 60s +s3: + disable_ssl: false + disable_cert_verification: true +cos: + url: "https://clickhouse-backup-1336113806.cos.na-ashburn.myqcloud.com" + secret_id: "${QA_TENCENT_SECRET_ID}" + secret_key: "${QA_TENCENT_SECRET_KEY}" + path: backup/{cluster}/{shard} + object_disk_path: object_disk/{cluster}/{shard} +api: + listen: :7171 + create_integration_tables: true + integration_tables_host: "clickhouse-backup" + allow_parallel: true diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index f38a15c1..f11d5871 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -131,6 +131,9 @@ services: # AlibabaCloud over S3 QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}" QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}" + # Tencent Cloud Object Storage + QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}" + QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}" # https://github.com/Altinity/clickhouse-backup/issues/691: AWS_ACCESS_KEY_ID: access_key AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key @@ -176,6 +179,9 @@ services: # AlibabaCloud over S3 QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}" QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}" + # Tencent Cloud Object Storage + QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}" + QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}" AWS_EC2_METADATA_DISABLED: "true" volumes: @@ -191,6 +197,7 @@ services: - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml + - ./config-cos.yml:/etc/clickhouse-backup/config-cos.yml.template - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index ade36fef..8b38fccd 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -180,7 +180,11 @@ services: # AlibabaCloud over S3 QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}" QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}" -# https://github.com/Altinity/clickhouse-backup/issues/691: + # Tencent Cloud Object Storage + QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}" + QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}" + + # https://github.com/Altinity/clickhouse-backup/issues/691: AWS_ACCESS_KEY_ID: access_key AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key volumes_from: @@ -226,6 +230,9 @@ services: # AlibabaCloud over S3 QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}" QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}" + # Tencent Cloud Object Storage + QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}" + QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}" # to avoid backward incompatibility ;( # https://t.me/clickhouse_ru/359960 @@ -246,6 +253,7 @@ services: - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml + - ./config-cos.yml:/etc/clickhouse-backup/config-cos.yml.template - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 3f7b4099..d0ee86b9 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -751,6 +751,18 @@ func TestIntegrationAlibabaOverS3(t *testing.T) { env.Cleanup(t, r) } +func TestIntegrationCOS(t *testing.T) { + if os.Getenv("QA_TENCENT_SECRET_KEY") == "" { + t.Skip("Skipping Tencent Cloud Object Storage integration tests... QA_TENCENT_SECRET_KEY missing") + return + } + env, r := NewTestEnvironment(t) + env.InstallDebIfNotExists(r, "clickhouse-backup", "gettext-base") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-cos.yml.template | envsubst > /etc/clickhouse-backup/config-cos.yml") + env.runMainIntegrationScenario(t, "COS", "config-cos.yml") + env.Cleanup(t, r) +} + func TestIntegrationGCS(t *testing.T) { if isTestShouldSkip("GCS_TESTS") { t.Skip("Skipping GCS integration tests...")