add test for COS, fix #1053

Signed-off-by: Slach <[email protected]>
Altinity · Jan 14, 2025 · 7893b8a · 7893b8a
1 parent 63afba7
commit 7893b8a
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 1 deletion.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -7,6 +7,7 @@ BUG FIXES
 - add alibaba/oss/aliyuncs worked config,
   fix [877](https://github.com/Altinity/clickhouse-backup/issues/877#issuecomment-2589164718),
   fix [505](https://github.com/Altinity/clickhouse-backup/issues/505#issuecomment-2589163706)
+- add test for COS, fix [1053](https://github.com/Altinity/clickhouse-backup/issues/1053)
 
 # v2.6.5
 

diff --git a/test/integration/config-cos.yml b/test/integration/config-cos.yml
@@ -0,0 +1,33 @@
+general:
+  remote_storage: cos
+  upload_concurrency: 4
+  download_concurrency: 4
+  restore_schema_on_cluster: "{cluster}"
+  allow_object_disk_streaming: true
+  object_disk_server_side_copy_concurrency: 2
+clickhouse:
+  host: clickhouse
+  port: 9440
+  username: backup
+  password: meow=& 123?*%# МЯУ
+  secure: true
+  skip_verify: true
+  sync_replicated_tables: true
+  restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN"
+  # restart_command: bash -c 'echo "FAKE RESTART"'
+  backup_mutations: true
+  timeout: 60s
+s3:
+  disable_ssl: false
+  disable_cert_verification: true
+cos:
+  url: "https://clickhouse-backup-1336113806.cos.na-ashburn.myqcloud.com"
+  secret_id: "${QA_TENCENT_SECRET_ID}"
+  secret_key: "${QA_TENCENT_SECRET_KEY}"
+  path: backup/{cluster}/{shard}
+  object_disk_path: object_disk/{cluster}/{shard}
+api:
+  listen: :7171
+  create_integration_tables: true
+  integration_tables_host: "clickhouse-backup"
+  allow_parallel: true
diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml
@@ -131,6 +131,9 @@ services:
       # AlibabaCloud over S3
       QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}"
       QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}"
+      # Tencent Cloud Object Storage
+      QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}"
+      QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}"
 # https://github.com/Altinity/clickhouse-backup/issues/691:
       AWS_ACCESS_KEY_ID: access_key
       AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key
@@ -176,6 +179,9 @@ services:
       #     AlibabaCloud over S3
       QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}"
       QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}"
+      #     Tencent Cloud Object Storage
+      QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}"
+      QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}"
 
       AWS_EC2_METADATA_DISABLED: "true"
     volumes:
@@ -191,6 +197,7 @@ services:
       - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml
       - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml
       - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml
+      - ./config-cos.yml:/etc/clickhouse-backup/config-cos.yml.template
       - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml
       - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml
       - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml

diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml
@@ -180,7 +180,11 @@ services:
       # AlibabaCloud over S3
       QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}"
       QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}"
-# https://github.com/Altinity/clickhouse-backup/issues/691:
+      # Tencent Cloud Object Storage
+      QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}"
+      QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}"
+
+      # https://github.com/Altinity/clickhouse-backup/issues/691:
       AWS_ACCESS_KEY_ID: access_key
       AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key
     volumes_from:
@@ -226,6 +230,9 @@ services:
       #     AlibabaCloud over S3
       QA_ALIBABA_ACCESS_KEY: "${QA_ALIBABA_ACCESS_KEY:-}"
       QA_ALIBABA_SECRET_KEY: "${QA_ALIBABA_SECRET_KEY:-}"
+      #     Tencent Cloud Object Storage
+      QA_TENCENT_SECRET_ID: "${QA_TENCENT_SECRET_ID:-}"
+      QA_TENCENT_SECRET_KEY: "${QA_TENCENT_SECRET_KEY:-}"
 
     # to avoid backward incompatibility ;(
     # https://t.me/clickhouse_ru/359960
@@ -246,6 +253,7 @@ services:
       - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml
       - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml
       - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml
+      - ./config-cos.yml:/etc/clickhouse-backup/config-cos.yml.template
       - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml
       - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml
       - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml

diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go
@@ -751,6 +751,18 @@ func TestIntegrationAlibabaOverS3(t *testing.T) {
 	env.Cleanup(t, r)
 }
 
+func TestIntegrationCOS(t *testing.T) {
+	if os.Getenv("QA_TENCENT_SECRET_KEY") == "" {
+		t.Skip("Skipping Tencent Cloud Object Storage integration tests... QA_TENCENT_SECRET_KEY missing")
+		return
+	}
+	env, r := NewTestEnvironment(t)
+	env.InstallDebIfNotExists(r, "clickhouse-backup", "gettext-base")
+	env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-cos.yml.template | envsubst > /etc/clickhouse-backup/config-cos.yml")
+	env.runMainIntegrationScenario(t, "COS", "config-cos.yml")
+	env.Cleanup(t, r)
+}
+
 func TestIntegrationGCS(t *testing.T) {
 	if isTestShouldSkip("GCS_TESTS") {
 		t.Skip("Skipping GCS integration tests...")