Skip to content

Commit

Permalink
add some features
Browse files Browse the repository at this point in the history 
1. update cos sdk
2. support  migrate csp
3. migrateUrl support skip head
4. migrateLocal support skip empty file and skip suffix
  • Loading branch information
@kitmanzheng
kitmanzheng committed Aug 8, 2019
1 parent 7378e98 commit f9582b6
Show file tree
Hide file tree
Showing 25 changed files with 1,166 additions and 291 deletions.
18 changes: 17 additions & 1 deletion conf/config.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ exeludes=
# 默认不设置, 表示不根据lastmodified时间进行筛选
# 适用于客户在更新文件的同时又在运行迁移工具, 不准备把正在更新的文件迁移上传到COS, 比如设置为300, 表示只上传更新了5分钟以上的文件
ignoreModifiedTimeLessThanSeconds=

#多个后缀用;隔开，例如:.txt;.tmp;
ignoreSuffix=
#on：忽略空文件，off：不忽略。默认不忽略
ignoreEmptyFile=

## 从阿里迁移到COS的配置分节
[migrateAli]
Expand Down Expand Up @@ -103,6 +106,9 @@ needSign=true
# 对于linux绝对路径, 如/a/b/c, 对于windows绝对路径，注意分隔符为两个反斜杠，如E:\\a\\b\\c
urllistPath=/data/mydata/url

#on：迁移时不去head url，off:会head url，默认off。不去head可能导致源更新了，迁移工具却不知道，适用于确定源不更新，加快迁移速度。
isSkipHead=

## 从AWS迁移到COS的配置分节
[migrateAws]
# aws的bucket
Expand Down Expand Up @@ -135,3 +141,13 @@ srcSecretId=xxxxxxxxxxx
srcSecretKey=yyyyyyyyyyyyyyyy
# 要迁移的cos路径的
srcCosPath=/

[migrateCsp]
bucket=xxx
accessKeyId=xxx
accessKeySecret=xxx
endPoint=xxx.com

#指定了uriList之后，就只迁uri列表中的数据，不指定则是自动list全部数据然后迁移。uri格式不带域名，如：/dir1/subdir2/a.txt
uriList=/data/uriList.txt
prefix=
Binary file modified dep/cos_migrate_tool-1.0-jar-with-dependencies.jar
View file
Binary file not shown.
89 changes: 77 additions & 12 deletions src/main/java/com/qcloud/cos_migrate_tool/app/App.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package com.qcloud.cos_migrate_tool.app;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -8,14 +14,17 @@
import com.qcloud.cos_migrate_tool.config.CopyBucketConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromAliConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromAwsConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromCompetitorConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromLocalConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromQiniuConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromUrllistConfig;
import com.qcloud.cos_migrate_tool.config.CopyFromCspConfig;
import com.qcloud.cos_migrate_tool.config.MigrateType;
import com.qcloud.cos_migrate_tool.meta.TaskStatics;
import com.qcloud.cos_migrate_tool.task.MigrateAliTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateAwsTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateCopyBucketTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateCspTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateLocalTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateQiniuTaskExecutor;
import com.qcloud.cos_migrate_tool.task.MigrateUrllistTaskExecutor;
Expand All @@ -40,6 +49,8 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
return new MigrateUrllistTaskExecutor((CopyFromUrllistConfig) config);
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_QINIU)) {
return new MigrateQiniuTaskExecutor((CopyFromQiniuConfig) config);
} else if (ConfigParser.instance.getMigrateType().equals(MigrateType.MIGRATE_FROM_CSP)) {
return new MigrateCspTaskExecutor((CopyFromCspConfig) config);
} else {
System.out.println("unknown migrate type");
}
Expand All @@ -48,26 +59,80 @@ private static TaskExecutor buildTaskExecutor(CommonConfig config) {
}

public static void main(String[] args) {

if (!ConfigParser.instance.parse()) {
return;
}

CommonConfig config = ConfigParser.instance.getConfig();
while (true) {
TaskStatics.instance.reset();

TaskExecutor taskExecutor = buildTaskExecutor(config);
taskExecutor.run();
taskExecutor.waitTaskOver();

if (!config.isDamonMode())
break;
String batchTaskPath = config.getBatchTaskPath();
if (!batchTaskPath.isEmpty()) {

FileInputStream inputStream = null;
try {
inputStream = new FileInputStream(batchTaskPath);
} catch (FileNotFoundException e) {
e.printStackTrace();
return;
}
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

String str = null;
try {
Thread.sleep(config.getDamonInterVal() * 1000);
} catch (InterruptedException e) {
log.error("the program is interrupted!", e);
break;
while (true) {
while ((str = bufferedReader.readLine()) != null) {
String[] task = str.split("\t");
if (!((CopyFromCompetitorConfig) config).setTask(task)) {
System.out.printf("task:%s invalid\n", str);
log.error("task:{} invalid", str);
continue;
}

TaskStatics.instance.reset();

TaskExecutor taskExecutor = buildTaskExecutor(config);
taskExecutor.run();
taskExecutor.waitTaskOver();
}

if (!config.isDamonMode())
break;

try {
Thread.sleep(config.getDamonInterVal() * 1000);
} catch (InterruptedException e) {
log.error("the program is interrupted!", e);
break;
}
}

inputStream.close();
bufferedReader.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return;
}

} else {

while (true) {
TaskStatics.instance.reset();

TaskExecutor taskExecutor = buildTaskExecutor(config);
taskExecutor.run();
taskExecutor.waitTaskOver();

if (!config.isDamonMode())
break;

try {
Thread.sleep(config.getDamonInterVal() * 1000);
} catch (InterruptedException e) {
log.error("the program is interrupted!", e);
break;
}
}
}
}
Expand Down
31 changes: 25 additions & 6 deletions src/main/java/com/qcloud/cos_migrate_tool/config/CommonConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class CommonConfig {
private String cosPath;
private boolean enableHttps;
private boolean entireFileMd5Attached;
private int taskExecutorNumber = 64;
private int taskExecutorNumber = 68;
private StorageClass storageClass = StorageClass.Standard;
private int smallFileExecutorNumber = 64;
private int bigFileExecutorNum = 4;
Expand All @@ -33,6 +33,30 @@ public class CommonConfig {
private String cosProxyHost = "";
private int cosProxyPort = -1;
private String encryptionType = "";
private String batchTaskPath = "";
private boolean realTimeCompare = false;

public void setRealTimeCompare(String realTimeCompare) throws IllegalArgumentException {
if (realTimeCompare.equalsIgnoreCase("on")) {
this.realTimeCompare = true;
} else if (realTimeCompare.equalsIgnoreCase("off")) {
this.realTimeCompare = false;
} else {
throw new IllegalArgumentException("invalid realTimeCompare config. only support on/off");
}
}

public boolean getRealTimeCompare() {
return this.realTimeCompare;
}

public void setBatchTaskPath(String batchTaskPath) {
this.batchTaskPath = batchTaskPath;
}

public String getBatchTaskPath() {
return this.batchTaskPath;
}

public void setEncryptionType(String encryptionType) {
if (!encryptionType.equals("sse-cos")) {
Expand Down Expand Up @@ -71,11 +95,6 @@ public String getBucketName() {

public void setBucketName(String bucketName) {
bucketName = bucketName.trim();
String parrtern = ".*-(125|100|20)[0-9]{3,}$";
if (!Pattern.matches(parrtern, bucketName)) {
throw new IllegalArgumentException(
"bucketName must contain appid. example: test-1250001000");
}
this.bucketName = bucketName;
}

Expand Down
Loading

0 comments on commit f9582b6

Please sign in to comment.