Skip to content

Commit

Permalink
output become more friendly
Browse files Browse the repository at this point in the history
  • Loading branch information
@kitmanzheng
kitmanzheng committed Nov 21, 2018
1 parent 5a367d4 commit 27634be
Show file tree
Hide file tree
Showing 10 changed files with 55 additions and 33 deletions.
2 changes: 1 addition & 1 deletion src/main/java/com/qcloud/cos_migrate_tool/meta/TaskStatics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class TaskStatics {
private TaskStatics() {}

public void setListFinished(boolean is_finished) {
this.list_finished = true;
this.list_finished = is_finished;
}

public boolean getListFinished() {
Expand Down
17 changes: 13 additions & 4 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAliTaskExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,8 @@ public void buildTask() {
com.qcloud.cos.utils.UrlEncoderUtils.urlDecode(s.getKey()),
s.getSize(), s.getETag(), smallFileTransferManager,
bigFileTransferManager, recordDb, semaphore);
try {

AddTask(task);
} catch (InterruptedException e) {
log.error(e.getMessage());
}
}
nextMarker = com.qcloud.cos.utils.UrlEncoderUtils
.urlDecode(objectListing.getNextMarker());
Expand All @@ -121,10 +118,22 @@ public void buildTask() {
} catch (OSSException e) {
log.error("list fail msg: {}", e.getMessage());
TaskStatics.instance.setListFinished(false);
if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
System.out.println(e.getMessage());
break;
}
} catch (ClientException e) {
log.error("list fail msg: {}", e.getMessage());
TaskStatics.instance.setListFinished(false);
if (e.getErrorCode().equalsIgnoreCase("AccessDenied")) {
System.out.println(e.getMessage());
break;
}
} catch (Exception e) {
log.error(e.getMessage());
TaskStatics.instance.setListFinished(false);
}
retry_num++;
} while (retry_num < 20);

}
Expand Down
8 changes: 3 additions & 5 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateAwsTaskExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,7 @@ public void buildTask() {
bigFileTransferManager, recordDb, semaphore);
log.info("list key: {}, size: {}, etag: {}", objectSummary.getKey(), objectSummary.getSize(), objectSummary.getETag());

try {
AddTask(task);
} catch (InterruptedException e) {
log.error(e.getMessage());
}

}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
Expand All @@ -125,6 +120,9 @@ public void buildTask() {
} catch (AmazonClientException ace) {
log.error("list fail AmazonClientException msg: {}", ace.getMessage().toString());
TaskStatics.instance.setListFinished(false);
} catch (Exception e) {
log.error(e.getMessage());
TaskStatics.instance.setListFinished(false);
}

}
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateCopyBucketTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,13 @@ public void doTask() {
}
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(new Region(srcRegion),
srcBucketName, srcKey, destBucketName, destKey);

if (srcBucketName.equals(destBucketName) && ( destKey.equals("/" + srcKey) || srcKey.equals(destKey)) ) {
ObjectMetadata newObjectMetadata = new ObjectMetadata();
newObjectMetadata.addUserMetadata("x-cos-metadata-directive", "Replaced");
copyObjectRequest.setNewObjectMetadata(newObjectMetadata);
}

copyObjectRequest.setSourceEndpointSuffix(srcEndpointSuffx);
try {
Copy copy = smallFileTransfer.copy(copyObjectRequest, srcCOSClient, null);
Expand Down
4 changes: 0 additions & 4 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateQiniuTaskExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,7 @@ public void buildTask() {
item.hash, smallFileTransferManager, bigFileTransferManager, recordDb,
semaphore);

try {
AddTask(task);
} catch (InterruptedException e) {
log.error("add task fail,msg:{}", e.getMessage());
}
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ public void doTask() {
headAttr = Downloader.instance.headFile(url, false);
} catch (Exception e) {
String printMsg = String.format("head url attr fail, url: %s", url);
System.err.println(printMsg);
log.error(printMsg, e);
System.err.println(e.toString());
log.error(e.toString(), printMsg);
TaskStatics.instance.addFailCnt();
return;
}
Expand Down
11 changes: 6 additions & 5 deletions src/main/java/com/qcloud/cos_migrate_tool/task/MigrateUrllistTaskExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,19 @@ public FileVisitResult visitFile(Path urllistPath, BasicFileAttributes attrs) th

MigrateUrllistTask task = new MigrateUrllistTask(config, line, url_path,
smallFileTransferManager, bigFileTransferManager, recordDb, semaphore);
try {
AddTask(task);
} catch (InterruptedException e) {
log.error("add task fail,msg:{}", e.getMessage());
}

AddTask(task);

}

TaskStatics.instance.setListFinished(true);

} catch (IOException e) {
log.error("read line fail,msg:{}", e.getMessage());
TaskStatics.instance.setListFinished(false);
} catch (Exception e) {
log.error(e.getMessage());
TaskStatics.instance.setListFinished(false);
}

try {
Expand Down
29 changes: 19 additions & 10 deletions src/main/java/com/qcloud/cos_migrate_tool/task/Task.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
public abstract class Task implements Runnable {
private Semaphore semaphore;
public static final Logger log = LoggerFactory.getLogger(Task.class);

protected static Semaphore mutex = new Semaphore(1);

protected TransferManager smallFileTransfer;
protected TransferManager bigFileTransfer;
Expand All @@ -38,6 +38,7 @@ public abstract class Task implements Runnable {
QUERY_RESULT query_result;



public Task(Semaphore semaphore, CommonConfig config, TransferManager smallFileTransfer,
TransferManager bigFileTransfer, RecordDb recordDb) {
super();
Expand All @@ -58,7 +59,7 @@ public boolean isExist(RecordElement recordElement) {
recordElement.buildValue());
return true;
}

return false;
}

Expand Down Expand Up @@ -192,7 +193,7 @@ private String uploadSmallFile(PutObjectRequest putObjectRequest) throws Interru

public String uploadFile(String bucketName, String cosPath, File localFile,
StorageClass storageClass, boolean entireMd5Attached, ObjectMetadata objectMetadata)
throws Exception {
throws Exception {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosPath, localFile);
putObjectRequest.setStorageClass(storageClass);

Expand Down Expand Up @@ -236,13 +237,21 @@ private void checkTimeWindows() throws InterruptedException {
if (minuteOfDay >= timeWindowBegin && minuteOfDay <= timeWindowEnd) {
return;
}
String printTips =
String.format("currentTime %s, wait next time window [%02d:%02d, %02d:%02d]",
dateTime.toString("yyyy-MM-dd HH:mm:ss"), timeWindowBegin / 60,
timeWindowBegin % 60, timeWindowEnd / 60, timeWindowEnd % 60);
System.out.println(printTips);
log.info(printTips);
Thread.sleep(60000);

if (mutex.tryAcquire()) {
String printTips = String.format(
"currentTime %s, wait next time window [%02d:%02d, %02d:%02d]",
dateTime.toString("yyyy-MM-dd HH:mm:ss"), timeWindowBegin / 60,
timeWindowBegin % 60, timeWindowEnd / 60, timeWindowEnd % 60);
System.out.println(printTips);
System.out.println(
"---------------------------------------------------------------------");
log.info(printTips);
Thread.sleep(60000);
mutex.release();
} else {
Thread.sleep(60000);
}
}
}

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/qcloud/cos_migrate_tool/task/TaskExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ public abstract class TaskExecutor {
protected COSClient cosClient;
protected TransferManager smallFileTransferManager;
protected TransferManager bigFileTransferManager;



enum RUN_MODE {
NORMAL, DUMP_REQUESTID, QUERY_REQUESTID;
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/qcloud/cos_migrate_tool/utils/Downloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ public HeadAttr headFile(String url, boolean qiniuDownLoadFlag) {
int http_statuscode = httpResponse.getStatusLine().getStatusCode();
if (http_statuscode < 200 || http_statuscode > 299) {
String errMsg = String.format(
"head failed, url: %s, httpResponse: %s, response_statuscode: %d", url,
httpResponse.toString(), http_statuscode);
"head failed, response_statuscode: %d, url: %s, httpResponse: %s, ", http_statuscode, url,
httpResponse.toString());
throw new Exception(errMsg);
}

Expand Down

0 comments on commit 27634be

Please sign in to comment.