From 6f14d59d394c88cd24835b071c80a3c47046b8f9 Mon Sep 17 00:00:00 2001 From: Laetitia Fesselier Date: Thu, 12 Oct 2023 11:22:35 -0400 Subject: [PATCH 1/5] [AWS S3] Copy file (#1016) --- python/lib/aws_s3.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/python/lib/aws_s3.py b/python/lib/aws_s3.py index 792488027..7c270da04 100644 --- a/python/lib/aws_s3.py +++ b/python/lib/aws_s3.py @@ -172,6 +172,32 @@ def delete_file(self, s3_object_name): except Exception as err: raise Exception(f"{s3_object_name} download failure = {format(err)}") + def copy_file(self, src_s3_object_name, dst_s3_object_name): + """ + Function to copy a s3 file or directory. + + :param src_s3_object_name: name of the source s3 file or directory + :type src_s3_object_name: str + :param dst_s3_object_name: name of the destination s3 file or directory + :type dst_s3_object_name: str + """ + + print(f"Copying {src_s3_object_name} to {dst_s3_object_name}") + + try: + (src_s3_bucket_name, src_s3_bucket, src_s3_file_name) = self.get_s3_object_path_part(src_s3_object_name) + (dst_s3_bucket_name, dst_s3_bucket, dst_s3_file_name) = self.get_s3_object_path_part(dst_s3_object_name) + for obj in src_s3_bucket.objects.filter(Prefix=src_s3_file_name): + subcontent = obj.key.replace(src_s3_file_name, "") + dst_s3_bucket.Object( + dst_s3_file_name + subcontent + ).copy_from( + CopySource=f'{obj.bucket_name}/{obj.key}' + ) + src_s3_bucket.Object(obj.key).delete() + except Exception as err: + raise Exception(f"{src_s3_object_name} => {dst_s3_object_name} copy failure = {format(err)}") + def get_s3_object_path_part(self, s3_object_name): """ Function to dissect a s3 file to extract the file prefix and the bucket name From 3c61ffcac1e9ecc4ad055b0ef5664d6470708193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9cile=20Madjar?= Date: Thu, 12 Oct 2023 11:23:03 -0400 Subject: [PATCH 2/5] Update VERSION to 24.1.12 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3d9da73e1..b9d88e029 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.1.11 +24.1.12 From 4eff7fbff0a7895b20735a54bdf4e3b300e63819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9cile=20Madjar?= Date: Thu, 12 Oct 2023 11:39:46 -0400 Subject: [PATCH 3/5] modify permission for extract eeg bids archive to be executable (#1018) --- python/extract_eeg_bids_archive.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 python/extract_eeg_bids_archive.py diff --git a/python/extract_eeg_bids_archive.py b/python/extract_eeg_bids_archive.py old mode 100644 new mode 100755 From af6591fc063a5896e253b2c98ee545e7814ae70b Mon Sep 17 00:00:00 2001 From: Laetitia Fesselier Date: Fri, 13 Oct 2023 13:46:34 -0400 Subject: [PATCH 4/5] aws-s3 copy-file optional delete (#1019) --- python/lib/aws_s3.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 python/lib/aws_s3.py diff --git a/python/lib/aws_s3.py b/python/lib/aws_s3.py old mode 100644 new mode 100755 index 7c270da04..5109e4319 --- a/python/lib/aws_s3.py +++ b/python/lib/aws_s3.py @@ -172,7 +172,7 @@ def delete_file(self, s3_object_name): except Exception as err: raise Exception(f"{s3_object_name} download failure = {format(err)}") - def copy_file(self, src_s3_object_name, dst_s3_object_name): + def copy_file(self, src_s3_object_name, dst_s3_object_name, delete = False): """ Function to copy a s3 file or directory. @@ -180,6 +180,8 @@ def copy_file(self, src_s3_object_name, dst_s3_object_name): :type src_s3_object_name: str :param dst_s3_object_name: name of the destination s3 file or directory :type dst_s3_object_name: str + :param delete: whether to delete the source file after the copy + :type delete: bool """ print(f"Copying {src_s3_object_name} to {dst_s3_object_name}") @@ -194,7 +196,8 @@ def copy_file(self, src_s3_object_name, dst_s3_object_name): ).copy_from( CopySource=f'{obj.bucket_name}/{obj.key}' ) - src_s3_bucket.Object(obj.key).delete() + if delete: + src_s3_bucket.Object(obj.key).delete() except Exception as err: raise Exception(f"{src_s3_object_name} => {dst_s3_object_name} copy failure = {format(err)}") From 69ff4c63fe18a688e05625f3f79bf9dcf7b197cd Mon Sep 17 00:00:00 2001 From: Laetitia Fesselier Date: Mon, 16 Oct 2023 14:58:33 -0400 Subject: [PATCH 5/5] [AWS S3] Upload_dir subdir fix (#1023) --- python/extract_eeg_bids_archive.py | 3 +-- python/lib/aws_s3.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/extract_eeg_bids_archive.py b/python/extract_eeg_bids_archive.py index ab43b018e..f85c98d12 100755 --- a/python/extract_eeg_bids_archive.py +++ b/python/extract_eeg_bids_archive.py @@ -200,7 +200,7 @@ def main(): """ If the suject/session/modality BIDS data already exists on the destination folder, delete it first - copying the data + before copying the data """ s3_obj.delete_file(s3_data_eeg_modality_path) @@ -222,7 +222,6 @@ def main(): copying the data """ imaging_io_obj.remove_dir(data_eeg_modality_path) - imaging_io_obj.copy_file(tmp_eeg_modality_path, data_eeg_modality_path) # Delete tmp location diff --git a/python/lib/aws_s3.py b/python/lib/aws_s3.py index 5109e4319..8df31c83e 100755 --- a/python/lib/aws_s3.py +++ b/python/lib/aws_s3.py @@ -80,7 +80,7 @@ def upload_dir(self, dir_name, s3_object_name, force = False): for (root, dirs, files) in os.walk(dir_name): for file in files: - s3_prefix = os.path.join(s3_file_name, root.replace(dir_name, ""), file) + s3_prefix = os.path.join(s3_file_name, root.replace(dir_name, "").lstrip('/'), file) s3_dest = os.path.join( 's3://', s3_bucket_name,