Skip to content

Commit 20c0ddd

Browse files
pahatzkostas-kou
pahatz
authored and
kostas-kou
committed
Handle multiple inbox buckets
1 parent b6fec5d commit 20c0ddd

1 file changed

Lines changed: 67 additions & 31 deletions

File tree

‎validator/validator.sh‎

Lines changed: 67 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ fi
103103
INBOX_ACCESS_KEY=""
104104
INBOX_SECRET_KEY=""
105105
HOST_BUCKET=""
106-
INBOX_BUCKET=""
106+
INBOX_BUCKETS=()
107107
METADATA_ACCESS_KEY=""
108108
METADATA_SECRET_KEY=""
109109
C4GH_PASSPHRASE=""
@@ -307,15 +307,15 @@ function get_credentials {
307307
HOST_BUCKET=$(vault kv get -field=endpoints bp-secrets/S3_keys/STO2 | cut -d, -f2)
308308
INBOX_ACCESS_KEY=$(vault kv get -field=access_key bp-secrets/S3_keys/STO2/inbox)
309309
INBOX_SECRET_KEY=$(vault kv get -field=secret_key bp-secrets/S3_keys/STO2/inbox)
310-
INBOX_BUCKET=$(s3cmd_command ls | cut -d'/' -f3 | grep -v "staging" | grep '^inbox-' | sort -r | head -n1)
310+
INBOX_BUCKETS=("inbox" "inbox-2024-01")
311311
METADATA_ACCESS_KEY=$(vault kv get -field=access_key bp-secrets/S3_keys/STO2/private)
312312
METADATA_SECRET_KEY=$(vault kv get -field=secret_key bp-secrets/S3_keys/STO2/private)
313313
METADATA_BUCKET=$(s3cmd_metadata ls | cut -d'/' -f3)
314314
elif [[ "$cluster" == "staging" ]]; then
315315
INBOX_ACCESS_KEY=$(vault kv get -field=access_key bp-secrets/S3_keys/STO2/inbox)
316316
INBOX_SECRET_KEY=$(vault kv get -field=secret_key bp-secrets/S3_keys/STO2/inbox)
317317
HOST_BUCKET=$(vault kv get -field=endpoints bp-secrets/S3_keys/STO2 | cut -d, -f2)
318-
INBOX_BUCKET="staging-inbox"
318+
INBOX_BUCKETS=("staging-inbox")
319319
METADATA_BUCKET="bigpicture-test-metadata"
320320
else
321321
cecho red "ERROR: Cluster name is not valid"
@@ -364,10 +364,14 @@ function validate_private_files {
364364
function validate_files {
365365
cecho yellow "Validating METADATA folder files ..."
366366
expected_metadata_content=("dataset" "policy" "image" "annotation" "observation" "sample" "staining")
367-
inbox_metadata_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/METADATA/ | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f3 | tr '\n' ' ')
367+
inbox_metadata_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
368+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/METADATA/" 2>/dev/null
369+
done | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f3 | sort -u | tr '\n' ' ')
368370
# Convert multiple lines to array
369371
read -a metadata_files <<<"$inbox_metadata_files"
370-
inbox_private_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/PRIVATE/ | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f3 | tr '\n' ' ')
372+
inbox_private_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
373+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/PRIVATE/" 2>/dev/null
374+
done | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f3 | sort -u | tr '\n' ' ')
371375
# Convert multiple lines to array
372376
read -a metadata_private_files <<<"$inbox_private_files"
373377
# if the folder ANNOTATIONS is missing, make sure annotation.xml and observer.xml are not present
@@ -416,7 +420,9 @@ function validate_structure {
416420
cecho yellow "Validating folder structure ..."
417421
annotations=true
418422
main_folder="$dataset"
419-
inbox_subfolders=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f2)
423+
inbox_subfolders=$(for bucket in "${INBOX_BUCKETS[@]}"; do
424+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" 2>/dev/null
425+
done | awk -F'_lifescience-ri.eu/|_elixir-europe.org/' '{print $2}' | cut -d'/' -f2 | sort -u)
420426
# Convert multiple lines to array
421427
IFS=$'\n' read -r -d '' -a subfolders <<<"$inbox_subfolders"$'\n'
422428
expected_subfolders=("METADATA" "IMAGES" "ANNOTATIONS" "PRIVATE" "LANDING_PAGE")
@@ -446,7 +452,9 @@ function validate_structure {
446452
expected_subfolders=("${expected_subfolders[@]}")
447453
fi
448454
else
449-
thumbnail_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/LANDING_PAGE/THUMBNAILS/ --recursive | wc -l)
455+
thumbnail_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
456+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/LANDING_PAGE/THUMBNAILS/" --recursive 2>/dev/null
457+
done | awk '{print $4}' | sort -u | wc -l)
450458
if [[ "$thumbnail_files" -eq 0 ]]; then
451459
cecho red "ERROR: THUMBNAILS folder is missing or empty" | tee -a general_errors.logs
452460
ERROR_STATUS=1
@@ -456,7 +464,9 @@ function validate_structure {
456464

457465
# Check if landing page folder is empty in case it exists
458466
if [[ "$LANDING_PAGE" == "true" ]]; then
459-
landing_page_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/LANDING_PAGE/ --recursive | wc -l)
467+
landing_page_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
468+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/LANDING_PAGE/" --recursive 2>/dev/null
469+
done | awk '{print $4}' | sort -u | wc -l)
460470
if [[ "$landing_page_files" -eq 0 ]]; then
461471
cecho red "ERROR: LANDING_PAGE folder is empty" | tee -a general_errors.logs
462472
ERROR_STATUS=1
@@ -502,13 +512,21 @@ function validate_structure {
502512
function get_xml_files {
503513
mkdir -p xml-files
504514
cecho yellow "Getting xml files ..."
505-
metadata_path=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ | grep -i METADATA | awk '{print $2}')
506-
private_path=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ | grep -i PRIVATE | awk '{print $2}')
507-
s3cmd_command get "$metadata_path" --recursive xml-files/ >/dev/null 2>&1
508-
s3cmd_command get "$private_path" --recursive xml-files/ >/dev/null 2>&1
515+
for bucket in "${INBOX_BUCKETS[@]}"; do
516+
metadata_path=$(s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" 2>/dev/null | grep -i METADATA | awk '{print $2}')
517+
private_path=$(s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" 2>/dev/null | grep -i PRIVATE | awk '{print $2}')
518+
if [[ -n "$metadata_path" ]]; then
519+
s3cmd_command get "$metadata_path" --recursive xml-files/ >/dev/null 2>&1
520+
fi
521+
if [[ -n "$private_path" ]]; then
522+
s3cmd_command get "$private_path" --recursive xml-files/ >/dev/null 2>&1
523+
fi
524+
done
509525
if [[ "$LANDING_PAGE" == "true" ]]; then
510-
landing_page_path=s3://"$INBOX_BUCKET"/"$user"/"$dataset"/LANDING_PAGE/landing_page.xml
511-
s3cmd_command get "$landing_page_path" --recursive xml-files/ >/dev/null 2>&1
526+
for bucket in "${INBOX_BUCKETS[@]}"; do
527+
landing_page_path="s3://${bucket}/${user}/${dataset}/LANDING_PAGE/landing_page.xml"
528+
s3cmd_command get "$landing_page_path" --recursive xml-files/ >/dev/null 2>&1
529+
done
512530
# Throw error if the landing_page.xml file does not exist
513531
if [[ ! -f xml-files/landing_page.xml.c4gh ]]; then
514532
cecho red "ERROR: landing_page.xml is missing" | tee -a general_errors.logs
@@ -666,7 +684,9 @@ function check_files {
666684
function check_file_sizes {
667685
cecho yellow "Checking file sizes ..."
668686
local bad_files
669-
bad_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ --recursive | \
687+
bad_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
688+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" --recursive 2>/dev/null
689+
done | sort -u | \
670690
awk -v min="$MIN_FILE_SIZE" '$3+0 <= min {print $4 " (size: " $3 ")"}')
671691
if [[ -n "$bad_files" ]]; then
672692
while IFS= read -r line; do
@@ -686,12 +706,14 @@ function check_file_sizes {
686706
# - If they are not equal, it prints the extra or missing files and exits
687707
function comparing_files {
688708
cecho yellow "Checking files ..."
689-
all_inbox_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ --recursive | awk '{print $4}')
709+
all_inbox_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
710+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" --recursive 2>/dev/null
711+
done | awk '{print $4}' | sort -u)
690712
# Strip the S3 prefix and .c4gh suffix so paths match the relative paths stored in metadata (e.g. IMAGES/IMAGE_xxx/file.dcm)
691713
# Pre-sort once so comm calls below don't need to re-sort the large list
692-
all_inbox_relative=$(echo "$all_inbox_files" | sed "s|s3://${INBOX_BUCKET}/${user}/${dataset}/||" | sed 's/\.c4gh$//' | sort)
714+
all_inbox_relative=$(echo "$all_inbox_files" | sed -E "s|s3://[^/]+/${user}/${dataset}/||" | sed 's/\.c4gh$//' | sort -u)
693715

694-
count_inbox_files=$(echo "$all_inbox_files" | grep -c "IMAGES")
716+
count_inbox_files=$(echo "$all_inbox_relative" | grep -c '^IMAGES/')
695717

696718
for file in xml-files/*.xml; do
697719
if [[ "$file" == *"image"* ]]; then
@@ -716,13 +738,13 @@ function comparing_files {
716738
elif [ "$count_inbox_files" -gt "$count_metadata_files" ]; then
717739
cecho red "ERROR: There are more files in the inbox than the ones that are referenced in metadata (inbox=$count_inbox_files, metadata=$count_metadata_files)" | tee -a general_errors.logs
718740
# Modify all_inbox_files to contain only the parts that are referenced in metadata
719-
inbox_images_files=$(echo "$all_inbox_files" | grep "/IMAGES/" | sed -E 's|.*(IMAGES/IMAGE_[^/]+/[^.]+(\.[^.]+)*\.dcm).*|\1|')
741+
inbox_images_files=$(echo "$all_inbox_relative" | grep '^IMAGES/IMAGE_')
720742
extra_inbox_relative=$(check_files "$inbox_images_files" "$new_metadata_files")
721743
extra_inbox_files=$(awk -F'\t' 'NR==FNR { if (NF) wanted[$1]=1; next } wanted[$2] { print $1 }' \
722744
<(printf '%s\n' "$extra_inbox_relative") \
723745
<(paste \
724746
<(echo "$all_inbox_files" | grep "/IMAGES/") \
725-
<(echo "$all_inbox_files" | grep "/IMAGES/" | sed -E 's|.*(IMAGES/IMAGE_[^/]+/[^.]+(\.[^.]+)*\.dcm).*|\1|')))
747+
<(echo "$all_inbox_files" | sed -E "s|s3://[^/]+/${user}/${dataset}/||" | sed 's/\.c4gh$//')))
726748
length_extra_files=$(echo "$extra_inbox_relative" | sed '/^$/d' | wc -l)
727749
missing_files_diff=$((count_inbox_files - count_metadata_files))
728750
if [ "$length_extra_files" -eq "$missing_files_diff" ]; then
@@ -746,9 +768,11 @@ function comparing_files {
746768

747769
# Check the thumbnails files
748770
if [[ "$LANDING_PAGE" == "true" ]]; then
749-
all_thumbnail_files=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/LANDING_PAGE/THUMBNAILS/ --recursive | awk '{print $4}')
750-
all_thumbnail_relative=$(echo "$all_thumbnail_files" | sed "s|s3://${INBOX_BUCKET}/${user}/${dataset}/LANDING_PAGE/THUMBNAILS/||" | sed 's/\.c4gh$//' | sort)
751-
count_inbox_thumbnail_files=$(echo "$all_thumbnail_files" | wc -l)
771+
all_thumbnail_files=$(for bucket in "${INBOX_BUCKETS[@]}"; do
772+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/LANDING_PAGE/THUMBNAILS/" --recursive 2>/dev/null
773+
done | awk '{print $4}' | sort -u)
774+
all_thumbnail_relative=$(echo "$all_thumbnail_files" | sed -E "s|s3://[^/]+/${user}/${dataset}/LANDING_PAGE/THUMBNAILS/||" | sed 's/\.c4gh$//' | sort -u)
775+
count_inbox_thumbnail_files=$(echo "$all_thumbnail_relative" | sed '/^$/d' | wc -l)
752776
metadata_thumbnail_files=$(xmllint --xpath '/LANDING_PAGE_SET/LANDING_PAGE/SAMPLE_IMAGE_FILES/SAMPLE_IMAGE_FILE/@filename' xml-files/landing_page.xml | awk -F= '{print $2}' | sed 's/"//g')
753777
count_metadata_thumbnail_files=$(echo "$metadata_thumbnail_files" | wc -l)
754778
if [[ "metadata_thumbnail_files" == "" ]]; then
@@ -757,7 +781,7 @@ function comparing_files {
757781
elif [ "$count_inbox_thumbnail_files" -lt "$count_metadata_thumbnail_files" ]; then
758782
cecho red "ERROR: There are more thumbnail files in metadata than the ones that exist in the inbox (inbox=$count_inbox_thumbnail_files, metadata=$count_metadata_thumbnail_files)" | tee -a general_errors.logs
759783
echo "The missing files in the inbox are:"
760-
missing_inbox_files=$(check_files "$metadata_thumbnail_files" "$all_thumbnail_files")
784+
missing_inbox_files=$(check_files "$metadata_thumbnail_files" "$all_thumbnail_relative")
761785
echo "$missing_inbox_files"
762786
ERROR_STATUS=1
763787
elif [ "$count_inbox_thumbnail_files" -gt "$count_metadata_thumbnail_files" ]; then
@@ -817,28 +841,39 @@ function move_private_metadata {
817841
cecho yellow "Moving metadata ..."
818842

819843
stable_id=$(cat dataset_id.txt)
820-
metadata_inbox_path=$(s3cmd_command ls s3://"$INBOX_BUCKET"/"$user"/"$dataset"/ | grep "PRIVATE" | awk '{print $2}')
844+
metadata_inbox_paths=$(for bucket in "${INBOX_BUCKETS[@]}"; do
845+
s3cmd_command ls "s3://${bucket}/${user}/${dataset}/" 2>/dev/null | grep "PRIVATE" | awk '{print $2}'
846+
done)
821847
metadata_bucket_path="s3://"$METADATA_BUCKET"/"$user"/"$stable_id"/"$dataset"/"PRIVATE"/"
822848

823849
cecho yellow "Moving PRIVATE folder in metadata bucket"
824850

825851
if [[ "$cluster" == "staging" ]]; then
826-
s3cmd_command mv "$metadata_inbox_path" "$metadata_bucket_path" --recursive
852+
while IFS= read -r metadata_inbox_path; do
853+
[[ -z "$metadata_inbox_path" ]] && continue
854+
s3cmd_command mv "$metadata_inbox_path" "$metadata_bucket_path" --recursive
855+
done <<< "$metadata_inbox_paths"
827856
metadata_size=$(s3cmd_command ls "$metadata_bucket_path" --recursive | awk '{print $3}')
828857
if [ -z "$metadata_size" ]; then
829858
cecho red "ERROR: Moving metadata failed"
830859
exit 1
831860
fi
832861
else
833862
mkdir -p "PRIVATE"
834-
s3cmd_command get "$metadata_inbox_path" --recursive "PRIVATE"/ >/dev/null 2>&1
863+
while IFS= read -r metadata_inbox_path; do
864+
[[ -z "$metadata_inbox_path" ]] && continue
865+
s3cmd_command get "$metadata_inbox_path" --recursive "PRIVATE"/ >/dev/null 2>&1
866+
done <<< "$metadata_inbox_paths"
835867
s3cmd_metadata put "PRIVATE"/ "$metadata_bucket_path" --recursive
836868
metadata_size=$(s3cmd_metadata ls "$metadata_bucket_path" --recursive | awk '{print $3}')
837869
if [ -z "$metadata_size" ]; then
838870
cecho red "ERROR: Moving metadata failed"
839871
exit 1
840872
fi
841-
s3cmd_command del "$metadata_inbox_path" --recursive
873+
while IFS= read -r metadata_inbox_path; do
874+
[[ -z "$metadata_inbox_path" ]] && continue
875+
s3cmd_command del "$metadata_inbox_path" --recursive
876+
done <<< "$metadata_inbox_paths"
842877
fi
843878

844879
cecho green "Done"
@@ -872,9 +907,10 @@ function modify_dataset {
872907
exit 1
873908
fi
874909

875-
s3cmd_command del s3://"$INBOX_BUCKET"/"$user"/"$dataset"/METADATA/dataset.xml.c4gh
876-
877-
s3cmd_command put xml-files/dataset.xml.c4gh s3://"$INBOX_BUCKET"/"$user"/"$dataset"/METADATA/dataset.xml.c4gh
910+
for bucket in "${INBOX_BUCKETS[@]}"; do
911+
s3cmd_command del "s3://${bucket}/${user}/${dataset}/METADATA/dataset.xml.c4gh"
912+
s3cmd_command put xml-files/dataset.xml.c4gh "s3://${bucket}/${user}/${dataset}/METADATA/dataset.xml.c4gh"
913+
done
878914

879915
cecho green "Done"
880916
}

0 commit comments

Comments
 (0)