Skip to content

Commit 0aa3606

Browse files
apredeusapredeus
authored
Merge pull request #12 from cellgeni/fix-issue11
Fixed an issue#11
2 parents 370c8f1 + 9b1a0a9 commit 0aa3606

2 files changed

Lines changed: 52 additions & 58 deletions

File tree

‎scripts/curl_ena_metadata.sh‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,16 @@
22

33
RUNS=$1
44

5-
if [[ ! -f $RUNS ]]
6-
then
7-
>&2 echo "ERROR: file $RUNS not found!"
5+
if [[ ! -f $RUNS ]]; then
6+
echo >&2 "ERROR: file $RUNS not found!"
87
exit 1
98
fi
109

11-
KK=`cat $RUNS`
10+
KK=$(cat $RUNS)
1211

13-
for i in $KK
14-
do
15-
>&2 echo "Processing run ID $i.."
16-
curl "https://www.ebi.ac.uk/ena/portal/api/filereport?accession=$i&result=read_run&fields=study_accession,secondary_study_accession,sample_accession,secondary_sample_accession,experiment_accession,experiment_alias,run_accession,run_alias,tax_id,scientific_name,fastq_ftp,submitted_ftp,sra_ftp&format=tsv&download=true&limit=0" 2> /dev/null | grep -v study_accession
12+
for i in $KK; do
13+
echo >&2 "Processing run ID $i.."
14+
curl "https://www.ebi.ac.uk/ena/portal/api/filereport?accession=$i&result=read_run&fields=study_accession,secondary_study_accession,sample_accession,secondary_sample_accession,experiment_accession,experiment_alias,run_accession,run_alias,tax_id,scientific_name,fastq_ftp,submitted_ftp,sra_ftp,bam_ftp&format=tsv&download=true&limit=0" 2>/dev/null | grep -v study_accession
1715
done
1816

19-
>&2 echo "CURL ENA METADATA: ALL DONE!"
17+
echo >&2 "CURL ENA METADATA: ALL DONE!"

‎scripts/parse_ena_metadata.sh‎

Lines changed: 45 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,79 +2,75 @@
22

33
SERIES=$1
44

5-
if [[ -f $SERIES.urls.list ]]
6-
then
7-
>&2 echo "WARNING: File '$SERIES.urls.list' exists! This should not happen; overwriting the file.."
5+
if [[ -f $SERIES.urls.list ]]; then
6+
echo >&2 "WARNING: File '$SERIES.urls.list' exists! This should not happen; overwriting the file.."
87
rm $SERIES.urls.list
9-
fi
8+
fi
109

11-
for i in `cat $SERIES.run.list`
12-
do
13-
TYPE="SRA" ## we always default to SRA. This could cause problems for very fresh datasets.
10+
for i in $(cat $SERIES.run.list); do
11+
TYPE="SRA" ## we always default to SRA. This could cause problems for very fresh datasets.
1412
LOC=""
15-
AEGZ=""
16-
if [[ $SERIES == E-MTAB-* ]]
17-
then
18-
AEGZ=`grep -w $i $SERIES.sdrf.txt | tr '\t' '\n' | grep "ftp://.*\.f.*q" | tr '\n' ';' | sed "s/;$//"`
19-
fi
20-
SPECIES=`grep -w $i $SERIES.ena.tsv | cut -f10`
21-
ENAGZ=`grep -w $i $SERIES.ena.tsv | cut -f11 | grep "_1\.fastq.gz" | grep "_2\.fastq.gz"` ## ENA formatting is strict
22-
ORIFQ=`grep -w $i $SERIES.ena.tsv | cut -f12 | grep "f.*q"` ## ppl name files *all kinds of random shiz*, really
23-
ORIBAM=`grep -w $i $SERIES.ena.tsv | cut -f12 | tr ';' '\n' | grep -v "\.bai" | grep "\.bam"` ## don't need the BAM index which is often there
24-
SRA=`grep -w $i $SERIES.ena.tsv | cut -f13`
25-
SRABAM=`curl -s "https://locate.ncbi.nlm.nih.gov/sdl/2/retrieve?acc=$i&accept-alternate-locations=yes" | jq -r '
13+
AEGZ=""
14+
if [[ $SERIES == E-MTAB-* ]]; then
15+
AEGZ=$(grep -w $i $SERIES.sdrf.txt | tr '\t' '\n' | grep "ftp://.*\.f.*q" | tr '\n' ';' | sed "s/;$//")
16+
fi
17+
SPECIES=$(grep -w $i $SERIES.ena.tsv | cut -f10)
18+
ENAGZ=$(grep -w $i $SERIES.ena.tsv | cut -f11 | grep "_1\.fastq.gz" | grep "_2\.fastq.gz") ## ENA formatting is strict
19+
ORIFQ=$(grep -w $i $SERIES.ena.tsv | cut -f12 | grep "f.*q" | grep ";") ## ppl name files *all kinds of random shiz*, really
20+
ORIBAM=$(grep -w $i $SERIES.ena.tsv | cut -f12 | tr ';' '\n' | grep -v "\.bai" | grep "\.bam") ## don't need the BAM index which is often there
21+
BAMFTP=$(grep -w $i $SERIES.ena.tsv | cut -f14 | tr ';' '\n' | grep -v "\.bai" | grep "\.bam") ## don't need the BAM index which is often there
22+
SRA=$(grep -w $i $SERIES.ena.tsv | cut -f13)
23+
SRABAM=$(curl -s "https://locate.ncbi.nlm.nih.gov/sdl/2/retrieve?acc=$i&accept-alternate-locations=yes" | jq -r '
2624
.result[].files[] |
2725
select(.name | contains("bam")) |
2826
.locations[] |
2927
select((.rehydrationRequired // false) == false and (.payRequired // false) == false) |
3028
.link
31-
'`
29+
')
3230

33-
if [[ $AEGZ != "" ]]
34-
then
31+
if [[ $AEGZ != "" ]]; then
3532
TYPE="ORIFQ"
3633
LOC=$AEGZ
37-
echo $AEGZ | tr ';' '\n' >> $SERIES.urls.list
38-
>&2 echo "Sample $i is available via ArrayExpress as paired-end fastq archive: $LOC"
39-
elif [[ $ENAGZ != "" ]]
40-
then
34+
echo $AEGZ | tr ';' '\n' >>$SERIES.urls.list
35+
echo >&2 "Sample $i is available via ArrayExpress as paired-end fastq archive: $LOC"
36+
elif [[ $ENAGZ != "" ]]; then
4137
TYPE="ENAFQ"
4238
LOC=$ENAGZ
43-
echo $ENAGZ | tr ';' '\n' >> $SERIES.urls.list
44-
>&2 echo "Sample $i is available via ENA as a paired-end fastq: $LOC"
45-
elif [[ $ORIFQ != "" ]]
46-
then
39+
echo $ENAGZ | tr ';' '\n' >>$SERIES.urls.list
40+
echo >&2 "Sample $i is available via ENA as a paired-end fastq: $LOC"
41+
elif [[ $ORIFQ != "" ]]; then
4742
TYPE="ORIFQ"
4843
LOC=$ORIFQ
49-
echo $ORIFQ | tr ';' '\n' >> $SERIES.urls.list
50-
>&2 echo "Sample $i is available via ENA as original submitter's fastq: $LOC"
51-
elif [[ $ORIBAM != "" ]]
52-
then
44+
echo $ORIFQ | tr ';' '\n' >>$SERIES.urls.list
45+
echo >&2 "Sample $i is available via ENA as original submitter's fastq: $LOC"
46+
elif [[ $ORIBAM != "" ]]; then
5347
TYPE="BAM"
5448
LOC=$ORIBAM
55-
echo $ORIBAM >> $SERIES.urls.list
56-
>&2 echo "Sample $i is available via ENA as an original submitter's BAM file: $LOC"
57-
elif [[ $SRABAM != "" ]]
58-
then
49+
echo $ORIBAM >>$SERIES.urls.list
50+
echo >&2 "Sample $i is available via ENA as an original submitter's BAM file: $LOC"
51+
elif [[ $SRABAM != "" ]]; then
5952
TYPE="BAM"
6053
LOC=$SRABAM
61-
echo $SRABAM >> $SERIES.urls.list
62-
>&2 echo "Sample $i is available via SRA as an original submitter's BAM file: $LOC"
63-
elif [[ $SRA != "" ]]
64-
then
54+
echo $SRABAM >>$SERIES.urls.list
55+
echo >&2 "Sample $i is available via SRA as an original submitter's BAM file: $LOC"
56+
elif [[ $BAMFTP != "" ]]; then
57+
TYPE="BAM"
58+
LOC=$BAMFTP
59+
echo $BAMFTP >>$SERIES.urls.list
60+
echo >&2 "Sample $i is available via SRA as an original submitter's BAM file: $LOC"
61+
elif [[ $SRA != "" ]]; then
6562
TYPE="SRA"
6663
LOC=$SRA
67-
echo $SRA >> $SERIES.urls.list
68-
>&2 echo "Sample $i is available via ENA as an SRA archive: $LOC"
64+
echo $SRA >>$SERIES.urls.list
65+
echo >&2 "Sample $i is available via ENA as an SRA archive: $LOC"
6966
else
70-
## means $SRA == "" for some reason - usually this is a failure of ENA, but not always
71-
SRA=`srapath $i`
67+
## means $SRA == "" for some reason - usually this is a failure of ENA, but not always
68+
SRA=$(srapath $i)
7269
LOC=$SRA
73-
>&2 echo "WARNING: No ENA ftp URL found for sample $i, using 'srapath' to get the (open) Amazon link to SRA archive.."
74-
echo $SRA >> $SERIES.urls.list
75-
>&2 echo "Sample $i is available via NCBI/Amazon as an SRA archive: $LOC"
70+
echo >&2 "WARNING: No ENA ftp URL found for sample $i, using 'srapath' to get the (open) Amazon link to SRA archive.."
71+
echo $SRA >>$SERIES.urls.list
72+
echo >&2 "Sample $i is available via NCBI/Amazon as an SRA archive: $LOC"
7673
fi
7774

7875
echo -e "$i\t$SPECIES\t$LOC\t$TYPE"
7976
done
80-

0 commit comments

Comments
 (0)