Skip to content

Commit 1bcf371

Browse files
Rub21Rub21
committed
Update docker base image for planet dump and history
1 parent ee57c16 commit 1bcf371

File tree

7 files changed

+107
-70
lines changed

7 files changed

+107
-70
lines changed

‎images/full-history/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.962.hee57c16
22

33
VOLUME /mnt/data
44
COPY ./start.sh /
5-
CMD /start.sh
5+
CMD /start.sh

‎images/full-history/start.sh

Lines changed: 77 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,97 @@
11
#!/usr/bin/env bash
22
set -e
3-
export VOLUME_DIR=/mnt/data
43

54
# osmosis tuning: https://wiki.openstreetmap.org/wiki/Osmosis/Tuning,https://lists.openstreetmap.org/pipermail/talk/2012-October/064771.html
65
if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then
7-
echo JAVACMD_OPTIONS=\"-server\" >~/.osmosis
6+
echo JAVACMD_OPTIONS="-server" >~/.osmosis
87
else
98
memory="${MEMORY_JAVACMD_OPTIONS//i/}"
10-
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
9+
echo JAVACMD_OPTIONS="-server -Xmx$memory" >~/.osmosis
1110
fi
1211

13-
# Fixing name for historical file
12+
export VOLUME_DIR=/mnt/data
13+
export PLANET_EPOCH_DATE="${PLANET_EPOCH_DATE:-2004-01-01}"
1414
date=$(date '+%y%m%d_%H%M')
15-
local_fullHistoryFile=$VOLUME_DIR/history-${date}.osh.pbf
16-
cloud_fullHistoryFile=planet/full-history/history-${date}.osh.pbf
1715

18-
# In case overwrite the file
19-
if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
20-
local_fullHistoryFile=$VOLUME_DIR/history-latest.osh.pbf
21-
cloud_fullHistoryFile=planet/full-history/history-latest.osh.pbf
16+
local_planetPBFFile=$VOLUME_DIR/planet-history-${date}.osm.pbf
17+
cloud_planetPBFFile=planet/planet-history-${date}.osm.pbf
18+
stateFile="$VOLUME_DIR/state.txt"
19+
dumpFile="$VOLUME_DIR/input-latest.dump"
20+
21+
22+
# If overwrite flag is enabled, use fixed filenames
23+
if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
24+
local_planetPBFFile=$VOLUME_DIR/planet-history-latest.osm.pbf
25+
cloud_planetPBFFile=planet/planet-history-latest.osm.pbf
2226
fi
2327

24-
# State file nname
25-
stateFile="$VOLUME_DIR/state.txt"
26-
osm_tmp_file="osm_tmp.osm"
28+
# ===============================
29+
# Download db .dump file
30+
# ===============================
31+
download_dump_file() {
32+
echo "Downloading db .dump file from cloud..."
33+
if [ "$CLOUDPROVIDER" == "aws" ]; then
34+
if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then
35+
temp_txt="$VOLUME_DIR/tmp_dump_url.txt"
36+
aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt"
37+
first_line=$(head -n 1 "$temp_txt")
38+
aws s3 cp "$first_line" "$dumpFile"
39+
else
40+
aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile"
41+
fi
42+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
43+
gsutil cp "$DUMP_CLOUD_URL" "$dumpFile"
44+
fi
45+
}
2746

28-
# Creating full history
29-
osmosis --read-apidb-change \
30-
host=$POSTGRES_HOST \
31-
database=$POSTGRES_DB \
32-
user=$POSTGRES_USER \
33-
password=$POSTGRES_PASSWORD \
34-
validateSchemaVersion=no \
35-
readFullHistory=yes \
36-
--write-xml-change \
37-
compressionMethod=auto \
38-
$osm_tmp_file
47+
# ===============================
48+
# Upload planet + state
49+
# ===============================
50+
upload_planet_file() {
51+
echo "Uploading history planet file and updating state.txt..."
3952

40-
# Convert file to PBF file
41-
osmium cat $osm_tmp_file -o $local_fullHistoryFile
42-
osmium fileinfo $local_fullHistoryFile
53+
if [ "$CLOUDPROVIDER" == "aws" ]; then
54+
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
55+
echo "$AWS_URL.s3.amazonaws.com/$cloud_planetPBFFile" > "$stateFile"
56+
aws s3 cp "$local_planetPBFFile" "$AWS_S3_BUCKET/$cloud_planetPBFFile" --acl public-read
57+
aws s3 cp "$stateFile" "$AWS_S3_BUCKET/planet/state.txt" --acl public-read
4358

44-
# Remove full-hitory osm file, keep only history-latest.osh.pbf files
45-
rm $osm_tmp_file
59+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
60+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_planetPBFFile" > "$stateFile"
61+
gsutil cp -a public-read "$local_planetPBFFile" "$GCP_STORAGE_BUCKET/$cloud_planetPBFFile"
62+
gsutil cp -a public-read "$stateFile" "$GCP_STORAGE_BUCKET/planet/state.txt"
63+
fi
64+
}
4665

47-
# AWS
48-
if [ $CLOUDPROVIDER == "aws" ]; then
49-
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
50-
echo "$AWS_URL.s3.amazonaws.com/$cloud_fullHistoryFile" >$stateFile
51-
# Upload history-planet.osm.pbf
52-
aws s3 cp $local_fullHistoryFile $AWS_S3_BUCKET/$cloud_fullHistoryFile --acl public-read
53-
# Upload state.txt
54-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/state.txt --acl public-read
55-
fi
66+
# ===============================
67+
# Generate planet file
68+
# ===============================
5669

57-
# Google Storage
58-
if [ $CLOUDPROVIDER == "gcp" ]; then
59-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_fullHistoryFile" >$stateFile
60-
# Upload history-planet.osm.pbf
61-
gsutil cp -a public-read $local_fullHistoryFile $GCP_STORAGE_BUCKET/$cloud_fullHistoryFile
62-
# Upload state.txt
63-
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/state.txt
64-
fi
70+
if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
71+
download_dump_file
72+
echo "Generating history planet file with planet-dump-ng..."
73+
export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
74+
planet-dump-ng \
75+
--dump-file "$dumpFile" \
76+
--history-pbf "$local_planetPBFFile"
6577

66-
# Azure
67-
if [ $CLOUDPROVIDER == "azure" ]; then
68-
# Save the path file
69-
echo "https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$AZURE_CONTAINER_NAME/$cloud_fullHistoryFile" >$stateFile
70-
# Upload history-planet.osm.pbf
71-
az storage blob upload \
72-
--container-name $AZURE_CONTAINER_NAME \
73-
--file $local_fullHistoryFile \
74-
--name $cloud_fullHistoryFile \
75-
--output table
76-
# Upload state.txt
77-
az storage blob upload \
78-
--container-name $AZURE_CONTAINER_NAME \
79-
--file $stateFile \
80-
--name planet/full-history/state.txt \
81-
--output table
78+
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
79+
echo "Generating history planet file with osmosis..."
80+
# Creating full history
81+
osmosis --read-apidb-change \
82+
host=$POSTGRES_HOST \
83+
database=$POSTGRES_DB \
84+
user=$POSTGRES_USER \
85+
password=$POSTGRES_PASSWORD \
86+
validateSchemaVersion=no \
87+
readFullHistory=yes \
88+
--write-xml-change \
89+
compressionMethod=auto \
90+
$local_planetPBFFile
91+
else
92+
echo "Error: Unknown PLANET_EXPORT_METHOD value. Use 'planet-dump-ng' or 'osmosis'."
93+
exit 1
8294
fi
95+
96+
# Upload results
97+
upload_planet_file

‎images/planet-dump/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.962.hee57c16
22

33
VOLUME /mnt/data
44
COPY ./start.sh /

‎images/planet-dump/start.sh

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ date=$(date '+%y%m%d_%H%M')
1515
local_planetPBFFile=$VOLUME_DIR/planet-${date}.osm.pbf
1616
cloud_planetPBFFile=planet/planet-${date}.osm.pbf
1717
stateFile="$VOLUME_DIR/state.txt"
18+
dumpFile="$VOLUME_DIR/input-latest.dump"
1819

1920
# If overwrite flag is enabled, use fixed filenames
2021
if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
@@ -26,16 +27,22 @@ fi
2627
# Download db .dump file
2728
# ===============================
2829
download_dump_file() {
29-
local_dumpFile="$VOLUME_DIR/input-latest.dump"
3030
echo "Downloading db .dump file from cloud..."
31-
3231
if [ "$CLOUDPROVIDER" == "aws" ]; then
33-
aws s3 cp "$DUMP_CLOUD_URL" "$local_dumpFile"
32+
if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then
33+
temp_txt="$VOLUME_DIR/tmp_dump_url.txt"
34+
aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt"
35+
first_line=$(head -n 1 "$temp_txt")
36+
aws s3 cp "$first_line" "$dumpFile"
37+
else
38+
aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile"
39+
fi
3440
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
35-
gsutil cp "$DUMP_CLOUD_URL" "$local_dumpFile"
41+
gsutil cp "$DUMP_CLOUD_URL" "$dumpFile"
3642
fi
3743
}
3844

45+
3946
# ===============================
4047
# Upload planet + state
4148
# ===============================
@@ -63,7 +70,7 @@ if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
6370
download_dump_file
6471
echo "Generating planet file with planet-dump-ng..."
6572
planet-dump-ng \
66-
--dump-file "$VOLUME_DIR/input-latest.dump" \
73+
--dump-file "$dumpFile" \
6774
--pbf "$local_planetPBFFile"
6875
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
6976
echo "Generating planet file with osmosis..."

‎osm-seed/templates/jobs/full-history-job.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ spec:
4545
value: {{ .Values.fullHistory.env.OVERWRITE_FHISTORY_FILE | quote}}
4646
- name: CLOUDPROVIDER
4747
value: {{ .Values.cloudProvider }}
48+
- name: PLANET_EXPORT_METHOD
49+
value: {{ .Values.fullHistory.env.PLANET_EXPORT_METHOD | quote}}
50+
- name: DUMP_CLOUD_URL
51+
value: {{ .Values.fullHistory.env.DUMP_CLOUD_URL | quote}}
52+
- name: PLANET_EPOCH_DATE
53+
value: {{ .Values.fullHistory.env.PLANET_EPOCH_DATE | quote}}
4854
# In case cloudProvider=aws
4955
{{- if eq .Values.cloudProvider "aws" }}
5056
- name: AWS_S3_BUCKET

‎osm-seed/templates/jobs/planet-dump-job.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ spec:
4949
value: {{ .Values.cloudProvider }}
5050
- name: OVERWRITE_PLANET_FILE
5151
value: {{ .Values.planetDump.env.OVERWRITE_PLANET_FILE | quote}}
52+
- name: PLANET_EXPORT_METHOD
53+
value: {{ .Values.planetDump.env.PLANET_EXPORT_METHOD | quote}}
54+
- name: DUMP_CLOUD_URL
55+
value: {{ .Values.planetDump.env.DUMP_CLOUD_URL | quote}}
56+
- name: PLANET_EPOCH_DATE
57+
value: {{ .Values.planetDump.env.PLANET_EPOCH_DATE | quote}}
5258
# In case cloudProvider=aws
5359
{{- if eq .Values.cloudProvider "aws" }}
5460
- name: AWS_S3_BUCKET

‎osm-seed/values.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,8 @@ fullHistory:
260260
schedule: "* * * * *"
261261
env:
262262
OVERWRITE_FHISTORY_FILE: false
263+
PLANET_EXPORT_METHOD: osmosis
264+
DUMP_CLOUD_URL: s3://osm-seed/db.dump
263265
resources:
264266
enabled: false
265267
requests:
@@ -343,6 +345,7 @@ planetDump:
343345
env:
344346
OVERWRITE_PLANET_FILE: false
345347
DUMP_CLOUD_URL : s3://osm-seed/db.dump
348+
PLANET_EXPORT_METHOD: osmosis
346349
resources:
347350
enabled: false
348351
requests:

0 commit comments

Comments
 (0)