Skip to content

Commit 1693d65

Browse files
Rub21Rub21
committed
Update docker base image for planet dump and history
1 parent ee57c16 commit 1693d65

File tree

4 files changed

+78
-65
lines changed

4 files changed

+78
-65
lines changed

‎images/full-history/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.962.hee57c16
22

33
VOLUME /mnt/data
44
COPY ./start.sh /
5-
CMD /start.sh
5+
CMD /start.sh

‎images/full-history/start.sh

Lines changed: 72 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,92 @@
11
#!/usr/bin/env bash
22
set -e
3-
export VOLUME_DIR=/mnt/data
43

54
# osmosis tuning: https://wiki.openstreetmap.org/wiki/Osmosis/Tuning,https://lists.openstreetmap.org/pipermail/talk/2012-October/064771.html
65
if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then
7-
echo JAVACMD_OPTIONS=\"-server\" >~/.osmosis
6+
echo JAVACMD_OPTIONS="-server" >~/.osmosis
87
else
98
memory="${MEMORY_JAVACMD_OPTIONS//i/}"
10-
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
9+
echo JAVACMD_OPTIONS="-server -Xmx$memory" >~/.osmosis
1110
fi
1211

13-
# Fixing name for historical file
12+
export VOLUME_DIR=/mnt/data
13+
export PLANET_EPOCH_DATE="${PLANET_EPOCH_DATE:-2004-01-01}"
1414
date=$(date '+%y%m%d_%H%M')
15-
local_fullHistoryFile=$VOLUME_DIR/history-${date}.osh.pbf
16-
cloud_fullHistoryFile=planet/full-history/history-${date}.osh.pbf
1715

18-
# In case overwrite the file
19-
if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
20-
local_fullHistoryFile=$VOLUME_DIR/history-latest.osh.pbf
21-
cloud_fullHistoryFile=planet/full-history/history-latest.osh.pbf
16+
local_planetPBFFile=$VOLUME_DIR/planet-history-${date}.osm.pbf
17+
cloud_planetPBFFile=planet/planet-history-${date}.osm.pbf
18+
stateFile="$VOLUME_DIR/state.txt"
19+
20+
dumpFile="$VOLUME_DIR/input-latest.dump"
21+
22+
23+
# If overwrite flag is enabled, use fixed filenames
24+
if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
25+
local_planetPBFFile=$VOLUME_DIR/planet-history-latest.osm.pbf
26+
cloud_planetPBFFile=planet/planet-history-latest.osm.pbf
2227
fi
2328

24-
# State file nname
25-
stateFile="$VOLUME_DIR/state.txt"
26-
osm_tmp_file="osm_tmp.osm"
29+
# ===============================
30+
# Download db .dump file
31+
# ===============================
32+
download_dump_file() {
33+
echo "Downloading db .dump file from cloud..."
2734

28-
# Creating full history
29-
osmosis --read-apidb-change \
30-
host=$POSTGRES_HOST \
31-
database=$POSTGRES_DB \
32-
user=$POSTGRES_USER \
33-
password=$POSTGRES_PASSWORD \
34-
validateSchemaVersion=no \
35-
readFullHistory=yes \
36-
--write-xml-change \
37-
compressionMethod=auto \
38-
$osm_tmp_file
35+
if [ "$CLOUDPROVIDER" == "aws" ]; then
36+
aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile"
37+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
38+
gsutil cp "$DUMP_CLOUD_URL" "$dumpFile"
39+
fi
40+
}
3941

40-
# Convert file to PBF file
41-
osmium cat $osm_tmp_file -o $local_fullHistoryFile
42-
osmium fileinfo $local_fullHistoryFile
42+
# ===============================
43+
# Upload planet + state
44+
# ===============================
45+
upload_planet_file() {
46+
echo "Uploading planet file and updating state.txt..."
4347

44-
# Remove full-hitory osm file, keep only history-latest.osh.pbf files
45-
rm $osm_tmp_file
48+
if [ "$CLOUDPROVIDER" == "aws" ]; then
49+
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
50+
echo "$AWS_URL.s3.amazonaws.com/$cloud_planetPBFFile" > "$stateFile"
51+
aws s3 cp "$local_planetPBFFile" "$AWS_S3_BUCKET/$cloud_planetPBFFile" --acl public-read
52+
aws s3 cp "$stateFile" "$AWS_S3_BUCKET/planet/state.txt" --acl public-read
4653

47-
# AWS
48-
if [ $CLOUDPROVIDER == "aws" ]; then
49-
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
50-
echo "$AWS_URL.s3.amazonaws.com/$cloud_fullHistoryFile" >$stateFile
51-
# Upload history-planet.osm.pbf
52-
aws s3 cp $local_fullHistoryFile $AWS_S3_BUCKET/$cloud_fullHistoryFile --acl public-read
53-
# Upload state.txt
54-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/state.txt --acl public-read
55-
fi
54+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
55+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_planetPBFFile" > "$stateFile"
56+
gsutil cp -a public-read "$local_planetPBFFile" "$GCP_STORAGE_BUCKET/$cloud_planetPBFFile"
57+
gsutil cp -a public-read "$stateFile" "$GCP_STORAGE_BUCKET/planet/state.txt"
58+
fi
59+
}
5660

57-
# Google Storage
58-
if [ $CLOUDPROVIDER == "gcp" ]; then
59-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_fullHistoryFile" >$stateFile
60-
# Upload history-planet.osm.pbf
61-
gsutil cp -a public-read $local_fullHistoryFile $GCP_STORAGE_BUCKET/$cloud_fullHistoryFile
62-
# Upload state.txt
63-
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/state.txt
64-
fi
61+
# ===============================
62+
# Generate planet file
63+
# ===============================
6564

66-
# Azure
67-
if [ $CLOUDPROVIDER == "azure" ]; then
68-
# Save the path file
69-
echo "https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$AZURE_CONTAINER_NAME/$cloud_fullHistoryFile" >$stateFile
70-
# Upload history-planet.osm.pbf
71-
az storage blob upload \
72-
--container-name $AZURE_CONTAINER_NAME \
73-
--file $local_fullHistoryFile \
74-
--name $cloud_fullHistoryFile \
75-
--output table
76-
# Upload state.txt
77-
az storage blob upload \
78-
--container-name $AZURE_CONTAINER_NAME \
79-
--file $stateFile \
80-
--name planet/full-history/state.txt \
81-
--output table
65+
if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
66+
download_dump_file
67+
echo "Generating planet file with planet-dump-ng..."
68+
export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
69+
planet-dump-ng \
70+
--dump-file "$dumpFile" \
71+
--pbf "$local_planetPBFFile"
72+
73+
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
74+
echo "Generating history planet file with osmosis..."
75+
# Creating full history
76+
osmosis --read-apidb-change \
77+
host=$POSTGRES_HOST \
78+
database=$POSTGRES_DB \
79+
user=$POSTGRES_USER \
80+
password=$POSTGRES_PASSWORD \
81+
validateSchemaVersion=no \
82+
readFullHistory=yes \
83+
--write-xml-change \
84+
compressionMethod=auto \
85+
$local_planetPBFFile
86+
else
87+
echo "Error: Unknown PLANET_EXPORT_METHOD value. Use 'planet-dump-ng' or 'osmosis'."
88+
exit 1
8289
fi
90+
91+
# Upload results
92+
upload_planet_file

‎images/planet-dump/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.962.hee57c16
22

33
VOLUME /mnt/data
44
COPY ./start.sh /

‎osm-seed/values.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,8 @@ fullHistory:
260260
schedule: "* * * * *"
261261
env:
262262
OVERWRITE_FHISTORY_FILE: false
263+
PLANET_EXPORT_METHOD: osmosis
264+
DUMP_CLOUD_URL: s3://osm-seed/db.dump
263265
resources:
264266
enabled: false
265267
requests:
@@ -343,6 +345,7 @@ planetDump:
343345
env:
344346
OVERWRITE_PLANET_FILE: false
345347
DUMP_CLOUD_URL : s3://osm-seed/db.dump
348+
PLANET_EXPORT_METHOD: osmosis
346349
resources:
347350
enabled: false
348351
requests:

0 commit comments

Comments
 (0)