Skip to content

Commit ee57c16

Browse files
Rub21Rub21
committed
Update scripts and continaer for planet-dump-ng
1 parent cf3a8ba commit ee57c16

File tree

4 files changed

+71
-52
lines changed

4 files changed

+71
-52
lines changed

‎compose/db-backup-restore.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
version: '3'
21
services:
32
#####################################################
43
## OSM Database backup and restore section
54
#####################################################
65
db-backup-restore:
7-
image: osmseed-backup-restore:v1
6+
image: rub21/osmseed-backup-restore:v1
87
build:
98
context: ../images/backup-restore
109
dockerfile: Dockerfile

‎images/osm-processor/Dockerfile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,11 @@ RUN set -ex \
4242
libprotobuf-dev \
4343
pkg-config \
4444
git \
45-
&& git clone https://github.com/zerebubuth/planet-dump-ng.git /opt/planet-dump-ng \
45+
&& git clone -b timestamp_epoch https://github.com/OpenHistoricalMap/planet-dump-ng.git /opt/planet-dump-ng \
4646
&& cd /opt/planet-dump-ng \
4747
&& ./autogen.sh \
4848
&& ./configure \
4949
&& make \
5050
&& ln -s /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng \
51-
&& rm -rf /var/lib/apt/lists/*
51+
&& rm -rf /var/lib/apt/lists/*
52+

‎images/planet-dump/start.sh

Lines changed: 66 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22
set -e
3-
export VOLUME_DIR=/mnt/data
43

54
# osmosis tuning: https://wiki.openstreetmap.org/wiki/Osmosis/Tuning,https://lists.openstreetmap.org/pipermail/talk/2012-October/064771.html
65
if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then
@@ -10,64 +9,83 @@ else
109
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
1110
fi
1211

13-
# Read the DB and create the planet osm file
12+
export VOLUME_DIR=/mnt/data
1413
date=$(date '+%y%m%d_%H%M')
14+
1515
local_planetPBFFile=$VOLUME_DIR/planet-${date}.osm.pbf
1616
cloud_planetPBFFile=planet/planet-${date}.osm.pbf
17+
stateFile="$VOLUME_DIR/state.txt"
1718

18-
# In case overwrite the file
19+
# If overwrite flag is enabled, use fixed filenames
1920
if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
2021
local_planetPBFFile=$VOLUME_DIR/planet-latest.osm.pbf
2122
cloud_planetPBFFile=planet/planet-latest.osm.pbf
2223
fi
2324

24-
stateFile="$VOLUME_DIR/state.txt"
25+
# ===============================
26+
# Download db .dump file
27+
# ===============================
28+
download_dump_file() {
29+
local_dumpFile="$VOLUME_DIR/input-latest.dump"
30+
echo "Downloading db .dump file from cloud..."
2531

26-
# Creating the replication file
27-
osmosis --read-apidb \
28-
host=$POSTGRES_HOST \
29-
database=$POSTGRES_DB \
30-
user=$POSTGRES_USER \
31-
password=$POSTGRES_PASSWORD \
32-
validateSchemaVersion=no \
33-
--write-pbf \
34-
file=$local_planetPBFFile
32+
if [ "$CLOUDPROVIDER" == "aws" ]; then
33+
aws s3 cp "$DUMP_CLOUD_URL" "$local_dumpFile"
34+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
35+
gsutil cp "$DUMP_CLOUD_URL" "$local_dumpFile"
36+
fi
37+
}
3538

36-
# AWS
37-
if [ $CLOUDPROVIDER == "aws" ]; then
38-
# Save the path file
39-
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
40-
echo "$AWS_URL.s3.amazonaws.com/$cloud_planetPBFFile" > $stateFile
41-
# Upload planet.osm.pbf file to s3
42-
aws s3 cp $local_planetPBFFile $AWS_S3_BUCKET/$cloud_planetPBFFile --acl public-read
43-
# Upload state.txt file to s3
44-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/state.txt --acl public-read
45-
fi
39+
# ===============================
40+
# Upload planet + state
41+
# ===============================
42+
upload_planet_file() {
43+
echo "Uploading planet file and updating state.txt..."
4644

47-
# gcp
48-
if [ $CLOUDPROVIDER == "gcp" ]; then
49-
# Save the path file
50-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_planetPBFFile" > $stateFile
51-
# Upload planet.osm.pbf file to cloud storage
52-
gsutil cp -a public-read $local_planetPBFFile $GCP_STORAGE_BUCKET/$cloud_planetPBFFile
53-
# Upload state.txt file to cloud storage
54-
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/state.txt
55-
fi
45+
if [ "$CLOUDPROVIDER" == "aws" ]; then
46+
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
47+
echo "$AWS_URL.s3.amazonaws.com/$cloud_planetPBFFile" > "$stateFile"
48+
aws s3 cp "$local_planetPBFFile" "$AWS_S3_BUCKET/$cloud_planetPBFFile" --acl public-read
49+
aws s3 cp "$stateFile" "$AWS_S3_BUCKET/planet/state.txt" --acl public-read
50+
51+
elif [ "$CLOUDPROVIDER" == "gcp" ]; then
52+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_planetPBFFile" > "$stateFile"
53+
gsutil cp -a public-read "$local_planetPBFFile" "$GCP_STORAGE_BUCKET/$cloud_planetPBFFile"
54+
gsutil cp -a public-read "$stateFile" "$GCP_STORAGE_BUCKET/planet/state.txt"
55+
fi
56+
}
57+
58+
# ===============================
59+
# Generate planet file
60+
# ===============================
5661

57-
# Azure
58-
if [ $CLOUDPROVIDER == "azure" ]; then
59-
# Save the path file
60-
echo "https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$AZURE_CONTAINER_NAME/$cloud_planetPBFFile" > $stateFile
61-
# Upload planet.osm.pbf file to blob storage
62-
az storage blob upload \
63-
--container-name $AZURE_CONTAINER_NAME \
64-
--file $local_planetPBFFile \
65-
--name $cloud_planetPBFFile \
66-
--output table
67-
# Upload state.txt file to blob storage
68-
az storage blob upload \
69-
--container-name $AZURE_CONTAINER_NAME \
70-
--file $stateFile \
71-
--name planet/state.txt \
72-
--output table
62+
if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
63+
download_dump_file
64+
echo "Generating planet file with planet-dump-ng..."
65+
planet-dump-ng \
66+
--dump-file "$VOLUME_DIR/input-latest.dump" \
67+
--pbf "$local_planetPBFFile"
68+
elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
69+
echo "Generating planet file with osmosis..."
70+
if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then
71+
echo JAVACMD_OPTIONS=\"-server\" > ~/.osmosis
72+
else
73+
memory="${MEMORY_JAVACMD_OPTIONS//i/}"
74+
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" > ~/.osmosis
75+
fi
76+
77+
osmosis --read-apidb \
78+
host=$POSTGRES_HOST \
79+
database=$POSTGRES_DB \
80+
user=$POSTGRES_USER \
81+
password=$POSTGRES_PASSWORD \
82+
validateSchemaVersion=no \
83+
--write-pbf \
84+
file=$local_planetPBFFile
85+
else
86+
echo "Error: Unknown PLANET_EXPORT_METHOD value. Use 'planet-dump-ng' or 'osmosis'."
87+
exit 1
7388
fi
89+
90+
# Upload results
91+
upload_planet_file

‎osm-seed/values.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ planetDump:
342342
schedule: "* * * * *"
343343
env:
344344
OVERWRITE_PLANET_FILE: false
345+
DUMP_CLOUD_URL : s3://osm-seed/db.dump
345346
resources:
346347
enabled: false
347348
requests:

0 commit comments

Comments
 (0)