Merge branch 'master' of https://github.com/openmaptiles/openmaptiles

nst-guide · Feb 6, 2020 · 9e029fe · 9e029fe
2 parents d325605 + ace7595
commit 9e029fe
Show file tree

Hide file tree

Showing 62 changed files with 545 additions and 407 deletions.
diff --git a/.gitignore b/.gitignore
@@ -16,9 +16,6 @@ quickstart.log
 # imput / output data
 data/*
 
-# input wikidata
-wikidata/*
-
 # generated source files
 build/*
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -32,15 +32,11 @@ Your pull request must:
  * Use clear commit messages.
  * Be possible to merge automatically.
 
-When you modify rules of importing data in `mapping.yaml` or `*.sql`, please update also:
+When you modify import data rules in `mapping.yaml` or `*.sql`, please update:
 
 1. field description in `[layer].yaml`
 2. comments starting with `#etldoc`
-3. if needed, generate new `mapping_diagram.png` or `etl_diagram.png` using commands below:
-```
-make mapping-graph-[layer]
-make etl-graph-[layer]
-```
+3. regenerate documentation graphs with `make generate-devdoc`
 4. update layer description on https://openmaptiles.org/schema/ (https://github.com/openmaptiles/www.openmaptiles.org/tree/master/layers)
 5. check if OMT styles are affected by the PR and if there is a need for style updates
 

diff --git a/Makefile b/Makefile
@@ -7,9 +7,11 @@ DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
 ifeq ($(TEST_MODE),yes)
-  COPY_TO_GIT=diff
+  # create images in ./build/devdoc and compare them to ./layers
+  GRAPH_PARAMS=./build/devdoc ./layers
 else
-  COPY_TO_GIT=cp
+  # update graphs in the ./layers dir
+  GRAPH_PARAMS=./layers
 endif
 
 .PHONY: all
@@ -36,9 +38,7 @@ help:
 	@echo "  make psql-vacuum-analyze             # PostgreSQL: VACUUM ANALYZE"
 	@echo "  make psql-analyze                    # PostgreSQL: ANALYZE"
 	@echo "  make generate-qareports              # generate reports [./build/qareports]"
-	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers  [./build/devdoc]"
-	@echo "  make etl-graph                       # hint for generating a single etl graph"
-	@echo "  make mapping-graph                   # hint for generating a single mapping graph"
+	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers  [./layers/...]"
 	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
 	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
 	@echo "  make clean-docker                    # remove docker containers, PG data volume"
@@ -65,7 +65,8 @@ build/mapping.yaml: build
 	docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@
 
 build/tileset.sql: build
-	docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@
+	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
+	docker-compose run $(DC_OPTS) openmaptiles-tools-latest generate-sql openmaptiles.yaml > $@
 
 .PHONY: clean
 clean:
@@ -172,41 +173,13 @@ start-postserve: db-start
 generate-qareports:
 	./qa/run.sh
 
-build/devdoc:
-	mkdir -p ./build/devdoc
-
-
-layers = $(notdir $(wildcard layers/*)) # all layers
-
-.PHONY: etl-graph
-etl-graph:
-	@echo 'Use'
-	@echo '   make etl-graph-[layer]	to generate etl graph for [layer]'
-	@echo '   example: make etl-graph-poi'
-	@echo 'Valid layers: $(layers)'
-
-# generate etl graph for a certain layer, e.g. etl-graph-building, etl-graph-place
-etl-graph-%: layers/% build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-etlgraph layers/$*/$*.yaml ./build/devdoc
-	@$(COPY_TO_GIT) ./build/devdoc/etl_$*.png layers/$*/etl_diagram.png
-
-
-mappingLayers = $(notdir $(patsubst %/mapping.yaml,%, $(wildcard layers/*/mapping.yaml))) # layers with mapping.yaml
-
-# generate mapping graph for a certain layer, e.g. mapping-graph-building, mapping-graph-place
-.PHONY: mapping-graph
-mapping-graph:
-	@echo 'Use'
-	@echo '   make mapping-graph-[layer]	to generate mapping graph for [layer]'
-	@echo '   example: make mapping-graph-poi'
-	@echo 'Valid layers: $(mappingLayers)'
-
-mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
-	@$(COPY_TO_GIT) ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png
-
 # generate all etl and mapping graphs
-generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(mappingLayers))
+.PHONY: generate-devdoc
+generate-devdoc:
+	mkdir -p ./build/devdoc && \
+	docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \
+			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
+			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
 
 .PHONY: import-sql-dev
 import-sql-dev:
@@ -226,9 +199,10 @@ list:
 download-geofabrik-list:
 	docker-compose run $(DC_OPTS) import-osm  ./download-geofabrik-list.sh
 
-.PHONY: download-wikidata
-download-wikidata:
-	mkdir -p wikidata && docker-compose run $(DC_OPTS) --entrypoint /usr/src/app/download-gz.sh import-wikidata
+.PHONY: import-wikidata
+import-wikidata:
+	# FIXME:   switch to openmaptiles-tools after v3.2+ is published
+	docker-compose run $(DC_OPTS) openmaptiles-tools-latest import-wikidata openmaptiles.yaml
 
 .PHONY: psql-list-tables
 psql-list-tables:

diff --git a/README.md b/README.md
@@ -316,16 +316,6 @@ docker-compose run import-lakelines
 docker-compose run import-osmborder
 ```
 
-**[Optional]**
-Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field.
-
-Beware that current [Wikidata dump](https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.gz) is more than 55GB, it takes time to download and import it. If you just want to have a quickstart on OpenMapTiles, just skip this step.
-
-```bash
-make download-wikidata
-docker-compose run import-wikidata
-```
-
 [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
 
 ```bash
@@ -340,6 +330,14 @@ wget http://download.geofabrik.de/europe/albania-latest.osm.pbf
 docker-compose run import-osm
 ```
 
+Import latest Wikidata. If an OSM feature has [Key:wikidata](https://wiki.openstreetmap.org/wiki/Key:wikidata), OpenMapTiles check corresponding item in Wikidata and use its [labels](https://www.wikidata.org/wiki/Help:Label) for languages listed in [openmaptiles.yaml](openmaptiles.yaml). So the generated vector tiles includes multi-languages in name field.
+
+This step uses [Wikidata Query Service](https://query.wikidata.org) to download just the Wikidata IDs that already exist in the database.
+
+```bash
+make import-wikidata
+```
+
 ### Work on Layers
 
 Each time you modify layer SQL code run `make` and `make import-sql`.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -67,27 +67,18 @@ services:
       - ./data:/import
       - ./build:/mapping
       - cache:/cache
-  import-sql:
-    # This target is obsolete, and was left for backwards compatibility
-    # Use openmaptiles-tools target instead
+  openmaptiles-tools:
     image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
-    command: import-sql
     env_file: .env
     networks:
       - postgres_conn
     volumes:
       - .:/tileset
       - ./build:/sql
-  import-wikidata:
-    image: "openmaptiles/import-wikidata:${TOOLS_VERSION}"
-    env_file: .env
-    command: import-wikidata
-    networks:
-      - postgres_conn
-    volumes:
-      - ./wikidata:/import
-  openmaptiles-tools:
-    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
+  openmaptiles-tools-latest:
+    # This target exists for experimental tools that have not yet been published.
+    # Do not use this for production.
+    image: "openmaptiles/openmaptiles-tools:latest"
     env_file: .env
     networks:
       - postgres_conn

diff --git a/layers/aerodrome_label/aerodrome_label.yaml b/layers/aerodrome_label/aerodrome_label.yaml
@@ -15,12 +15,23 @@ layer:
           [`aerodrome`](http://wiki.openstreetmap.org/wiki/Proposed_features/Aerodrome)
           and `aerodrome:type` tags.
       values:
-      - international
-      - public
-      - regional
-      - military
-      - private
-      - other
+        international:
+          aerodrome: 'international'
+          aerodrome_type: 'international'
+        public:
+          aerodrome: 'public'
+          aerodrome_type: ['%public%', 'civil']
+        regional:
+          aerodrome: 'regional'
+          aerodrome_type: 'regional'
+        military:
+          aerodrome: 'military'
+          aerodrome_type: '%military%'
+          military: 'airfield'
+        private:
+          aerodrome: 'private'
+          aerodrome_type: 'private'
+        other:
     iata: 3-character code issued by the IATA.
     icao: 4-letter code issued by the ICAO.
     ele: Elevation (`ele`) in meters.

diff --git a/layers/aerodrome_label/layer.sql b/layers/aerodrome_label/layer.sql
@@ -27,27 +27,7 @@ $$
     COALESCE(NULLIF(name_de, ''), name, name_en) AS name_de,
     tags,
     CASE
-      WHEN aerodrome = 'international'
-          OR aerodrome_type = 'international'
-        THEN 'international'
-      WHEN
-          aerodrome = 'public'
-          OR aerodrome_type LIKE '%public%'
-          OR aerodrome_type = 'civil'
-        THEN 'public'
-      WHEN
-          aerodrome = 'regional'
-          OR aerodrome_type = 'regional'
-        THEN 'regional'
-      WHEN
-          aerodrome = 'military'
-          OR aerodrome_type LIKE '%military%'
-          OR military = 'airfield'
-        THEN 'military'
-      WHEN
-          aerodrome = 'private'
-          OR aerodrome_type = 'private'
-        THEN 'private'
+      %%FIELD_MAPPING: class %%
       ELSE 'other'
     END AS class,
     NULLIF(iata, '') AS iata,
@@ -56,5 +36,6 @@ $$
     round(substring(ele from E'^(-?\\d+)(\\D|$)')::int*3.2808399)::int AS ele_ft
   FROM osm_aerodrome_label_point
   WHERE geometry && bbox AND zoom_level >= 10;
-
-$$ LANGUAGE SQL IMMUTABLE;
+$$
+LANGUAGE SQL
+IMMUTABLE PARALLEL SAFE;
diff --git a/layers/aerodrome_label/mapping.yaml b/layers/aerodrome_label/mapping.yaml
@@ -3,7 +3,7 @@ tables:
   # etldoc: imposm3 -> osm_aerodrome_label_point
   aerodrome_label_point:
     type: geometry
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

diff --git a/layers/aeroway/layer.sql b/layers/aeroway/layer.sql
@@ -40,4 +40,6 @@ RETURNS TABLE(geometry geometry, class text, ref text) AS $$
         FROM osm_aeroway_polygon WHERE zoom_level >= 14
     ) AS zoom_levels
     WHERE geometry && bbox;
-$$ LANGUAGE SQL IMMUTABLE;
+$$
+LANGUAGE SQL IMMUTABLE
+PARALLEL SAFE;
diff --git a/layers/aeroway/mapping.yaml b/layers/aeroway/mapping.yaml
@@ -50,7 +50,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_polygon
   aeroway_polygon:
     type: polygon
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id
@@ -67,7 +67,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_linestring
   aeroway_linestring:
     type: linestring
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id