CGIMAP container (#300)

* Add cgmap container

* Correct the misspelled name

* Add helm template for cgimap

* Update cgimap dockerfile

* Update tiler-imposm dockerfile

* Update python

* Update cgimap container

* Update cgimap container

* Update container for cgimap

* Build cgimap image using multistage

* Update cgimap config

Update relese tag

Pass env vars in web container for cgimap

Update release tag

Update tag name

Update release tags

* Update release tags

* Add memcache server for cgimap

* Update cgimap start and github action

* Add --daemon for cgimap

* Set empty values for planetFiles.image

* Disable cgimap  in chartpress

* Use chartpress==2.3.0

* Add HorizontalPodAutoscaler for cgimap

* Update web-api container

* Fix apache config

* Add PG password for tiler-db config as env var

* Update cgimap dockerfile using code form original repo

* Enable resources option for requests and limits - cgimap

* Evaluate memoryUtilization for web container - autoscaling

---------

Co-authored-by: Rub21 <[email protected]>

Author: Rub21

.github/workflows/chartpress.yaml

@@ -4,44 +4,42 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-      - name: Setup Git
-        run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "Github Action"
-      - name: Install Chartpress and Python modules
-        run: |
-          pip install -v chartpress six yq
-      - name: Set env vars for publish charts
-        run: |
-          GIT_BRANCH_NAME=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
-          GIT_BRANCH_NAME=${GIT_BRANCH_NAME/\//_}
-          OSM_SEED_VERSION=$(yq -r .appVersion ./osm-seed/Chart.yaml)
-          SHORT_GITHUB_SHA=${GITHUB_SHA::7}
-          echo "GIT_BRANCH_NAME=$GIT_BRANCH_NAME" >> $GITHUB_ENV
-          echo "OSM_SEED_VERSION=$OSM_SEED_VERSION" >> $GITHUB_ENV
-          echo "SHORT_GITHUB_SHA=$SHORT_GITHUB_SHA" >> $GITHUB_ENV
-      - name: Run Chartpress
-        env:
-          GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }}
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
-        run: |
-            if [[ $GIT_BRANCH_NAME = "main" ]]
-            then
-                chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}
-            elif [[ $GIT_BRANCH_NAME = "develop" ]]
-            then
-                chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA
-            else
-                chartpress --push --publish-chart
-            fi
-            
+    - uses: actions/checkout@v1
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+    - name: Setup git
+      run: git config --global user.email "[email protected]" && git config --global user.name "Github Action"
+    - name: Install Chartpress and some python modules
+      run: |
+        pip install -v chartpress==2.3.0 && pip install six && pip install yq
+    - name: Set env vars for publish charts
+      run: |
+        GIT_BRANCH_NAME=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+        GIT_BRANCH_NAME=${GIT_BRANCH_NAME/\//_}
+        OSM_SEED_VERSION=$(yq -r .appVersion ./osm-seed/Chart.yaml)
+        SHORT_GITHUB_SHA=${GITHUB_SHA::7}
+        echo "GIT_BRANCH_NAME=$GIT_BRANCH_NAME" >> $GITHUB_ENV
+        echo "OSM_SEED_VERSION=$OSM_SEED_VERSION" >> $GITHUB_ENV
+        echo "SHORT_GITHUB_SHA=$SHORT_GITHUB_SHA" >> $GITHUB_ENV
+    - name: Run Chartpress
+      env:
+        GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+      run: |
+          if [[ $GIT_BRANCH_NAME = "main" ]]
+          then
+              chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}
+          elif [[ $GIT_BRANCH_NAME = "develop" ]]
+          then
+              chartpress --push --publish-chart --tag ${OSM_SEED_VERSION}-dev.h$SHORT_GITHUB_SHA
+          else
+              chartpress --push --publish-chart
+          fi
+          

chartpress.yaml

@@ -30,8 +30,6 @@ charts:
       valuesPath: tilerServer.image
     tasking-manager-api:
       valuesPath: tmApi.image
-    # tiler-visor:
-    #   valuesPath: tilerVisor.image
     nominatim:
       valuesPath: nominatimApi.image
     overpass-api:
@@ -48,4 +46,5 @@ charts:
       valuesPath: osmchaDb.image
     planet-files:
       valuesPath: planetFiles.image
-
+    # cgimap:
+    #   valuesPath: cgimap.image

compose/cgimap.yml

@@ -0,0 +1,18 @@
+version: '3'
+services:
+  # #####################################################
+  # ## cgmap section
+  # #####################################################
+  cgimap:
+    image: osmseed-cgimap:v1
+    build:
+      context: ../images/cgimap
+      dockerfile: Dockerfile
+    ports:
+    - '80:80'
+    volumes:
+    - ../data/cgimap-data:/apps/data/
+    # command: >
+    #   /bin/bash -c " ./start.sh"
+    env_file:
+    - ../envs/.env.db

images/cgimap/Dockerfile

@@ -0,0 +1,45 @@
+FROM debian:bookworm-slim AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -qq && \
+    apt-get install -y gcc g++ make cmake \
+    libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
+    libboost-program-options-dev libcrypto++-dev libyajl-dev \
+    libpqxx-dev zlib1g-dev libfmt-dev \
+    postgresql-15 postgresql-server-dev-all dpkg-dev file ca-certificates git \
+    --no-install-recommends && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+    
+# Clone application
+ENV CGIMAP_GITSHA=8ea707e10aeab5698e6859856111816d75354592
+RUN git clone https://github.com/zerebubuth/openstreetmap-cgimap.git /app \
+&& git checkout $CGIMAP_GITSHA 
+
+# Compile, install and remove source
+RUN mkdir build && cd build && \
+    CXXFLAGS="-flto=auto -ffat-lto-objects -flto=auto -ffat-lto-objects -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release && \
+    make -j${nproc} && \
+    ctest --output-on-failure && \
+    cmake --build . -t package
+
+FROM debian:bookworm-slim
+
+COPY --from=builder /app/build/*.deb /app_deb/
+
+RUN apt-get update -qq && \
+    apt install --no-install-recommends -y /app_deb/*.deb postgresql-client procps && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 61000 cgimap && \
+    useradd -g 61000 -l -M -s /bin/false -u 61000 cgimap
+
+EXPOSE 8000
+
+COPY ./*.sh .
+
+CMD ["/start.sh"]

images/cgimap/README.md

@@ -0,0 +1,22 @@
+# openstreetmap-cgimap
+
+This container is built using the configuration from Zerebubuth's OpenStreetMap CGImap GitHub repository, with minor modifications.
+
+
+# Build and up 
+
+```sh
+docker compose -f compose/cgimap.yml build
+docker compose -f compose/cgimap.yml up
+```
+
+Note: Ensure that you are running PostgreSQL on your local machine. For example:
+
+
+```sh
+kubectl port-forward staging-db-0 5432:5432
+```
+
+Check results:
+
+http://localhost/api/0.6/map?bbox=-77.09529161453248,-12.071898885565846,-77.077374458313,-12.066474684936727

images/cgimap/liveness.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+pgrep -f openstreetmap-cgimap > /dev/null
+cgimap_status=$?
+
+# Check PostgreSQL connection
+PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 1;" > /dev/null 2>&1
+postgres_status=$?
+
+# Exit code logic
+if [ $cgimap_status -eq 0 ] && [ $postgres_status -eq 0 ]; then
+  echo "cgimap and PostgreSQL are healthy"
+  exit 0
+else
+  [ $cgimap_status -ne 0 ] && echo "cgimap not running" >&2
+  [ $postgres_status -ne 0 ] && echo "cannot connect to PostgreSQL" >&2
+  exit 1
+fi

images/cgimap/start.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+export PGPASSWORD=$POSTGRES_PASSWORD
+export CGIMAP_HOST=$POSTGRES_HOST
+export CGIMAP_DBNAME=$POSTGRES_DB
+export CGIMAP_USERNAME=$POSTGRES_USER
+export CGIMAP_PASSWORD=$POSTGRES_PASSWORD
+export CGIMAP_OAUTH_HOST=$POSTGRES_HOST
+export CGIMAP_UPDATE_HOST=$POSTGRES_HOST
+# Export CGIMAP configuration
+export CGIMAP_LOGFILE="/var/www/log/cgimap.log"
+export CGIMAP_MEMCACHE=$MEMCACHE_SERVER
+# Average number of bytes/s to allow each client
+export CGIMAP_RATELIMIT="204800"
+# Maximum debt in MB to allow each client before rate limiting
+export CGIMAP_MAXDEBT="2048"  
+export CGIMAP_MAP_AREA="0.25"
+export CGIMAP_MAP_NODES="100000"
+export CGIMAP_MAX_WAY_NODES="2000"
+export CGIMAP_MAX_RELATION_MEMBERS="32000"
+# export CGIMAP_RATELIMIT_UPLOAD="true"
+export CGIMAP_MODERATOR_RATELIMIT="1048576"
+export CGIMAP_MODERATOR_MAXDEBT="2048"
+
+if [[ "$WEBSITE_STATUS" == "database_readonly" || "$WEBSITE_STATUS" == "api_readonly" ]]; then
+  export CGIMAP_DISABLE_API_WRITE="true"
+fi
+
+if [[ "$WEBSITE_STATUS" == "database_offline" || "$WEBSITE_STATUS" == "api_offline" ]]; then
+  echo "Website is $WEBSITE_STATUS. No action required for cgimap service."
+else
+  # PostgreSQL options to disable certain joins
+  export PGOPTIONS="-c enable_mergejoin=false -c enable_hashjoin=false"
+  # Display current PostgreSQL settings
+  psql -h $POSTGRES_HOST -U $POSTGRES_USER -c "SHOW enable_mergejoin;"
+  psql -h $POSTGRES_HOST -U $POSTGRES_USER -c "SHOW enable_hashjoin;"
+  # Start the cgimap service
+  /usr/local/bin/openstreetmap-cgimap --port=8000 --daemon --instances=10
+fi

images/tiler-imposm/Dockerfile

@@ -22,13 +22,26 @@ RUN apt-get install -y \
     software-properties-common && \
     rm -rf /var/lib/apt/lists/*
 
-# # Install python
-RUN add-apt-repository ppa:deadsnakes/ppa && \
+# Install Python 3.6
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository -y ppa:ubuntu-toolchain-r/test && \
     apt-get update && \
-    apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \
-    rm -rf /var/lib/apt/lists/* && \
-    python3 -m pip install pip --upgrade && \
-    python3 -m pip install wheel
+    apt-get install -y gcc-8 g++-8 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 && \
+    apt-get install -y build-essential wget libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev && \
+    wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz && \
+    tar xzf Python-3.6.15.tgz && \
+    cd Python-3.6.15 && \
+    ./configure --enable-optimizations && \
+    make altinstall && \
+    cd .. && \
+    rm -rf Python-3.6.15 Python-3.6.15.tgz && \
+    apt-get install -y python3-pip && \
+    python3.6 -m pip install pip --upgrade && \
+    python3.6 -m pip install wheel && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install postgresql-client
 RUN apt-get update && apt-get install -y postgresql-client && \