adding misc

Author: Melanie Kneisel

svcs/data/misc/Cargo.toml

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+[package]
+name = "osmingester"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1"
+clap = { version="3", features = ["derive"] }
+env_logger = "0"
+itertools = "0"
+log = "0"
+openssl = "0"
+postgres = "0"
+postgres-openssl = "0"
+reqwest = { version = "0", features = ["blocking", "stream"] }

svcs/data/misc/Dockerfile.ingest

@@ -0,0 +1,43 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as builder
+
+ENV OSM2PGSQL_VERSION=1.6.0
+
+RUN tdnf repolist --refresh
+RUN tdnf install -y ca-certificates-microsoft wget tar gzip
+RUN tdnf install -y build-essential cmake boost-devel expat-devel \
+    bzip2-devel postgresql-devel lua-devel glibc
+
+RUN mkdir /osm2pgsql && wget -q -O -  https://github.com/openstreetmap/osm2pgsql/archive/refs/tags/$OSM2PGSQL_VERSION.tar.gz | tar -xvz --strip-components=1 -C /osm2pgsql
+
+RUN mkdir /osm2pgsql/build
+WORKDIR /osm2pgsql/build
+
+RUN cmake ..
+RUN make
+RUN make install
+
+FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as rustbuilder
+
+RUN tdnf repolist --refresh
+RUN tdnf install -y ca-certificates-microsoft build-essential
+RUN tdnf install -y rust openssl-devel
+
+RUN mkdir /build /build/osmingester /build/osmingester/src
+COPY Cargo.toml Cargo.lock /build/
+COPY osmingester/Cargo.toml /build/osmingester
+COPY osmingester/src/main.rs /build/osmingester/src
+WORKDIR /build
+RUN cargo build --release
+
+FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as final
+
+RUN tdnf install -y ca-certificates boost-devel expat-devel \
+  bzip2-devel postgresql-libs lua lua-devel gdb
+RUN mkdir /tools
+
+COPY --from=builder /usr/local/bin/osm2pgsql /usr/local/bin/osm2pgsql-replication /tools/
+COPY --from=rustbuilder /build/target/release/osmingester /tools
+WORKDIR /work

svcs/data/misc/README.md

@@ -0,0 +1,2 @@
+
+This directory contains material that has not be tested or used in production but might be helpful when bringing up the services portion of Soundscape.

svcs/data/misc/ingester.yaml

@@ -0,0 +1,77 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-ingester-service
+  labels:
+    app: {{ .Chart.Name }}
+    tier: ingestion
+    role: ingester-service
+spec:
+  selector:
+    app: {{ .Chart.Name }}
+    tier: ingestion
+    role: ingester
+  clusterIP: None
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Chart.Name}}-ingester
+spec:
+  serviceName: {{ .Chart.Name }}-ingester-service
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+      tier: ingestion
+      role: ingester
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}
+        tier: ingestion
+        role: ingester
+    spec:
+      serviceAccountName: {{ .Chart.Name}}-ingester-serviceaccount
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: {{ .Chart.Name }}-ingester
+          image: {{ .Values.containerRegistry }}/{{ .Values.ingesterImage }}:{{ required "A valid .Values.soundscapeImageVersion entry required!" .Values.soundscapeImageVersion }}
+          command: ["/tools/osmingester"]
+          args: ["--osm2pgsql", "/tools/osm2pgsql", "--pbf", "/config/pbf", "--mapper", "/config/soundscape.lua", "--provision-sql", "/config/provision.sql", "--provision-sql", "/config/postgis-vt-util.sql", "--provision-sql", "/config/tilefunc.sql", "--work", "/work", "--dsn", "/secret/dsn2"]
+          securityContext:
+            allowPrivilegeEscalation: false
+          volumeMounts:
+            - mountPath: /work
+              name: {{ .Chart.Name }}-ingester-volume
+            - mountPath: /config
+              name: {{ .Chart.Name }}-ingester-config-volume
+              readOnly: true
+            - mountPath: /secret
+              name: {{ .Chart.Name }}-database-volume
+              readOnly: true
+      volumes:
+        - name: {{ .Chart.Name }}-ingester-config-volume
+          configMap:
+            name: {{ .Chart.Name }}-ingester-config
+        - name: {{ .Chart.Name }}-database-volume
+          secret:
+            secretName: {{ .Chart.Name }}-database
+  updateStrategy:
+    type: RollingUpdate
+  volumeClaimTemplates:
+    - metadata:
+        name: {{ .Chart.Name }}-ingester-volume
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        storageClassName: {{ .Chart.Name }}-sc
+        resources:
+          requests:
+            storage: {{ .Values.config.ingester.workingStorage }}