Implement a Persistent Job Scheduler

Closes: #1486
samply · Mar 30, 2024 · 008f489 · 008f489
1 parent 5662408
commit 008f489
Show file tree

Hide file tree

Showing 148 changed files with 3,982 additions and 1,158 deletions.
diff --git a/.clj-kondo/root/config.edn b/.clj-kondo/root/config.edn
@@ -71,6 +71,7 @@
     blaze.executors ex
     blaze.fhir.spec.references fsr
     blaze.fhir.structure-definition-repo sdr
+    blaze.job-scheduler js
     blaze.middleware.fhir.db db
     blaze.rest-api.header header
     blaze.scheduler sched

diff --git a/.github/distributed-test/credentials b/.github/distributed-test/credentials
@@ -0,0 +1 @@
+password
diff --git a/.github/distributed-test/docker-compose.yml b/.github/distributed-test/docker-compose.yml
@@ -1,67 +1,109 @@
 services:
-  zookeeper:
-    image: "docker.io/bitnami/zookeeper:3@sha256:8104cf1ff552cb2c0270ed71c8d2aed31ac4b69cb109695871a2d08193dea7c1"
-    volumes:
-    - "zookeeper-data:/bitnami"
-    environment:
-      ALLOW_ANONYMOUS_LOGIN: "yes"
-
   kafka:
-    image: "docker.io/bitnami/kafka:2@sha256:2096ad73e2cbe3b615bcb7953b19f021cde62011435465c08707d5cfd9ecc9da"
-    hostname: "kafka"
+    image: "apache/kafka:3.7.0"
     environment:
-      KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181"
-      ALLOW_PLAINTEXT_LISTENER: "yes"
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CLIENT:SSL"
-      KAFKA_CFG_LISTENERS: "CLIENT://:9092"
-      KAFKA_CFG_ADVERTISED_LISTENERS: "CLIENT://kafka:9092"
-      KAFKA_INTER_BROKER_LISTENER_NAME: "CLIENT"
-      KAFKA_CERTIFICATE_PASSWORD: "password"
-      KAFKA_CFG_TLS_TYPE: "JKS"
-      KAFKA_TLS_TRUSTSTORE_FILE: "/opt/bitnami/kafka/config/certs/kafka.truststore.jks"
+      KAFKA_NODE_ID: 1
+      CLUSTER_ID: '5L6g3nShT-eMCtK--X86sw'
+      # KRaft
+      KAFKA_PROCESS_ROLES: "broker,controller"
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:29093'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
+      # Listeners
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL"
+      KAFKA_LISTENERS: "SSL://:9092,CONTROLLER://:29093,SSL-INTERNAL://:19093"
+      KAFKA_ADVERTISED_LISTENERS: "SSL://kafka:9092,SSL-INTERNAL://kafka:19093"
+      KAFKA_SSL_KEYSTORE_FILENAME: "kafka.keystore.jks"
+      KAFKA_SSL_KEY_CREDENTIALS: "credentials"
+      KAFKA_SSL_KEYSTORE_CREDENTIALS: "credentials"
+      KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks"
+      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "credentials"
+      KAFKA_SSL_CLIENT_AUTH: 'required'
       # It's important to create the tx topic ourselves, because it needs to use
       # LogAppendTime timestamps
-      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+    volumes:
+    - "./kafka.keystore.jks:/etc/kafka/secrets/kafka.keystore.jks:ro"
+    - "./kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks:ro"
+    - "./credentials:/etc/kafka/secrets/credentials:ro"
+    - "kafka-data:/var/lib/kafka/data"
+    healthcheck:
+      test: nc -z localhost 9092 || exit -1
+      start_period: 15s
+      interval: 5s
+      timeout: 10s
+      retries: 10
+
+  kafka-topic-creator-main:
+    image: "apache/kafka:3.7.0"
+    command: "/opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:9092 
+      --command-config /etc/kafka/kafka-init.conf --create --if-not-exists 
+      --topic tx-main --partitions 1 --replication-factor 1 
+      --config message.timestamp.type=LogAppendTime --config retention.ms=-1"
     volumes:
-    - "./kafka.keystore.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro"
-    - "./kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro"
-    - "kafka-data:/bitnami"
+    - "./kafka-init.conf:/etc/kafka/kafka-init.conf:ro"
+    - "./kafka-topic-creator.keystore.jks:/etc/kafka/secrets/kafka-topic-creator.keystore.jks:ro"
+    - "./kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks:ro"
     depends_on:
-    - zookeeper
+      kafka:
+        condition: service_healthy
 
-  kafka-topic-creator:
-    image: "docker.io/bitnami/kafka:2@sha256:2096ad73e2cbe3b615bcb7953b19f021cde62011435465c08707d5cfd9ecc9da"
-    command: "/opt/bitnami/kafka/bin/kafka-init.sh"
+  kafka-topic-creator-admin:
+    image: "apache/kafka:3.7.0"
+    command: "/opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:9092 
+      --command-config /etc/kafka/kafka-init.conf --create --if-not-exists 
+      --topic tx-admin --partitions 1 --replication-factor 1 
+      --config message.timestamp.type=LogAppendTime --config retention.ms=-1"
     volumes:
-    - "./kafka-init.sh:/opt/bitnami/kafka/bin/kafka-init.sh:ro"
-    - "./kafka-init.conf:/opt/bitnami/kafka/config/kafka-init.conf:ro"
-    - "./kafka-topic-creator.keystore.jks:/opt/bitnami/kafka/config/certs/kafka-topic-creator.keystore.jks:ro"
-    - "./kafka.truststore.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro"
+    - "./kafka-init.conf:/etc/kafka/kafka-init.conf:ro"
+    - "./kafka-topic-creator.keystore.jks:/etc/kafka/secrets/kafka-topic-creator.keystore.jks:ro"
+    - "./kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks:ro"
     depends_on:
-    - zookeeper
-    - kafka
+      kafka:
+        condition: service_healthy
 
   cassandra-1:
-    image: "docker.io/bitnami/cassandra:3@sha256:60b13bf8f0a01ad6db72689be114c8fabf9134c7a2b5ba6d855b79ad457344e9"
-    hostname: "cassandra-1"
+    image: "cassandra:4.1.4"
     volumes:
-    - "./cassandra-init.cql:/docker-entrypoint-initdb.d/cassandra-init.cql:ro"
-    - "cassandra-1-data:/bitnami"
+    - "cassandra-1-data:/var/lib/cassandra"
     environment:
       CASSANDRA_SEEDS: "cassandra-1,cassandra-2"
-      CASSANDRA_PASSWORD_SEEDER: "yes"
       MAX_HEAP_SIZE: "512M"
       HEAP_NEWSIZE: "100M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-e", "describe keyspaces"]
+      start_period: 45s
+      interval: 5s
+      timeout: 10s
+      retries: 10
 
   cassandra-2:
-    image: "docker.io/bitnami/cassandra:3@sha256:60b13bf8f0a01ad6db72689be114c8fabf9134c7a2b5ba6d855b79ad457344e9"
-    hostname: "cassandra-2"
+    image: "cassandra:4.1.4"
     volumes:
-    - "cassandra-2-data:/bitnami"
+    - "cassandra-2-data:/var/lib/cassandra"
     environment:
       CASSANDRA_SEEDS: "cassandra-1,cassandra-2"
       MAX_HEAP_SIZE: "512M"
       HEAP_NEWSIZE: "100M"
+    healthcheck:
+      test: ["CMD", "cqlsh", "-e", "describe keyspaces"]
+      start_period: 45s
+      interval: 5s
+      timeout: 10s
+      retries: 10
+
+  cassandra-init-data:
+    image: "cassandra:4.1.4"
+    command: "cqlsh -f /scripts/cassandra-init.cql"
+    environment:
+      CQLSH_HOST: "cassandra-1"
+    volumes:
+    - "./cassandra-init.cql:/scripts/cassandra-init.cql:ro"
+    depends_on:
+      cassandra-1:
+        condition: service_healthy
+      cassandra-2:
+        condition: service_healthy
 
   blaze-1:
     image: "blaze:latest"
@@ -80,6 +122,7 @@ services:
       DB_CASSANDRA_REQUEST_TIMEOUT: "60000"
       DB_CASSANDRA_MAX_CONCURRENT_REQUESTS: "128"
       DB_RESOURCE_CACHE_SIZE: "100000"
+      ENABLE_ADMIN_API: "true"
       LOG_LEVEL: "debug"
     ports:
     - "8081:8081"
@@ -88,10 +131,9 @@ services:
     - "./kafka.truststore.jks:/app/kafka.truststore.jks:ro"
     - "blaze-1-data:/app/data"
     depends_on:
-    - kafka-topic-creator
-    - cassandra-1
-    - cassandra-2
-    restart: unless-stopped
+    - kafka-topic-creator-main
+    - kafka-topic-creator-admin
+    - cassandra-init-data
 
   blaze-2:
     image: "blaze:latest"
@@ -110,6 +152,7 @@ services:
       DB_CASSANDRA_REQUEST_TIMEOUT: "60000"
       DB_CASSANDRA_MAX_CONCURRENT_REQUESTS: "128"
       DB_RESOURCE_CACHE_SIZE: "100000"
+      ENABLE_ADMIN_API: "true"
       LOG_LEVEL: "debug"
     ports:
     - "8082:8081"
@@ -118,10 +161,9 @@ services:
     - "./kafka.truststore.jks:/app/kafka.truststore.jks:ro"
     - "blaze-2-data:/app/data"
     depends_on:
-    - kafka-topic-creator
-    - cassandra-1
-    - cassandra-2
-    restart: unless-stopped
+    - kafka-topic-creator-main
+    - kafka-topic-creator-admin
+    - cassandra-init-data
 
   ingress:
     image: "haproxy:2.8@sha256:d533c21b446c45e47dc5b89933df1e256ec09976dc5e83a4c1bea209c2baeeaa"
@@ -134,7 +176,6 @@ services:
     - blaze-2
 
 volumes:
-  zookeeper-data:
   kafka-data:
   cassandra-1-data:
   cassandra-2-data:

diff --git a/.github/distributed-test/kafka-init.conf b/.github/distributed-test/kafka-init.conf
@@ -1,6 +1,6 @@
 security.protocol=SSL
-ssl.truststore.location=/opt/bitnami/kafka/config/certs/kafka.truststore.jks
+ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks
 ssl.truststore.password=password
-ssl.keystore.location=/opt/bitnami/kafka/config/certs/kafka-topic-creator.keystore.jks
+ssl.keystore.location=/etc/kafka/secrets/kafka-topic-creator.keystore.jks
 ssl.keystore.password=password
 ssl.key.password=password
diff --git a/.github/distributed-test/kafka-init.sh b/.github/distributed-test/kafka-init.sh
diff --git a/.github/scripts/jobs/re-index/create-invalid-job.sh b/.github/scripts/jobs/re-index/create-invalid-job.sh
@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+. "$SCRIPT_DIR/../../util.sh"
+
+BASE="http://localhost:8080/fhir"
diff --git a/.github/scripts/util.sh b/.github/scripts/util.sh
@@ -25,3 +25,11 @@ create() {
 update() {
   curl -XPUT -s -H 'Accept: application/fhir+json' -H "Content-Type: application/fhir+json" -d @- -o /dev/null "$1"
 }
+
+create() {
+  curl -s -H 'Accept: application/fhir+json' -H "Content-Type: application/fhir+json" -d @- "$1"
+}
+
+update() {
+  curl -XPUT -s -H 'Accept: application/fhir+json' -H "Content-Type: application/fhir+json" -d @- -o /dev/null "$1"
+}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -307,7 +307,7 @@ jobs:
       run: docker run --name blaze -d -e JAVA_TOOL_OPTIONS=-Xmx2g -p 8080:8080 -p 8081:8081 -v blaze-data:/app/data blaze:latest
       if: ${{ matrix.variant == 'standalone' }}
 
-    - name: Run Zookeeper, Kafka, Cassandra and Blaze
+    - name: Run Kafka, Cassandra and Blaze
       run: docker-compose -f .github/distributed-test/docker-compose.yml up -d
       if: ${{ matrix.variant == 'distributed' }}
 
@@ -327,10 +327,6 @@ jobs:
       run: docker logs blaze
       if: ${{ matrix.variant == 'standalone' }}
 
-    - name: Docker Logs Zookepper
-      run: docker-compose -f .github/distributed-test/docker-compose.yml logs zookeeper
-      if: ${{ matrix.variant == 'distributed' }}
-
     - name: Docker Logs Kafka
       run: docker-compose -f .github/distributed-test/docker-compose.yml logs kafka
       if: ${{ matrix.variant == 'distributed' }}
@@ -1450,7 +1446,7 @@ jobs:
     - name: Load Blaze Image
       run: docker load --input /tmp/blaze.tar
 
-    - name: Run Zookeeper, Kafka, Cassandra and Blaze
+    - name: Run Kafka, Cassandra and Blaze
       run: docker-compose -f .github/distributed-test/docker-compose.yml up -d
 
     - name: Wait for Blaze 1
@@ -1459,9 +1455,6 @@ jobs:
     - name: Wait for Blaze 2
       run: .github/scripts/wait-for-url.sh http://localhost:8082/metrics
 
-    - name: Docker Logs Zookepper
-      run: docker-compose -f .github/distributed-test/docker-compose.yml logs zookeeper
-
     - name: Docker Logs Kafka
       run: docker-compose -f .github/distributed-test/docker-compose.yml logs kafka
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,9 @@
-# Update the SHA by calling crane digest eclipse-temurin:17-jre-jammy
+FROM mcr.microsoft.com/dotnet/sdk:6.0 as fhir-packages
+
+RUN dotnet tool install -g firely.terminal
+RUN /root/.dotnet/tools/fhir install hl7.fhir.r4.core 4.0.1
+RUN /root/.dotnet/tools/fhir install hl7.fhir.xver-extensions 0.1.0
+
 FROM eclipse-temurin:17-jre-jammy@sha256:2da160772ec16d9d6a0c71585cf87b689dbbda531dc002de1856d8970cd0daf3
 
 RUN apt-get update && apt-get upgrade -y && \
@@ -7,8 +12,13 @@ RUN apt-get update && apt-get upgrade -y && \
     apt-get autoremove -y && apt-get clean && \
     rm -rf /var/lib/apt/lists/
 
+RUN groupadd -g 1001 blaze
+RUN useradd -u 1001 -g 1001 --create-home blaze
+
 RUN mkdir -p /app/data && chown 1001:1001 /app/data
 COPY target/blaze-0.25.0-standalone.jar /app/
+COPY --from=fhir-packages /root/.fhir/packages /home/blaze/.fhir/packages/
+RUN chown -R 1001:1001 /home/blaze/.fhir
 
 WORKDIR /app
 USER 1001
@@ -18,5 +28,8 @@ ENV STORAGE="standalone"
 ENV INDEX_DB_DIR="/app/data/index"
 ENV TRANSACTION_DB_DIR="/app/data/transaction"
 ENV RESOURCE_DB_DIR="/app/data/resource"
+ENV ADMIN_INDEX_DB_DIR="/app/data/admin-index"
+ENV ADMIN_TRANSACTION_DB_DIR="/app/data/admin-transaction"
+ENV ADMIN_RESOURCE_DB_DIR="/app/data/admin-resource"
 
 CMD ["java", "-jar",  "blaze-0.25.0-standalone.jar"]
diff --git a/docs/consistency/cassandra-init.cql b/docs/consistency/cassandra-init.cql
@@ -1,5 +1,3 @@
-ALTER KEYSPACE system_auth WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor' : 3};
-
 CREATE KEYSPACE blaze WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor' : 3};
 CREATE TABLE blaze.resources (hash text PRIMARY KEY, content blob);
 CREATE TABLE blaze.clauses ("token" text PRIMARY KEY, content blob);
diff --git a/docs/consistency/cassandra-init.sh b/docs/consistency/cassandra-init.sh
diff --git a/docs/consistency/credentials b/docs/consistency/credentials
@@ -0,0 +1 @@
+password