run-metadb.sh

#!/bin/bash

LOGGING_ENABLED=0
ORIGINAL_METADB_RUN_MODE=$METADB_RUN_MODE

# Determine if we're logging or not
# TODO: Add Log Rotation functionality
if [ ! -z "$LOG_FILE_PATH" ]; then
  if [ ! -f "$LOG_FILE_PATH" ]; then
    touch "$LOG_FILE_PATH"
    echo "[$(date +%F' '%T)] INFO: Initializing MetaDB Docker Edition." | tee -a "$LOG_FILE_PATH"
    if [ $? -ne 0 ]; then
      echo "[$(date +%F' '%T)] FATAL: Cannot write to log file at path ${LOG_FILE_PATH}!"
      exit 1
    fi
  else
    echo "" >> "$LOG_FILE_PATH"
    echo "[$(date +%F' '%T)] INFO: Initializing MetaDB Docker Edition." | tee -a "$LOG_FILE_PATH"
    if [ $? -ne 0 ]; then
      echo "[$(date +%F' '%T)] FATAL: Cannot write to log file at path ${LOG_FILE_PATH}!"
      exit 1
    fi
  fi

  LOGGING_ENABLED=1
fi

# Helper function for stdout and file logging
function log()
{
  STR="[$(date +%F' '%T)] $1"
  if [ $LOGGING_ENABLED -ne 0 ]; then
    echo $STR | tee -a "$LOG_FILE_PATH"
  else
    echo $STR
  fi
}

# Trap exit method in case of exit while running initial sync.
function clean_quit()
{
  log "INFO: Received exit signal, stopping MetaDB Instance."
  if [ -f "$DATA_DIR/metadb.pid" ]; then
    /usr/bin/metadb stop -D "$DATA_DIR"
  fi

  if [ ! -z "$TAIL_PID" ]; then
    kill $TAIL_PID
  fi
  exit 0
}
trap clean_quit SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM

#---------------
################
##Script Start##
################
#---------------
if [ "$VERBOSE_LOGGING" = "true" ]; then
  log "DEBUG: Testing if $DATA_DIR/metadb.conf exists"
fi

# Check if conf file exists, if not then create one with ENV variables
if [ ! -f "$DATA_DIR/metadb.conf" ]; then
  log "INFO: $DATA_DIR/metadb.conf does NOT exist, using BACKEND_ Envrionment Variables."
  mkdir -p "$DATA_DIR"
  touch "$DATA_DIR/metadb.conf"
  if [ $? -ne 0 ]; then
    log "ERROR: Could not write to $DATA_DIR/metadb.conf. Setting DATA_DIR to /etc/metadb."
    DATA_DIR="/etc/metadb"
    mkdir -p "$DATA_DIR"
    touch "$DATA_DIR/metadb.conf"
    if [ $? -ne 0 ]; then
      log "FATAL: Could not write to DATA_DIR nor /etc/metadb!"
      exit 1
    fi
  fi
  
  # Sanity check ENV variables
  if [ -z "$BACKEND_DB_HOST" ]; then
    log "FATAL: BACKEND_DB_HOST must be set, OR a valid metadb.conf must be mounted to ${DATA_DIR}/metadb.conf."
    exit 1
  fi
  if [ -z "$BACKEND_DB_PORT" ]; then
    log "WARN: BACKEND_DB_PORT is not set, defaulting to 5432."
    BACKEND_DB_PORT=5432
  fi
  if [ -z "$BACKEND_PG_DATABASE" ]; then
    log "WARN: BACKEND_PG_DATABASE is not set, defaulting to 'metadb'."
    BACKEND_PG_DATABASE="metadb"
  fi
  if [ -z "$BACKEND_PG_USER" ]; then
    log "WARN: BACKEND_PG_USER is not set, defaulting to 'metadb'."
    BACKEND_PG_USER="metadb"
  fi
  if [ -z "$BACKEND_PG_USER_PASSWORD" ]; then
    log "FATAL: BACKEND_PG_USER_PASSWORD must be set, OR a valid metadb.conf must be mounted to ${DATA_DIR}/metadb.conf."
    exit 1
  fi
  if [ -z "$BACKEND_PG_SUPERUSER" ] || [ -z "$BACKEND_PG_SUPERUSER_PASSWORD" ]; then
    log "WARN: BACKEND_PG_SUPERUSER and/or BACKEND_PG_SUPERUSER_PASSWORD is not set, defaulting to '${BACKEND_PG_USER}' user and its password."
    BACKEND_PG_SUPERUSER=$BACKEND_PG_USER
    BACKEND_PG_SUPERUSER_PASSWORD=$BACKEND_PG_USER_PASSWORD
  fi
  if [ -z "$BACKEND_PG_SSLMODE" ]; then
    log "WARN: BACKEND_PG_SSLMODE is not set, defaulting to 'prefer'."
    BACKEND_PG_SSLMODE="prefer"
  fi
  
  # Write out metadb.conf file
  if [ "$VERBOSE_LOGGING" = "true" ]; then
    log "DEBUG: Generated metadb.conf:"
    log "DEBUG: [main]"
    log "DEBUG: host = ${BACKEND_DB_HOST}"
    log "DEBUG: port = ${BACKEND_DB_PORT}"
    log "DEBUG: database = ${BACKEND_PG_DATABASE}"
    log "DEBUG: superuser = ${BACKEND_PG_SUPERUSER}"
    log "DEBUG: superuser_password = <redacted>"
    log "DEBUG: systemuser = ${BACKEND_PG_USER}"
    log "DEBUG: systemuser_password = <redacted>"
    log "DEBUG: sslmode = ${BACKEND_PG_SSLMODE}"
  fi
  
  echo "[main]
  host = $BACKEND_DB_HOST
  port = $BACKEND_DB_PORT
  database = $BACKEND_PG_DATABASE
  superuser = $BACKEND_PG_SUPERUSER
  superuser_password = $BACKEND_PG_SUPERUSER_PASSWORD
  systemuser = $BACKEND_PG_USER
  systemuser_password = $BACKEND_PG_USER_PASSWORD
  sslmode = $BACKEND_PG_SSLMODE" > "$DATA_DIR/metadb.conf"
  
  if [ $? -ne 0 ]; then
    log "FATAL: Failed to write autogenerated metadb.conf file to ${DATA_DIR}/metadb.conf!"
    exit 1
  fi
else
  if [ "$VERBOSE_LOGGING" = "true" ]; then
    log "DEBUG: Configuration detected at ${DATA_DIR}/metadb.conf. Running sanity checks."
  fi

  # Read in existing config and make sure all required options are present
  source <(grep = "$DATA_DIR/metadb.conf" | sed 's/ *= */=/g')

  BACKEND_DB_HOST=$host
  BACKEND_DB_PORT=$port
  BACKEND_PG_DATABASE=$database
  BACKEND_PG_SUPERUSER=$superuser
  BACKEND_PG_SUPERUSER_PASSWORD=$superuser_password
  BACKEND_PG_USER=$systemuser
  BACKEND_PG_USER_PASSWORD=$systemuser_password
  BACKEND_PG_SSLMODE=$sslmode

  if [ -z "$BACKEND_DB_HOST" ]; then
    log "FATAL: 'host' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
  if [ -z "$BACKEND_DB_PORT" ]; then
    log "FATAL: 'port' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
  if [ -z "$BACKEND_PG_DATABASE" ]; then
    log "FATAL: 'database' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
  if [ -z "$BACKEND_PG_SUPERUSER" ]; then
    log "FATAL: 'superuser' entry must be set in $DATA_DIR/metadb.conf! Hint: Try setting it to the same as 'systemuser'."
    exit 1
  fi
  if [ -z "$BACKEND_PG_SUPERUSER_PASSWORD" ]; then
    log "FATAL: 'superuser_password' entry must be set in $DATA_DIR/metadb.conf! Hint: Try setting it to the same as 'systemuser_password'."
    exit 1
  fi
  if [ -z "$BACKEND_PG_USER" ]; then
    log "FATAL: 'systemuser' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
  if [ -z "$BACKEND_PG_USER_PASSWORD" ]; then
    log "FATAL: 'systemuser_password' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
  if [ -z "$BACKEND_PG_SSLMODE" ]; then
    log "FATAL: 'sslmode' entry must be set in $DATA_DIR/metadb.conf!"
    exit 1
  fi
fi

# Check runtime variables
if [ -z "$METADB_PORT" ]; then
  log "WARN: METADB_PORT is not set, defaulting to '8550'."
  METADB_PORT=8550
fi
if [ -z "$MEM_LIMIT_GB" ]; then
  log "WARN: MEM_LIMIT_GB is not set, defaulting to '4'."
  MEM_LIMIT_GB=4
fi
if [ -z "$METADB_RUN_MODE" ]; then
  log "WARN: METADB_RUN_MODE is not set, defaulting to 'start'."
  METADB_RUN_MODE="start"
elif [ ! "$METADB_RUN_MODE" = "start" ] && [ ! "$METADB_RUN_MODE" = "sync" ] && [ ! "$METADB_RUN_MODE" = "endsync" ] && [ ! "$METADB_RUN_MODE" = "upgrade" ] && [ ! "$METADB_RUN_MODE" = "migrate" ]; then
  log "WARN: METADB_RUN_MODE is set to invalid option '$METADB_RUN_MODE'. Valid options are 'start', 'sync', 'endsync', 'upgrade', and 'migrate'. Defaulting to 'start'."
  METADB_RUN_MODE="start"
fi

# Clear old PID file (if exists)
if [ -f "$DATA_DIR/metadb.pid" ]; then
  if [ "$VERBOSE_LOGGING" = "true" ]; then
    log "DEBUG: ${DATA_DIR}/metadb.pid detected upon start. Deleting metadb.pid."
  fi

  rm -f "$DATA_DIR/metadb.pid"
fi

# Create Data Source Object if Initializing new MetaDB Instance
PGPASSWORD="$BACKEND_PG_USER_PASSWORD"
export PGPASSWORD

if [[ -z $(psql -h "$BACKEND_DB_HOST" -p $BACKEND_DB_PORT -d "$BACKEND_PG_DATABASE" -U "$BACKEND_PG_USER" -c "\dt metadb.init") ]]; then
  INIT_FLAG=1
else
  INIT_FLAG=0
fi

PGPASSWORD=""
export PGPASSWORD

if [ $INIT_FLAG -eq 1 ]; then
  log "INFO: Initializing MetaDB and attempting to initialize Kafka Connector."
  /usr/bin/metadb start -D "$DATA_DIR" -l "$DATA_DIR/metadb-init.log" --port $METADB_PORT --debug --memlimit $MEM_LIMIT_GB &
  sleep 5

  if [ ! -d /proc/$(cat "$DATA_DIR/metadb.pid") ]; then
    log "FATAL: metadb process failed to start!"
    exit 1
  fi
  
  if [ $LOGGING_ENABLED -ne 0 ]; then
    tail -f "$DATA_DIR/metadb-init.log" | tee -a "$LOG_FILE_PATH" &
    TAIL_PID=$!
  else
    tail -f "$DATA_DIR/metadb-init.log" &
    TAIL_PID=$!
  fi

  if [ -f "$SQL_INIT_SCRIPT_PATH" ]; then
    log "INFO: Running SQL Init Script at $SQL_INIT_SCRIPT_PATH"
    psql -X -h localhost -d metadb -p $METADB_PORT -f "$SQL_INIT_SCRIPT_PATH"
  fi

  log "INFO: Registering Kafka Connector at ${KAFKA_BROKERS} with topic(s) ${KAFKA_TOPICS}, and consumer group(s) ${KAFKA_CONSUMER_GROUP}."

  if [[ "$ADD_SCHEMA_PREFIX" == *"_" ]] && ! [[ "$FOLIO_TENANT_NAME" == *"_" ]]; then
    FOLIO_TENANT_NAME="${FOLIO_TENANT_NAME}_"
  fi
  if ! [[ "$ADD_SCHEMA_PREFIX" == *"_" ]] && [[ "$FOLIO_TENANT_NAME" == *"_" ]]; then
    ADD_SCHEMA_PREFIX="${ADD_SCHEMA_PREFIX}_"
  fi

  PSQL_LINE="CREATE DATA SOURCE sensor TYPE kafka OPTIONS (module 'folio'"
  if [ -z "$KAFKA_BROKERS" ]; then
    log "WARN: KAFKA_BROKERS is not set, defaulting to 'kafka:9092'."
    KAFKA_BROKERS="kafka:9092"
  fi
  PSQL_LINE=$PSQL_LINE", brokers '$KAFKA_BROKERS'"
  if [ -z "$KAFKA_TOPICS" ]; then
    log "WARN: KAFKA_TOPICS is not set, defaulting to '^metadb_sensor_1.'."
    KAFKA_TOPICS="^metadb_sensor_1."
  fi
  PSQL_LINE=$PSQL_LINE", topics '$KAFKA_TOPICS'"
  if [ -z "$KAFKA_CONSUMER_GROUP" ]; then
    log "WARN: KAFKA_CONSUMER_GROUP is not set, defaulting to 'metadb_sensor_1_1'."
    KAFKA_CONSUMER_GROUP="metadb_sensor_1_1"
  fi
  PSQL_LINE=$PSQL_LINE", consumer_group '$KAFKA_CONSUMER_GROUP'"
  if [ -z "$KAFKA_SECURITY" ]; then
    log "WARN: KAFKA_SECURITY is not set, defaulting to 'plaintext'."
    KAFKA_SECURITY="plaintext"
  fi
  if [ ! "$KAFKA_SECURITY" = "plaintext" ] && [ ! "$KAFKA_SECURITY" = "ssl" ]; then
    log "WARN: KAFKA_SECURITY is set to invalid value '$KAFKA_SECURITY'. Valid options are 'plaintext' and 'ssl'. Defaulting to 'plaintext'."
    KAFKA_SECURITY="plaintext"
  fi
  PSQL_LINE=$PSQL_LINE", security '$KAFKA_SECURITY'"
  if [ ! -z "$FOLIO_TENANT_NAME" ]; then
    PSQL_LINE=$PSQL_LINE", trim_schema_prefix '$FOLIO_TENANT_NAME'"
  fi
  if [ ! -z "$ADD_SCHEMA_PREFIX" ]; then
    PSQL_LINE=$PSQL_LINE", add_schema_prefix '$ADD_SCHEMA_PREFIX'"
  fi
  if [ ! -z "$SCHEMA_STOP_FILTER" ]; then
    PSQL_LINE=$PSQL_LINE", schema_stop_filter '$SCHEMA_STOP_FILTER'"
  fi
  PSQL_LINE=$PSQL_LINE");"

  log "INFO: Registering Kafka Connector with the following command: \"$PSQL_LINE\""

  psql -X -h localhost -d metadb -p $METADB_PORT -c "$PSQL_LINE"
  log "INFO: Running initial synchronization with Kafka Connect sensor (this will take awhile). Once the sync is complete (check number of rows in tables) then change METADB_RUN_MODE to 'endsync'."

  INIT_SYNC_FLAG=0
  while [ $INIT_SYNC_FLAG -le 0 ]
  do
    INIT_SYNC_FLAG=$(cat "$DATA_DIR/metadb-init.log" | grep "snapshot complete" | wc -l)
    sleep 1
  done

  log "INFO: Initial snapshot completed. Stopping MetaDB Server."

  /usr/bin/metadb stop -D "$DATA_DIR"
  kill $TAIL_PID
  TAIL_PID=''

  if [ "$SLEEP_AFTER_TASK" = "true" ]; then
    log "INFO: MetaDB Initial Sync Complete. Setting container to sleep to prevent task from unintentionally re-running because SLEEP_AFTER_TASK is set to 'true'. Change the METADB_RUN_MODE variable to 'start' to start MetaDB."
    sleep 999999
    exit 0
  else
    METADB_RUN_MODE="endsync"
  fi
fi

# Run MetaDB
if [ "$METADB_RUN_MODE" = "upgrade" ]; then
  EX_LINE="/usr/bin/metadb upgrade -D $DATA_DIR"

  log "INFO: Starting MetaDB Upgrade Task."
  if [ "$FORCE_RUN" = "true" ]; then
    EX_LINE=$EX_LINE" --force"
  fi

  if [ $LOGGING_ENABLED -ne 0 ]; then
    $EX_LINE 2>&1 | tee -a "$LOG_FILE_PATH"
  else
    $EX_LINE
  fi

  if [ ! -d /proc/$(cat "$DATA_DIR/metadb.pid") ]; then
    log "FATAL: metadb process failed to start!"
    exit 1
  fi

  if [ "$SLEEP_AFTER_TASK" = "true" ]; then
    log "INFO: MetaDB Upgrade Complete. Setting container to sleep to prevent task from unintentionally re-running because SLEEP_AFTER_TASK is set to 'true'. Change the METADB_RUN_MODE variable to 'start' to start MetaDB."
    sleep 999999
    exit 0
  else
    log "INFO: MetaDB Upgrade Complete. Running MetaDB with METADB_RUN_MODE variable set to 'start'. Recommended to change the METADB_RUN_MODE variable value to 'start' and restarting the container when convenient."
    METADB_RUN_MODE="start"
  fi
fi

if [ "$METADB_RUN_MODE" = "sync" ]; then
  EX_LINE="/usr/bin/metadb sync -D $DATA_DIR --source sensor"

  log "Starting MetaDB Sync Task (source: sensor)"
  if [ "$FORCE_RUN" = "true" ]; then
    EX_LINE=$EX_LINE" --force"
  fi

  if [ $LOGGING_ENABLED -ne 0 ]; then
    $EX_LINE 2>&1 | tee -a "$LOG_FILE_PATH"
  else
    $EX_LINE
  fi

  if [ ! -d /proc/$(cat "$DATA_DIR/metadb.pid") ]; then
    log "FATAL: metadb process failed to start!"
    exit 1
  fi

  if [ "$SLEEP_AFTER_TASK" = "true" ]; then
    log "INFO: MetaDB Sync Complete. Setting container to sleep to prevent task from unintentionally re-running because SLEEP_AFTER_TASK is set to 'true'. Change the METADB_RUN_MODE variable to 'start' to start MetaDB."
    sleep 999999
    exit 0
  else
    log "INFO: MetaDB pre-sync tasks complete. Running MetaDB with METADB_RUN_MODE variable set to 'start'."
    METADB_RUN_MODE="start"
  fi
fi

if [ "$METADB_RUN_MODE" = "endsync" ]; then
  EX_LINE="/usr/bin/metadb endsync -D $DATA_DIR --source sensor"

  log "INFO: Starting MetaDB Endsync Task (source: sensor)"
  if [ "$FORCE_RUN" = "true" ]; then
    EX_LINE=$EX_LINE" --force"
  fi

  if [ $LOGGING_ENABLED -ne 0 ]; then
    $EX_LINE 2>&1 | tee -a "$LOG_FILE_PATH"
  else
    $EX_LINE
  fi

  if [ ! -d /proc/$(cat "$DATA_DIR/metadb.pid") ]; then
    log "FATAL: metadb process failed to start!"
    exit 1
  fi

  if [ "$SLEEP_AFTER_TASK" = "true" ]; then
    log "INFO: MetaDB Endsync Complete. Setting container to sleep to prevent task from unintentionally re-running because SLEEP_AFTER_TASK is set to 'true'. Change the METADB_RUN_MODE variable to 'start' to start MetaDB."
    sleep 9999999
    exit 0
  else
    log "INFO: MetaDB Endsync Complete. Running MetaDB with METADB_RUN_MODE variable set to 'start'. Recommended to change the METADB_RUN_MODE variable value to 'start' and restarting the container when convenient."
    METADB_RUN_MODE="start"
  fi
fi

if [ "$METADB_RUN_MODE" = "migrate" ]; then
  EX_LINE="/usr/bin/metadb migrate -D $DATA_DIR --source sensor --ldpconf $LDP_CONF_FILE_PATH"

  if [ ! -f "$LDP_CONF_FILE_PATH" ]; then
    log "FATAL: METADB_RUN_MODE set to 'migrate' yet the LDP Configuration JSON File specified by LDP_CONF_FILE_PATH (value: ${LDP_CONF_FILE_PATH}) does not exist!"
    exit 1
  fi

  log "INFO: Starting MetaDB migration from LDP using configuration file ${LDP_CONF_FILE_PATH}."
  if [ $LOGGING_ENABLED -ne 0 ]; then
    $EX_LINE 2>&1 | tee -a "$LOG_FILE_PATH"
  else
    $EX_LINE
  fi

  if [ ! -d /proc/$(cat "$DATA_DIR/metadb.pid") ]; then
    log "FATAL: metadb process failed to start!"
    exit 1
  fi

  if [ "$SLEEP_AFTER_TASK" = "true" ]; then
    log "INFO: MetaDB migration from LDP complete. Setting container to sleep to prevent task from unintentionally re-running because SLEEP_AFTER_TASK is set to 'true'. Change the METADB_RUN_MODE variable to 'start' to start MetaDB."
    sleep 9999999
    exit 0
  else
    log "INFO: MetaDB migration from LDP complete. Running MetaDB with METADB_RUN_MODE variable set to 'start'. Recommended to change the METADB_RUN_MODE variable value to 'start' and restarting the container when convenient."
    METADB_RUN_MODE="start"
  fi
fi

if [ "$METADB_RUN_MODE" = "start" ]; then
  EX_LINE="/usr/bin/metadb start -D $DATA_DIR --port $METADB_PORT --memlimit $MEM_LIMIT_GB"

  log "INFO: Starting MetaDB Server in normal mode."
  if [ "$VERBOSE_LOGGING" = "true" ]; then
    EX_LINE=$EX_LINE" --debug"
  fi

  if [ $LOGGING_ENABLED -ne 0 ]; then
    exec $EX_LINE 2>&1 | tee -a "$LOG_FILE_PATH"
  else
    exec $EX_LINE
  fi
fi