run_outlier_finder.sh

#!/bin/bash -e

# NB: -e makes script to fail if internal script fails (for example when --run is enabled)

#######################################
##         CHECK ARGS
#######################################
NARGS="$#"
echo "INFO: NARGS= $NARGS"

if [ "$NARGS" -lt 1 ]; then
	echo "ERROR: Invalid number of arguments...see script usage!"
  echo ""
	echo "**************************"
  echo "***     USAGE          ***"
	echo "**************************"
 	echo "$0 [ARGS]"
	echo ""
	echo "=========================="
	echo "==    ARGUMENT LIST     =="
	echo "=========================="
	echo "*** MANDATORY ARGS ***"
	echo "--inputfile=[FILENAME] - Input file name (.json) containing images to be processed."
	echo ""

	echo "*** OPTIONAL ARGS ***"
	echo "=== INPUT OPTIONS ==="
	echo "--datalist-key=[KEY] - Dictionary key name to be read in input datalist. Default: data"
	echo "--selcols=[COLS] - Data column ids to be selected from input data, separated by commas"
	
	echo "=== IFOREST OPTIONS ==="
	echo "--predict - Predict data encoding using input IsolationForest model. Default: False"
	echo "--model=[MODEL] - IsolationForest model filename (.h5) when using predict mode"
	echo "--nestimators=[NESTIMATORS] - Number of forest trees to fit. Default: 100"
	echo "--max-features=[MAX_FEATURES] - Number of max features used in each forest tree. Default: 1"
	echo "--max-samples=[MAX_SAMPLES] - Number of max samples used in each forest tree. -1 means auto options, e.g. 256 entries, otherwise it is the fraction of total available entries. Default: -1"
	echo "--contamination=[CONTAMINATION] - Fraction of outliers expected [0,0.5]. If None set it to auto. Default: None"
	echo "--anomaly-thr=[ANOMALY_THR] - Threshold in anomaly score above which observation is set as outlier. Default: 0.9"
	echo "--run-scan - Run parameter optimization scan before run. Default: false"
	echo "--scan-nestimators - Scan n_estimators parameter. Default: false"
	echo "--scan-maxfeatures - Scan max_features parameter . Default: false"
	echo "--scan-maxsamples - Scan max_samples parameter. Default: false"
	echo "--scan-contamination - Scan contamination parameter. Default: false"
	echo "--random-state=[RND] - Model random state . Default: None"	
	echo ""
	
	echo "=== DATA PRE-PROCESSING OPTIONS ==="
	echo "--normalize - Apply minmax normalization to images "
	echo "--scalerfile=[SCALER_FILE] - Load and use data transform stored in this file (.sav)."
	echo "--classid-label-map=[DICT] - Class ID label dictionary. Will take labels from input dictionary label field if left empty. Default: empty"
	#echo "--objids-excluded-in-train=[OBJIDS] - Source ids not included for training as considered unknown classes. Default: -1,0"		
	echo ""
	
	echo "=== SAVE OPTIONS ==="
	echo "--no-save-ascii - Disable save output to ascii format "
	echo "--no-save-json - Disable save output to json format  "
	echo "--no-save-model - Disable save model to file "
	echo "--no-save-features - Disable save features to file "
	echo "--outfile=[FILENAME] - Name of output file. Default: outlier_data.dat"
	echo "--outfile-json=[FILENAME] - Name of output file in json format. Default: outlier_data.json"
	echo "--save-labels-in-ascii - Save class labels to ascii. Default: False (save class ids) "
	
	echo "=== RUN OPTIONS ==="
	echo "--run - Run the generated run script on the local shell. If disabled only run script will be generated for later run."	
	echo "--scriptdir=[SCRIPT_DIR] - Job directory where to find scripts (default=/usr/bin)"
	echo "--modeldir=[MODEL_DIR] - Job directory where to find model & weight files (default=/opt/models)"
	echo "--jobdir=[JOB_DIR] - Job directory where to run (default=pwd)"
	echo "--outdir=[OUTPUT_DIR] - Output directory where to put run output file (default=pwd)"
	echo "--waitcopy - Wait a bit after copying output files to output dir (default=no)"
	echo "--copywaittime=[COPY_WAIT_TIME] - Time to wait after copying output files (default=30)"
	echo "--no-logredir - Do not redirect logs to output file in script "	
	echo "=========================="
  exit 1
fi


#######################################
##         PARSE ARGS
#######################################
# - Run options
JOB_DIR=""
JOB_OUTDIR=""
SCRIPT_DIR="/usr/bin"
RUN_SCRIPT=false
WAIT_COPY=false
COPY_WAIT_TIME=30
REDIRECT_LOGS=true

# - Input options
DATALIST=""
DATALIST_GIVEN=false
DATALIST_KEY="data"
SELCOLS=""

# - IsolationForest options
PREDICT=""
MODEL=""
NESTIMATORS=100
MAX_FEATURES=1
MAX_SAMPLES=-1
CONTAMINATION=""
ANOMALY_THR=0.9
RUN_SCAN=""
SCAN_NESTIMATORS=""
SCAN_MAXFEATURES=""
SCAN_MAXSAMPLES=""
SCAN_CONTAMINATION=""
RANDOM_STATE_OPT=""

# - Data pre-processing options
NORMALIZE=""
SCALERFILE=""
CLASSID_LABEL_MAP=""
#OBJS_EXCLUDED_IN_TRAIN="0,1"

# - Save options
OUTFILE="outlier_data.dat"
OUTFILE_JSON="outlier_data.json"
NO_SAVE_ASCII=""
NO_SAVE_JSON=""
NO_SAVE_MODEL=""
NO_SAVE_FEATURES=""
SAVE_LABELS=""

for item in "$@"
do
	case $item in 
		# **************************
		# **   MANDATORY
		# **************************
		# - INPUT OPTIONS 	
    --inputfile=*)
    	DATALIST=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`		
			if [ "$DATALIST" != "" ]; then
				DATALIST_GIVEN=true
			fi
    ;;
    --datalist-key=*)
    	DATALIST_KEY=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --selcols=*)
    	SELCOLS=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    
    # **************************
		# **   OPTIONAL OPTIONS 
		# **************************
		# - IsolationForest options
		--predict*)
    	PREDICT="--predict"
    ;;
		--model=*)
    	MODEL=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --nestimators=*)
    	NESTIMATORS=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --max-features=*)
    	MAX_FEATURES=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --max-samples=*)
    	MAX_SAMPLES=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --contamination=*)
    	CONTAMINATION=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --anomaly-thr=*)
    	ANOMALY_THR=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --run-scan*)
    	RUN_SCAN="--run_scan"
    ;;
    --scan-nestimators*)
    	SCAN_NESTIMATORS="--scan_nestimators"
    ;;
    --scan-maxfeatures*)
    	SCAN_MAXFEATURES="--scan_maxfeatures"
    ;;
    --scan-maxsamples*)
    	SCAN_MAXSAMPLES="--scan_maxsamples"
    ;;
    --scan-contamination*)
    	SCAN_CONTAMINATION="--scan_contamination"
    ;;
    --random-state=*)
    	RANDOM_STATE=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    	RANDOM_STATE_OPT="--random_state=$RANDOM_STATE "
    ;;
    
		# - PREPROC OPTIONS
		--normalize*)
    	NORMALIZE="--normalize"
    ;;
    --scalerfile=*)
    	SCALERFILE=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --classid-label-map=*)
    	CLASSID_LABEL_MAP=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
    ;;
#    --objids-excluded-in-train=*)
#    	OBJS_EXCLUDED_IN_TRAIN=`echo $item | sed 's/[-a-zA-Z0-9]*=//'`
#    ;;
    
    # - SAVE OPTIONS
    --no-save-ascii*)
    	NO_SAVE_ASCII="--no_save_ascii"
    ;;
    --no-save-json*)
    	NO_SAVE_JSON="--no_save_json"
    ;;
    --no-save-model*)
    	NO_SAVE_MODEL="--no_save_model"
    ;;
    --no-save-features*)
    	NO_SAVE_FEATURES="--no_save_features"
    ;;
    --outfile=*)
    	OUTFILE=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --outfile-json=*)
    	OUTFILE_JSON=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --save-labels-in-ascii*)
    	SAVE_LABELS="--save_labels_in_ascii" 
    ;;
    
	
		# - RUN OPTIONS
    --run*)
    	RUN_SCRIPT=true
    ;;
    --scriptdir=*)
    	SCRIPT_DIR=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    
    --outdir=*)
    	JOB_OUTDIR=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
		--waitcopy*)
    	WAIT_COPY=true
    ;;
		--copywaittime=*)
    	COPY_WAIT_TIME=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --jobdir=*)
    	JOB_DIR=`echo $item | /bin/sed 's/[-a-zA-Z0-9]*=//'`
    ;;
    --no-logredir*)
			REDIRECT_LOGS=false
		;;
    
    *)
    # Unknown option
    echo "ERROR: Unknown option ($item)...exit!"
    exit 1
    ;;
	esac
done


## Check arguments parsed
if [ "$DATALIST_GIVEN" = false ]; then
  echo "ERROR: Missing or empty DATALIST args (hint: you must specify it)!"
  exit 1
fi

if [ "$JOB_DIR" = "" ]; then
  echo "WARN: Empty JOB_DIR given, setting it to pwd ($PWD) ..."
	JOB_DIR="$PWD"
fi

if [ "$JOB_OUTDIR" = "" ]; then
  echo "WARN: Empty JOB_OUTDIR given, setting it to pwd ($PWD) ..."
	JOB_OUTDIR="$PWD"
fi

#######################################
##   SET OPTIONS
#######################################
INPUT_OPTS="--inputfile=$DATALIST --datalist_key=$DATALIST_KEY --selcols=$SELCOLS "
#PREPROC_OPTS="$NORMALIZE --scalerfile=$SCALERFILE --classid_label_map=$CLASSID_LABEL_MAP --objids_excluded_in_train=$OBJS_EXCLUDED_IN_TRAIN "
PREPROC_OPTS="$NORMALIZE --scalerfile=$SCALERFILE --classid_label_map=$CLASSID_LABEL_MAP "
IFOREST_OPTS="--modelfile=$MODEL $PREDICT --n_estimators=$NESTIMATORS --max_features=$MAX_FEATURES --max_samples=$MAX_SAMPLES --contamination=$CONTAMINATION --anomaly_thr=$ANOMALY_THR $RUN_SCAN $SCAN_NESTIMATORS $SCAN_MAXFEATURES $SCAN_MAXSAMPLES $SCAN_CONTAMINATION $RANDOM_STATE_OPT "
SAVE_OPTS="--outfile=$OUTFILE --outfile_json=$OUTFILE_JSON $SAVE_LABELS $NO_SAVE_ASCII $NO_SAVE_JSON $NO_SAVE_MODEL $NO_SAVE_FEATURES "

#######################################
##   DEFINE GENERATE EXE SCRIPT FCN
#######################################
# - Set shfile
shfile="run_outlier_finder.sh"

generate_exec_script(){

	local shfile=$1
	
	
	echo "INFO: Creating sh file $shfile ..."
	( 
			echo "#!/bin/bash -e"
			
      echo " "
      echo " "

      echo 'echo "*************************************************"'
      echo 'echo "****         PREPARE JOB                     ****"'
      echo 'echo "*************************************************"'

      echo " "
       
      echo "echo \"INFO: Entering job dir $JOB_DIR ...\""
      echo "cd $JOB_DIR"

			echo " "

      echo 'echo "*************************************************"'
      echo 'echo "****         RUN CLASSIFIER                  ****"'
      echo 'echo "*************************************************"'
				
			EXE="python $SCRIPT_DIR/find_outliers.py" 
			ARGS="$INPUT_OPTS $PREPROC_OPTS $IFOREST_OPTS $SAVE_OPTS "
			CMD="$EXE $ARGS"

			echo "date"
			echo ""
		
			echo "echo \"INFO: Running outlier search ...\""
			
			if [ $REDIRECT_LOGS = true ]; then			
      	echo "$CMD >> $logfile 2>&1"
			else
				echo "$CMD"
      fi
      
			echo " "

			echo 'JOB_STATUS=$?'
			echo 'echo "Outlier search run terminated with status=$JOB_STATUS"'

			echo "date"

			echo " "

      echo 'echo "*************************************************"'
      echo 'echo "****         COPY DATA TO OUTDIR             ****"'
      echo 'echo "*************************************************"'
      echo 'echo ""'
			
			if [ "$JOB_DIR" != "$JOB_OUTDIR" ]; then
				echo "echo \"INFO: Copying job outputs in $JOB_OUTDIR ...\""
				echo "ls -ltr $JOB_DIR"
				echo " "

				echo "# - Copy output data"
				echo 'tab_count=`ls -1 *.dat 2>/dev/null | wc -l`'
				echo 'if [ $tab_count != 0 ] ; then'
				echo "  echo \"INFO: Copying output table file(s) to $JOB_OUTDIR ...\""
				echo "  cp *.dat $JOB_OUTDIR"
				echo "fi"

				echo " "
				
				echo 'tab_count=`ls -1 *.json 2>/dev/null | wc -l`'
				echo 'if [ $tab_count != 0 ] ; then'
				echo "  echo \"INFO: Copying output json file(s) to $JOB_OUTDIR ...\""
				echo "  cp *.json $JOB_OUTDIR"
				echo "fi"
				
				echo " "
		
				echo "# - Show output directory"
				echo "echo \"INFO: Show files in $JOB_OUTDIR ...\""
				echo "ls -ltr $JOB_OUTDIR"

				echo " "

				echo "# - Wait a bit after copying data"
				echo "#   NB: Needed if using rclone inside a container, otherwise nothing is copied"
				if [ $WAIT_COPY = true ]; then
           echo "sleep $COPY_WAIT_TIME"
        fi
	
			fi

      echo " "
      echo " "
      
      echo 'echo "*** END RUN ***"'

			echo 'exit $JOB_STATUS'

 	) > $shfile

	chmod +x $shfile
}
## close function generate_exec_script()

###############################
##    RUN OUTLIER FINDER
###############################
# - Check if job directory exists
if [ ! -d "$JOB_DIR" ] ; then 
  echo "INFO: Job dir $JOB_DIR not existing, creating it now ..."
	mkdir -p "$JOB_DIR" 
fi

# - Moving to job directory
echo "INFO: Moving to job directory $JOB_DIR ..."
cd $JOB_DIR

# - Generate run script
echo "INFO: Creating run script file $shfile ..."
generate_exec_script "$shfile"

# - Launch run script
if [ "$RUN_SCRIPT" = true ] ; then
	echo "INFO: Running script $shfile to local shell system ..."
	$JOB_DIR/$shfile
fi


echo "*** END SUBMISSION ***"