create-batch

#!/usr/bin/env python

version = "18.04.17"

import sys, os, re
# Check necessary envvars
if 'CMSSW_VERSION' not in os.environ:
    print "ERROR: You have to set up CMSSW. Run cmsenv in your working directory."
    sys.exit()

import stat, shutil
from getopt import gnu_getopt as getopt
from math import ceil
from datetime import datetime
import imp
import cPickle

def usage():
    print sys.argv[0], " : create pbs/condor jobs"
    print " Usage: create-batch cmsRun --cfg CONFIG_cfg.py --jobName JOBNAME --fileList fileList.txt --maxFiles MAXFILE"
    print ""
    print "        create-batch bash YOUR_BASH_script.sh YOUR OTHER ARGUMENTS --jobName JOBNAME --nJobs NJOBS --transferFiles YOUR_OUTPUT_FILES"
    print ""
    print "  Mandatory options :"
    print "   --jobName  NAME                  Name of job"
    print "   --fileList DATA_FILES            File list text file"
    print "   --maxFiles N                     Maximum number of files per job"
    print "   --nJobs    N                     Number of Job sections"
    print "   --cfg      CONFIG_FILE_cfg.py    Configuration file"
    print "  Optional :"
    print "   --queue QUEUE_NAME               Set the batch queue name"
    print "   --secondFileList DATA_FILES      Secondary file list text file"
    print "   -n                               Do not submit jobs to batch"
    print "   --transferDest OUTPUT_LOCATION   OUTPUT DIRECTORY (/store will be assumed to SE)"
    print "   -G                               Disable Grid certificate"
    print "   --maxEvent N                     Maximum number of events per job (-1 by default)"
    print "   --transferFiles                  Additional files to transfer"
    print "   -T                               Automatically transfer new files by archiveing them"
    print "   --customise CUSTOMISE_cfg.py     Configuration file for customization"
    print "   --args                           general arguments"
    print "   --firstRun N                     For MC: run number"
    print "   -B                               Rebuild whole package before job starts"
    print "  Optional, condor-specific :"
    print "   --blacklist HOST1,HOST2,...      Remove specific hosts"
    print "   --whitelist HOST1,HOST2,...      Remove specific hosts"
    sys.exit()

import FWCore.ParameterSet.Config as cms
class TheSiteConfig:
    def __init__(self, **kwargs):
        hostName = os.environ["HOSTNAME"]
        userName = os.environ["USER"]

        self.site = ""
        self.scheduler = ""

        queue = None
        ## Load the site default configuration
        if hostName.startswith("lxplus") or hostName.endswith("cern.ch"):
            self.site = "CERN"
            self.scheduler = "LSB"
            self.xrdSrv = 'eoscms'
            self.xrdBase = '/eos/cms'
            self.localBase = '/eos/cms'
            queue = "8nh"
        elif 'sscc.uos' in hostName:
            self.site = "UOS"
            self.scheduler = "CONDOR"
            #self.xrdSrv = 'cmsxrootd.sscc.uos.ac.kr/'
            #self.xrdBase = '/'
            self.mkdirCmd = "hadoop fs -mkdir -p "
            #self.localBase = ''
        elif hostName.startswith('ccp'): ## CCP should be BEFORE KNU T2/T3
            self.site = "CCP"
            self.scheduler = "PBS"
            queue = "long"
        elif 'knu.ac.kr' in hostName: ## KNU T2/T3
            self.site = "KNU"
            self.scheduler = "PBS"
            self.xrdSrv = 'cluster142.knu.ac.kr'
            self.xrdBase = '/'
            self.localBase = '/pnfs/knu.ac.kr/data/cms'
            queue = "cms"
        elif hostName.endswith('sdfarm.kr'):
            self.site = "KISTI"
            self.scheduler = "CONDOR"
            self.xrdSrv = 'cms-xrdr.private.lo:2094/'
            self.xrdBase = '/xrd'
            self.localBase = '/xrootd_user/%s/xrootd' % userName
            self.mkdirCmd = "gfal-mkdir gsiftp://cms-se.sdfarm.kr//xrootd"
        else:
            print "!!! This site is not supported"
            sys.exit()

        if 'queue' in kwargs: queue = kwargs['queue']

        ## Determine transfer command, default transfer command by simple mv command
        ## NOTE: kwargs['dest'] must be given in this script. exception in this line means there's unexpected modification
        self.dest = kwargs['dest']
        if self.dest.startswith('/store/') and not 'sscc.uos' in hostName: self.dest = 'root://%s/%s%s' % (self.xrdSrv, self.xrdBase, self.dest)
        if self.dest.startswith('/hdfs/') and 'sscc.uos' in hostName:
            self.dest = self.dest.replace('/hdfs','')
            
        ## Make absolute path
        if '://' not in self.dest: self.dest = os.path.abspath(self.dest)
        self.transferCmd = 'xrdcp -d 1 -f' if self.dest.startswith('root://') else 'mv'
        # hadoop copy for uos
        if  'sscc.uos' in hostName:
            self.transferCmd = 'hadoop fs -put'
        
        ## Determine submit command
        if self.scheduler == "CONDOR":
            self.submitCmd = 'condor_submit submit.jds'
        else:
            self.submitCmd = ""
            if queue: self.submitCmd = ' -q %s ' % queue
            if   self.scheduler == "LSB"   : self.submitCmd = 'bsub ' + self.submitCmd
            elif self.scheduler == "PBS"   : self.submitCmd = 'qsub ' + self.submitCmd + '-N'

class TheJobConfig:
    def __init__(self, cmd, opts):
        self.cmd = cmd
        self.maxEvent = -1

        self.doSubmit = ('-n' not in opts) ## Submit jobs by default unless -n is given
        self.doGrid = ('-G' not in opts) ## Carry the grid certificate by default unless -G is given
        self.doArchive = ('-T' in opts)
        self.doRebuild = ('-b' in opts)

        self.blacklist = opts['--blacklist'].split(',') if '--blacklist' in opts else []
        self.whitelist = opts['--whitelist'].split(',') if '--whitelist' in opts else []
        self.outFileNames = opts['--transferFiles'].split(',') if '--transferFiles' in opts else []
        self.additional_options = opts['--args'] if '--args' in opts else ""

        ## Initialise configuration, set site-dependent default values
        kwargs = {}
        if '--queue' in opts: kwargs['queue'] = opts['--queue']

        if self.cmd.split()[0] != 'cmsRun':
            if '--nJobs' not in opts:
                print 'ERROR: --nJobs is mandatory option for custom shell command'
                usage()
            self.nSection = int(opts['--nJobs'])
            self.cmd = "%s %s $SECTION" % (self.cmd, self.additional_options)
        else:
            ## (pre) load cfg file to determine the cmssw job type
            if '--cfg' not in opts:
                print 'ERROR: --cfg is mandatory option'
                usage()

            self.cfgFileName = opts['--cfg']
            if not os.path.exists(self.cfgFileName):
                print "ERROR: Cannot find config file", self.cfgFileName
                sys.exit()

            cout = sys.stdout
            sys.stdout = open("/dev/null", "w")
            sys.argv=[]
            sys.argv.extend(self.cmd.split())
            sys.argv.extend(self.additional_options.split())
            self.process = imp.load_source("process", self.cfgFileName).process
            sys.stdout = cout
            source = self.process.source
            if source.type_() == "EmptySource":
                if '--maxEvent' not in opts or '--nJobs' not in opts:
                    print "ERROR: --maxEvent, --nJobs are necessary for the EmptySource"
                    sys.exit()
                self.maxEvent = int(opts['--maxEvent'])
                self.nSection = int(opts['--nJobs'])
                self.firstRun = int(opts['--firstRun']) if '--firstRun' in opts else 1
            elif source.type_() == "PoolSource":
                if '--maxEvent' in opts: self.maxEvent = int(opts['--maxEvent'])
                fileList = opts['--fileList']
                if not os.path.exists(fileList):
                    print "ERROR: Cannot find file list", fileList
                    sys.exit()

                ## Collect root files
                self.files = []
                for f in open(fileList).readlines():
                    f = f.strip().strip('\',"')
                    if len(f) < 5: continue
                    if '#' == f[0] or '.root' != f[-5:]: continue
                    self.files.append(f)
                nFiles = len(self.files)
                if nFiles == 0:
                    print "ERROR: Empty dataset."
                    sys.exit()
                ## Sort files by its key "i", filename_[i]_[j]_hash.root
                #files.sort(key=lambda f: int(f.split('/')[-1].split('_')[1]))

                ## Collect secondary input files
                self.secondFiles = []
                if '--secondFileList' in opts:
                    secondFileList = opts['--secondFileList']
                    for f in open(secondFileList).readlines():
                        f = f.strip().strip('\',"')
                        if len(f) < 5: continue
                        if '#' == f[0] or '.root' != f[-5:]: continue
                        self.secondFiles.append(f)

                if '--maxFiles' in opts:
                    self.maxFiles = int(opts['--maxFiles'])
                elif '--nJobs' in opts:
                    nJobs = int(opts['--nJobs'])
                    self.maxFiles = max(1, len(self.files)/nJobs)
                else:
                    print "ERROR: No maxFiles nor nJobs among the option"
                    sys.exit()

            ## Load customisation if given
            self.customise = None
            if '--customise' in opts:
                customiseFile = opts['--customise']
                if os.path.exists(customiseFile):
                    tmpObj = imp.load_source('customise', customiseFile)
                    if hasattr(tmpObj, 'customise'): self.customise = tmpObj.customise
                    else: print "Cannot find customise(process) function in the cfg ", customiseFile

        ## Override by options
        try:
            self.jobName = opts['--jobName']
            self.jobDir = os.path.abspath(self.jobName)
            self.cmsswBase = os.environ["CMSSW_BASE"]
            self.jobBase =  os.path.dirname(self.jobDir).replace(os.path.dirname(self.cmsswBase), '').strip('/')

            if os.path.isdir(self.jobDir):
                print "ERROR: Output directory already exists."
                sys.exit()
        except:
            print "!!! Error parsing options"
            usage()

        ## Default destination to the workdir
        kwargs['dest'] = opts['--transferDest'] if '--transferDest' in opts else self.jobDir
        if self.doArchive: self.outFileNames.append("result_%s.tgz" % self.jobName)

        self.config = TheSiteConfig(**kwargs)
        if self.config.dest == self.jobDir: self.config.transferCmd = ''

    def initialiseWorkspace(self):
        ## Prepare working directory
        print "@@ Preparing batch jobs in", self.jobName, "..."
        os.makedirs(self.jobDir)
        dest = self.config.dest
        if self.config.transferCmd == 'mv' and not os.path.exists(dest):
            print "@@ Preparing output scratch directory in", dest, "..."
            os.system("mkdir -p %s" % dest)
            os.system("chmod go+w %s" % dest)
            #os.makedirs(dest)

        elif 'xrdcp' in self.config.transferCmd:
            print "@@ Preparing output directory in", self.config.dest, "..."

            destStorage, destDir = self.config.dest[7:].split('/',1)
            if destStorage == self.config.xrdSrv and \
               self.config.site in ("KNU", "CERN"):
                if '/store/' in destDir: destDir = '/store/'+destDir.split('/store/', 1)[-1]
                if not os.path.exists("%s/%s" % (self.config.localBase, destDir)):
                    os.makedirs("%s/%s" % (self.config.localBase, destDir))
            elif hasattr(self.config, 'mkdirCmd'):
                if '/store/' in destDir: destDir = '/store/'+destDir.split('/store/', 1)[-1]
                print self.config.mkdirCmd+destDir
                os.system(self.config.mkdirCmd+destDir)

        if hasattr(self, 'process'):
            ## Load cfg file
            print "@@ Loading python cfg..."
            cout = sys.stdout
            sys.stdout = open("%s/log.txt" % self.jobDir, "w")
            sys.argv=[]
            sys.argv.extend(self.cmd.split())
            sys.argv.extend(self.additional_options.split())

            process = self.process
            if not hasattr(process, 'maxEvents'): process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(-1))
            if not hasattr(process.maxEvents, 'input'): process.maxEvents.input = cms.untracked.int32(-1)
            process.maxEvents.input = self.maxEvent
            ## Customise the cfg
            if self.customise != None: process = self.customise(process)
            sys.stdout = cout

            ## Memorise to modify output file names
            print "@@ Setting output modules..."
            outFileModes = {}
            if hasattr(process, 'TFileService'):
                outFileModes['TFileService'] = process.TFileService.fileName.value()

            for modName in process.outputModules_():
                outFileModes[modName] = getattr(process, modName).fileName.value()

            for f in outFileModes.values():
                f = re.sub(r'^file:', '', f)
                if self.config.site in ("UOS"):
                    self.outFileNames.append(f)
                else :
                    self.outFileNames.append(f)


            pklFileName = "%s/job_cfg.pkl" % (self.jobDir)
            cPickle.dump(process, open(pklFileName, "w"))

            ## Split files into jobs and write python cfg
            print "@@ Splitting jobs..."
            if hasattr(self, 'files'):
                nFiles = len(self.files)
                self.nSection = int(ceil(1.0*nFiles/self.maxFiles))
                for section in range(self.nSection):
                    begin = section*self.maxFiles
                    end = min(begin+self.maxFiles, nFiles)

                    cfgFileName = "%s/job_%03d_cfg.py" % (self.jobDir, section)
                    cfgOut = open(cfgFileName, "w")
                    print>>cfgOut, """#!/usr/bin/env python
import cPickle
process = cPickle.load(open("job_cfg.pkl")) """

                    print>>cfgOut, """process.source.fileNames = %s """ % (self.files[begin:end])
                    ## Add secondary files if requested
                    if len(self.secondFiles) > 0:
                        print>>cfgOut, """process.source.secondaryFileNames = cms.untracked.vstring(%s)""" % (self.secondFiles[:])
                    #process.source.fileNames = self.files[begin:end]
                    #for modName in outFileModes:
                    #    getattr(process, modName).fileName = "%s_%03d.root" % (outFileModes[modName][:-5], section)

            else:
                for section in range(self.nSection):
                    cfgFileName = "%s/job_%03d_cfg.py" % (self.jobDir, section)
                    cfgOut = open(cfgFileName, "w")
                    print>>cfgOut, """#!/usr/bin/env python
import FWCore.ParameterSet.Config as cms
import cPickle
process = cPickle.load(open("job_cfg.pkl")) """

                    print>>cfgOut, """process.source.firstRun = cms.untracked.uint32(%s)""" % (self.firstRun)
                    print>>cfgOut, """process.source.firstLuminosityBlock = cms.untracked.uint32(%s)""" % (section+1)

                    for shift, p in enumerate(process.RandomNumberGeneratorService.parameterNames_()):
                        print>>cfgOut, """process.RandomNumberGeneratorService.%s.initialSeed = %s""" % (p,section+1000+shift+1000*self.firstRun)

        ## Make scripts
        if   self.config.scheduler == "LSB": self.makeLSBJob()
        elif self.config.scheduler == "PBS": self.makePBSJob()
        elif self.config.scheduler == "CONDOR": self.makeCondorJob()

    def makeLSBJob(self):

        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ $# != 1 ]; then
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi
SECTION=${{@: -1}}
FSECTION=`printf %03d $SECTION`

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}
fi
source $CMS_PATH/cmsset_default.sh

tar xzf {0}/job.tar.gz
cd {1}
scram build ProjectRename
eval `scram runtime -sh`
cd $CMSSW_BASE/src
""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'])
        if self.doRebuild:
            print>>fout, """
scram build clean
scram build vclean"""
        print>>fout, """
scram build -j
cd -
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
echo BEGIN `date` {2} {1} >> {0}/submit.log
echo {2} {1}
touch ___started___job___
time {2} {1}
EXITCODE=$?
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` {2} {1} >> {0}/submit.log
""".format(self.jobDir, self.additional_options, self.cmd)
        if self.doArchive:
            print>>fout, """
    tar -czf ../result_{0}.tgz -N "`date -r ___started___job___`" .
    mv ../result_{0}.tgz ./
    readlink -f result_{0}.tgz
""".format(self.jobName)
        print>>fout, """
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` {2} {1} >> {0}/submit.log
    exit 1
fi""".format(self.jobDir, self.additional_options, self.cmd)
        print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=(%s)
    DESTDIR=%s
    echo ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        echo ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        echo ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
done""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        for section in range(self.nSection):
            print>>fout, "{0} -J {1}.{2} -oo {3}/job_{2:03d}.log run_{4}.sh {2}".format(self.config.submitCmd, self.jobName, section, self.jobDir, self.jobName.replace('/','_'))
        fout = None
        os.chmod(submitFileName, 0755)

    def makePBSJob(self):
        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ _$PBS_ARRAYID != '_' ]; then
    FSECTION=`printf %03d $PBS_ARRAYID`
    SECTION=$PBS_ARRAYID
elif [ _$SECTION != '_' ]; then
    exprt FSECTION=`printf %03d $SECTION`
else
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}
fi
source $CMS_PATH/cmsset_default.sh

hostname
whoami
## Prepare workdirectory
mkdir -p /tmp/${{USER}}/PBS_${{PBS_JOBID}}
cd /tmp/${{USER}}/PBS_${{PBS_JOBID}}
tar xzf {0}/job.tar.gz
cd {1}
scram build ProjectRename
eval `scram runtime -sh`
cd $CMSSW_BASE/src
""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'])
        if self.doRebuild:
            print>>fout, """
scram build clean
scram build vclean"""
        print>>fout, """
scram build -j
cd -
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
echo BEGIN `date` {2} {1} >> {0}/submit.log
echo {2} {1}
touch ___started___job___
time {2} {1}
EXITCODE=$?
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` {2} {1} >> {0}/submit.log
""".format(self.jobDir, self.additional_options, self.cmd)
        if self.doArchive:
            print>>fout, """
    tar -czf ../result_{0}.tgz -N "`date -r ___started___job___`" .
    mv ../result_{0}.tgz ./
    readlink -f result_{0}.tgz
""".format(self.jobName)
        print>>fout, """
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` {2} {1} >> {0}/submit.log
    rm -rf /tmp/${{USER}}/PBS_${{PBS_JOBID}}
    exit 1
fi""".format(self.jobDir, self.additional_options, self.cmd)
        print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=(%s)
    DESTDIR=%s
    ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
done

## Clean up current workarea
rm -rf /tmp/${USER}/PBS_${PBS_JOBID}""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        for section in range(self.nSection):
            print>>fout, "{0} {1}_{2} run_{3}.sh -vSECTION={2}".format(self.config.submitCmd, self.jobName, section, self.jobName.replace('/','_'))
        fout = None
        os.chmod(submitFileName, 0755)

    def makeCondorJob(self):
        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ $# != 1 ]; then
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi
SECTION=${{@: -1}}
FSECTION=`printf %03d $SECTION`

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}
fi
source $CMS_PATH/cmsset_default.sh

hostname
whoami
tar xzf job.tar.gz
cd {1}
cd -
cd {3}/src
scram build ProjectRename
eval `scram runtime -sh`
""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'], os.environ['CMSSW_VERSION'])
        if self.doRebuild:
            print>>fout, """
scram build clean
scram build vclean"""
        print>>fout, """
scram build -j
cd -
cd {1}
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'],self.additional_options)
        print>>fout,"""
echo BEGIN `date` {2} {1} #>> {0}/submit.log
echo {2} {1}
touch ___started___job___
time {2} {1}
EXITCODE=$?
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` {2} {1} #>> {0}/submit.log
""".format(self.jobDir, self.additional_options, self.cmd)
        if self.doArchive:
            print>>fout, """
    tar -czf ../result_{0}.tgz -N "`date -r ___started___job___`" .
    mv ../result_{0}.tgz ./
    readlink -f result_{0}.tgz
""".format(self.jobName)
        print>>fout,"""
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` {2} {1} #>> {0}/submit.log
    exit 1
fi""".format(self.jobDir, self.additional_options, self.cmd)
        if self.config.transferCmd != '':
            print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=(%s)
    DESTDIR=%s
    ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        echo "trying: " ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        ${TRANSFERCMD[@]} ${FILE} ${DESTDIR}/${PREFIX}_${FSECTION}.${EXT}
    fi
done""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` # >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        print>>fout, self.config.submitCmd
        fout = None
        os.chmod(submitFileName, 0755)

        # Condor jobs
        jdlOut = open("%s/submit.jds" % self.jobDir, "w")
        print>>jdlOut, "# Job description file for condor job", self.jobName
        print>>jdlOut, """executable = run_{0}.sh
universe   = vanilla
arguments  = $(Process)
JobBatchName = {0}

log = condor.log

getenv     = True
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
output = job_$(Process).log
error = job_$(Process).err
transfer_input_files = job.tar.gz""".format(self.jobName.replace('/','_'))
        if self.config.transferCmd == '':
            print>>jdlOut, "transfer_output_files = ", (",".join([os.path.join(self.jobBase, x) for x in self.outFileNames]))
            remapStrs = ["{0}.{1}={0}_$(Process).{1}".format('.'.join(x.split('.')[:-1]), x.split('.')[-1]) for x in self.outFileNames]
            print>>jdlOut, 'transfer_output_remaps = "%s"' % (';'.join(remapStrs))

        requirements = []

        ## Special care for the Singularity environment at KISTI
        if self.config.site == "KISTI":
            osgVersion = "osgvo-el%s" % os.environ["SCRAM_ARCH"].split('_',1)[0][-1]
            print>>jdlOut, '''
accounting_group=group_cms
+Tag = "create-batch v%s"
+SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/%s:latest"'''% (version, osgVersion)
            requirements.append("HasSingularity == true")

        if len(self.blacklist) > 0:
            requirements.extend(["Machine != \""+x+"\"" for x in self.blacklist])
        if len(self.whitelist) > 0:
            white_requirements = ["Machine == \""+x+"\"" for x in self.whitelist]
            requirements.append("(%s)" % (" || ".join(white_requirements)))
        if len(requirements) > 0:
            req = "requirements = ", (" && ".join(requirements))
            print>>jdlOut, " ".join(req)
        print>>jdlOut, "queue",  self.nSection
        jdlOut = None

    def archive(self):
        ## Archive libraries and other stuffs
        tmpFile = open("%s/.create-batch" % self.jobDir, "w")
        tmpFile.close()

        print "@@ Archive files for job submission..."
        exclCommon = "--exclude tmp --exclude 'job.tar' --exclude 'job.tar.gz' --exclude-vcs --exclude-tag-all=.requestcache "
        os.system("tar cf {0}/job.tar {1} --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude-tag-all=.create-batch "
                 + " --exclude={0}/src/*/*/data --exclude=*.root".format(self.cmsswBase))
        os.system("tar rf {0}/job.tar {1}/src/*/*/data --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude-tag-all=.create-batch ")
        os.system("tar rf {0}/job.tar {1}/src/*/*/python {1}/src/*/*/interface {1}/src/*/*/src {1}/src/*/*/plugins {1}/src/*/*/BuildFile* --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude-tag-all=.create-batch ")
        if self.cmsswBase in self.jobDir:
            os.system("tar rf {0}/job.tar {0} --xform='s:{1}:{2}:' --xform='s:{3}::' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase), os.path.basename(self.jobDir))
                      + exclCommon + " --exclude=*.root")
        else:
            os.system("tar rf {0}/job.tar {0} --xform='s:{1}:{2}:' --xform='s:{3}:{4}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase), self.jobDir, self.jobBase)
                      + exclCommon + " --exclude=*.root")

        
        ## Checking voms proxy
        if self.doGrid:
            print "@@ Checking grid certificate to access files..."
            if os.system("voms-proxy-info -exists --valid 8:00") != 0:
                os.system("voms-proxy-init -voms cms --valid 144:00")
            uid = os.getuid()
            os.system("tar rf {0}/job.tar {1} --xform='s:{1}:{2}/proxy.x509:' -P ".format(self.jobDir, "/tmp/x509up_u%d" % uid, os.path.basename(self.cmsswBase)))

        os.system("gzip %s/job.tar" % self.jobDir)
        #os.system("rm -f %s/job_*_cfg.py" % self.jobDir)

    def submit(self):
        if self.doSubmit:
            print "@@ Submitting jobs..."
            if self.config.site == "UOS":
              if os.environ['HOSTNAME'].startswith("gate.") :
                os.system("cd %s;./submit.sh; cd -" % self.jobDir)
              else :
                os.system("ssh gate 'cd %s;./submit.sh'" % self.jobDir)
            elif self.config.site == "CERN" and not os.environ['HOSTNAME'].startswith("lxplus"):
                os.system("ssh lxplus.cern.ch 'cd %s;./submit.sh'" % self.jobDir)
            else:
                os.system("cd %s;./submit.sh" % self.jobDir)
        else:
            print "@@ Jobs are prepared. You can submit jobs with following command:"
            print "cd %s;./submit.sh" % (self.jobDir)

if __name__ == '__main__':
    # Parse arguments
    if len(sys.argv) < 2: usage()
    try:
        opts, args = getopt(sys.argv[1:], 'nGT', ['jobName=', 'fileList=', 'maxFiles=', 'nJobs=', 'cfg=',
                                                 'maxEvent=', 'queue=', 'transferDest=', 'transferFiles=',
                                                 'args=', 'secondFileList=', 'customise=', 'firstRun=',
                                                 'blacklist=','whitelist='])
        opts = dict(opts)
    except:
        print "!!! Error parsing arguments"
        usage()

    ## Create job configure
    if len(args) == 0 or (args[0] == 'cmsRun' and len(args) == 1):
        args = ['cmsRun', 'job_${FSECTION}_cfg.py']
    jobConfig = TheJobConfig(' '.join(args), opts)
    jobConfig.initialiseWorkspace()
    jobConfig.archive()
    jobConfig.submit()

    print "@@ Done."