create-batch

#!/usr/bin/env python

import sys, os, re
# Check necessary envvars
if 'CMSSW_VERSION' not in os.environ:
    print "ERROR: You have to set up CMSSW. Run cmsenv in your working directory."
    sys.exit()

import stat, shutil
from getopt import gnu_getopt as getopt
from math import ceil
from datetime import datetime
import imp
import cPickle

def usage():
    print sys.argv[0], " : create pbs jobs"
    print "  Mandatory options :"
    print "   --jobName  NAME                  Name of job"
    print "   --fileList DATA_FILES            File list text file"
    print "   --maxFiles N                     Maximum number of files per job"
    print "   --nJobs    N                     Number of Job sections"
    print "   --cfg      CONFIG_FILE_cfg.py    Configuration file"
    print "  Optional :"
    print "   --queue QUEUE_NAME               Set the batch queue name"
    print "   --secondFileList DATA_FILES      Secondary file list text file"
    print "   -n                               Do not submit jobs to batch"
    print "   --transferDest OUTPUT_LOCATION   OUTPUT DIRECTORY (/store will be assumed to SE)"
    print "   -G                               Disable Grid certificate"
    print "   --maxEvent N                     Maximum number of events per job (-1 by default)"
    print "   --transferFiles                  Additional files to transfer"
    print "   --customise CUSTOMISE_cfg.py     Configuration file for customization"
    print "   --args                           general arguments"
    print "   --firstRun N                     For MC: run number"
    print "  Optional, condor-specific :"
    print "   --blacklist HOST1,HOST2,...      Remove specific hosts"
    print "   --whitelist HOST1,HOST2,...      Remove specific hosts"
    sys.exit()

# Parse arguments
if len(sys.argv) < 2: usage()
try:
    opts, args = getopt(sys.argv[1:], 'ng', ['jobName=', 'fileList=', 'maxFiles=', 'nJobs=', 'cfg=',
                                             'maxEvent=', 'queue=', 'transferDest=', 'transferFiles=',
                                             'args=', 'secondFileList=', 'customise=', 'firstRun=',
                                             'blacklist=','whitelist='])
    opts = dict(opts)
except:
    print "!!! Error parsing arguments"
    usage()

import FWCore.ParameterSet.Config as cms
class TheSiteConfig:
    def __init__(self, **kwargs):
        hostName = os.environ["HOSTNAME"]
        userName = os.environ["USER"]

        self.site = ""
        self.scheduler = ""

        queue = None
        ## Load the site default configuration
        if hostName.startswith("lxplus") or hostName.endswith("cern.ch"):
            self.site = "CERN"
            self.scheduler = "LSB"
            self.xrdSrv = 'eoscms'
            self.xrdBase = '/eos/cms'
            self.localBase = '/eos/cms'
            queue = "8nh"
        elif 'sscc.uos' in hostName:
            self.site = "UOS"
            self.scheduler = "CONDOR"
            #queue = "batch"
        elif hostName.startswith('ccp'): ## CCP should be BEFORE KNU T2/T3
            self.site = "CCP"
            self.scheduler = "PBS"
            queue = "long"
        elif 'knu.ac.kr' in hostName: ## KNU T2/T3
            self.site = "KNU"
            self.scheduler = "PBS"
            self.xrdSrv = 'cluster142.knu.ac.kr'
            self.xrdBase = '/'
            self.localBase = '/pnfs/knu.ac.kr/data/cms'
            queue = "cms"
        elif hostName.endswith('sdfarm.kr'):
            self.site = "KISTI"
            self.scheduler = "CONDOR"
            self.xrdSrv = 'cms-xrdr.sdfarm.kr:1094/'
            self.xrdBase = '/xrd'
            self.localBase = '/xrootd_user/%s/xrootd' % userName
        else:
            print "!!! This site is not supported"
            sys.exit()

        if 'queue' in kwargs: queue = kwargs['queue']

        ## Determine transfer command, default transfer command by simple mv command
        ## NOTE: kwargs['dest'] must be given in this script. exception in this line means there's unexpected modification
        self.dest = kwargs['dest']
        if self.dest.startswith('/store/'): self.dest = 'root://%s/%s%s' % (self.xrdSrv, self.xrdBase, self.dest)
        ## Make absolute path
        if '://' not in self.dest: self.dest = os.path.abspath(self.dest)
        self.transferCmd = 'xrdcp' if self.dest.startswith('root://') else 'mv'

        ## Determine submit command
        if self.scheduler == "CONDOR":
            self.submitCmd = 'condor_submit submit.jds'
        else:
            if   self.scheduler == "LSB"   : self.submitCmd = 'bsub'
            elif self.scheduler == "PBS"   : self.submitCmd = 'qsub -N'

            if queue: self.submitCmd += ' -q %s' % queue

class TheJobConfig:
    def __init__(self, opts):
        self.maxEvent = -1

        self.doSubmit = ('-n' not in opts) ## Submit jobs by default unless -n is given
        self.doGrid = ('-G' not in opts) ## Carry the grid certificate by default unless -G is given

        self.blacklist = opts['--blacklist'].split(',') if '--blacklist' in opts else []
        self.whitelist = opts['--whitelist'].split(',') if '--whitelist' in opts else []
        self.outFileNames = opts['--transferFiles'].split(',') if '--transferFiles' in opts else []
        self.additional_options = opts['--args'] if '--args' in opts else ""

        ## Initialise configuration, set site-dependent default values
        kwargs = {}
        if '--queue' in opts: kwargs['queue'] = opts['--queue']

        ## (pre) load cfg file to determine the cmssw job type
        if '--cfg' not in opts:
            print 'ERROR: --cfg is mandatory option'
            usage()

        self.cfgFileName = opts['--cfg']
        if not self.cfgFileName.endswith('.py'):
            print "ERROR: Python config file is needed"
            usage()
        if not os.path.exists(self.cfgFileName):
            print "ERROR: Cannot find config file", self.cfgFileName
            sys.exit()

        cout = sys.stdout
        sys.stdout = open("/dev/null", "w")
        sys.argv=[]
        sys.argv.extend(["cmsRun", ".py"])
        sys.argv.extend(self.additional_options.split())
        self.process = imp.load_source("process", self.cfgFileName).process
        sys.stdout = cout
        source = self.process.source
        if source.type_() == "EmptySource":
            if '--maxEvent' not in opts or '--nJobs' not in opts:
                print "ERROR: --maxEvent, --nJobs are necessary for the EmptySource"
                sys.exit()
            self.maxEvent = int(opts['--maxEvent'])
            self.nSection = int(opts['--nJobs'])
            self.firstRun = int(opts['--firstRun']) if '--firstRun' in opts else 1
        elif source.type_() == "PoolSource":
            if '--maxEvent' in opts: self.maxEvent = int(opts['--maxEvent'])
            fileList = opts['--fileList']
            if not os.path.exists(fileList):
                print "ERROR: Cannot find file list", fileList
                sys.exit()

            ## Collect root files
            self.files = []
            for f in open(fileList).readlines():
                f = f.strip().strip('\',"')
                if len(f) < 5: continue
                if '#' == f[0] or '.root' != f[-5:]: continue
                self.files.append(f)
            nFiles = len(self.files)
            if nFiles == 0:
                print "ERROR: Empty dataset."
                sys.exit()
            ## Sort files by its key "i", filename_[i]_[j]_hash.root
            #files.sort(key=lambda f: int(f.split('/')[-1].split('_')[1]))

            ## Collect secondary input files
            self.secondFiles = []
            if '--secondFileList' in opts:
                secondFileList = opts['--secondFileList']
                for f in open(secondFileList).readlines():
                    f = f.strip().strip('\',"')
                    if len(f) < 5: continue
                    if '#' == f[0] or '.root' != f[-5:]: continue
                    self.secondFiles.append(f)

            if '--maxFiles' in opts:
                self.maxFiles = int(opts['--maxFiles'])
            elif '--nJobs' in opts:
                nJobs = int(opts['--nJobs'])
                self.maxFiles = max(1, len(self.files)/nJobs)
            else:
                print "ERROR: No maxFiles nor nJobs among the option"
                sys.exit()

        ## Override by options
        try:
            self.jobName = opts['--jobName']
            self.jobDir = os.path.abspath(self.jobName)
            self.cmsswBase = os.environ["CMSSW_BASE"]
            self.jobBase =  self.jobDir.replace(os.path.dirname(self.cmsswBase), '').strip('/')

            if os.path.isdir(self.jobDir):
                print "ERROR: Output directory already exists."
                sys.exit()
        except:
            print "!!! Error parsing options"
            usage()

        ## Load customisation if given
        self.customise = None
        if '--customise' in opts:
            customiseFile = opts['--customise']
            if os.path.exists(customiseFile):
                tmpObj = imp.load_source('customise', customiseFile)
                if hasattr(tmpObj, 'customise'): self.customise = tmpObj.customise
                else: print "Cannot find customise(process) function in the cfg ", customiseFile

        ## Default destination to the workdir
        kwargs['dest'] = opts['--transferDest'] if '--transferDest' in opts else self.jobDir

        self.config = TheSiteConfig(**kwargs)
        if self.config.dest == self.jobDir: self.config.transferCmd = ''

    def initialiseWorkspace(self):
        ## Prepare working directory
        print "@@ Preparing batch jobs in", self.jobName, "..."
        os.makedirs(self.jobDir)
        dest = self.config.dest
        if self.config.transferCmd == 'mv' and not os.path.exists(dest):
            print "@@ Preparing output scratch directory in", dest, "..."
            os.makedirs(dest)
            if self.config.site == "KISTI": os.system("chmod go+w %s" % dest)

        elif self.config.transferCmd == 'xrdcp':
            print "@@ Preparing output directory in", self.config.dest, "..."

            destStorage, destDir = self.config.dest[7:].split('/',1)
            if destStorage == self.config.xrdSrv and \
               self.config.site in ("KNU", "CERN"):
                if '/store/' in destDir: destDir = '/store/'+destDir.split('/store/', 1)[-1]
                if not os.path.exists("%s/%s" % (self.config.localBase, destDir)):
                    os.makedirs("%s/%s" % (self.config.localBase, destDir))

            else:
                print "gfal-mkdir %s" % dest
                os.system("gfal-mkdir %s" % dest)

        ## Load cfg file
        print "@@ Loading python cfg..."
        cout = sys.stdout
        sys.stdout = open("%s/log.txt" % self.jobDir, "w")
        sys.argv=[]
        sys.argv.extend(["cmsRun", ".py"])
        sys.argv.extend(self.additional_options.split())
        process = self.process
        if not hasattr(process, 'maxEvents'): process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(-1))
        if not hasattr(process.maxEvents, 'input'): process.maxEvents.input = cms.untracked.int32(-1)
        process.maxEvents.input = self.maxEvent
        ## Customise the cfg
        if self.customise != None: process = self.customise(process)
        sys.stdout = cout

        ## Memorise to modify output file names
        print "@@ Setting output modules..."
        outFileModes = {}
        if hasattr(process, 'TFileService'):
            outFileModes['TFileService'] = process.TFileService.fileName.value()

        for modName in process.outputModules_():
            outFileModes[modName] = getattr(process, modName).fileName.value()

        for f in outFileModes.values():
            f = re.sub(r'^file:', '', f)            
            self.outFileNames.append(f)
            
        pklFileName = "%s/job_cfg.pkl" % (self.jobDir)        
        cPickle.dump(process, open(pklFileName, "w"))
        
        ## Split files into jobs and write python cfg
        print "@@ Splitting jobs..."
        if hasattr(self, 'files'):
            nFiles = len(self.files)
            self.nSection = int(ceil(1.0*nFiles/self.maxFiles))
            for section in range(self.nSection):                
                begin = section*self.maxFiles
                end = min(begin+self.maxFiles, nFiles)

                cfgFileName = "%s/job_%03d_cfg.py" % (self.jobDir, section)
                cfgOut = open(cfgFileName, "w")
                print>>cfgOut, """#!/usr/bin/env python
import cPickle
process = cPickle.load(open("job_cfg.pkl")) """
                
                print>>cfgOut, """process.source.fileNames = %s """ % (self.files[begin:end])
                ## Add secondary files if requested                
                if len(self.secondFiles) > 0:
                    print>>cfgOut, """process.source.secondaryFileNames = cms.untracked.vstring(%s)""" % (self.secondFiles[:])
                #process.source.fileNames = self.files[begin:end]
                #for modName in outFileModes:
                #    getattr(process, modName).fileName = "%s_%03d.root" % (outFileModes[modName][:-5], section)

        else:
            for section in range(self.nSection):
                cfgFileName = "%s/job_%03d_cfg.py" % (self.jobDir, section)
                cfgOut = open(cfgFileName, "w")
                print>>cfgOut, """#!/usr/bin/env python
import FWCore.ParameterSet.Config as cms
import cPickle
process = cPickle.load(open("job_cfg.pkl")) """
                
                print>>cfgOut, """process.source.firstRun = cms.untracked.uint32(%s)""" % (self.firstRun)
                print>>cfgOut, """process.source.firstLuminosityBlock = cms.untracked.uint32(%s)""" % (section+1)
                
                for shift, p in enumerate(process.RandomNumberGeneratorService.parameterNames_()):
                    print>>cfgOut, """process.RandomNumberGeneratorService.%s.initialSeed = %s""" % (p,section+1000+shift+1000*self.firstRun)
                    
        ## Make scripts
        if   self.config.scheduler == "LSB": self.makeLSBJob()
        elif self.config.scheduler == "PBS": self.makePBSJob()
        elif self.config.scheduler == "CONDOR": self.makeCondorJob()

    def makeLSBJob(self):

        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ $# != 1 ]; then
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi
SECTION=`printf %03d $1`

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}  
fi
source $CMS_PATH/cmsset_default.sh

tar xzf {0}/job.tar.gz
cd {1}
scram build ProjectRename
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
echo BEGIN `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
echo cmsRun job_${{SECTION}}_cfg.py {3}
time cmsRun job_${{SECTION}}_cfg.py {3}
EXITCODE=$?
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
    exit 1
fi""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'],self.additional_options)
        print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=%s
    DESTDIR=%s
    echo ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        echo ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        echo ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
done""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        for section in range(self.nSection):
            print>>fout, "{0} -J {1}.{2} -oo {3}/job_{2:03d}.log run_{4}.sh {2}".format(self.config.submitCmd, self.jobName, section, self.jobDir, self.jobName.replace('/','_'))
        fout = None
        os.chmod(submitFileName, 0755)

    def makePBSJob(self):
        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ _$PBS_ARRAYID != '_' ]; then
    SECTION=`printf %03d $PBS_ARRAYID`
elif [ _$SECTION != '_' ]; then
    SECTION=`printf %03d $SECTION`
else
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}  
fi
source $CMS_PATH/cmsset_default.sh

hostname
## Prepare workdirectory
mkdir -p /tmp/${{USER}}/PBS_${{PBS_JOBID}}
cd /tmp/${{USER}}/PBS_${{PBS_JOBID}}
tar xzf {0}/job.tar.gz
cd {1}
scram build ProjectRename
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
echo BEGIN `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
time cmsRun job_${{SECTION}}_cfg.py {3}
EXITCODE=$?
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` cmsRun job_${{SECTION}}_cfg.py >> {0}/submit.log
    rm -rf /tmp/${{USER}}/PBS_${{PBS_JOBID}}
    exit 1
fi""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'],self.additional_options)
        print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=%s
    DESTDIR=%s
    ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
done

## Clean up current workarea
rm -rf /tmp/${USER}/PBS_${PBS_JOBID}""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        for section in range(self.nSection):
            print>>fout, "{0} {1}_{2} run_{3}.sh -vSECTION={2}".format(self.config.submitCmd, self.jobName, section, self.jobName.replace('/','_'))
        fout = None
        os.chmod(submitFileName, 0755)

    def makeCondorJob(self):
        ## Write run script
        print "@@ Writing run script..."
        runFileName = "%s/run_%s.sh" % (self.jobDir, self.jobName.replace('/','_'))
        fout = open(runFileName, "w")

        print>>fout, """#!/bin/bash
if [ $# != 1 ]; then
    echo "JOB SECTION NUMBER IS MISSING!!!"
    exit 1
fi
SECTION=`printf %03d $1`

if [ _$CMS_PATH == _ ]; then
  export CMS_PATH={2}  
fi
source $CMS_PATH/cmsset_default.sh

hostname
tar xzf job.tar.gz
cd {1}""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'],self.additional_options)
        print>>fout,"""
scram build ProjectRename
eval `scram runtime -sh`
if [ -f $CMSSW_BASE/proxy.x509 ]; then
    export X509_USER_PROXY=$CMSSW_BASE/proxy.x509
fi
echo BEGIN `date` cmsRun job_${{SECTION}}_cfg.py #>> {0}/submit.log
ls -al
time cmsRun job_${{SECTION}}_cfg.py {3}
EXITCODE=$?
ls -al
if [ $EXITCODE == 0 ]; then
    echo ENDED `date` cmsRun job_${{SECTION}}_cfg.py #>> {0}/submit.log
else
    rm -f core.*
    echo TERMINATED_$EXITCODE `date` cmsRun job_${{SECTION}}_cfg.py #>> {0}/submit.log
    exit 1
fi""".format(self.jobDir, self.jobBase, os.environ['CMS_PATH'],self.additional_options)
        if self.config.transferCmd != '':
            print>>fout, """
for FILE in %s; do
    EXT=${FILE##*.}
    PREFIX=${FILE%%.${EXT}}
    TRANSFERCMD=%s
    DESTDIR=%s
    ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. retry in 5 seconds"
        sleep 5
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
    if [ $? -ne 0 ]; then
        echo "Failed to copy file.. (trial2) retry in 30 seconds"
        sleep 30
        ${TRANSFERCMD} ${FILE} ${DESTDIR}/${PREFIX}_${SECTION}.${EXT}
    fi
done""" % (' '.join(self.outFileNames), self.config.transferCmd, self.config.dest)
        print>>fout, "echo FINISHED `date` # >> %s/submit.log" % self.jobDir
        fout = None
        os.chmod(runFileName, 0755)

        ## Write submit script
        fout = open("%s/submit.log" % self.jobDir, "w")
        print>>fout, "SUBMIT STARTED", self.jobName
        fout = None
        os.chmod("%s/submit.log" % self.jobDir, 0666)

        print "@@ Writing submit script..."
        submitFileName = "%s/submit.sh" % self.jobDir
        fout = open(submitFileName, "w")
        print>>fout, "#!/bin/bash"
        print>>fout, self.config.submitCmd
        fout = None
        os.chmod(submitFileName, 0755)

        # Condor jobs
        jdlOut = open("%s/submit.jds" % self.jobDir, "w")
        print>>jdlOut, "# Job description file for condor job", self.jobName
        print>>jdlOut, """executable = run_%s.sh
universe   = vanilla
arguments  = $(Process)

log = condor.log

getenv     = True
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
output = job_$(Process).log
error = job_$(Process).err
transfer_input_files = job.tar.gz""" % self.jobName.replace('/','_')
        if self.config.transferCmd == '':
            print>>jdlOut, "transfer_output_files = ", (",".join([os.path.join(self.jobBase, x) for x in self.outFileNames]))
            remapStrs = ["{0}.{1}={0}_$(Process).{1}".format('.'.join(x.split('.')[:-1]), x.split('.')[-1]) for x in self.outFileNames]
            print>>jdlOut, 'transfer_output_remaps = "%s"' % (';'.join(remapStrs))

        requirements = []

        ## Special care for the Singularity environment at KISTI
        if self.config.site == "KISTI":
            print>>jdlOut, '''
accounting_group=group_cms
+SingularityImage = "/cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el6:latest"
+SingularityBind = "/cvmfs, /cms, /share"'''
            requirements.append("HasSingularity == true")

        if len(self.blacklist) > 0:
            requirements.extend(["Machine != \""+x+"\"" for x in self.blacklist])
        if len(self.whitelist) > 0: 
            white_requirements = ["Machine == \""+x+"\"" for x in self.whitelist]
            requirements.append("(%s)" % (" || ".join(white_requirements)))
        if len(requirements) > 0:
            req = "requirements = ", (" && ".join(requirements))
            print>>jdlOut, " ".join(req)
        print>>jdlOut, "queue",  self.nSection
        jdlOut = None

    def archive(self):
        ## Archive libraries and other stuffs
        tmpFile = open("%s/.create-batch" % self.jobDir, "w")
        tmpFile.close()

        print "@@ Archive files for job submission..." 
        exclCommon = "--exclude tmp --exclude 'job.tar' --exclude 'job.tar.gz' --exclude-vcs --exclude-tag-all=.requestcache "
        os.system("tar cf {0}/job.tar {1} --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude-tag-all=.create-batch "
                 + " --exclude={0}/src/*/*/data --exclude=*.root".format(self.cmsswBase))
        os.system("tar rf {0}/job.tar {1}/src/*/*/data --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude-tag-all=.create-batch ")
        os.system("tar rf {0}/job.tar {0} --xform='s:{1}:{2}:' -P ".format(self.jobDir, self.cmsswBase, os.path.basename(self.cmsswBase))
                 + exclCommon + " --exclude=*.root")

        ## Checking voms proxy
        if self.doGrid:
            print "@@ Checking grid certificate to access files..."
            if os.system("voms-proxy-info -exists --valid 8:00") != 0:
                os.system("voms-proxy-init -voms cms --valid 144:00")
            uid = os.getuid()
            os.system("tar rf {0}/job.tar {1} --xform='s:{1}:{2}/proxy.x509:' -P ".format(self.jobDir, "/tmp/x509up_u%d" % uid, os.path.basename(self.cmsswBase)))

        os.system("gzip %s/job.tar" % self.jobDir)
        #os.system("rm -f %s/job_*_cfg.py" % self.jobDir)

    def submit(self):
        if self.doSubmit:
            print "@@ Submitting jobs..."
            if self.config.site == "UOS":
              if os.environ['HOSTNAME'].startswith("gate") :
                os.system("cd %s;./submit.sh; cd -" % self.jobDir)
              else :
                os.system("ssh uosaf0008 'cd %s;./submit.sh'" % self.jobDir)
            elif self.config.site == "CERN" and not os.environ['HOSTNAME'].startswith("lxplus"):
                os.system("ssh lxplus.cern.ch 'cd %s;./submit.sh'" % self.jobDir)
            else:
                os.system("cd %s;./submit.sh" % self.jobDir)
        else:
            print "@@ Jobs are prepared. You can submit jobs with following command:"
            print "cd %s;./submit.sh" % (self.jobDir)

## Create job configure
jobConfig = TheJobConfig(opts)
jobConfig.initialiseWorkspace()
jobConfig.archive()
jobConfig.submit()

print "@@ Done."