Skip to content

Hua work dsupdt #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zaihuaji merged 2 commits into from
Jan 8, 2026
Merged

Hua work dsupdt #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9468a7b
remove spaces before :
Jan 8, 2026
9e8f440
v2.0.3
Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "rda_python_dsupdt"
version = "2.0.2"
version = "2.0.3"
authors = [
{ name="Zaihua Ji", email="[email protected]" },
]
Expand Down
24 changes: 12 additions & 12 deletions src/rda_python_dsupdt/dsupdt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def __init__(self):
self.TEMPINFO = {}
self.TOPMSG = self.SUBJECT = self.ACTSTR = None
self.ALLCNT = 0
self.DEFTYPES = {'WT' : 'D', 'ST' : 'P', 'QT' : 'B'}
self.DEFTYPES = {'WT': 'D', 'ST': 'P', 'QT': 'B'}

# main function to run dsupdt
def read_parameters(self):
Expand Down Expand Up @@ -387,7 +387,7 @@ def unlock_control_info(self):
# get update info of local and remote files owned by login name
def get_update_info(self):
if 'DS' in self.params:
dsids = {'dsid' : [self.params['DS']]}
dsids = {'dsid': [self.params['DS']]}
dscnt = 1
else:
tname = "dlupdt"
Expand Down Expand Up @@ -639,7 +639,7 @@ def file_update(self, locrec, logact, caching = 0):
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
# create a empty record remote file
rcnt = 1
rmtrecs = {'lindex' : [lindex], 'dindex' : [0]}
rmtrecs = {'lindex': [lindex], 'dindex': [0]}
rflds = ['remotefile', 'serverfile', 'download', 'begintime', 'endtime', 'tinterval']
for rfld in rflds: rmtrecs[rfld] = [None]
if rcnt == 1:
Expand Down Expand Up @@ -1291,7 +1291,7 @@ def archive_data_file(self, lfile, locrec, tempinfo, eidx):
if ainfo['vindex']: growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
tempinfo['ainfo'] = None # clean the archive info recorded earlier
else:
ainfo = {'archived' : 0, 'note' : None} # reference to empty hash
ainfo = {'archived': 0, 'note': None} # reference to empty hash
self.pglog("{}: start {} for {}".format(lfile, locrec['action'], tempinfo['einfo']), self.PGOPT['emllog'])
options = locrec['options'] if locrec['options'] else ""
act = locrec['action']
Expand Down Expand Up @@ -1428,9 +1428,9 @@ def get_tempinfo(self, locrec, locinfo, eidx = 0):
val = self.PGOPT['UCNTL']['validint']
else:
val = None
tempinfo = {'AT' : None, 'DC' : None, 'ED' : [], 'EH' : [], 'VI' : None,
'VD' : None, 'VH' : None, 'CVD' : None, 'NX' : None, 'FQ' : None,
'QU' : None, 'EP' : 0, 'RS' : -1, 'AQ' : None}
tempinfo = {'AT': None, 'DC': None, 'ED': [], 'EH': [], 'VI': None,
'VD': None, 'VH': None, 'CVD': None, 'NX': None, 'FQ': None,
'QU': None, 'EP': 0, 'RS': -1, 'AQ': None}
if val: val = self.get_control_time(val, "Valid Internal")
if val:
tempinfo['VI'] = val
Expand Down Expand Up @@ -1506,7 +1506,7 @@ def get_tempinfo(self, locrec, locinfo, eidx = 0):
dhour = self.diffdatehour(vdate, vhour, locrec['missdate'], locrec['misshour'])
if dhour > 0:
if dhour > 240:
record = {'missdate' : None, 'misshour' : None}
record = {'missdate': None, 'misshour': None}
self.pgupdt("dlupdt", record, "lindex = {}".format(locrec['lindex']))
else:
vdate = locrec['missdate']
Expand Down Expand Up @@ -1576,7 +1576,7 @@ def file_archive_info(self, lfile, locrec, tempinfo):
if tempinfo['ainfo'] != None: return tempinfo['ainfo']
edate = tempinfo['edate']
ehour = tempinfo['ehour']
ainfo = {'archcnt' : 0, 'archived' : 0, 'present' : 0, 'vindex' : 0, 'types' : {}, 'note' : None}
ainfo = {'archcnt': 0, 'archived': 0, 'present': 0, 'vindex': 0, 'types': {}, 'note': None}
growing = self.is_growing_file(locrec['locfile'], tempinfo['FQ'])
if growing:
if tempinfo['NX']:
Expand Down Expand Up @@ -1790,8 +1790,8 @@ def check_newer_file(self, dcmd, cfile, ainfo):
finfo = self.check_local_file(cfile, 33, self.PGOPT['wrnlog'])
if not finfo: return 3 # download if can not check newer
else:
finfo = {'isfile' : 0, 'checksum' : ainfo['chksm'], 'data_size' : ainfo['asize'],
'date_modified' : ainfo['adate'], 'time_modified' : ainfo['atime']}
finfo = {'isfile': 0, 'checksum': ainfo['chksm'], 'data_size': ainfo['asize'],
'date_modified': ainfo['adate'], 'time_modified': ainfo['atime']}
cinfo = self.check_server_file(dcmd, 33, cfile)
if not cinfo:
sact = self.get_download_action(dcmd)
Expand Down Expand Up @@ -1938,7 +1938,7 @@ def check_locfile_status(self, locrec):
if rcnd and self.pgget("drupdt", "", loccnd):
return self.pglog("{}: NO remote file record matched for {}".format(locinfo, rcnd), self.PGOPT['emlerr'])
rcnt = 1 # create a empty record remote file
rmtrecs = {'lindex' : locrec['lindex'], 'remotefile' : None, 'serverfile' : None}
rmtrecs = {'lindex': locrec['lindex'], 'remotefile': None, 'serverfile': None}
if rcnt == 1:
if 'RF' in self.params and len(self.params['RF']) == 1 and not (rmtrecs['remotefile'][0] and self.params['RF'][0] == rmtrecs['remotefile'][0]):
rmtrecs['remotefile'][0] = self.params['RF'][0]
Expand Down
12 changes: 1 addition & 11 deletions src/rda_python_dsupdt/dsupdt.usg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ ControlIndex<:>Specialist<:>ParentIndex<:>Action<:>Frequency<:>ControlOffset<:>C
D -(DT|DataTime) time data updated to, format: YYY-MM-DD HH:NN:SS
H -(HN|HostName) hostnames this cindex can/cannot be processed on
S -(SB|SBatchOptions) additional PBS batch options
Q -(QS|QSubOptions) additional SLURM batch options
Q -(QS|QSubOptions) additional PBS batch options
Y -(CC|CarbonCopy) carbon copies for additional email addresses
X -(XC|ExecuteCommand) Command to be executed after successful update

Expand Down Expand Up @@ -687,7 +687,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(VS|ValidSize) MinSizeForValidFile]
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(HO|HourOffset) TimeZoneHourOffset]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(CC|CarbonCopy) Cc'dEmailAddresses]
Expand Down Expand Up @@ -811,7 +810,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(VS|ValidSize) MinSizeForValidFile]
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(HO|HourOffset) TimeZoneHourOffset]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(CC|CarbonCopy) Cc'dEmailAddresses]
Expand Down Expand Up @@ -883,7 +881,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(CH|CurrentHour) CurrentHour] # provide if different than current
[-(VS|ValidSize) MinSizeForValidFile]
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(CC|CarbonCopy) Cc'dEmailAddresses]
Expand Down Expand Up @@ -959,7 +956,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(CH|CurrentHour) CurrentHour] # provide if different than current
[-(VS|ValidSize) MinSizeForValidFile]
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(HO|HourOffset) TimeZoneHourOffset]
Expand Down Expand Up @@ -1024,7 +1020,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(CH|CurrentHour) CurrentHour] # provide if different than current
[-(VS|ValidSize) MinSizeForValidFile]
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(CC|CarbonCopy) Cc'dEmailAddresses]
Expand Down Expand Up @@ -1087,7 +1082,6 @@ LocalIndex<:>LocalFile<:>RemoteFile<:>ExecOrder<:>DownloadOrder<:>ServerFile<:>D
[-(CD|CurrentDate) CurrentDate] # provide if different than current
[-(CH|CurrentHour) CurrentHour] # provide if different than current
[-(PL|ProcessLimit) MaxNumberOfChildProcesses]
[-(SB|SbatchOptions) SlurmBatchOptions]
[-(QS|QsubOptions) PBSBatchOptions]
[-(BP|BatchProcess) [BatchProcessHosts]]
[-(CC|CarbonCopy) Cc'dEmailAddresses]
Expand Down Expand Up @@ -1717,10 +1711,6 @@ Information options are used to pass information, one or multiple values, into
actions -SL (-SetLocalFile). Another way of reordering the local files is to
provide explicitly order index values per Info option -XO (-ExecOrder).

-SB or -SbatchOptions, (Alias: -SlurmOptions), specifies options to execute dsupdt
as a batch job via sbatch on SLURM nodes. The sbatch options must be quoted when
prsented on command line, such as, -QS '-t 24:00:00'.

-SF or -ServerFile, only used to set file names on Remote servers when they
are different from the remote file names.

Expand Down
Loading