From 479f5f001459505f6dfa4e935c44f4bd6d807da8 Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Thu, 23 Jan 2025 15:48:32 -0600 Subject: [PATCH 1/3] use .pgpass file for db passwords --- src/rda_python_common/PgDBI.py | 46 ++++++++++++++++++++++++++-------- src/rda_python_common/PgLOG.py | 40 ++++++++++++++--------------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/src/rda_python_common/PgDBI.py b/src/rda_python_common/PgDBI.py index 840abd5..4aecf10 100644 --- a/src/rda_python_common/PgDBI.py +++ b/src/rda_python_common/PgDBI.py @@ -37,9 +37,11 @@ # hard coded db ports for dbnames DBPORTS = { - 'default' : 0 # skip default port number 5432 + 'default' : 5432 } +DBPASS = {} + # hard coded db names for given schema names DBNAMES = { 'ivaddb' : 'ivaddb', @@ -79,7 +81,6 @@ def SETPGDBI(name, value): PGDBI[name] = PgLOG.get_environment(name, value) -SETPGDBI('CDHOST', 'rda-db.ucar.edu') # common domain for db host for master server SETPGDBI('DEFDB', 'rdadb') SETPGDBI("DEFSC", 'dssdb') SETPGDBI('DEFHOST', PgLOG.PGLOG['PSQLHOST']) @@ -88,7 +89,7 @@ def SETPGDBI(name, value): SETPGDBI("DBNAME", PGDBI['DEFDB']) SETPGDBI("SCNAME", PGDBI['DEFSC']) SETPGDBI("LNNAME", PGDBI['DEFSC']) -SETPGDBI("PWNAME", PGDBI['DEFSC']) +SETPGDBI("PWNAME", None) SETPGDBI("DBHOST", (os.environ['DSSDBHOST'] if os.environ.get('DSSDBHOST') else PGDBI['DEFHOST'])) SETPGDBI("DBPORT", 0) SETPGDBI("ERRLOG", PgLOG.LOGERR) # default error logact @@ -242,12 +243,12 @@ def set_scname(dbname = None, scname = None, lnname = None, pwname = None, dbhos PGDBI['DBNAME'] = dbname changed = 1 if scname and scname != PGDBI['SCNAME']: - PGDBI['PWNAME'] = PGDBI['LNNAME'] = PGDBI['SCNAME'] = scname + PGDBI['LNNAME'] = PGDBI['SCNAME'] = scname changed = 1 if lnname and lnname != PGDBI['LNNAME']: - PGDBI['PWNAME'] = PGDBI['LNNAME'] = lnname + PGDBI['LNNAME'] = lnname changed = 1 - if pwname and pwname != PGDBI['PWNAME']: + if pwname != PGDBI['PWNAME']: PGDBI['PWNAME'] = pwname changed = 1 if dbhost and dbhost != PGDBI['DBHOST']: @@ -475,7 +476,8 @@ def pgbatch(sqlfile, foreground = 0): dbhost = 'localhost' if PGDBI['DBSHOST'] == PgLOG.PGLOG['HOSTNAME'] else PGDBI['DBHOST'] options = "-h {} -p {}".format(dbhost, PGDBI['DBPORT']) - os.environ['PGPASSWORD'] = PGDBI['PWNAME'] + pwname = get_pgpass_password() + os.environ['PGPASSWORD'] = pwname options += " -U {} {}".format(PGDBI['LNNAME'], PGDBI['DBNAME']) if not sqlfile: return options @@ -512,14 +514,14 @@ def pgconnect(reconnect = 0, pgcnt = 0, autocommit = True): while True: config = {'database' : PGDBI['DBNAME'], - 'user' : PGDBI['LNNAME'], - 'password' : PGDBI['PWNAME']} + 'user' : PGDBI['LNNAME']} if PGDBI['DBSHOST'] == PgLOG.PGLOG['HOSTNAME']: config['host'] = 'localhost' else: - config['host'] = PGDBI['DBHOST'] if PGDBI['DBHOST'] else PGDBI['CDHOST'] + config['host'] = PGDBI['DBHOST'] if PGDBI['DBHOST'] else PGDBI['DEFHOST'] if not PGDBI['DBPORT']: PGDBI['DBPORT'] = get_dbport(PGDBI['DBNAME']) if PGDBI['DBPORT']: config['port'] = PGDBI['DBPORT'] + config = ['password'] = get_pgpass_password() sqlstr = "psycopg2.connect(**{})".format(config) if PgLOG.PGLOG['DBGLEVEL']: PgLOG.pgdbg(1000, sqlstr) @@ -2216,3 +2218,27 @@ def pgname(str, sign = None): nstr = '"{}"'.format(nstr) return nstr + +# +# get a postgres password for given host, port, dbname, usname +# +def get_pgpass_password(): + + if PGDBI['PWNAME']: return PGDBI['PWNAME'] + if not DBPASS: read_pgpass() + return DBPASS.get((PGDBI['DBSHOST'], PGDBI['DBPORT'], PGDBI['DBNAME'], PGDBI['USNAME'])) + +# +# Reads the .pgpass file and returns a dictionary of credentials. +# +def read_pgpass(): + + try: + with open(PgLOG.PGLOG['DSSHOME'] + '/.pgpass', "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): continue + dbhost, dbport, dbname, usname, pwname = line.split(":") + DBPASS[(dbhost, dbport, dbname, usname)] = pwname + except FileNotFoundError: + pass diff --git a/src/rda_python_common/PgLOG.py b/src/rda_python_common/PgLOG.py index a69d29d..e02cb24 100644 --- a/src/rda_python_common/PgLOG.py +++ b/src/rda_python_common/PgLOG.py @@ -109,7 +109,7 @@ 'DSIDCHRS' : "d", 'DOSHELL' : False, 'NEWDSID' : True, - 'BCHHOSTS' : "SLURM:PBS", + 'BCHHOSTS' : "PBS", 'HOSTTYPE' : 'dav', # default HOSTTYPE 'EMLMAX' : 256, # up limit of email line count 'PGBATCH' : '', # current batch service name, SLURM or PBS @@ -1219,14 +1219,14 @@ def set_common_pglog(): PGLOG['NOTAROOT'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['BACKROOT']]) PGLOG['NOTBROOT'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['ARCHROOT']]) PGLOG['ALLROOTS'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['ARCHROOT'], PGLOG['BACKROOT']]) - SETPGLOG("USRHOME", "PGUSRHOME") - SETPGLOG("DSSHOME", "PGDSSHOME") - SETPGLOG("ADDPATH", "PGADDPATH") - SETPGLOG("ADDLIB", "PGADDLIB") - SETPGLOG("OTHPATH", "PGOTHPATH") - SETPGLOG("PSQLHOME", "PGPSQLHOME") - SETPGLOG("DSGHOSTS", "PGDSGHOSTS") - SETPGLOG("DSIDCHRS", "PGDSIDCHRS") + SETPGLOG("USRHOME", "/glade/u/home") + SETPGLOG("DSSHOME", "/glade/u/home/rdadata") + SETPGLOG("ADDPATH", "") + SETPGLOG("ADDLIB", "") + SETPGLOG("OTHPATH", "") + SETPGLOG("PSQLHOME", "/usr/pgsql-15") + SETPGLOG("DSGHOSTS", "") + SETPGLOG("DSIDCHRS", "d") if not os.getenv('HOME'): os.environ['HOME'] = "{}/{}".format(PGLOG['USRHOME'], PGLOG['CURUID']) SETPGLOG("HOMEBIN", os.environ.get('HOME') + "/bin") @@ -1279,7 +1279,7 @@ def set_common_pglog(): SETPGLOG("PUSGDIR", PGLOG['DSSDBHM']+"/prog_usage") # path to program usage files SETPGLOG("DSSURL", "https://rda.ucar.edu") # current dss web URL SETPGLOG("RQSTURL", "/datasets/request") # request URL path - SETPGLOG("WEBSERVERS", "PGWEBSERVERS") # webserver names for Web server + SETPGLOG("WEBSERVERS", "rda-web-prod01.ucar.edu:rda-web-test01.ucar.edu") # webserver names for Web server PGLOG['WEBHOSTS'] = PGLOG['WEBSERVERS'].split(':') if PGLOG['WEBSERVERS'] else [] SETPGLOG("DBMODULE", '') SETPGLOG("LOCDATA", "/data") @@ -1288,21 +1288,21 @@ def set_common_pglog(): SETPGLOG("DSSWEB", PGLOG['LOCDATA']+"/web") SETPGLOG("DSWHOME", PGLOG['DSSWEB']+"/datasets") # datast web root path PGLOG['HOMEROOTS'] = "{}|{}".format(PGLOG['DSSHOME'], PGLOG['DSWHOME']) - SETPGLOG("DSSDATA", "PGDSSDATA") # dss data root path + SETPGLOG("DSSDATA", "/glade/campaign/collections/rda") # dss data root path SETPGLOG("DSDHOME", PGLOG['DSSDATA']+"/data") # dataset data root path SETPGLOG("DECSHOME", PGLOG['DSSDATA']+"/decsdata") # dataset decsdata root path SETPGLOG("DSHHOME", PGLOG['DECSHOME']+"/helpfiles") # dataset help root path SETPGLOG("UPDTWKP", PGLOG['DSSDATA']+"/work") # dsupdt work root path SETPGLOG("TRANSFER", PGLOG['DSSDATA']+"/transfer") # dss transfer partition SETPGLOG("RQSTHOME", PGLOG['TRANSFER']+"/dsrqst") # dsrqst home - SETPGLOG("DSAHOME", "PGDSAHOME") # dataset data alternate root path - SETPGLOG("RQSTALTH", "PGRQSTALTH") # alternate dsrqst path - SETPGLOG("GPFSHOST", "PGGPFSHOST") # empty if writable to glade - SETPGLOG("PSQLHOST", "PGPSQLHOST") # host name for postgresql server - SETPGLOG("SLMHOSTS", "PGSLMHOSTS") # host names for SLURM server - SETPGLOG("PBSHOSTS", "PGPBSHOSTS") # host names for PBS server - SETPGLOG("CHKHOSTS", "PGCHKHOSTS") # host names for dscheck daemon - SETPGLOG("PVIEWHOST", "PGPVIEWHOST") # host name for view only postgresql server + SETPGLOG("DSAHOME", "") # dataset data alternate root path + SETPGLOG("RQSTALTH", "") # alternate dsrqst path + SETPGLOG("GPFSHOST", "") # empty if writable to glade + SETPGLOG("PSQLHOST", "rda-db.ucar.edu") # host name for postgresql server + SETPGLOG("SLMHOSTS", "cheyenne:casper") # host names for SLURM server + SETPGLOG("PBSHOSTS", "cheyenne:casper") # host names for PBS server + SETPGLOG("CHKHOSTS", "") # host names for dscheck daemon + SETPGLOG("PVIEWHOST", "rda-pgdb-02.ucar.edu") # host name for view only postgresql server SETPGLOG("FTPUPLD", PGLOG['TRANSFER']+"/rossby") # ftp upload path PGLOG['GPFSROOTS'] = "{}|{}|{}".format(PGLOG['DSDHOME'], PGLOG['UPDTWKP'], PGLOG['RQSTHOME']) @@ -1311,7 +1311,7 @@ def set_common_pglog(): os.environ['history'] = '0' # set tmp dir - SETPGLOG("TMPPATH", "PGTMPPATH") + SETPGLOG("TMPPATH", "/glade/campaign/collections/rda/work/ptmp") if not PGLOG['TMPPATH']: PGLOG['TMPPATH'] = "/data/ptmp" SETPGLOG("TMPDIR", '') From d1609a0cf4376279ea8d57d2ad9344643a7fe83d Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Thu, 23 Jan 2025 15:51:40 -0600 Subject: [PATCH 2/3] version to 1.0.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5473532..c77a78a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "1.0.1" +version = "1.0.4" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] From 4d0b7b40b039caf2805754400cc5f0a0825c1503 Mon Sep 17 00:00:00 2001 From: Zaihua Ji Date: Thu, 23 Jan 2025 16:08:02 -0600 Subject: [PATCH 3/3] Update PgDBI.py --- src/rda_python_common/PgDBI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rda_python_common/PgDBI.py b/src/rda_python_common/PgDBI.py index 4aecf10..6659edd 100644 --- a/src/rda_python_common/PgDBI.py +++ b/src/rda_python_common/PgDBI.py @@ -521,7 +521,7 @@ def pgconnect(reconnect = 0, pgcnt = 0, autocommit = True): config['host'] = PGDBI['DBHOST'] if PGDBI['DBHOST'] else PGDBI['DEFHOST'] if not PGDBI['DBPORT']: PGDBI['DBPORT'] = get_dbport(PGDBI['DBNAME']) if PGDBI['DBPORT']: config['port'] = PGDBI['DBPORT'] - config = ['password'] = get_pgpass_password() + config['password'] = get_pgpass_password() sqlstr = "psycopg2.connect(**{})".format(config) if PgLOG.PGLOG['DBGLEVEL']: PgLOG.pgdbg(1000, sqlstr)