Merge pull request #6 from NCAR/hua-work-common

Hua work common

Author: zaihuaji

pyproject.toml

@@ -4,7 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "1.0.2"
+version = "1.0.4"
+
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

src/rda_python_common/PgDBI.py

@@ -37,9 +37,11 @@
 
 # hard coded db ports for dbnames
 DBPORTS = {
-  'default' : 0    # skip default port number 5432
+  'default' : 5432
 }
 
+DBPASS = {}
+
 # hard coded db names for given schema names
 DBNAMES = {
    'ivaddb' : 'ivaddb',
@@ -79,7 +81,6 @@
 def SETPGDBI(name, value):
    PGDBI[name] = PgLOG.get_environment(name, value)
 
-SETPGDBI('CDHOST', 'rda-db.ucar.edu')  # common domain for db host for master server
 SETPGDBI('DEFDB', 'rdadb')
 SETPGDBI("DEFSC", 'dssdb')
 SETPGDBI('DEFHOST', PgLOG.PGLOG['PSQLHOST'])
@@ -88,7 +89,7 @@ def SETPGDBI(name, value):
 SETPGDBI("DBNAME", PGDBI['DEFDB'])
 SETPGDBI("SCNAME", PGDBI['DEFSC'])
 SETPGDBI("LNNAME", PGDBI['DEFSC'])
-SETPGDBI("PWNAME", PGDBI['DEFSC'])
+SETPGDBI("PWNAME", None)
 SETPGDBI("DBHOST", (os.environ['DSSDBHOST'] if os.environ.get('DSSDBHOST') else PGDBI['DEFHOST']))
 SETPGDBI("DBPORT", 0)
 SETPGDBI("ERRLOG", PgLOG.LOGERR)   # default error logact
@@ -242,12 +243,12 @@ def set_scname(dbname = None, scname = None, lnname = None, pwname = None, dbhos
       PGDBI['DBNAME'] = dbname
       changed = 1
    if scname and scname != PGDBI['SCNAME']:
-      PGDBI['PWNAME'] = PGDBI['LNNAME'] = PGDBI['SCNAME'] = scname
+      PGDBI['LNNAME'] = PGDBI['SCNAME'] = scname
       changed = 1
    if lnname and lnname != PGDBI['LNNAME']:
-      PGDBI['PWNAME'] = PGDBI['LNNAME'] = lnname
+      PGDBI['LNNAME'] = lnname
       changed = 1
-   if pwname and pwname != PGDBI['PWNAME']:
+   if pwname != PGDBI['PWNAME']:
       PGDBI['PWNAME'] = pwname
       changed = 1
    if dbhost and dbhost != PGDBI['DBHOST']:
@@ -475,7 +476,8 @@ def pgbatch(sqlfile, foreground = 0):
 
    dbhost = 'localhost' if PGDBI['DBSHOST'] == PgLOG.PGLOG['HOSTNAME'] else PGDBI['DBHOST'] 
    options = "-h {} -p {}".format(dbhost, PGDBI['DBPORT'])
-   os.environ['PGPASSWORD'] = PGDBI['PWNAME']
+   pwname = get_pgpass_password()
+   os.environ['PGPASSWORD'] = pwname
    options += " -U {} {}".format(PGDBI['LNNAME'], PGDBI['DBNAME'])
 
    if not sqlfile: return options
@@ -512,14 +514,14 @@ def pgconnect(reconnect = 0, pgcnt = 0, autocommit = True):
 
    while True:
       config = {'database' : PGDBI['DBNAME'],
-                    'user' : PGDBI['LNNAME'],
-                'password' : PGDBI['PWNAME']}
+                    'user' : PGDBI['LNNAME']}
       if PGDBI['DBSHOST'] == PgLOG.PGLOG['HOSTNAME']:
          config['host'] = 'localhost'
       else:
-         config['host'] = PGDBI['DBHOST'] if PGDBI['DBHOST'] else PGDBI['CDHOST']
+         config['host'] = PGDBI['DBHOST'] if PGDBI['DBHOST'] else PGDBI['DEFHOST']
          if not PGDBI['DBPORT']: PGDBI['DBPORT'] = get_dbport(PGDBI['DBNAME'])
       if PGDBI['DBPORT']: config['port'] = PGDBI['DBPORT'] 
+      config['password'] = get_pgpass_password()
 
       sqlstr = "psycopg2.connect(**{})".format(config)
       if PgLOG.PGLOG['DBGLEVEL']: PgLOG.pgdbg(1000, sqlstr)
@@ -2216,3 +2218,27 @@ def pgname(str, sign = None):
           nstr = '"{}"'.format(nstr)
 
    return nstr
+
+#
+# get a postgres password for given host, port, dbname, usname
+#
+def get_pgpass_password():
+
+   if PGDBI['PWNAME']: return PGDBI['PWNAME']
+   if not DBPASS: read_pgpass()  
+   return DBPASS.get((PGDBI['DBSHOST'], PGDBI['DBPORT'], PGDBI['DBNAME'], PGDBI['USNAME']))
+
+#
+# Reads the .pgpass file and returns a dictionary of credentials.
+#
+def read_pgpass():
+
+   try:
+      with open(PgLOG.PGLOG['DSSHOME'] + '/.pgpass', "r") as f:
+         for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"): continue
+            dbhost, dbport, dbname, usname, pwname = line.split(":")
+            DBPASS[(dbhost, dbport, dbname, usname)] = pwname
+   except FileNotFoundError:
+      pass

src/rda_python_common/PgLOG.py

@@ -109,7 +109,7 @@
    'DSIDCHRS' : "d",
    'DOSHELL' : False,
    'NEWDSID' : True,
-   'BCHHOSTS' : "SLURM:PBS",
+   'BCHHOSTS' : "PBS",
    'HOSTTYPE' : 'dav',    # default HOSTTYPE
    'EMLMAX' : 256,       # up limit of email line count 
    'PGBATCH' : '',       # current batch service name, SLURM or PBS
@@ -1219,14 +1219,14 @@ def set_common_pglog():
    PGLOG['NOTAROOT'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['BACKROOT']])
    PGLOG['NOTBROOT'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['ARCHROOT']])
    PGLOG['ALLROOTS'] = '|'.join([PGLOG['OLDAROOT'], PGLOG['OLDBROOT'], PGLOG['ARCHROOT'], PGLOG['BACKROOT']])
-   SETPGLOG("USRHOME", "PGUSRHOME")
-   SETPGLOG("DSSHOME", "PGDSSHOME")
-   SETPGLOG("ADDPATH", "PGADDPATH")
-   SETPGLOG("ADDLIB",  "PGADDLIB")
-   SETPGLOG("OTHPATH", "PGOTHPATH")
-   SETPGLOG("PSQLHOME", "PGPSQLHOME")
-   SETPGLOG("DSGHOSTS", "PGDSGHOSTS")
-   SETPGLOG("DSIDCHRS", "PGDSIDCHRS")
+   SETPGLOG("USRHOME", "/glade/u/home")
+   SETPGLOG("DSSHOME", "/glade/u/home/rdadata")
+   SETPGLOG("ADDPATH", "")
+   SETPGLOG("ADDLIB",  "")
+   SETPGLOG("OTHPATH", "")
+   SETPGLOG("PSQLHOME", "/usr/pgsql-15")
+   SETPGLOG("DSGHOSTS", "")
+   SETPGLOG("DSIDCHRS", "d")
 
    if not os.getenv('HOME'): os.environ['HOME'] = "{}/{}".format(PGLOG['USRHOME'], PGLOG['CURUID'])
    SETPGLOG("HOMEBIN", os.environ.get('HOME') + "/bin")
@@ -1279,7 +1279,7 @@ def set_common_pglog():
    SETPGLOG("PUSGDIR", PGLOG['DSSDBHM']+"/prog_usage")  # path to program usage files
    SETPGLOG("DSSURL",  "https://rda.ucar.edu")          # current dss web URL
    SETPGLOG("RQSTURL", "/datasets/request")              # request URL path
-   SETPGLOG("WEBSERVERS", "PGWEBSERVERS")                 # webserver names for Web server
+   SETPGLOG("WEBSERVERS", "rda-web-prod01.ucar.edu:rda-web-test01.ucar.edu")                 # webserver names for Web server
    PGLOG['WEBHOSTS'] = PGLOG['WEBSERVERS'].split(':') if PGLOG['WEBSERVERS'] else []
    SETPGLOG("DBMODULE", '')
    SETPGLOG("LOCDATA", "/data")
@@ -1288,21 +1288,21 @@ def set_common_pglog():
    SETPGLOG("DSSWEB",  PGLOG['LOCDATA']+"/web")
    SETPGLOG("DSWHOME", PGLOG['DSSWEB']+"/datasets")     # datast web root path
    PGLOG['HOMEROOTS'] = "{}|{}".format(PGLOG['DSSHOME'], PGLOG['DSWHOME'])
-   SETPGLOG("DSSDATA", "PGDSSDATA")                     # dss data root path
+   SETPGLOG("DSSDATA", "/glade/campaign/collections/rda")                     # dss data root path
    SETPGLOG("DSDHOME", PGLOG['DSSDATA']+"/data")        # dataset data root path
    SETPGLOG("DECSHOME", PGLOG['DSSDATA']+"/decsdata")   # dataset decsdata root path
    SETPGLOG("DSHHOME", PGLOG['DECSHOME']+"/helpfiles")  # dataset help root path
    SETPGLOG("UPDTWKP", PGLOG['DSSDATA']+"/work")        # dsupdt work root path
    SETPGLOG("TRANSFER", PGLOG['DSSDATA']+"/transfer")   # dss transfer partition
    SETPGLOG("RQSTHOME", PGLOG['TRANSFER']+"/dsrqst")    # dsrqst home
-   SETPGLOG("DSAHOME",  "PGDSAHOME")                    # dataset data alternate root path
-   SETPGLOG("RQSTALTH", "PGRQSTALTH")                   # alternate dsrqst path
-   SETPGLOG("GPFSHOST", "PGGPFSHOST")                   # empty if writable to glade
-   SETPGLOG("PSQLHOST", "PGPSQLHOST")                   # host name for postgresql server
-   SETPGLOG("SLMHOSTS", "PGSLMHOSTS")                   # host names for SLURM server
-   SETPGLOG("PBSHOSTS", "PGPBSHOSTS")                   # host names for PBS server
-   SETPGLOG("CHKHOSTS", "PGCHKHOSTS")                   # host names for dscheck daemon
-   SETPGLOG("PVIEWHOST", "PGPVIEWHOST")                   # host name for view only postgresql server
+   SETPGLOG("DSAHOME",  "")                    # dataset data alternate root path
+   SETPGLOG("RQSTALTH", "")                   # alternate dsrqst path
+   SETPGLOG("GPFSHOST", "")                   # empty if writable to glade
+   SETPGLOG("PSQLHOST", "rda-db.ucar.edu")                   # host name for postgresql server
+   SETPGLOG("SLMHOSTS", "cheyenne:casper")                   # host names for SLURM server
+   SETPGLOG("PBSHOSTS", "cheyenne:casper")                   # host names for PBS server
+   SETPGLOG("CHKHOSTS", "")                   # host names for dscheck daemon
+   SETPGLOG("PVIEWHOST", "rda-pgdb-02.ucar.edu")                   # host name for view only postgresql server
    SETPGLOG("FTPUPLD",  PGLOG['TRANSFER']+"/rossby")    # ftp upload path
    PGLOG['GPFSROOTS'] = "{}|{}|{}".format(PGLOG['DSDHOME'], PGLOG['UPDTWKP'], PGLOG['RQSTHOME'])
 
@@ -1311,7 +1311,7 @@ def set_common_pglog():
    os.environ['history'] = '0'
 
    # set tmp dir
-   SETPGLOG("TMPPATH", "PGTMPPATH")
+   SETPGLOG("TMPPATH", "/glade/campaign/collections/rda/work/ptmp")
    if not PGLOG['TMPPATH']: PGLOG['TMPPATH'] = "/data/ptmp"
 
    SETPGLOG("TMPDIR", '')