Merge pull request #30 from EarthScope/lk-ph5-fixes

PH5 fixes

Author: gillian-earthscope

README.md

@@ -323,8 +323,7 @@ path to a directory containing miniSEED files (_See: "Using Local Data Files", b
 * `station_url:` should indicate a metadata location as an FDSN web service alias, EARTHSCOPE (an alias of 'IRIS'),
 the EarthScope PH5 web service alias 'IRISPH5',
 an explicit URL, or a path to a file containing metadata in [StationXML](https://www.fdsn.org/xml/station/) format 
-([schema](https://www.fdsn.org/xml/station/fdsn-station-1.0.xsd)). For web services, this should point to the same place as 
-`dataselect_url` (e.g. `https://service.earthscope.org`). For local metadata, StationXML is read at the channel level and any 
+([schema](https://www.fdsn.org/xml/station/fdsn-station-1.0.xsd)). If both `dataselect_url` and `station_url` point to web services, they should point to the same location (e.g. `https://service.earthscope.org`). For local metadata, StationXML is read at the channel level and any 
 response information is ignored. Local instrument response (if used) is expected to be in RESP file format and specified 
 in the `resp_dir` entry (see below). If neither webservices or StationXML is available for metadata, the `station_url` entry 
 should be left unspecified (blank). In this case, metrics that do not require metadata will still be calculated. Metrics that 
@@ -344,7 +343,7 @@ orientation_check*). The EarthScope event service is deprecated and event_urls d
 redirect to the USGS event service at https://earthquake.usgs.gov.
 
 * `resp_dir:` should be unspecified or absent if local response files are not used. The default behavior
- is to retrieve response information from the EarthScope web service [Evalresp](https://service.earthscope.org/irisws/evalresp/1/). 
+ is to retrieve response information from the EarthScope web service [Evalresp](https://service.earthscope.org/irisws/evalresp/1/). If the Earthscope PH5 service (IRISPH5) is the source for the `station_url`, then it will default to the [ph5 Evalresp] (https://service.earthscope.org/ph5ws/evalresp/1/) instead. 
 To use local instrument responses instead of [Evalresp](https://service.earthscope.org/irisws/evalresp/1/),
  this parameter should indicate a path to a directory containing response files 
 in [RESP](https://ds.iris.edu/ds/nodes/dmc/data/formats/resp/) format. Local response files are expected to be 

ispaq/PSD_metrics.py

@@ -103,13 +103,12 @@ def do_psd(concierge,starttime,endtime):
             # Run the PSD metric ----------------------------------------
             if any(key in function_metadata for key in ("PSD","PSDText")) :
                 try:
-                    evalresp = None
-                    if (concierge.resp_dir):   # if resp_dir: run evalresp on local RESP file instead of web service
-                        sampling_rate = utils.get_slot(r_stream, 'sampling_rate')
-                        evalresp = utils.getSpectra(r_stream, sampling_rate, "PSD", concierge)
-                    if (concierge.dataselect_type == "ph5ws"):
+                    try:
                         sampling_rate = utils.get_slot(r_stream, 'sampling_rate')
                         evalresp = utils.getSpectra(r_stream, sampling_rate, "PSD", concierge)
+                    except Exception as e:
+                            logger.warning('"PSD_metric" metric calculation failed for %s: %s' % (av.snclId, e))
+
                     # get corrected PSD
                     try:
                         (df, PSDcorrected, PDF) = irismustangmetrics.apply_PSD_metric(concierge, r_stream, evalresp=evalresp)

ispaq/concierge.py

@@ -140,37 +140,34 @@ def __init__(self, user_request=None, logger=None):
             if (user_request.event_url == "IRIS" or user_request.event_url.upper() == "EARTHSCOPE"):
                 user_request.event_url = "USGS"
 
-        # Add dataselect clients and URLs or reference a local file
+        ## Add dataselect clients and URLs or reference a local file
         self.dataselect_type = None
         if user_request.dataselect_url in URL_MAPPINGS.keys():
             # Get data from FDSN dataselect service
             self.dataselect_url = URL_MAPPINGS[user_request.dataselect_url]
             self.dataselect_type = "fdsnws"
+
+            if user_request.dataselect_url == "IRISPH5":
+                self.dataselect_type = "ph5ws"
+
             try:
-                self.dataselect_client = Client(self.dataselect_url)
+                self.dataselect_client = Client(user_request.dataselect_url)
             except Exception as e:
                 err_msg = e
                 self.logger.critical(err_msg)
                 raise SystemExit
 
-            if user_request.station_url is not None:    
-                if user_request.station_url != user_request.dataselect_url:
-                    self.logger.warning("Station_url should be the same as dataselect_url when retrieving data from FDSN webservices. Station_url '%s' does not match dataselect_url '%s'" 
-                                         % (user_request.station_url, user_request.dataselect_url))
-
-        elif user_request.dataselect_url == "IRISPH5":
-            self.dataselect_url = "https://service.earthscope.org"
-            self.dataselect_type = "ph5ws"
-            self.dataselect_client = "PH5"
-
             if user_request.station_url is not None:
                if user_request.station_url != user_request.dataselect_url:
-                   self.logger.warning("Station_url should be the same as dataselect_url when retreiving data from IRIS PH5 webservices. Station_url '%s' does not match dataselect_url '%s'"
-                                        % (user_request.station_url, user_request.dataselect_url))
+                   self.logger.warning("Station_url should be the same as dataselect_url when retrieving data from FDSN or PH5 web services. Station_url '%s' does not match dataselect_url '%s'")
 
         elif "http://" in user_request.dataselect_url or "https://" in user_request.dataselect_url:
             self.dataselect_url = user_request.dataselect_url
             self.dataselect_type = "fdsnws"
+
+            if "ph5ws" in user_request.dataselect_url:
+                self.dataselect_type = "ph5ws"
+                
             try:
                 self.dataselect_client = Client(self.dataselect_url)
             except Exception as e:
@@ -180,7 +177,7 @@ def __init__(self, user_request=None, logger=None):
 
             if user_request.station_url is not None:
                 if user_request.station_url != user_request.dataselect_url:
-                    self.logger.warning("Station_url should be the same as dataselect_url when retreiving data from FDSN webservices. Station_url '%s' does not match dataselect_url '%s'" 
+                    self.logger.warning("Station_url should be the same as dataselect_url when retrieving data from FDSN webservices. Station_url '%s' does not match dataselect_url '%s'" 
                                          % (user_request.station_url, user_request.dataselect_url))
 
         else:
@@ -193,43 +190,48 @@ def __init__(self, user_request=None, logger=None):
                 self.logger.critical(err_msg)
                 raise SystemExit
 
-        # Add station clients and URLs or reference a local file
+        ## Add station clients and URLs or reference a local file
+        self.station_type = None
         if user_request.station_url is None:
             if ("http://" in self.dataselect_url or "https://" in self.dataselect_url):
                 self.station_url = self.dataselect_url
+                self.station_type = self.dataselect_type
                 self.logger.info("Using station_url = %s" % self.dataselect_url)
-                if (self.dataselect_type == "ph5ws"):
-                    self.station_type = "ph5ws"
-                    self.station_client = "PH5"
-                else:
-                    try:
-                        self.station_client = Client(self.station_url)
-                    except Exception as e:
-                        self.logger.warning(e)
-                        self.logger.info("Metrics that require metadata information cannot be calculated")
-                        self.station_url = None
-                        self.station_client = None
+
+                try:
+                    self.station_client = Client(self.station_url)
+                except Exception as e:
+                    self.logger.warning(e)
+                    self.logger.info("Metrics that require metadata information cannot be calculated")
+                    self.station_url = None
+                    self.station_client = None
             else:
                 self.logger.info("No station_url found")
                 self.logger.info("Metrics that require metadata information cannot be calculated")
                 self.station_url = None
                 self.station_client = None
         elif user_request.station_url in URL_MAPPINGS.keys():
             self.station_url = URL_MAPPINGS[user_request.station_url]
+            self.station_type = "fdsnws"
+            
+            if user_request.station_url == "IRISPH5":
+                self.station_type = "ph5ws"
+
             try:
-                self.station_client = Client(self.station_url)
+                self.station_client = Client(user_request.station_url)
             except Exception as e:
                 self.logger.warning(e)
                 self.logger.info("Metrics that require metadata information cannot be calculated")
                 self.station_url = None
                 self.station_client = None
-        elif user_request.station_url == "IRISPH5":
-            self.station_url = "https://service.earthscope.org"
-            self.station_type = "ph5ws"
-            self.station_client = "PH5"
 
         elif "http://" in user_request.station_url or "https://" in user_request.station_url:
             self.station_url = user_request.station_url
+            self.station_type = 'fdsnws'
+
+            if "ph5ws" in user_request.station_url:
+                self.station_type = 'ph5ws'
+
             try:
                 self.station_client = Client(self.station_url)
             except Exception as e:
@@ -381,9 +383,9 @@ def __init__(self, user_request=None, logger=None):
         self.filtered_availability = None
 
         # Add local response files if used
-        if user_request.resp_dir is None:                  # use irisws/evalresp
-            self.resp_dir = None                           # use irisws/evalresp
-        elif user_request.resp_dir in URL_MAPPINGS.keys(): # use irisws/evalresp
+        if user_request.resp_dir is None:                  # use EarthScope evalresp web service
+            self.resp_dir = None                           # use EarthScope evalresp web service
+        elif user_request.resp_dir in URL_MAPPINGS.keys(): # use EarthScope evalresp web service
             self.resp_dir = None 
         else:
             if os.path.exists(os.path.abspath(user_request.resp_dir)):   
@@ -398,7 +400,9 @@ def __init__(self, user_request=None, logger=None):
         self.logger.debug("metric_names %s", self.metric_names)
         self.logger.debug("sncl_patterns %s", self.sncl_patterns)
         self.logger.debug("dataselect_url %s", self.dataselect_url)
+        self.logger.debug("dataselect_type %s", self.dataselect_type)
         self.logger.debug("station_url %s", self.station_url)
+        self.logger.debug("station_type %s", self.station_type)
         self.logger.debug("event_url %s", self.event_url)
         self.logger.debug("resp_dir %s", self.resp_dir)
         self.logger.debug("output %s", self.output)
@@ -753,6 +757,8 @@ def get_availability(self, metric,
             else:
                 _channel = channel
 
+            
+            
             _sncl_pattern = self.get_sncl_pattern(_network, _station, _location, _channel)
 
 
@@ -773,7 +779,7 @@ def get_availability(self, metric,
 
                 if df is None:
                     continue 
-            elif self.station_client == "PH5":
+            elif self.station_type == "ph5ws":
                 self.logger.debug("read IRISPH5 station web services %s/%s for %s,%s,%s,%s,%s,%s" % (self.station_url,self.station_type,_network, _station, _location, _channel, _starttime.strftime('%Y.%j'), _endtime.strftime('%Y.%j')))
                 try:
                     df = irisseismic.getAvailability(self.station_url,self.station_type,network=_network, station=_station,
@@ -972,14 +978,14 @@ def get_availability(self, metric,
                         next
                 df = df[df.dist.str.contains("KEEP")]
             df = df.drop('dist', 1)
+            
 
             # Append this dataframe
             if df.shape[0] == 0:
                 self.logger.debug("No SNCLS found matching '%s' (sncl_format=%s)" % (_sncl_pattern,self.sncl_format))
             else:
                 #if df.snclId not in sncl_pattern_dataframes[:].snclId:
                 sncl_pattern_dataframes.append(df)	# tack the dataframes together
-
         # END of sncl_patterns loop --------------------------------------------
 
         if len(sncl_pattern_dataframes) == 0:

ispaq/ispaq.py

@@ -15,7 +15,7 @@
 from _ast import Try
 from numpy.random import sample
 
-__version__ = "3.2.1"
+__version__ = "3.3.0"
 
 # dictionary of currently defined ISPAQ metric groups and business logic
 # for comparison with R package IRISMustangMetrics/ISPAQUtils.R json

ispaq/utils.py

@@ -648,7 +648,7 @@ def getSpectra(st, sampling_rate, metric, concierge):
 
                 try:
                     evalResp = evresp.getEvalresp(localFiles, network, station, location, channel, starttime,
-                                       minfreq, maxfreq, nfreq, units.upper(), output.upper(), "LOG", debugMode)
+                                    minfreq, maxfreq, nfreq, units.upper(), output.upper(), "LOG", debugMode)
                 except Exception as e:
                     raise 
 
@@ -657,12 +657,17 @@ def getSpectra(st, sampling_rate, metric, concierge):
         if evalResp is None:
             raise EvalrespException('No RESP file found at %s[.txt] or %s[.txt]' % (localFile,localFile2))
 
-    else:    
-        # calling the web service 
-        concierge.logger.debug('calling IRIS evalresp web service')
+    else:   
+        ## Use web services
         try:
-            evalResp = irisseismic.getEvalresp(concierge.dataselect_url, concierge.dataselect_type, network, station, location, channel, starttime,
-                                       minfreq, maxfreq, nfreq, units.lower(), output.lower())
+            if concierge.station_type == None:
+                concierge.logger.debug(f'calling irisws evalresp web service')
+                evalResp = irisseismic.getEvalresp(network=network, station=station, location=location, channel=channel, time=starttime,
+                                       minfreq=minfreq, maxfreq=maxfreq, nfreq=nfreq, units=units.lower(), output=output.lower())
+            else:
+                concierge.logger.debug(f'calling {concierge.station_url} - {concierge.station_type} evalresp web service')
+                evalResp = irisseismic.getEvalresp(client_url=concierge.station_url, client_type=concierge.station_type,network=network, station=station, location=location, channel=channel, time=starttime,
+                                       minfreq=minfreq, maxfreq=maxfreq, nfreq=nfreq, units=units.lower(), output=output.lower())
         except Exception as e:
             raise
     return(evalResp)