Merge pull request #28 from matyasselmeci/wip/sw3305-fixprobes

Fix & simplify CA & CRL verification probes (SOFTWARE-3305)

Author: matyasselmeci

rsv-metrics/libexec/probes/cacert-verify-probe

@@ -182,7 +182,7 @@ Additionally it makes sure that all IGTF CAs are installed.
       self.return_ok(out_msg)
 
   def Check_Local_CA(self, certdir, errorHrs):
-    "Function to compare CA certificates against the list at GOC"
+    "Function to compare CA certificates against the list at OSG"
     source = {}
     found_cas = []
     sha256 = {}
@@ -195,34 +195,8 @@ Additionally it makes sure that all IGTF CAs are installed.
     if not os.path.isdir(certdir):
       self.return_unknown("ERROR: CA Certs Directory %s does not exist.. Setting metric to unknown." % certdir)
 
-    #Check if the file CA cers are installed from ITB or OSG, new or old format
-    local_urls = ["http://repo.opensciencegrid.org/pacman/cadist/cacerts_sha256sum-new.txt"]
-    ca_format_type = 0
-    ca_index = os.path.join(certdir, "INDEX.txt")
-    if os.path.isfile(ca_index):
-      try: 
-        lines = open(ca_index).readlines()
-        for i in lines:
-          if i.find("IndexTypeVersion") >= 0:
-            try:
-              ca_format_type = int(i.split()[2])
-            except IndexError:
-              # When the line with IndexTypeVersion is missing, the type is 0 (default value)
-              pass
-      except (OSError, ValueError):
-        pass
-    grid_type = rsvprobe.get_grid_type()
-    if not grid_type in [0, 1]:
-      self.return_unknown("ERROR: Unknown grid type %s. Setting metric to unknown." % grid_type) 
-    #ca_format_type can be only 0,1:
-    cert_ext = ".0"
-    local_url = local_urls[0]
-    if ca_format_type == 1:
-      cert_ext = ".pem"
-      local_url = local_urls[1]
-    elif ca_format_type != 0:
-      self.return_unknown("ERROR: CA Certs Directory %s has unknown certificate type %s. Setting metric to unknown." % (certdir, ca_format_type))
-    self.add_message("Grid type is %s. CA format type is %s. URL with CA distribution: %s" % (rsvprobe.get_grid_type_string(grid_type), ca_format_type, local_url))
+    # Step 1: Download the sha256sums for CAs in the OSG
+    cert_ext = ".pem"
     cert_files = rsvprobe.list_directory(certdir, [cert_ext])
     if not cert_files:
       self.return_unknown("ERROR: CA Certs Directory %s contains no certificate files (*.%s).. Setting metric to unknown." % (certdir, cert_ext))
@@ -234,28 +208,23 @@ Additionally it makes sure that all IGTF CAs are installed.
         self.add_message("Could not calculate sha256sums of your CA file %s. This may result in a probe error. Please verify the file." % f)
         # eliminate error? file may have been deleted in the mean time, just skip?
         pass
-    # Download the checksum file from OSG/ITB cache
-    lines = rsvprobe.get_http_doc(local_url)
+    # Download the checksum file from OSG repo
+    self.add_message("URL with CA distribution: %s" % rsvprobe.CA_CERT_HASH_URL)
+    lines = rsvprobe.get_http_doc(rsvprobe.CA_CERT_HASH_URL)
     if not lines:
-      self.return_unknown("Could not download the sha256sum for CA list from OSG (%s) or the file is empty. Unable to verify CAs." % local_url)
+      self.return_unknown("Could not download the sha256sum for CA list from OSG (%s) or the file is empty."
+                          " Unable to verify CAs." % rsvprobe.CA_CERT_HASH_URL)
     for line in lines:
       line.strip()
       values = line.split()
       local_hash = values[1].split('.')[0].strip()
       sha256[local_hash] = values[0]
 
-    # Step 2: Get the list of Certs included in OSG from GOC website.
-    #chdir($working_dir);
-    local_urls = ["http://repo.opensciencegrid.org/pacman/cadist/INDEX.txt",
-                  "http://repo.opensciencegrid.org/pacman/cadist/INDEX-new.txt"]
-    local_url = local_urls[ca_format_type]
-    lines = rsvprobe.get_http_doc(local_url)
+    # Step 2: Get the list of Certs included in OSG from OSG repo
+    lines = rsvprobe.get_http_doc(rsvprobe.CA_CERT_INDEX_URL)
     if not lines:
-      self.return_unknown("Could not download the CA list from OSG (%s) or the list is empty. Unable to verify CAs." % local_url)
-    ## Old format (remove 1 #):
-    ## Hash         Source                   URL                                      Accreditation
-    ##--------------------------------------------------------------------------------------------------------
-    #09ff08b7     CNRS2-Projets            http://igc.services.cnrs.fr/GRID-FR/       I
+      self.return_unknown("Could not download the CA list from OSG (%s) or the list is empty. Unable to verify CAs." %
+                          rsvprobe.CA_CERT_INDEX_URL)
     ## New format (remove 1 #):
     ## OldHash      NewHash      CAfile                   CAURL                                       Version   Accreditation   
     ##--------------------------------------------------------------------------------------------------------
@@ -265,10 +234,8 @@ Additionally it makes sure that all IGTF CAs are installed.
       if not line or line.startswith('#'):
         continue
       line_content = line.split()
-      chash = line_content[0]
-      if ca_format_type == 1:
-        # New CA format type we use file names
-        chash = line_content[2].split('.')[0]
+      # New CA format type we use file names
+      chash = line_content[2].split('.')[0]
       source[chash] = line_content[-1]
 
     # Step 3: Check the CAs to ensure that sha256sums match-up

rsv-metrics/libexec/probes/crl-freshness-probe

@@ -113,75 +113,34 @@ Additionally it makes sure that all IGTF CAs are installed.
       certdir = rsvprobe.get_ca_dir()
     self.verify_CRL(certdir, self.warningHrs, self.errorHrs)
 
-  def get_osg_CAs(self, ca_format_type, grid_type):
-    "Download and parce the CA list from OSG GOC"
-    if not grid_type in [0, 1]:
-      self.return_unknown("ERROR: Unknown grid type %s. Setting metric to unknown." % grid_type)
-    if not ca_format_type in [0, 1]:
-      self.return_unknown("ERROR: CA Certs Directory has unknown certificate type %s. Setting metric to unknown." % 
-                          ca_format_type)
+  def get_osg_CAs(self):
+    "Download and parse the CA list from OSG repo"
     source_name = {} # list of crl files name -> data (hash0, hash1, accreditation)
     source_hash0 = {} # Old hash -> accreditation
     source_hash1 = {} # New hash -> accreditation
-    local_urls = [["http://repo.opensciencegrid.org/pacman/cadist/INDEX.txt"]
-                  ["http://repo.opensciencegrid.org/pacman/cadist/INDEX-new.txt"]
-                 ]
-    local_url = local_urls[ca_format_type][grid_type]
-    # Download the CA list file from OSG/ITB cache
-    lines = rsvprobe.get_http_doc(local_url)
+    # Download the CA list file
+    lines = rsvprobe.get_http_doc(rsvprobe.CA_CERT_INDEX_URL)
     if not lines:
-      self.return_unknown("Could not download the CA list from OSG (%s) or the file is empty. Unable to verify CRLs." % local_url)
+      self.return_unknown("Could not download the CA list from OSG (%s) or the file is empty."
+                          " Unable to verify CRLs." % rsvprobe.CA_CERT_INDEX_URL)
     ## Parsing of the CA list
-    ## type 1 format:
-    ## OldHash      NewHash      CAfile                   CAURL                                       Version   Accreditation   
+    ## OldHash      NewHash      CAfile                   CAURL                                       Version   Accreditation
     ##--------------------------------------------------------------------------------------------------------
     #75680d2e     ee64a828     AAACertificateServices.pem https://www.terena.org/activities/tcs/         1.41   I
     #3c58f906     157753a5     AddTrust-External-CA-Root.pem http://www.comodo.com/                         1.41   I
-    ## type 0 format:
-    ## Hash         Source                   URL                                      Accreditation
-    ##--------------------------------------------------------------------------------------------------------
-    #09ff08b7     CNRS2-Projets            http://igc.services.cnrs.fr/GRID-FR/       I
-    #0a12b607     UGRID                    https://ca.ugrid.org/                      I
     for line in lines:
       if not line or line.startswith('#'):
         continue
       line_content = line.split()  
-      if ca_format_type == 1:
-        # New CA format type, both old hash (hash0) and new hash (hash1), source name has extension (filename)
-        source_hash0[line_content[0]] = line_content[-1]
-        source_hash1[line_content[1]] = line_content[-1]
-        source_name[line_content[2].split('.')[0]] = { 'hash': line_content[0],
-                                                       'newhash': line_content[1],
-                                                       'accreditation': line_content[-1]
-                                                     }
-      else: # assuming type 0
-        # Assuming CA format type (type 0), only old hash (hash0), source name has no extension
-        source_hash0[line_content[0]] = line_content[-1]
-        source_name[line_content[1]] = { 'hash': line_content[0],
-                                         'newhash': None,
-                                         'accreditation': line_content[-1]
-                                       }
-    return source_name, source_hash0, source_hash1 
+      # New CA format type, both old hash (hash0) and new hash (hash1), source name has extension (filename)
+      source_hash0[line_content[0]] = line_content[-1]
+      source_hash1[line_content[1]] = line_content[-1]
+      source_name[line_content[2].split('.')[0]] = { 'hash': line_content[0],
+                                                     'newhash': line_content[1],
+                                                     'accreditation': line_content[-1]
+                                                   }
+    return source_name, source_hash0, source_hash1
 
-  def parse_ca_index(self, certdir):
-    "Parse INDEX.txt in CA certs directory and retrieve IndexTypeVersion, default to 0"
-    ca_format_type = 0
-    ca_index = os.path.join(certdir, "INDEX.txt")
-    if os.path.isfile(ca_index):
-      try: 
-        lines = open(ca_index).readlines()
-        for i in lines:
-          if i.find("IndexTypeVersion") >= 0:
-            try:
-              ca_format_type = int(i.split()[2])
-            except IndexError:
-              # When missing it is 0 (default value)
-              pass
-      except (OSError, ValueError):
-        # If I don't find the index file or don't find the keyword IndexTypeVersion assume type 0
-        pass
-    return ca_format_type
- 
   def verify_CRL(self, certdir, warnHrs, errHrs):
     """Check_Freshness_Local_CRL: Checks the last time when the CRLs were successfully downloaded
 parameters :
@@ -192,15 +151,9 @@ errHrs:    Number hours since the failing downloads before an error is issued
 """    
     #status_code = 0 # Return status code as expected by RSV for summaryData
     found_crls = []
-    ca_format_type = 0
 
     # Step 1: Get the list of Certs included in OSG from GOC website.
-    ca_format_type = self.parse_ca_index(certdir)
-
-    #Check if the file CA certs are installed from ITB
-    # grid_type: 0 - osg; 1 - itb
-    grid_type = rsvprobe.get_grid_type()
-    source_name, source_hash0, source_hash1 = self.get_osg_CAs(ca_format_type, grid_type)
+    source_name, source_hash0, source_hash1 = self.get_osg_CAs()
 
     # Step 2: Get and Check time stamps on the CRL files to ensure that it has been recently downloaded
     error_count = 0
@@ -211,8 +164,7 @@ errHrs:    Number hours since the failing downloads before an error is issued
       self.return_unknown("ERROR: CA Certs Directory %s contains no CRL files (*.r0). Aborting the probe (UNKOWN status)." % 
                           certdir)     
     source_hash_list = source_hash0.keys()
-    if ca_format_type == 1:
-      source_hash_list += source_hash1.keys()
+    source_hash_list += source_hash1.keys()
     for i in crl_files:
       local_hash = os.path.basename(i).split('.')[0] #remove the extension
       # List of CRLs found.
@@ -232,13 +184,12 @@ errHrs:    Number hours since the failing downloads before an error is issued
           pass # not in hash0
         # keep the following outside form the except to cover the case when the hash h1 is the old hash of CA cert c1
         # and new hash of CA cert c2. Is this possible?
-        if not is_igtf_cert and ca_format_type == 1:
+        if not is_igtf_cert:
           try:
             if source_hash1[local_hash].find('I') >= 0:
               is_igtf_cert = True
           except KeyError:
             pass # not in hash1
-        if not is_igtf_cert:
           if self.verbose:
             self.add_ok("EGEE test and CRL file for CA with hash %s that is not in IGTF, ignored." % local_hash)
           continue       
@@ -265,25 +216,23 @@ errHrs:    Number hours since the failing downloads before an error is issued
         # Ignore non IGTF CAs for wlcg probe
         continue
       # continue if found CRL
-      if ca_format_type == 1 and ca_info['newhash'] in found_crls:
+      if ca_info['newhash'] in found_crls:
         continue
       if ca_info['hash'] in found_crls:
         continue
-      # continue if the CA file has been removed
-      # if ca_format_type == 1 then both CA files must have been removed to continue
+      # continue if the both CA files have been removed
       if not os.path.isfile(os.path.join(certdir, "%s.0" % ca_info['hash'])):
-        if ca_format_type != 1 or not os.path.isfile(os.path.join(certdir, "%s.0" % ca_info['newhash'])):
+        if not os.path.isfile(os.path.join(certdir, "%s.0" % ca_info['newhash'])):
           continue
-      warning = ("MISSING: CRL file for %s (%s/%s, type %s) is missing. OSG policy requires CRL for every CA distributed by OSG." %
-        (name, ca_info['newhash'], ca_info['hash'], ca_format_type))
+      warning = ("MISSING: CRL file for %s (%s/%s) is missing. OSG policy requires CRL for every CA distributed by OSG." %
+        (name, ca_info['newhash'], ca_info['hash']))
       if missing_count == 0:
         warning += (
           "\n- Has fetch-crl run recently?  (See /etc/cron.d/fetch-crl*)\n"
           "- See fetch-crl output (mailed to root by cron or sent to syslog), "
           "or try running fetch-crl by hand to inspect its output.\n"
           "- More info on troubleshooting this probe can be found at: "
-          "https://www.opensciencegrid.org/bin/view/Documentation/Release3/RsvProbeCrlFreshnessProbe\n"
-          "- For more help, submit a ticket to the GOC at: https://ticket.opensciencegrid.org/")
+          "- For more help, send email to help@opensciencegrid.org")
       self.add_warning(warning)
       missing_count += 1
 

rsv-metrics/libexec/probes/rsvprobe.py

@@ -111,107 +111,6 @@ def get_http_doc(url, quote=True):
   ret = f.readlines()
   return ret
 
-def get_config_val(req_key, req_section=None): 
-  """Get the value of an option from a section of OSG configuration in both Pacman and RPM installations. Return None if option is not found."""
-  confini_fname = None
-  confini_fname_list = []
-  if os.getenv("OSG_LOCATION"):
-    confini_fname_list.append(os.path.join(os.getenv("OSG_LOCATION"), "osg/etc/config.ini"))
-  if os.getenv("VDT_LOCATION"):
-    confini_fname_list.append(os.path.join(os.getenv("VDT_LOCATION"), "osg/etc/config.ini"))
-  for i in confini_fname_list:
-    if os.path.isfile(i):
-      confini_fname = i
-  if not confini_fname:
-    # Assume new OSG 3    
-    # Using NEW osg-configure code/API 
-    try:
-      from osg_configure.modules import configfile
-    except:
-      return None
-    # necassary for exception raised by osg_configure
-    import ConfigParser
-    try:
-      config = configfile.read_config_files() 
-    except IOError:
-      return None
-    if req_section:
-      try:
-        ret = config.get(req_section, req_key)
-      except ConfigParser.NoSectionError:
-        return None
-    else:
-      for section in config.sections():
-        if config.has_option(section, req_key):
-          return config.get(req_section, req_key)
-      try:
-        ret = config.defaults()[req_key]
-      except KeyError:
-        return None
-    return ret
-  # Continue old Pacman installation
-  # Behaves like the old probe: no variable substitution in config.ini
-  try:
-    f = open(confini_fname)
-  except (OSError, IOError):
-    # unable to find config.ini
-    return None
-  # comments at end of section line are OK
-  # comment at end of key = val line are not OK
-  SECTION = re.compile('^\s*\[\s*([^\]]*)\s*\]\s*(?:#.*)$')
-  PARAM   = re.compile('^\s*(\w+)\s*=\s*(.*)\s*$')
-  COMMENT = re.compile('^\s*#.*$')  
-  if not req_section:
-    in_section = True
-  else:
-    in_section = False
-  for line in f:
-    if COMMENT.match(line): continue
-    if req_section:
-      m = SECTION.match(line) 
-      if m:
-        section = m.groups()
-        if section == req_section:
-          in_section = True
-        else:
-          if in_section:
-            # assume sections are all differents (same section not coming back later in file)
-            break
-          in_section = False
-        continue
-    if in_section:          
-      m = PARAM.match(line)
-      if m:
-        key, val = m.groups()
-        return val
-      continue
-    # malformed line (not matching comment, section header or key=val)
-  # key not found (in section)
-  return None  
-
-# Returns the grid type:
-# 1 for itb (OSG-ITB)
-# 0 for production (OSG) or anything else
-def get_grid_type():
-  "Return 1 for OSG-ITB sites, 0 otherwise (production sites)"
-  # Equivalent of config.ini parsing in perl probe:
-  # cat $1/osg/etc/config.ini | sed -e's/ //g'| grep '^group=OSG-ITB' &>/dev/null
-  grid_type = get_config_val("group", "Site Information")
-  if grid_type:
-    if grid_type.strip() == "OSG-ITB":
-      return 1
-  return 0 
-
-def get_grid_type_string(gtype=-1):
-  "Return the translation of the gtype provided 1:OSG-ITB, 0:OSG, or the value from Site_Information"
-  if gtype == 0:
-    grid_type = "OSG"
-  elif gtype == 1:
-    grid_type = "OSG-ITB"
-  else:
-    grid_type = get_config_val("group", "Site Information")
-  return grid_type
-
 def get_temp_dir():
   "Return the a temporary directory to store data across executions."
   # Should I create a directory per user?
@@ -718,3 +617,6 @@ def describe(self):
     return ret
 
 EMPTY_METRIC = RSVMetric('UNKNOWN', 'UNKNOWN')
+
+CA_CERT_HASH_URL = "http://repo.opensciencegrid.org/cadist/cacerts_sha256sum-new.txt"
+CA_CERT_INDEX_URL = "http://repo.opensciencegrid.org/cadist/INDEX.txt"