Add SciMeta validation to GMN

Author: rogerdahl

README.md

@@ -198,7 +198,8 @@ Download the source from GitHub:
 
 Add the DataONE packages to the Python path, and install their dependencies:
 
-    $ sh $D1ROOT/dev_tools/develop.sh
+    cd ~/d1_python
+    sudo dev_tools/src/d1_dev/setup-all.py --root . develop
 
 Run the following commands, except, change the "createuser" line to:
 
@@ -208,7 +209,7 @@ Run the following commands, except, change the "createuser" line to:
 
 Run the following commands (all sections), except, change the location for openssl.cnf, so the line that copies it becomes:
 
-    $ sudo cp /home/dahl/d1_python/d1_mn_generic/src/deployment/openssl.cnf .
+    $ sudo cp <your_d1_python_path>/d1_mn_generic/src/deployment/openssl.cnf .
 
   https://pythonhosted.org/dataone.generic_member_node/setup-local-authn-ca.html
 

client_cli/src/d1_cli/tests/test_session.py

@@ -22,7 +22,6 @@
 """
 from __future__ import absolute_import
 
-import os
 import StringIO
 import sys
 import uuid
@@ -134,17 +133,16 @@ def test_1100(self):
     assert len(out) > 100
     assert type(out) is str
 
-  def test_1110(self):
+  def test_1110(self, tmpdir):
     """Session is successfully saved and then loaded (pickled and unpickled)"""
-    tmp_pickle = './pickle.tmp'
-    try:
-      os.unlink(tmp_pickle)
-    except OSError:
-      pass
+    tmp_pickle_path = str(tmpdir.join('session.pickle'))
+    print type(tmp_pickle_path)
+    print dir(tmp_pickle_path)
+    print tmp_pickle_path
     s1 = session.Session(nodes, format_ids)
     u = str(uuid.uuid1())
     s1.set('rights-holder', u)
-    s1.save(tmp_pickle)
+    s1.save(tmp_pickle_path)
     s2 = session.Session(nodes, format_ids)
-    s2.load(tmp_pickle)
+    s2.load(tmp_pickle_path)
     assert s2.get('rights-holder') == u

conftest.py

@@ -51,7 +51,9 @@
 import django.db
 import django.db.utils
 
-DEFAULT_DEBUG_PYCHARM_BIN_PATH = os.path.expanduser('~/bin/JetBrains/pycharm')
+DEFAULT_DEBUG_PYCHARM_BIN_PATH = os.path.expanduser(
+  '~/bin/JetBrains/pycharm.sh'
+)
 D1_SKIP_LIST = 'skip_passed/list'
 D1_SKIP_COUNT = 'skip_passed/count'
 
@@ -240,6 +242,8 @@ def _open_error_in_pycharm(call):
     logging.debug('Unable to find location of error')
     return
   try:
+    assert os.path.isfile(DEFAULT_DEBUG_PYCHARM_BIN_PATH), \
+      'Path to PyCharm is incorrect'
     subprocess.call(
       [DEFAULT_DEBUG_PYCHARM_BIN_PATH, '--line', str(src_line), str(src_path)]
     )

dev_tools/src/d1_dev/setup-all.py

@@ -41,6 +41,7 @@
 PKG_PATH_LIST = [
   'dev_tools',
   'lib_common',
+  'lib_scimeta',
   'lib_client',
   'client_cli',
   'client_onedrive',

gmn/src/d1_gmn/app/scimeta.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+
+# This work was created by participants in the DataONE project, and is
+# jointly copyrighted by participating institutions in DataONE. For
+# more information on DataONE, see our web site at http://dataone.org.
+#
+#   Copyright 2009-2016 DataONE
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for Science Metadata
+"""
+
+from __future__ import absolute_import
+
+import d1_scimeta.xml_schema
+
+import d1_common.const
+import d1_common.date_time
+import d1_common.types
+import d1_common.types.dataoneTypes
+import d1_common.types.dataoneTypes_v2_0
+import d1_common.types.exceptions
+import d1_common.util
+import d1_common.wrap.access_policy
+import d1_common.xml
+
+import django.conf
+
+
+def assert_valid(sysmeta_pyxb, sciobj_path):
+  """Validate file at {sciobj_path} against schema selected via formatId and
+  raise InvalidRequest if invalid
+
+  Validation is only performed when:
+
+  - SciMeta validation is enabled
+  - and Object size is below size limit for validation
+  - and formatId designates object as a Science Metadata object which is recognized
+    and parsed by DataONE CNs
+  - and XML Schema (XSD) files for formatId are present on local system
+  """
+  if not (
+      _is_validation_enabled() and
+      _is_installed_scimeta_format_id(sysmeta_pyxb)
+  ):
+    return
+
+  if _is_above_size_limit(sysmeta_pyxb):
+    if _is_action_accept():
+      return
+    else:
+      raise d1_common.types.exceptions.InvalidRequest(
+        0, 'Science Metadata file is above size limit for validation and this '
+        'node has been configured to reject unvalidated Science Metadata '
+        'files. For more information, see the SCIMETA_VALIDATE* settings. '
+        'size={} size_limit={}'.format(
+          sysmeta_pyxb.size, django.conf.settings.SCIMETA_VALIDATION_MAX_SIZE
+        )
+      )
+
+  with open(sciobj_path, 'rb') as f:
+    try:
+      d1_scimeta.xml_schema.validate(sysmeta_pyxb.formatId, f.read())
+    except d1_scimeta.xml_schema.SciMetaValidationError as e:
+      raise d1_common.types.exceptions.InvalidRequest(0, str(e))
+
+
+def _is_validation_enabled():
+  return django.conf.settings.SCIMETA_VALIDATION_ENABLED
+
+
+def _is_installed_scimeta_format_id(sysmeta_pyxb):
+  return d1_scimeta.xml_schema.is_installed_scimeta_format_id(
+    sysmeta_pyxb.formatId
+  )
+
+
+def _is_above_size_limit(sysmeta_pyxb):
+  return (
+    django.conf.settings.SCIMETA_VALIDATION_MAX_SIZE == -1 or
+    sysmeta_pyxb.size > django.conf.settings.SCIMETA_VALIDATION_MAX_SIZE
+  )
+
+
+def _is_action_accept():
+  return django.conf.settings.SCIMETA_VALIDATION_OVER_SIZE_ACTION == 'accept'

gmn/src/d1_gmn/app/settings_default.py

@@ -112,6 +112,10 @@
 
 RESOURCE_MAP_CREATE = 'block'
 
+SCIMETA_VALIDATION_ENABLED = True
+SCIMETA_VALIDATION_MAX_SIZE = 100 * 1024**2
+SCIMETA_VALIDATION_OVER_SIZE_ACTION = 'reject'
+
 PROXY_MODE_BASIC_AUTH_ENABLED = False
 PROXY_MODE_BASIC_AUTH_USERNAME = ''
 PROXY_MODE_BASIC_AUTH_PASSWORD = ''

gmn/src/d1_gmn/app/startup.py

@@ -43,12 +43,61 @@ class GMNStartupChecks(django.apps.AppConfig):
   name = 'd1_gmn.app.startup'
 
   def ready(self):
-    self._check_cert_file('CLIENT_CERT_PATH')
-    self._check_cert_file('CLIENT_CERT_PRIVATE_KEY_PATH')
+    self._assert_readable_file_if_set('CLIENT_CERT_PATH')
+    self._assert_readable_file_if_set('CLIENT_CERT_PRIVATE_KEY_PATH')
+
+    self._assert_is_type('SCIMETA_VALIDATION_ENABLED', bool)
+    self._assert_is_type('SCIMETA_VALIDATION_MAX_SIZE', int)
+    self._assert_is_in(
+      'SCIMETA_VALIDATION_OVER_SIZE_ACTION', ('reject', 'accept')
+    )
+
     self._warn_unsafe_for_prod()
     self._check_resource_map_create()
     self._create_sciobj_store_root()
 
+  def _assert_is_type(self, setting_name, valid_type):
+    v = getattr(django.conf.settings, setting_name, None)
+    if not isinstance(v, valid_type):
+      self.raise_config_error(setting_name, valid_type)
+
+  def _assert_is_in(self, setting_name, valid_list):
+    if getattr(django.conf.settings, setting_name, None) not in valid_list:
+      self.raise_config_error(setting_name, valid_list)
+
+  def _assert_readable_file_if_set(self, setting_name):
+    v = getattr(django.conf.settings, setting_name, None)
+    if v is None:
+      return
+    self._assert_is_type(setting_name, str)
+    if not os.path.isfile(v):
+      self.raise_config_error(
+        setting_name, v, str, valid_str='a path to a readable file'
+      )
+    try:
+      with open(v, 'r') as f:
+        f.read(1)
+    except EnvironmentError as e:
+      self.raise_config_error(
+        setting_name, v, str, valid_str='a path to a readable file. error="{}"'
+        .format(str(e))
+      )
+
+  def raise_config_error(self, setting_name, cur_val, exp_type, valid_str=None):
+    valid_str = valid_str or (
+      'a whole number' if isinstance(exp_type, int) else 'a number'
+      if isinstance(exp_type, float) else 'a string'
+      if isinstance(exp_type, str) else 'True or False' if
+      isinstance(exp_type, bool) else 'one of {}'.format(', '.join(exp_type))
+      if isinstance(exp_type,
+                    (list, tuple)) else 'of type {}'.format(exp_type.__name__)
+    )
+    msg_str = u'Configuration error: Setting {} must be {}. current="{}"'.format(
+      setting_name, valid_str, str(cur_val)
+    )
+    logging.error(msg_str)
+    raise django.core.exceptions.ImproperlyConfigured(msg_str)
+
   def _warn_unsafe_for_prod(self):
     """Warn on settings that are not safe for production"""
     safe_settings_list = [
@@ -65,21 +114,6 @@ def _warn_unsafe_for_prod(self):
           'safe="{}"'.format(setting_str, setting_current, setting_safe)
         )
 
-  def _check_cert_file(self, cert_pem_setting):
-    cert_pem_path = getattr(django.conf.settings, cert_pem_setting, None)
-    if cert_pem_path is None:
-      logging.warn(
-        'Certificate path not set. setting="{}"'.format(cert_pem_setting)
-      )
-      return
-    try:
-      self._assert_readable_file(cert_pem_path)
-    except ValueError as e:
-      raise django.core.exceptions.ImproperlyConfigured(
-        u'Configuration error: Invalid certificate path. '
-        u'setting="{}". msg="{}"'.format(cert_pem_setting, str(e))
-      )
-
   def _check_resource_map_create(self):
     if (
       django.conf.settings.RESOURCE_MAP_CREATE not in RESOURCE_MAP_CREATE_MODE_LIST
@@ -144,15 +178,3 @@ def _create_sciobj_store_root(self):
           d1_gmn.app.sciobj_store.get_gmn_version(),
         )
       )
-
-  def _assert_readable_file(self, file_path):
-    if not os.path.isfile(file_path):
-      raise ValueError('Not a valid file path. path="{}"'.format(file_path))
-    try:
-      with open(file_path, 'r') as f:
-        f.read(1)
-    except EnvironmentError as e:
-      raise ValueError(
-        'Unable to read file. path="{}" error="{}"'.
-        format(file_path, e.message)
-      )

gmn/src/d1_gmn/app/views/create.py

@@ -26,6 +26,7 @@
 import d1_gmn.app.event_log
 import d1_gmn.app.resource_map
 import d1_gmn.app.revision
+import d1_gmn.app.scimeta
 import d1_gmn.app.sciobj_store
 import d1_gmn.app.sysmeta
 import d1_gmn.app.util
@@ -75,6 +76,7 @@ def create_sciobj(request, sysmeta_pyxb):
       _create_resource_map(pid, request, sciobj_path, sysmeta_pyxb, url)
     else:
       _save_sciobj_bytes_from_request(request, sciobj_path)
+      d1_gmn.app.scimeta.assert_valid(sysmeta_pyxb, sciobj_path)
 
   d1_gmn.app.sysmeta.create_or_update(sysmeta_pyxb, url)
 

gmn/src/d1_gmn/settings_template.py

@@ -424,6 +424,53 @@
 # object is created by the Resource Map subject.
 RESOURCE_MAP_CREATE = 'block'
 
+# Validate Science Metadata objects against local XML Schema (XSD)
+# True (default):
+# - When a SciMeta format that is recognized and parsed by CNs is
+#   received in MNStorage.create() or MNStorage.update(), GMN rejects the
+#   operation and returns an InvalidRequest with details to the client if the
+#   object is not well formed, valid, and matching the SysMeta formatId.
+# False:
+# - SciMeta objects are not validated. This is not recommended, as any invalid
+#   objects will be rejected by the CN during synchronization.
+# Notes:
+# - Objects affected by this setting have formatType of METADATA in the CN's
+# objectFormatList.
+# - Actual validation is performed by the d1_sciobj package, which may not
+#   support all SciMeta formats. Validation is silently skipped for any
+#   unsupported formats.
+# - Objects that are stored remotely (using GMN's proxy support), are not
+#   validated.
+SCIMETA_VALIDATION_ENABLED = True
+
+# The maximum size in bytes of SciMeta objects received in MNStorage.create()
+# and MNStorage.update() that will be validated. SciMeta objects larger than
+# this size are not validated and are handled according to the
+# SCIMETA_VALIDATION_OVER_SIZE_ACTION setting.
+#
+# This setting applies only when SCIMETA_VALIDATION is set to True.
+#
+# As SciMeta documents are read into memory for validation, limiting the maximum
+# size of objects that will be validated helps reduce the chance of the server
+# running out of memory.
+#
+# E.g.: 100 MiB = 1024**2 (default)
+# To validate SciMeta of any size, set to -1 (not recommended).
+SCIMETA_VALIDATION_MAX_SIZE = 100 * 1024**2
+
+# The action to perform for SciMeta objects received in MNStorage.create()
+# and MNStorage.update() larger than size set in SCIMETA_VALIDATION_MAX_SIZE.
+#
+# This setting applies only when SCIMETA_VALIDATION_ENABLED is set to True and
+# SCIMETA_VALIDATION_MAX_SIZE is not set to -1.
+#
+# - 'reject' (default): SciMeta over Max Size is rejected and GMN returns an
+#   InvalidRequest with explanation to the client.
+# - 'accept': SciMeta over Max Size is accepted but not validated. This is not
+#   recommended, as any invalid objects will later be rejected by the CN during
+#   synchronization.
+SCIMETA_VALIDATION_OVER_SIZE_ACTION = 'reject'
+
 # GMN implements a vendor specific extension for MNStorage.create(). Instead of
 # providing an object for GMN to manage, the object can be left empty and the
 # URL of the object on a 3rd party server be provided instead. In that case, GMN