Misc smaller refactoring

Author: rogerdahl

CHANGELOG.md

@@ -21,15 +21,15 @@
 
   * Added more validation of GMN settings
   * Added migrations to generate Postgres indexes for default sorting  
-    * Suprisingly, Django does not generate indexes for default sort ordering specified when using the ORM
+    * Surprisingly, Django does not generate indexes for default sort ordering specified when using the ORM
 
   * Reconfigured logging to take advantage of Django's support for rotating logs, much like logrotate does. Maximum space to use for logging is now a config setting.
   * Other usability improvements (see log)
 
 * GMN deployment 
 
   * Improved and streamline GMN install procedures, including
-    * Refactored install so that it can be accomplished by in two stages, where only the first stage needs to be performed by account with sudo access
+    * Refactored install so that it can be accomplished in two stages, where only the first stage needs to be performed by account with sudo access
 
   * Improved and cleaned up Apache conf file
     * Move from Rewrite to Redirect / Alias

README.md

@@ -296,8 +296,6 @@ Edit `~/.pypirc`:
 
 #### Updating dependencies
 
-TODO: Move from pip to pipenv. https://docs.pipenv.org/
-
 Update all packages managed by pip:
 
     $ cd d1_python
@@ -328,17 +326,15 @@ Commit and push the changes, and check the build on Travis.
 
 After successful build, clone a fresh copy, which will be used for building the release packages:
 
-    $ cd ~
-    $ rm -rf ~/d1_python_build
-    $ git clone [email protected]:DataONEorg/d1_python.git d1_python_build
-
 Building the release packages from a fresh clone is a simple way of ensuring that only tracked files are released. It is a workaround for the way setuptools works, which is basically that it vacuums up everything that looks like a Python script in anything that looks like a package, which makes it easy to publish local files by accident.
 
 Build and publish the packages:
 
-    cd ~/d1_python_build
-    setup-all.py --root . bdist_wheel upload
-
+    $ cd
+    $ rm -rf ~/d1_python_build
+    $ git clone [email protected]:DataONEorg/d1_python.git d1_python_build
+    $ cd ~/d1_python_build
+    $ python3 ./dev_tools/src/d1_dev/setup-all.py --root . bdist_wheel upload
 
 ### Building the documentation
 

gmn/src/d1_gmn/deployment/gmn3-ssl.conf

@@ -9,7 +9,7 @@
     define gmn_base             /mn
 
     ServerName                  ${server_name}
-    WSGIPythonHome              ${d1_root}/gmn_venv_py3
+    WSGIPythonHome              ${gmn_venv}
 
     <VirtualHost *:80>
         Redirect /              https://${server_name}/
@@ -52,10 +52,11 @@
         WSGIDaemonProcess       gmn3 user=gmn processes=2 threads=25 python-path=${gmn_pkg}
         WSGIProcessGroup        gmn3
 
-        Alias /robots.txt               ${gmn_root}/app/static/robots.txt
-        Alias /static/                  ${gmn_root}/app/static/
-        RedirectMatch "^/?$"            ${gmn_base}/home
+        Alias /robots.txt       ${gmn_root}/app/static/robots.txt
+        Alias /static/          ${gmn_root}/app/static/
+        RedirectMatch "^/?$"    ${gmn_base}/home
         RedirectMatch "^${gmn_base}/?$" ${gmn_base}/v2/node
 
+        AddType                 text/xsl .xsl
     </VirtualHost>
 </IfModule>

gmn/src/d1_gmn/tests/gmn_direct.py

@@ -132,7 +132,7 @@ def get_log_records(version_tag, pid=None, start=None, count=None):
 
 def _add_query(query_dict, url_path):
   if query_dict:
-    url_str = '{}?{}'.format(url_path, d1_common.url.urlencode(query_dict))
+    url_str = '{}?{}'.format(url_path, d1_common.url.encodePathElement().urlencode(query_dict))
   else:
     url_str = url_path
   return url_str

gmn/src/d1_gmn/wsgi.py

@@ -25,7 +25,9 @@
 
 import django
 import django.core.handlers.wsgi
+import django.http
 import django.utils
+import django.utils.datastructures
 
 os.environ['DJANGO_SETTINGS_MODULE'] = 'd1_gmn.settings'
 

lib_common/src/d1_common/resource_map.py

@@ -43,7 +43,7 @@
 import d1_common.url
 import d1_common.util
 
-# RDFLib wrappers around the namespaces. Others are defined by RDFLib
+# RDFLib wrappers around the namespaces. Additional ones are defined by RDFLib
 DCTERMS = rdflib.Namespace(d1_common.const.ORE_NAMESPACE_DICT['dcterms'])
 CITO = rdflib.Namespace(d1_common.const.ORE_NAMESPACE_DICT['cito'])
 ORE = rdflib.Namespace(d1_common.const.ORE_NAMESPACE_DICT['ore'])
@@ -133,7 +133,7 @@ def oreInitialize(self, pid, ore_software_id=d1_common.const.ORE_SOFTWARE_ID):
     self.add((ore, DCTERMS.identifier, rdflib.term.Literal(pid)))
     self.add((ore, DCTERMS.creator, rdflib.term.Literal(ore_software_id)))
     # Add an empty aggregation
-    ag = rdflib.URIRef(d1_common.url.joinPathElements(oid, 'aggregation'))
+    ag = rdflib.URIRef(oid + '#aggregation')
     self.add((ore, ORE.describes, ag))
     self.add((ag, rdflib.RDF.type, ORE.Aggregation))
     self.add((ORE.Aggregation, rdflib.RDFS.isDefinedBy, ORE.term('')))

lib_common/src/d1_common/tests/test_url.py

@@ -25,46 +25,27 @@
 
 import d1_test.d1_test_case
 import d1_test.sample
+import d1_test.test_files
 
-HERE_DIR_PATH = os.path.abspath(os.path.dirname(__file__))
+# HERE_DIR_PATH = os.path.abspath(os.path.dirname(__file__))
 
 
 class TestUrl(d1_test.d1_test_case.D1TestCase):
-  unicode_str_list = d1_test.sample.load_utf8_to_str(
-    'testUnicodeStrings.utf8.txt'
-  )
+  # unicode_str_list = d1_test.test_files.load_utf8_to_str(
+  #   'testUnicodeStrings.utf8.txt'
+  # )
 
   def test_1000(self):
     """encodePathElement()"""
-    for row in self.unicode_str_list.splitlines():
-      assert isinstance(row, str)
-      parts = row.split('\t')
-      if len(parts) > 1:
-        v = parts[0]
-        if v.startswith('common') or v.startswith('path'):
-          e = parts[1].strip()
-          assert e == d1_common.url.encodePathElement(v)
+    for did, enc_did in d1_test.test_files.TRICKY_IDENTIFIER_LIST:
+      if did.startswith('common') or did.startswith('path'):
+        assert enc_did == d1_common.url.encodePathElement(did)
 
   def test_1010(self):
     """encodeQueryElement()"""
-    for row in self.unicode_str_list.splitlines():
-      parts = row.split('\t')
-      if len(parts) > 1:
-        v = parts[0]
-        if v.startswith('common') or v.startswith('query'):
-          e = parts[1].strip()
-          assert e == d1_common.url.encodeQueryElement(v)
-
-  def test_1020(self):
-    """urlencode()"""
-    data = [('a', '"#<>[]^`{}|'), ('b', '-&=&='),
-            ('c', 'http://example.com/data/mydata?row=24')]
-    expected = (
-      'a=%22%23%3C%3E%5B%5D%5E%60%7B%7D%7C&b=-%26%3D%26%3D&c='
-      'http://example.com/data/mydata?row%3D24'
-    )
-    test = d1_common.url.urlencode(data)
-    assert test == expected
+    for did, enc_did in d1_test.test_files.TRICKY_IDENTIFIER_LIST:
+      if did.startswith('common') or did.startswith('path'):
+        assert enc_did == d1_common.url.encodeQueryElement(did)
 
   def test_1030(self):
     """stripElementSlashes()"""

test_utilities/src/d1_test/d1_test_case.py

@@ -19,22 +19,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utilities for unit- and integration tests"""
-
+import codecs
 import collections
 import contextlib
 import datetime
 import gc
 import inspect
 import io
+import json
 import logging
 import os
 import random
+import re
 import resource
 import sys
 import tempfile
 import traceback
 import xml
 
+import d1_common
 import decorator
 import mock
 import psutil
@@ -54,6 +57,7 @@
 import d1_test.instance_generator.date_time
 import d1_test.instance_generator.system_metadata
 import d1_test.sample
+import d1_test.test_files
 
 CN_URL = d1_common.const.URL_DATAONE_ROOT
 
@@ -79,6 +83,7 @@
 }
 
 
+
 @contextlib.contextmanager
 def capture_std():
   """Capture stdout and stderr.
@@ -208,11 +213,11 @@ def temp_sparse_file(gib=0, mib=0, kib=0, b=0):
 
 
 @contextlib.contextmanager
-def temp_file_name():
+def temp_file_name(suffix=None):
   """Provide a file path that can be used as the location of a temporary file,
   and delete any file written to the path on exit
   """
-  with tempfile.NamedTemporaryFile() as f:
+  with tempfile.NamedTemporaryFile(suffix=suffix) as f:
     temp_file_path = f.name
   yield temp_file_path
   try:
@@ -236,8 +241,9 @@ def memory_limit(max_mem_bytes):
     current_used_bytes = process.memory_info().vms
     limit_bytes = current_used_bytes + max_mem_bytes
     logging.debug(
-      'Setting memory limit. current={:,} bytes, limit={:,} bytes'.
-      format(current_used_bytes, limit_bytes)
+      'Setting memory limit. current={:,} bytes, limit={:,} bytes'.format(
+        current_used_bytes, limit_bytes
+      )
     )
     resource.setrlimit(
       resource.RLIMIT_AS,
@@ -262,6 +268,11 @@ class D1TestCase(object):
   def sample(self):
     return d1_test.sample
 
+  @property
+  def test_files(self):
+    return d1_test.test_files
+
+
   @staticmethod
   def deserialize_and_check(doc, raises_pyxb_exc=False):
     try:
@@ -312,19 +323,91 @@ def touch(module_path, times=None):
     with open(module_path, 'a'):
       os.utime(module_path, times)
 
+
+  #
+  # Exception handling
+  #
+
+  # @staticmethod
+  # def get_d1_test_case_location(exc_traceback=None):
+  #   """Return the abs path and line number of the unit test that directly or
+  #   indirectly triggered the exception
+  #   - Use the exception currently being handled if exc_traceback is None,
+  #   else use exc_traceback.
+  #   - The unit test must be a method in a class that derives from D1TestCase.
+  #   """
+  #   tb_frame = D1TestCase.get_last_d1_test_case_frame(exc_traceback)
+  #   return tb_frame.f_code.co_filename, tb_frame.f_lineno #+ 1
+
+    # exc_traceback = D1TestCase.get_and_check_traceback(exc_traceback)
+    # location_tup = None
+    # while exc_traceback:
+    #   if D1TestCase.frame_is_d1_test_case_method(exc_traceback):
+    #     co = exc_traceback.tb_frame.f_code
+    #     location_tup = co.co_filename, co.co_firstlineno + 1
+    #   exc_traceback = exc_traceback.tb_next
+    # if location_tup:
+    #   return location_tup
+    # raise Exception(
+    #   "Exception was not triggered in a D1TestCase unit test method"
+    # )
+
+
+  @staticmethod
+  def get_d1_test_case_location(exc_traceback=None):
+    """Return the last stack frame that holds a D1TestCase unit test method
+    - Use the exception currently being handled if exc_traceback is None,
+    else use exc_traceback.
+    - The unit test must be a method in a class that derives from D1TestCase.
+    """
+    exc_traceback = D1TestCase.get_and_check_traceback(exc_traceback)
+    location_tup = None
+    while exc_traceback:
+      if D1TestCase.frame_is_d1_test_case_method(exc_traceback):
+        location_tup = (
+          exc_traceback.tb_frame.f_code.co_filename,
+          exc_traceback.tb_lineno,
+        )
+
+      exc_traceback = exc_traceback.tb_next
+    if not location_tup:
+      raise Exception(
+        "Exception was not triggered in a D1TestCase unit test method"
+      )
+    return location_tup
+
+
+  @staticmethod
+  def frame_is_d1_test_case_method(exc_traceback):
+    try:
+      return isinstance(exc_traceback.tb_frame.f_locals['self'],
+                        D1TestCase)
+    except KeyError:
+      return False
+
+
+  @staticmethod
+  def get_and_check_traceback(tb):
+    tb = tb or sys.exc_info()[2]
+    assert hasattr(tb, "tb_next"), \
+      "Not a valid traceback. Must be a native Python traceback, not a " \
+      "pytest.Traceback."
+    return tb
+
+  #
+  # Misc
+  #
+
   @staticmethod
   @contextlib.contextmanager
   def mock_ssl_download(cert_obj):
     """Simulate successful cert download by catching call to
     ssl.SSLSocket.getpeercert() and returning {cert_obj} in DER format.
     """
     cert_der = d1_common.cert.x509.get_cert_der(cert_obj)
-    with mock.patch(
-        'd1_common.cert.x509.ssl.SSLSocket.connect'
-    ) as mock_connect:
-      with mock.patch(
-          'd1_common.cert.x509.ssl.SSLSocket.getpeercert'
-      ) as mock_getpeercert:
+    with mock.patch('d1_common.cert.x509.ssl.SSLSocket.connect') as mock_connect:
+      with mock.patch('d1_common.cert.x509.ssl.SSLSocket.getpeercert'
+                      ) as mock_getpeercert:
         mock_getpeercert.return_value = cert_der
         yield mock_connect, mock_getpeercert
 
@@ -426,7 +509,6 @@ def prep_node_list(self, node_list, tag_str, num_nodes=5):
         for _ in range(num_nodes)
       ]
 
-
 #===============================================================================
 
 # import logging, logging.config, colorstreamhandler
@@ -496,3 +578,4 @@ def get_test_module_name():
     module_name = os.path.splitext(os.path.split(module_path)[1])[0]
     if module_name.startswith('test_') and func_name.startswith('test_'):
       return module_name
+

test_utilities/src/d1_test/utilities/generate_test_subject_certs.py

@@ -247,6 +247,7 @@ def create_session_extension(subject, persons, groups):
 
 
 def create_certificate(
+    # TODO: Fix and test
     req, xxx_todo_changeme, serial, xxx_todo_changeme1, digest="md5"
 ):
   """Generate a certificate given a certificate request.

utilities/src/d1_util/jwt_token_tasks.py

@@ -99,11 +99,11 @@ def find_valid_combinations(cert_file_name_list, jwt_file_name_list):
   successfully validated with the cert.
   """
   for cert_file_name in cert_file_name_list:
-    cert_pem = '' # self.sample.load_utf8_to_str(cert_file_name)
+    cert_pem = '' # self.test_files.load_utf8_to_str(cert_file_name)
     cert_obj = d1_common.cert.x509.deserialize_pem(cert_pem)
     # d1_common.cert.x509.log_cert_info(logging.info, 'CERT', cert_obj)
     for jwt_file_name in jwt_file_name_list:
-      jwt_bu64 = '' # self.sample.load_utf8_to_str(jwt_file_name)
+      jwt_bu64 = '' # self.test_files.load_utf8_to_str(jwt_file_name)
       # d1_common.cert.jwt.log_jwt_bu64_info(logging.info, 'JWT', jwt_bu64)
       is_ok = False
       try: