Move PyCharm debug support into test framework

+ Move to using the PyCharm Diff & Merge tool for viewing sample mismatches

Author: rogerdahl

README.md

@@ -135,16 +135,22 @@ This module is typically used in tests running under django.test.TestCase and re
 
 #### Command line switches
 
-We have added some custom functionality to pytest which can be enabled to launching pytest with the following switches:
+We have added some custom functionality to pytest which can be enabled by launching pytest with the following switches:
 
-  * `--sample-ask`: Enable a mode that invokes `kdiff3` to display diffs and, after user confirmation, can automatically update or write new test sample documents on mismatches.
+  * `--sample-ask`: Enable a mode that display diffs and, after user confirmation, can automatically update or write new test sample documents on mismatches.
 
-  * `parameterize_dict`: Support for parameterizing test functions by adding a dict class member containing parameter sets.
-
-  * `--pycharm`: Attempt to move the cursor in PyCharm to the location of the test of failure.
+  * `--pycharm`:
+   
+    * Automatically open files where errors occur and move the cursor to the line of the error
+    
+    * Show syntax highlighted diffs for scripts and data files using PyCharm's powerful diff viewer
+    
+    * Also requires the path to the PyCharm binary to be configured in `DEBUG_PYCHARM_BIN_PATH` in `./conftest.py`. 
 
   * See `./conftest.py` for implementation and notes.
 
+  * `parameterize_dict`: Support for parameterizing test functions by adding a dict class member containing parameter sets.
+
 Note: None of these switches can be used when running tests in parallel with xdist (`-n`, `--dist`, `--tx`).
 
 #### Debugging tests with PyCharm

conftest.py

@@ -19,6 +19,7 @@
 # limitations under the License.
 """pytest setup and customization
 """
+import d1_test.pycharm
 import logging
 import multiprocessing
 import os
@@ -52,9 +53,8 @@
 import django.db.transaction
 import django.db.utils
 
-DEFAULT_DEBUG_PYCHARM_BIN_PATH = os.path.expanduser(
-  '~/bin/JetBrains/pycharm.sh'
-)
+
+
 D1_SKIP_LIST = 'skip_passed/list'
 D1_SKIP_COUNT = 'skip_passed/count'
 
@@ -260,30 +260,9 @@ def _print_skip_list():
 
 def _open_error_in_pycharm(call):
   """Attempt to open error locations in PyCharm. Use with --exitfirst (-x)"""
-  # src_path, src_line, func_name = rep.location
   src_path = call.excinfo.traceback[-1].path
   src_line = call.excinfo.traceback[-1].lineno + 1
-  logging.info('src_path="{}", src_line={}'.format(src_path, src_line))
-  if src_path == '<string>':
-    logging.debug('Unable to find location of error')
-    return
-  try:
-    assert os.path.isfile(DEFAULT_DEBUG_PYCHARM_BIN_PATH), \
-      'Path to PyCharm is incorrect'
-    subprocess.call(
-      [DEFAULT_DEBUG_PYCHARM_BIN_PATH, '--line', str(src_line), str(src_path)]
-    )
-  except subprocess.CalledProcessError as e:
-    logging.warning(
-      'Unable to open in PyCharm. error="{}" src_path="{}", src_line={}'.
-      format(str(e), src_path, src_line)
-    )
-  else:
-    logging.debug(
-      'Opened in PyCharm. src_path="{}", src_line={}'.
-      format(src_path, src_line)
-    )
-
+  d1_test.pycharm.open_and_set_cursor(src_path, src_line)
 
 # Fixtures
 

dev_tools/src/d1_dev/trigger-pre-commit.py

@@ -36,6 +36,7 @@
 there are more errors or S to skip directly to the next file.
 """
 
+import d1_test.pycharm
 import argparse
 import logging
 import os
@@ -49,9 +50,6 @@
 import d1_common.iter.dir
 import d1_common.util
 
-# Path to PyCharm, for automatically moving to errors in the IDE
-DEFAULT_PYCHARM_BIN_PATH = os.path.expanduser('~/bin/JetBrains/pycharm.sh')
-
 
 def main():
   parser = argparse.ArgumentParser(
@@ -69,8 +67,7 @@ def main():
     '--ignore-invalid', action='store_true', help='Ignore invalid paths'
   )
   parser.add_argument(
-    '--pycharm', action='store', default=DEFAULT_PYCHARM_BIN_PATH,
-    help='Path to PyCharm binary. Set to "" to disable PyCharm integration'
+    '--pycharm', action='store_true', help='Enable PyCharm integration'
   )
   parser.add_argument(
     '--debug', action='store_true', help='Debug level logging'
@@ -89,7 +86,7 @@ def main():
   trigger_path_list = sorted(
     set(specified_file_path_list).intersection(tracked_and_modified_path_list)
   )
-  trigger_all_pre_commit_hooks(trigger_path_list)
+  trigger_all_pre_commit_hooks(args, trigger_path_list)
 
 
 def get_specified_file_path_list(args):
@@ -108,16 +105,16 @@ def get_specified_file_path_list(args):
   return specified_file_path_list
 
 
-def trigger_all_pre_commit_hooks(trigger_path_list):
+def trigger_all_pre_commit_hooks(args, trigger_path_list):
   for trigger_path in trigger_path_list:
     logging.info('Checking file. path="{}"'.format(trigger_path))
     while True:
-      recheck_bool = trigger_pre_commit_hook(trigger_path)
+      recheck_bool = trigger_pre_commit_hook(args, trigger_path)
       if not recheck_bool:
         break
 
 
-def trigger_pre_commit_hook(trigger_path):
+def trigger_pre_commit_hook(args, trigger_path):
   try:
     res_str = subprocess.check_output(
       [
@@ -138,11 +135,11 @@ def trigger_pre_commit_hook(trigger_path):
     m = re.search(r'\.py:(\d+):', res_line)
     if m:
       logging.info('Error: {}'.format(res_line))
-      if DEFAULT_PYCHARM_BIN_PATH is not None:
-        open_exception_location_in_pycharm(trigger_path, m.group(1))
+      if args.pycharm:
+        d1_test.pycharm.open_and_set_cursor(trigger_path, m.group(1))
       while True:
         action_str = input(
-          'Opened in PyCharm. Recheck: Enter, Skip file: s Enter: '
+          'Recheck: Enter, Skip file: s Enter: '
         )
         if action_str == '':
           return True
@@ -168,23 +165,6 @@ def get_git_staged_files(repo):
     yield file_path
 
 
-def open_exception_location_in_pycharm(src_path, src_line_num):
-  try:
-    subprocess.call([
-      DEFAULT_PYCHARM_BIN_PATH, '--line', src_line_num, src_path
-    ])
-  except subprocess.CalledProcessError as e:
-    logging.warning(
-      'PyCharm debugging is enabled but opening the location of the exception '
-      'in PyCharm failed. error="{}" src_path="{}", src_line={}'.format(
-        str(e), src_path, src_line_num
-      )
-    )
-  else:
-    logging.debug(
-      'Opened location of exception in PyCharm. src_path="{}", src_line={}'
-      .format(src_path, src_line_num)
-    )
 
 
 if __name__ == '__main__':

dev_tools/src/d1_dev/util.py

@@ -31,6 +31,7 @@
 import redbaron
 import redbaron.nodes
 
+import d1_test.pycharm
 
 def are_files_equal(old_file, new_file):
   with open(old_file, 'rb') as old_f:
@@ -94,7 +95,9 @@ def diff_update_file(module_path, module_str, show_diff=False, dry_run=False):
     if show_diff:
       try:
         tmp_file.seek(0)
-        subprocess.check_call(['kdiff3', module_path, tmp_file.name])
+        # subprocess.check_call(['kdiff3', module_path, tmp_file.name])
+        d1_test.pycharm.diff(module_path, tmp_file.name)
+
         # Running from the console
         # subprocess.check_call(['condiff.sh', module_path, tmp_file.name])
       except subprocess.CalledProcessError:

gmn/src/d1_gmn/app/gmn.py

@@ -128,7 +128,6 @@ def _warn_unsafe_for_prod(self):
     safe_settings_list = [
       ('DEBUG', False),
       ('DEBUG_GMN', False),
-      ('DEBUG_PYCHARM', False),
       ('STAND_ALONE', False),
       ('DATABASES.default.ATOMIC_REQUESTS', True),
       ('SECRET_KEY', '<Do not modify this placeholder value>'),

gmn/src/d1_gmn/app/middleware/exception_handler.py

@@ -102,8 +102,6 @@ def _log_dataone_exception(self, e):
 
   def _handle_internal_exception(self, request):
     self._log_internal_exception()
-    if django.conf.settings.DEBUG_PYCHARM:
-      self._open_exception_location_in_pycharm()
     if django.conf.settings.DEBUG:
       return self._django_html_exception_page()
     else:
@@ -157,37 +155,3 @@ def _traceback_to_trace_info(self):
       trace_info_list.append('Value: {}'.format(exc_value))
     return trace_info_list
 
-  # PyCharm debugging
-
-  def _open_exception_location_in_pycharm(self):
-    src_path, src_line_num = self._get_project_location()
-    try:
-      subprocess.call([
-        os.path.expanduser(django.conf.settings.DEBUG_PYCHARM_BIN_PATH),
-        '--line', src_line_num, src_path
-      ])
-    except subprocess.CalledProcessError as e:
-      logging.warning(
-        'PyCharm debugging is enabled but opening the location of the exception '
-        'in PyCharm failed. error="{}" src_path="{}", src_line={}'.format(
-          str(e), src_path, src_line_num
-        )
-      )
-    else:
-      logging.info(
-        'Opened location of exception in PyCharm. src_path="{}", src_line={}'.
-        format(src_path, src_line_num)
-      )
-
-  def _get_project_location(self):
-    """Return the abs path and line number of the line of project code that was
-    being executed when the exception was raised.
-    """
-    exc_type, exc_value, exc_traceback = sys.exc_info()
-    location_tup = ()
-    while exc_traceback:
-      co = exc_traceback.tb_frame.f_code
-      if co.co_filename.startswith(django.conf.settings.BASE_DIR):
-        location_tup = co.co_filename, str(exc_traceback.tb_lineno)
-      exc_traceback = exc_traceback.tb_next
-    return location_tup

gmn/src/d1_gmn/app/settings_default.py

@@ -33,8 +33,6 @@
 
 DEBUG = False
 DEBUG_GMN = False
-DEBUG_PYCHARM = False
-DEBUG_PYCHARM_BIN = 'pycharm.sh'
 DEBUG_ECHO_REQUEST = False
 DEBUG_PROFILE_SQL = False
 

gmn/src/d1_gmn/deployment/migrate_v1_to_v2.sh

@@ -38,7 +38,6 @@ cp ${v1s} ${v2s}
 sed -ri "s/django.utils.log.NullHandler/logging.NullHandler/" ${v2s}
 
 sed -ri "s/GMN_DEBUG/DEBUG_GMN/" ${v2s}
-sed -ri "/DEBUG_GMN\s*=/a\DEBUG_PYCHARM = False" ${v2s}
 
 sed -ri "s/^(PUBLIC_OBJECT_LIST\s*=).*/\1 True/" ${v2s}
 sed -ri "s/^(PUBLIC_LOG_RECORDS\s*=).*/\1 True/" ${v2s}

gmn/src/d1_gmn/settings_template.py

@@ -63,22 +63,6 @@
 # - Use for production.
 DEBUG_GMN = False
 
-# Enable PyCharm debugging.
-# True:
-# - If GMN encounters an unhandled internal exception, GMN will attempt to move
-#   the cursor in the PyCharm IDE to the code that was being executed when the
-#   exception was raised. The exception is then handled as normal.
-# False (default):
-# - GMN handles exceptions as normal.
-DEBUG_PYCHARM = False
-
-# Path to the PyCharm IDE binary.
-# - Only used if DEBUG_PYCHARM = True.
-# - If PyCharm is in the path, can typically left at 'pycharm.sh'
-# - If PyCharm is not in path, can be set to an absolute path. E.g.
-#   '~/JetBrains/pycharm'
-DEBUG_PYCHARM_BIN = 'pycharm.sh'
-
 # Enable request echo.
 # True:
 # - GMN will not process any requests. Instead, it will echo the requests

gmn/src/d1_gmn/settings_test.py

@@ -36,8 +36,6 @@
 
 DEBUG = True
 DEBUG_GMN = True
-DEBUG_PYCHARM = False
-DEBUG_PYCHARM_BIN = 'pycharm.sh'
 DEBUG_ECHO_REQUEST = False
 DEBUG_PROFILE_SQL = False