Python Code Cleanup

.travis.yml

@@ -1,8 +1,7 @@
 dist: xenial
 language: python
 python:
-  - 3.5
-  - 3.8
+  - '3.8'
 before_install:
   - sudo apt-get update
   - sudo xargs -a apt-packages.txt apt-get install --fix-missing

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # ML API documentation build configuration file, created by
 # sphinx-quickstart on Fri Mar  1 09:51:10 2013.
@@ -45,8 +44,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'ML'
-copyright = u'2013, edX'
+project = 'ML'
+copyright = '2013, edX'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -188,8 +187,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'ML.tex', u'ML API Documentation',
-   u'edX', 'manual'),
+  ('index', 'ML.tex', 'ML API Documentation',
+   'edX', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -218,8 +217,8 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'ml', u'ML Documentation',
-     [u'edX'], 1)
+    ('index', 'ml', 'ML Documentation',
+     ['edX'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -232,8 +231,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'ML', u'ML Documentation',
-   u'edX', 'ML', 'One line description of project.',
+  ('index', 'ML', 'ML Documentation',
+   'edX', 'ML', 'One line description of project.',
    'Miscellaneous'),
 ]
 

ease/essay_set.py

@@ -22,7 +22,7 @@
 MAXIMUM_ESSAY_LENGTH = 20000
 
 
-class EssaySet(object):
+class EssaySet:
     def __init__(self, essaytype="train"):
         """
         Initialize variables and check essay set type
@@ -71,7 +71,7 @@ def add_essay(self, essay_text, essay_score, essay_generated=0):
             essay_text = str(essay_text)
         except:
             # Nothing needed here, will return error in any case.
-            log.exception("Invalid type for essay score : {0} or essay text : {1}".format(type(essay_score), type(essay_text)))
+            log.exception("Invalid type for essay score : {} or essay text : {}".format(type(essay_score), type(essay_text)))
 
         if isinstance(essay_score, int) and isinstance(essay_text, str)\
                 and (essay_generated == 0 or essay_generated == 1):

ease/feature_extractor.py

@@ -28,7 +28,7 @@
 NGRAM_PATH = base_path + "data/good_pos_ngrams.p"
 ESSAY_CORPUS_PATH = util_functions.ESSAY_CORPUS_PATH
 
-class FeatureExtractor(object):
+class FeatureExtractor:
     def __init__(self):
         self._good_pos_ngrams = self.get_good_pos_ngrams()
         self.dict_initialized = False

ease/grade.py

@@ -56,7 +56,7 @@ def grade(grader_data,submission):
         grader_set.add_essay(str(submission),0)
         grader_set.update_prompt(str(grader_data['prompt']))
     except Exception:
-        error_message = "Essay could not be added to essay set:{0}".format(submission)
+        error_message = f"Essay could not be added to essay set:{submission}"
         log.exception(error_message)
         results['errors'].append(error_message)
         has_error=True
@@ -137,7 +137,7 @@ def grade_generic(grader_data, numeric_features, textual_features):
     try:
         grader_set.add_row(numeric_features, textual_features,0)
     except Exception:
-        error_msg = "Row could not be added to predictor set:{0} {1}".format(numeric_features, textual_features)
+        error_msg = f"Row could not be added to predictor set:{numeric_features} {textual_features}"
         log.exception(error_msg)
         results['errors'].append(error_msg)
         has_error=True

ease/model_creator.py

@@ -104,7 +104,7 @@ def get_cv_error(clf,feats,scores):
         results['success']=True
     except ValueError as ex:
         # If this is hit, everything is fine.  It is hard to explain why the error occurs, but it isn't a big deal.
-        msg = u"Not enough classes (0,1,etc) in each cross validation fold: {ex}".format(ex=ex)
+        msg = f"Not enough classes (0,1,etc) in each cross validation fold: {ex}"
         log.debug(msg)
     except:
         log.exception("Error getting cv error estimates.")

ease/predictor_extractor.py

@@ -27,7 +27,7 @@
 
 log = logging.getLogger(__name__)
 
-class PredictorExtractor(object):
+class PredictorExtractor:
     def __init__(self):
         self._extractors = []
         self._initialized = False

ease/predictor_set.py

@@ -1,4 +1,3 @@
-
 import numpy
 import nltk
 import sys
@@ -16,7 +15,7 @@
 
 log=logging.getLogger(__name__)
 
-class PredictorSet(object):
+class PredictorSet:
     def __init__(self, essaytype = "train"):
         """
         Initialize variables and check essay set type
@@ -70,7 +69,7 @@ def add_row(self, numeric_features, textual_features, target):
             try:
                 numeric_features[i] = float(numeric_features[i])
             except:
-                error_message = "Numeric feature {0} not numeric.".format(numeric_features[i])
+                error_message = "Numeric feature {} not numeric.".format(numeric_features[i])
                 log.exception(error_message)
                 raise util_functions.InputError(numeric_features, error_message)
 
@@ -79,7 +78,7 @@ def add_row(self, numeric_features, textual_features, target):
             try:
                 textual_features[i] = str(textual_features[i].encode('ascii', 'ignore'))
             except:
-                error_message = "Textual feature {0} not string.".format(textual_features[i])
+                error_message = "Textual feature {} not string.".format(textual_features[i])
                 log.exception(error_message)
                 raise util_functions.InputError(textual_features, error_message)
 

ease/tests/test_model_accuracy.py

@@ -1,4 +1,3 @@
-
 import unittest
 import os
 from ease import create, grade
@@ -47,7 +46,7 @@ def load_data(self):
         directories.sort()
         #We need to have both a postive and a negative folder to classify
         if len(directories)!=2:
-            raise Exception("Need a pos and a neg directory in {0}".format(self.pathname))
+            raise Exception(f"Need a pos and a neg directory in {self.pathname}")
 
         neg = self.load_text_files(directories[0])
         pos = self.load_text_files(directories[1])
@@ -125,7 +124,7 @@ def grade(self, submission):
         else:
             return grade.grade_generic(self.model_data, submission.get('numeric_values', []), submission.get('textual_values', []))
 
-class GenericTest(object):
+class GenericTest:
     loader = DataLoader
     data_path = ""
     expected_kappa_min = 0

ease/tests/test_spellcheck.py

@@ -1,7 +1,6 @@
-
 from unittest import TestCase
 from nose.tools import assert_equal
-from mock import patch
+from unittest.mock import patch
 from ease.util_functions import spell_correct
 
 

ease/util_functions.py

@@ -27,7 +27,7 @@
 ESSAY_CORPUS_PATH = base_path + "data/essaycorpus.txt"
 ESSAY_COR_TOKENS_PATH = base_path + "data/essay_cor_tokens.p"
 
-class AlgorithmTypes(object):
+class AlgorithmTypes:
     """
     Defines what types of algorithm can be used
     """

requirements/base.txt

@@ -5,8 +5,6 @@
 #    make upgrade
 #
 fisher==0.1.9             # via -r requirements/base.in
-importlib-metadata==1.7.0  # via path
-lxml==4.5.2               # via -r requirements/base.in
-path==13.1.0              # via -r requirements/base.in
-pytz==2020.1              # via -r requirements/base.in
-zipp==1.2.0               # via importlib-metadata
+lxml==4.6.2               # via -r requirements/base.in
+path==15.0.1              # via -r requirements/base.in
+pytz==2020.5              # via -r requirements/base.in

requirements/constraints.txt

@@ -8,6 +8,9 @@
 # pin when possible.  Writing an issue against the offending project and
 # linking to it here is good.
 
+# Common constraints for edx repos
+-c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
+
 # TODO: Many pinned dependencies should be unpinned and/or moved to this constraints file.
 
 # mock version 4.0.0 drops support for python 3.5

requirements/doc.txt

@@ -5,35 +5,32 @@
 #    make upgrade
 #
 alabaster==0.7.12         # via sphinx
-babel==2.8.0              # via sphinx
-certifi==2020.6.20        # via requests
-chardet==3.0.4            # via requests
+babel==2.9.0              # via sphinx
+certifi==2020.12.5        # via requests
+chardet==4.0.0            # via requests
 django-sphinx-autodoc==0.2  # via -r requirements/doc.in
 docutils==0.16            # via sphinx
 fisher==0.1.9             # via -r requirements/base.txt
 idna==2.10                # via requests
 imagesize==1.2.0          # via sphinx
-importlib-metadata==1.7.0  # via -r requirements/base.txt, path
 jinja2==2.11.2            # via sphinx
-lxml==4.5.2               # via -r requirements/base.txt
+lxml==4.6.2               # via -r requirements/base.txt
 markupsafe==1.1.1         # via jinja2
-packaging==20.4           # via sphinx
-path==13.1.0              # via -r requirements/base.txt
-pygments==2.6.1           # via sphinx
+packaging==20.8           # via sphinx
+path==15.0.1              # via -r requirements/base.txt
+pygments==2.7.4           # via sphinx
 pyparsing==2.4.7          # via packaging
-pytz==2020.1              # via -r requirements/base.txt, babel
-requests==2.24.0          # via sphinx
-six==1.15.0               # via packaging
-snowballstemmer==2.0.0    # via sphinx
-sphinx==3.1.2             # via -r requirements/doc.in
+pytz==2020.5              # via -r requirements/base.txt, babel
+requests==2.25.1          # via sphinx
+snowballstemmer==2.1.0    # via sphinx
+sphinx==3.4.3             # via -r requirements/doc.in
 sphinxcontrib-applehelp==1.0.2  # via sphinx
 sphinxcontrib-devhelp==1.0.2  # via sphinx
 sphinxcontrib-htmlhelp==1.0.3  # via sphinx
 sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.4  # via sphinx
-urllib3==1.25.9           # via requests
-zipp==1.2.0               # via -r requirements/base.txt, importlib-metadata
+urllib3==1.26.3           # via requests
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/pip_tools.txt

@@ -5,8 +5,7 @@
 #    make upgrade
 #
 click==7.1.2              # via pip-tools
-pip-tools==5.2.1          # via -r requirements/pip_tools.in
-six==1.15.0               # via pip-tools
+pip-tools==5.5.0          # via -r requirements/pip_tools.in
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/production.txt

@@ -6,12 +6,11 @@
 #
 -e git+https://github.com/edx/nltk.git@3.0.3#egg=nltk  # via -r requirements/production.in
 fisher==0.1.9             # via -r requirements/base.txt
-importlib-metadata==1.7.0  # via -r requirements/base.txt, path
-joblib==0.14.1            # via scikit-learn
-lxml==4.5.2               # via -r requirements/base.txt
+joblib==1.0.0             # via scikit-learn
+lxml==4.6.2               # via -r requirements/base.txt
 numpy==1.16.5             # via -c requirements/constraints.txt, scikit-learn, scipy
-path==13.1.0              # via -r requirements/base.txt
-pytz==2020.1              # via -r requirements/base.txt
-scikit-learn==0.22.2.post1  # via -r requirements/production.in
-scipy==1.4.1              # via -r requirements/production.in, scikit-learn
-zipp==1.2.0               # via -r requirements/base.txt, importlib-metadata
+path==15.0.1              # via -r requirements/base.txt
+pytz==2020.5              # via -r requirements/base.txt
+scikit-learn==0.24.1      # via -r requirements/production.in
+scipy==1.6.0              # via -r requirements/production.in, scikit-learn
+threadpoolctl==2.1.0      # via scikit-learn

requirements/test.txt

@@ -5,29 +5,24 @@
 #    make upgrade
 #
 astroid==2.4.2            # via pylint
-attrs==19.3.0             # via pytest
-coverage==5.2             # via -r requirements/test.in, pytest-cov
+attrs==20.3.0             # via pytest
+coverage==5.4             # via -r requirements/test.in, pytest-cov
 fisher==0.1.9             # via -r requirements/base.txt
-importlib-metadata==1.7.0  # via -r requirements/base.txt, path, pluggy, pytest
-isort==4.3.21             # via pylint
+iniconfig==1.1.1          # via pytest
+isort==5.7.0              # via pylint
 lazy-object-proxy==1.4.3  # via astroid
-lxml==4.5.2               # via -r requirements/base.txt
+lxml==4.6.2               # via -r requirements/base.txt
 mccabe==0.6.1             # via pylint
 mock==3.0.5               # via -c requirements/constraints.txt, -r requirements/test.in
-more-itertools==8.4.0     # via pytest
-packaging==20.4           # via pytest
-path==13.1.0              # via -r requirements/base.txt
-pathlib2==2.3.5           # via pytest
+packaging==20.8           # via pytest
+path==15.0.1              # via -r requirements/base.txt
 pluggy==0.13.1            # via pytest
-py==1.9.0                 # via pytest
-pylint==2.5.3             # via -r requirements/test.in
+py==1.10.0                # via pytest
+pylint==2.6.0             # via -r requirements/test.in
 pyparsing==2.4.7          # via packaging
-pytest-cov==2.10.0        # via -r requirements/test.in
-pytest==5.4.3             # via pytest-cov
-pytz==2020.1              # via -r requirements/base.txt
-six==1.15.0               # via astroid, mock, packaging, pathlib2
-toml==0.10.1              # via pylint
-typed-ast==1.4.1          # via astroid
-wcwidth==0.2.5            # via pytest
+pytest-cov==2.11.1        # via -r requirements/test.in
+pytest==6.2.2             # via pytest-cov
+pytz==2020.5              # via -r requirements/base.txt
+six==1.15.0               # via astroid, mock
+toml==0.10.2              # via pylint, pytest
 wrapt==1.12.1             # via astroid
-zipp==1.2.0               # via -r requirements/base.txt, importlib-metadata

requirements/travis.txt

@@ -6,40 +6,33 @@
 #
 -e git+https://github.com/edx/nltk.git@3.0.3#egg=nltk  # via -r requirements/travis.in
 astroid==2.4.2            # via -r requirements/test.txt, pylint
-attrs==19.3.0             # via -r requirements/test.txt, pytest
-coverage==5.2             # via -r requirements/test.txt, pytest-cov
+attrs==20.3.0             # via -r requirements/test.txt, pytest
+coverage==5.4             # via -r requirements/test.txt, pytest-cov
 cycler==0.10.0            # via matplotlib
 fisher==0.1.9             # via -r requirements/test.txt
-importlib-metadata==1.7.0  # via -r requirements/test.txt, path, pluggy, pytest
-isort==4.3.21             # via -r requirements/test.txt, pylint
-joblib==0.14.1            # via scikit-learn
-kiwisolver==1.1.0         # via matplotlib
+iniconfig==1.1.1          # via -r requirements/test.txt, pytest
+isort==5.7.0              # via -r requirements/test.txt, pylint
+joblib==1.0.0             # via scikit-learn
+kiwisolver==1.3.1         # via matplotlib
 lazy-object-proxy==1.4.3  # via -r requirements/test.txt, astroid
-lxml==4.5.2               # via -r requirements/test.txt
-matplotlib==3.0.3         # via -r requirements/travis.in
+lxml==4.6.2               # via -r requirements/test.txt
+matplotlib==3.3.4         # via -r requirements/travis.in
 mccabe==0.6.1             # via -r requirements/test.txt, pylint
 mock==3.0.5               # via -c requirements/constraints.txt, -r requirements/test.txt
-more-itertools==8.4.0     # via -r requirements/test.txt, pytest
 numpy==1.16.5             # via -c requirements/constraints.txt, -r requirements/travis.in, matplotlib, scikit-learn, scipy
-packaging==20.4           # via -r requirements/test.txt, pytest
-path==13.1.0              # via -r requirements/test.txt
-pathlib2==2.3.5           # via -r requirements/test.txt, pytest
+packaging==20.8           # via -r requirements/test.txt, pytest
+path==15.0.1              # via -r requirements/test.txt
+pillow==8.1.0             # via matplotlib
 pluggy==0.13.1            # via -r requirements/test.txt, pytest
-py==1.9.0                 # via -r requirements/test.txt, pytest
-pylint==2.5.3             # via -r requirements/test.txt
+py==1.10.0                # via -r requirements/test.txt, pytest
+pylint==2.6.0             # via -r requirements/test.txt
 pyparsing==2.4.7          # via -r requirements/test.txt, matplotlib, packaging
-pytest-cov==2.10.0        # via -r requirements/test.txt
-pytest==5.4.3             # via -r requirements/test.txt, pytest-cov
+pytest-cov==2.11.1        # via -r requirements/test.txt
+pytest==6.2.2             # via -r requirements/test.txt, pytest-cov
 python-dateutil==2.8.1    # via matplotlib
-pytz==2020.1              # via -r requirements/test.txt
+pytz==2020.5              # via -r requirements/test.txt
 scikit-learn==0.22.2.post1  # via -r requirements/travis.in
-scipy==1.4.1              # via -r requirements/travis.in, scikit-learn
-six==1.15.0               # via -r requirements/test.txt, astroid, cycler, mock, packaging, pathlib2, python-dateutil
-toml==0.10.1              # via -r requirements/test.txt, pylint
-typed-ast==1.4.1          # via -r requirements/test.txt, astroid
-wcwidth==0.2.5            # via -r requirements/test.txt, pytest
+scipy==1.6.0              # via -r requirements/travis.in, scikit-learn
+six==1.15.0               # via -r requirements/test.txt, astroid, cycler, mock, python-dateutil
+toml==0.10.2              # via -r requirements/test.txt, pylint, pytest
 wrapt==1.12.1             # via -r requirements/test.txt, astroid
-zipp==1.2.0               # via -r requirements/test.txt, importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools