Merge pull request #2556 from nexB/improve-copyright-detection

Improve copyright detection

Author: pombredanne

CHANGELOG.rst

@@ -39,6 +39,11 @@ Copyright detection:
  - The data structure in the JSON is now using consistently named attributes as
    opposed to a plain value.
 
+ - The copyright detection speed has been significantly improved with the tests
+   taking roughly 1/2 of the time to run. This is achieved mostly by replacing
+   NLTK with a the minimal and simplified subset we need in a new library named
+   pygmars.
+
 
 Package detection:
 ~~~~~~~~~~~~~~~~~~

requirements.txt

@@ -34,7 +34,6 @@ license-expression==1.2
 lxml==4.6.3
 MarkupSafe==2.0.1
 more-itertools==8.8.0
-nltk==3.4.5
 normality==2.2.2
 packageurl-python==0.9.4
 packaging==20.9
@@ -50,6 +49,7 @@ publicsuffix2==2.20191221
 pyahocorasick==1.4.2
 pycparser==2.20
 Pygments==2.9.0
+pygmars==0.5.0
 pymaven-patch==0.3.0
 pyparsing==2.4.7
 PyYAML==5.4.1

setup-mini.cfg

@@ -76,7 +76,6 @@ install_requires =
     license_expression >= 1.0
     lxml >= 4.6.3, < 5.0.0
     MarkupSafe >= 1.0
-    nltk >= 3.2, !=3.6, < 4.0
     packageurl_python >= 0.9.0
     packaging > 20
     pdfminer.six >= 20200101
@@ -86,6 +85,7 @@ install_requires =
     plugincode >= 21.1.21
     publicsuffix2
     pyahocorasick >= 1.4.0, < 1.5
+    pygmars
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0, < 3.0.0

setup.cfg

@@ -76,7 +76,6 @@ install_requires =
     license_expression >= 1.0
     lxml >= 4.6.3, < 5.0.0
     MarkupSafe >= 1.0
-    nltk >= 3.2, !=3.6, < 4.0
     packageurl_python >= 0.9.0
     packaging > 20
     pdfminer.six >= 20200101
@@ -86,6 +85,7 @@ install_requires =
     plugincode >= 21.1.21
     publicsuffix2
     pyahocorasick >= 1.4.0, < 1.5
+    pygmars
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0, < 3.0.0