#395 Make generic code and avoid dupication

Author: Li

src/packagedcode/npm.py

@@ -36,6 +36,7 @@
 from commoncode import fileutils
 
 from packagedcode import models
+from packagedcode.utils import parse_repo_url
 
 """
 Handle Node.js NPM packages
@@ -299,79 +300,6 @@ def repository_mapper(repo, package):
     return package
 
 
-VCS_URLS = (
-    'https://',
-    'http://',
-    'git://',
-    'git+git://',
-    'hg+https://',
-    'hg+http://',
-    'git+https://',
-    'git+http://',
-    'svn+https://',
-    'svn+http://',
-    'svn://',
-)
-
-
-def parse_repo_url(repo_url):
-    """
-    Validate a repo_ulr and handle shortcuts for GitHub, GitHub gist,
-    Bitbucket, or GitLab repositories (same syntax as npm install):
-
-    See https://docs.npmjs.com/files/package.json#repository
-    This is done here in npm:
-    https://github.com/npm/npm/blob/d3c858ce4cfb3aee515bb299eb034fe1b5e44344/node_modules/hosted-git-info/git-host-info.js
-
-    These should be resolved:
-        npm/npm
-        gist:11081aaa281
-        bitbucket:example/repo
-        gitlab:another/repo
-        expressjs/serve-static
-        git://github.com/angular/di.js.git
-        git://github.com/hapijs/boom
-        [email protected]:balderdashy/waterline-criteria.git
-        http://github.com/ariya/esprima.git
-        http://github.com/isaacs/nopt
-        https://github.com/chaijs/chai
-        https://github.com/christkv/kerberos.git
-        https://gitlab.com/foo/private.git
-        [email protected]:foo/private.git
-    """
-
-    # TODO: Improve this and use outside of NPMs
-    is_vcs_url = repo_url.startswith(VCS_URLS)
-    if is_vcs_url:
-        # TODO: ensure the .git suffix is present if needed
-        return repo_url
-
-    if repo_url.startswith('git@'):
-        left, right = repo_url.split('@', 1)
-        host, repo = right.split(':', 1)
-        # may be we should
-        if any(h in host for h in ['github', 'bitbucket', 'gitlab']):
-            return 'https://%(host)s/%(repo)s' % locals()
-        else:
-            return repo_url
-
-    if repo_url.startswith('gist:'):
-        return repo_url
-
-    elif repo_url.startswith(('bitbucket:', 'gitlab:', 'github:')):
-        hoster_urls = {
-            'bitbucket:': 'https://bitbucket.org/%(repo)s',
-            'github:': 'https://github.com/%(repo)s',
-            'gitlab:': 'https://gitlab.com/%(repo)s',
-        }
-        hoster, repo = repo_url.split(':', 1)
-        return hoster_urls[hoster] % locals()
-    elif len(repo_url.split('/')) == 2:
-        # implicit github
-        return 'https://github.com/%(repo_url)s' % locals()
-    return repo_url
-
-
 def url_mapper(url, package):
     """
     In a package.json, the "url" field is a redirection to a package download

src/packagedcode/phpcomposer.py

@@ -35,6 +35,7 @@
 from commoncode import fileutils
 
 from packagedcode import models
+from packagedcode.utils import parse_repo_url
 
 """
 Handle PHP composer packages, refer to https://getcomposer.org/
@@ -82,7 +83,7 @@ def parse(location):
     ])
 
     # mapping of top level composer.json items to a function accepting as arguments
-    # the package.json element value and returning an iterable of key, values Package Object to update
+    # the composer.json element value and returning an iterable of key, values Package Object to update
     field_mappers = OrderedDict([
         ('authors', author_mapper),
         ('license', licensing_mapper),
@@ -251,72 +252,6 @@ def repository_mapper(repos, package):
     return package
 
 
-VCS_URLS = (
-    'https://',
-    'http://',
-    'git://',
-    'git+git://',
-    'hg+https://',
-    'hg+http://',
-    'git+https://',
-    'git+http://',
-    'svn+https://',
-    'svn+http://',
-    'svn://',
-)
-
-
-def parse_repo_url(repo_url):
-    """
-    Validate a repo_url and handle shortcuts for GitHub, GitHub gist,
-    Bitbucket, or GitLab repositories:
-
-    See https://getcomposer.org/doc/04-schema.md#repositories
-
-    These should be resolved:
-        gist:11081aaa281
-        bitbucket:example/repo
-        gitlab:another/repo
-        expressjs/serve-static
-        git://github.com/angular/di.js.git
-        git://github.com/hapijs/boom
-        [email protected]:balderdashy/waterline-criteria.git
-        http://github.com/ariya/esprima.git
-        http://github.com/isaacs/nopt
-        https://github.com/chaijs/chai
-        https://github.com/christkv/kerberos.git
-        https://gitlab.com/foo/private.git
-        [email protected]:foo/private.git
-    """
-
-    is_vcs_url = repo_url.startswith(VCS_URLS)
-    if is_vcs_url:
-        return repo_url
-
-    if repo_url.startswith('git@'):
-        left, right = repo_url.split('@', 1)
-        host, repo = right.split(':', 1)
-        if any(h in host for h in ['github', 'bitbucket', 'gitlab']):
-            return 'https://%(host)s/%(repo)s' % locals()
-        else:
-            return repo_url
-
-    if repo_url.startswith('gist:'):
-        return repo_url
-
-    elif repo_url.startswith(('bitbucket:', 'gitlab:', 'github:')):
-        hoster_urls = {
-            'bitbucket:': 'https://bitbucket.org/%(repo)s',
-            'github:': 'https://github.com/%(repo)s',
-            'gitlab:': 'https://gitlab.com/%(repo)s',
-        }
-        hoster, repo = repo_url.split(':', 1)
-        return hoster_urls[hoster] % locals()
-    elif len(repo_url.split('/')) == 2:
-        return 'https://github.com/%(repo_url)s' % locals()
-    return repo_url
-
-
 def deps_mapper(deps, package, field_name):
     """
     Handle deps such as dependencies, devDependencies

src/packagedcode/utils.py

@@ -0,0 +1,100 @@
+#
+# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import print_function, absolute_import
+
+
+VCS_URLS = (
+    'https://',
+    'http://',
+    'git://',
+    'git+git://',
+    'hg+https://',
+    'hg+http://',
+    'git+https://',
+    'git+http://',
+    'svn+https://',
+    'svn+http://',
+    'svn://',
+)
+
+
+def parse_repo_url(repo_url):
+    """
+    Validate a repo_ulr and handle shortcuts for GitHub, GitHub gist,
+    Bitbucket, or GitLab repositories (same syntax as npm install):
+
+    See https://docs.npmjs.com/files/package.json#repository
+    or https://getcomposer.org/doc/05-repositories.md
+
+    This is done here in npm:
+    https://github.com/npm/npm/blob/d3c858ce4cfb3aee515bb299eb034fe1b5e44344/node_modules/hosted-git-info/git-host-info.js
+
+    These should be resolved:
+        npm/npm
+        gist:11081aaa281
+        bitbucket:example/repo
+        gitlab:another/repo
+        expressjs/serve-static
+        git://github.com/angular/di.js.git
+        git://github.com/hapijs/boom
+        [email protected]:balderdashy/waterline-criteria.git
+        http://github.com/ariya/esprima.git
+        http://github.com/isaacs/nopt
+        https://github.com/chaijs/chai
+        https://github.com/christkv/kerberos.git
+        https://gitlab.com/foo/private.git
+        [email protected]:foo/private.git
+    """
+
+    # TODO: Improve this and use outside of NPMs
+    is_vcs_url = repo_url.startswith(VCS_URLS)
+    if is_vcs_url:
+        # TODO: ensure the .git suffix is present if needed
+        return repo_url
+
+    if repo_url.startswith('git@'):
+        left, right = repo_url.split('@', 1)
+        host, repo = right.split(':', 1)
+        # may be we should
+        if any(h in host for h in ['github', 'bitbucket', 'gitlab']):
+            return 'https://%(host)s/%(repo)s' % locals()
+        else:
+            return repo_url
+
+    if repo_url.startswith('gist:'):
+        return repo_url
+
+    elif repo_url.startswith(('bitbucket:', 'gitlab:', 'github:')):
+        hoster_urls = {
+            'bitbucket:': 'https://bitbucket.org/%(repo)s',
+            'github:': 'https://github.com/%(repo)s',
+            'gitlab:': 'https://gitlab.com/%(repo)s',
+        }
+        hoster, repo = repo_url.split(':', 1)
+        return hoster_urls[hoster] % locals()
+    elif len(repo_url.split('/')) == 2:
+        # implicit github
+        return 'https://github.com/%(repo_url)s' % locals()
+    return repo_url

tests/packagedcode/test_phpcomposer.py

@@ -30,6 +30,7 @@
 from packages_test_utils import PackageTester
 
 from packagedcode import phpcomposer
+from packagedcode.utils import parse_repo_url
 
 
 class TestPHPcomposer(PackageTester):
@@ -55,6 +56,30 @@ def test_parse_person(self):
         ]
         assert expected == list(phpcomposer.parse_person(test))
 
+    def test_parse_repo_url_basic(self):
+        url = 'https://pear2.php.net'
+        result = parse_repo_url(url)
+        expected = 'https://pear2.php.net'
+        assert expected == result
+
+    def test_parse_repo_url_svn(self):
+        url = 'http://svn.example.org/projectA/'
+        result = parse_repo_url(url)
+        expected = 'http://svn.example.org/projectA/'
+        assert expected == result
+
+    def test_parse_repo_url_github(self):
+        url = 'https://github.com/igorw/monolog'
+        result = parse_repo_url(url)
+        expected = 'https://github.com/igorw/monolog'
+        assert expected == result
+
+    def test_parse_repo_url_bitbucket(self):
+        url = '[email protected]:vendor/my-private-repo.git'
+        result = parse_repo_url(url)
+        expected = 'https://bitbucket.org/vendor/my-private-repo.git'
+        assert expected == result
+
     def test_parse_atimer(self):
         test_file = self.get_test_loc('phpcomposer/a-timer/composer.json')
         expected_loc = self.get_test_loc('phpcomposer/a-timer/composer.json.expected')