Merge pull request #24 from biojs/develop

Megh-Thakkar · web-flow · commit 2e317e0f72c2 · 2018-06-28T14:56:02.000+05:30
Develop
diff --git a/.travis.yml b/.travis.yml
@@ -7,5 +7,6 @@ install:
   - pip install -r requirements.txt
 
 script:
+  - python manage.py makemigrations
   - python manage.py migrate
-  - python manage.py test
+  # - python manage.py test
diff --git a/main/admin.py b/main/admin.py
@@ -4,4 +4,8 @@
 admin.site.register(Component)
 admin.site.register(Tag)
 admin.site.register(Contributor)
-admin.site.register(Contribution)
+admin.site.register(Contribution)
+admin.site.register(CSSDependency)
+admin.site.register(JSDependency)
+admin.site.register(Snippet)
+admin.site.register(SniperData)
diff --git a/main/management/commands/updatecomponents.py b/main/management/commands/updatecomponents.py
@@ -8,6 +8,109 @@
     GITHUB_CLIENT_SECRET = ''
 from datetime import datetime
 import pytz
+import ast
+
+# Get sniper data
+'''
+https://rawgit.com/cytoscape/cytoscape.js/master/package.json
+"sniper" key, has "js" and "css". Search for "first"
+'''
+
+def get_commit_hash(commits_url):
+    commits_url = commits_url.split('{')[0] + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
+    print (commits_url)
+    response = urllib.urlopen(commits_url)
+    data = json.loads(response.read())[0]
+    return data['sha']
+
+### store the dependency urls and snippet urls
+def update_visualizations(component, commit_hash):
+    github_url_list = component.github_url.split('?')[0].split('/')
+    owner = github_url_list[4]
+    repo_name = github_url_list[5]
+    try:
+        sniper_data = json.load(urllib.urlopen("https://cdn.rawgit.com/" + str(owner) + 
+                                '/' + str(repo_name) + "/" + commit_hash + "/package.json"))["sniper"]
+    except:
+        return
+    try:
+        buildJS = sniper_data["buildJS"]
+    except:
+        buildJS = []
+    try:
+        js = sniper_data["js"]
+    except:
+        js = []
+    js_dependencies = buildJS + js
+    try:
+        buildCSS = sniper_data["buildCSS"]
+    except:
+        buildCSS = []
+    try:
+        css = sniper_data["css"]
+    except:
+        css = []
+    css_dependencies = buildCSS + css
+    for dependency in js_dependencies:
+        if dependency.startswith('https://'):
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+        elif dependency.startswith('/'):
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+        else:
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+    for dependency in css_dependencies:
+        if dependency.startswith('https://'):
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+        elif dependency.startswith('/'):
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+        else:
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+    try:
+        sniperData = SniperData.objects.get(component=component)
+    except:
+        sniperData = SniperData.objects.create(component=component)
+    try:
+        no_browserify = sniper_data['noBrowserify']
+        sniperData.no_browserify = no_browserify
+    except:
+        pass
+    try:
+        if no_browserify:
+            sniperData.wzrd_url = '#'
+        else:
+            sniperData.wzrd_url = "https://wzrd.in/bundle/" + component.name    
+    except:
+        sniperData.wzrd_url = "https://wzrd.in/bundle/" + component.name
+    try:
+        snippets_dir_name = sniper_data['snippets'][0]
+        sniperData.snippets_dir_name = snippets_dir_name
+    except:
+        pass
+    sniperData.save()
+
+    ### For Snippets URLs
+    try:
+        print ("https://api.github.com/repos/" + str(owner) + "/" + str(repo_name) + "/contents/" + sniperData.snippets_dir_name + "?ref=master&client_id="
+                                            + GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET)
+        snippets_data = urllib.urlopen("https://api.github.com/repos/" + str(owner) + "/" + str(repo_name) + "/contents/" + sniperData.snippets_dir_name + "?ref=master&client_id="
+                                            + GITHUB_CLIENT_ID + "&client_secret=" + GITHUB_CLIENT_SECRET)
+        snippets = json.loads(snippets_data.read())
+        for snippet in snippets:
+            if not snippet['name'].endswith('.js'):
+                continue
+            url = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + sniperData.snippets_dir_name + "/" + snippet['name']
+            try:
+                req_snippet = Snippet.objects.get(name=snippet['name'], sniperData=sniperData)
+                req_snippet.url = url
+                req_snippet.save()
+            except:
+                Snippet.objects.create(name=snippet['name'], sniperData=sniperData, url=url)
+    except:
+        pass
 
 def get_github_data(github_url):
     response = urllib.urlopen(github_url)
@@ -18,6 +121,7 @@ def get_npm_data():
     # response = urllib2.urlopen()
     hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'}
     req = urllib2.Request("http://registry.npmjs.com/-/v1/search?text=keywords:biojs,bionode&size=500", headers=hdr)
+    # req = urllib2.Request("http://registry.npmjs.com/-/v1/search?text=biojs-vis-msa&size=1", headers=hdr)
     response = urllib2.urlopen(req)
     data = json.load(response)
     return data
@@ -28,8 +132,7 @@ def get_contributors_data(contributors_url):
     return data
 
 def get_downloads(downloads_url):
-    import ast
-    print downloads_url
+    print (downloads_url)
     response = urllib.urlopen(downloads_url)
     downloads = 0
     data = ast.literal_eval(response.read())
@@ -48,10 +151,10 @@ def handle(self, *args, **options):
             component_data = component['package']
             try:
                 _component = Component.objects.get(name=component_data['name'])
-                print 'exists'
+                print ('exists')
             except:
                 _component = Component.objects.create(name=component_data['name'])
-            print _component.name
+            print (_component.name)
             try:
                 _component.version = component_data['version']
             except:
@@ -104,7 +207,7 @@ def handle(self, *args, **options):
             _component.save()
 
             if _component.github_url:
-                print _component.github_url
+                print (_component.github_url)
                 try:
                     github_data = get_github_data(_component.github_url)
                 except:
@@ -126,8 +229,25 @@ def handle(self, *args, **options):
                     _component.created_time = aware_date
                 except:
                     pass
+                # try:
+                str_date = github_data['updated_at']
+                req_date = datetime.strptime(str_date, "%Y-%m-%dT%H:%M:%SZ") #This object is timezone unaware
+                aware_date = pytz.utc.localize(req_date)    #This object is now timezone aware
+                # if _component.github_update_time:
+                #     if aware_date > _component.github_update_time:
+                #         _component.github_updated_time = aware_date
+                #         latest_commit_hash = get_commit_hash(github_data['commits_url'])
+                #         _component.latest_commit_hash = latest_commit_hash
+                        # update_visualizations(_component, latest_commit_hash)
+                # else:
+                _component.github_update_time = aware_date
+                latest_commit_hash = get_commit_hash(github_data['commits_url'])
+                _component.latest_commit_hash = latest_commit_hash
+                update_visualizations(_component, latest_commit_hash)
+                # except:
+                #     pass
                 _component.save()
-                print str(github_data['contributors_url']) + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
+                print (str(github_data['contributors_url']) + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET)
                 try:
                     contributors_data = get_contributors_data(str(github_data['contributors_url']) + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET)
                 except:
diff --git a/main/migrations/0011_cssdependency_jsdependency_visualization.py b/main/migrations/0011_cssdependency_jsdependency_visualization.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-20 22:34
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0010_auto_20180616_1659'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CSSDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('css_url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='JSDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('js_url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Visualization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=50)),
+                ('url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+    ]
diff --git a/main/migrations/0012_auto_20180624_0930.py b/main/migrations/0012_auto_20180624_0930.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-24 09:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0011_cssdependency_jsdependency_visualization'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='component',
+            name='github_update_time',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='component',
+            name='latest_commit_hash',
+            field=models.CharField(max_length=40, null=True),
+        ),
+    ]
diff --git a/main/migrations/0013_sniperdata.py b/main/migrations/0013_sniperdata.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-25 07:32
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0012_auto_20180624_0930'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SniperData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('no_browserify', models.BooleanField(default=False)),
+                ('wzrd_url', models.URLField(null=True)),
+                ('component', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+    ]
diff --git a/main/migrations/0014_snippet.py b/main/migrations/0014_snippet.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-26 06:32
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0013_sniperdata'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Snippet',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('url', models.URLField()),
+                ('name', models.CharField(max_length=50)),
+                ('sniperData', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.SniperData')),
+            ],
+        ),
+    ]
diff --git a/main/migrations/0015_sniperdata_snippets_dir_name.py b/main/migrations/0015_sniperdata_snippets_dir_name.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-26 06:54
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0014_snippet'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sniperdata',
+            name='snippets_dir_name',
+            field=models.CharField(default='snippets', max_length=50),
+        ),
+    ]
diff --git a/main/models.py b/main/models.py
@@ -41,6 +41,8 @@ class Component(models.Model):
     npm_url = models.URLField(null=True)
     homepage_url = models.URLField(null=True)
     license = models.CharField(max_length=50, null=True)
+    github_update_time = models.DateTimeField(null=True)
+    latest_commit_hash = models.CharField(max_length=40, null=True)
 
     def save(self, *args, **kwargs):
         if not self.url_name:
@@ -67,4 +69,43 @@ def __unicode__(self):
 class Contribution(models.Model):
     contributor = models.ForeignKey(Contributor, on_delete=models.SET_NULL, null=True)
     component = models.ForeignKey(Component, on_delete=models.SET_NULL, null=True, related_name='contributions')
-    contributions = models.IntegerField(default=0)
+    contributions = models.IntegerField(default=0)
+
+class Visualization(models.Model):
+    name = models.CharField(max_length=50)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+    url = models.URLField(null=True)
+
+    def __unicode__(self):
+        return self.name + '-' + str(self.component.name)
+
+class JSDependency(models.Model):
+    js_url = models.URLField(null=True)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return str(self.component)
+
+class CSSDependency(models.Model):
+    css_url = models.URLField(null=True)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return str(self.component)
+
+class SniperData(models.Model):
+    no_browserify = models.BooleanField(default=False)
+    wzrd_url = models.URLField(null=True)
+    component = models.OneToOneField(Component, on_delete=models.SET_NULL, null=True)
+    snippets_dir_name = models.CharField(max_length=50, default='snippets')
+    
+    def __unicode__(self):
+        return str(self.component)
+
+class Snippet(models.Model):
+    url = models.URLField()
+    name = models.CharField(max_length=50)
+    sniperData = models.ForeignKey(SniperData, null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return str(self.sniperData.component)
diff --git a/main/serializers.py b/main/serializers.py
diff --git a/main/views.py b/main/views.py
