Models and management command to handle sniper dependencies

Author: Megh-Thakkar

main/admin.py

@@ -4,4 +4,6 @@
 admin.site.register(Component)
 admin.site.register(Tag)
 admin.site.register(Contributor)
-admin.site.register(Contribution)
+admin.site.register(Contribution)
+admin.site.register(CSSDependency)
+admin.site.register(JSDependency)

main/management/commands/updatecomponents.py

@@ -8,6 +8,83 @@
     GITHUB_CLIENT_SECRET = ''
 from datetime import datetime
 import pytz
+import ast
+
+# Get sniper data
+'''
+https://rawgit.com/cytoscape/cytoscape.js/master/package.json
+"sniper" key, has "js" and "css". Search for "first"
+'''
+
+def get_commit_hash(commits_url):
+    # import ast
+    
+    commits_url = commits_url.split('{')[0] + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
+    print commits_url
+    response = urllib.urlopen(commits_url)
+    data = json.loads(response.read())[0]
+    return data['sha']
+
+def update_visualizations(component, commit_hash):
+    github_url_list = component.github_url.split('?')[0].split('/')
+    owner = github_url_list[4]
+    repo_name = github_url_list[5]
+    # print "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/package.json"
+    try:
+        sniper_data = json.load(urllib.urlopen("https://cdn.rawgit.com/" + str(owner) + 
+                                '/' + str(repo_name) + "/" + commit_hash + "/package.json"))["sniper"]
+    except:
+        return
+    try:
+        buildJS = sniper_data["buildJS"]
+    except:
+        buildJS = []
+    try:
+        js = sniper_data["js"]
+    except:
+        js = []
+    js_dependencies = buildJS + js
+    try:
+        buildCSS = sniper_data["buildCSS"]
+    except:
+        buildCSS = []
+    try:
+        css = sniper_data["css"]
+    except:
+        css = []
+    css_dependencies = buildCSS + css
+    for dependency in js_dependencies:
+        if dependency.startswith('https://'):
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+        elif dependency.startswith('/'):
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+        else:
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
+            dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
+    for dependency in css_dependencies:
+        if dependency.startswith('https://'):
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+        elif dependency.startswith('/'):
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+        else:
+            dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
+            dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
+    try:
+        sniperData = SniperData.objects.get(component=component)
+    except:
+        sniperData = SniperData.objects.create(component=component)
+    try:
+        no_browserify = sniper_data['noBrowserify']
+        sniperData.no_browserify = no_browserify
+        if no_browserify:
+            sniperData.wzrd_url = '#'
+        else:
+            sniperData.wzrd_url = "https://wzrd.in/bundle/" + component.name
+    except:
+        pass
+    sniperData.save()
 
 def get_github_data(github_url):
     response = urllib.urlopen(github_url)
@@ -18,6 +95,7 @@ def get_npm_data():
     # response = urllib2.urlopen()
     hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'}
     req = urllib2.Request("http://registry.npmjs.com/-/v1/search?text=keywords:biojs,bionode&size=500", headers=hdr)
+    # req = urllib2.Request("http://registry.npmjs.com/-/v1/search?text=biojs-vis-msa&size=1", headers=hdr)
     response = urllib2.urlopen(req)
     data = json.load(response)
     return data
@@ -28,7 +106,7 @@ def get_contributors_data(contributors_url):
     return data
 
 def get_downloads(downloads_url):
-    import ast
+    # import ast
     print downloads_url
     response = urllib.urlopen(downloads_url)
     downloads = 0
@@ -126,6 +204,23 @@ def handle(self, *args, **options):
                     _component.created_time = aware_date
                 except:
                     pass
+                # try:
+                str_date = github_data['updated_at']
+                req_date = datetime.strptime(str_date, "%Y-%m-%dT%H:%M:%SZ") #This object is timezone unaware
+                aware_date = pytz.utc.localize(req_date)    #This object is now timezone aware
+                # if _component.github_update_time:
+                #     if aware_date > _component.github_update_time:
+                #         _component.github_updated_time = aware_date
+                #         latest_commit_hash = get_commit_hash(github_data['commits_url'])
+                #         _component.latest_commit_hash = latest_commit_hash
+                        # update_visualizations(_component, latest_commit_hash)
+                # else:
+                _component.github_update_time = aware_date
+                latest_commit_hash = get_commit_hash(github_data['commits_url'])
+                _component.latest_commit_hash = latest_commit_hash
+                update_visualizations(_component, latest_commit_hash)
+                # except:
+                #     pass
                 _component.save()
                 print str(github_data['contributors_url']) + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
                 try:

main/migrations/0011_cssdependency_jsdependency_visualization.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-20 22:34
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0010_auto_20180616_1659'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CSSDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('css_url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='JSDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('js_url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Visualization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=50)),
+                ('url', models.URLField(null=True)),
+                ('component', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+    ]

main/migrations/0012_auto_20180624_0930.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-24 09:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0011_cssdependency_jsdependency_visualization'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='component',
+            name='github_update_time',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AddField(
+            model_name='component',
+            name='latest_commit_hash',
+            field=models.CharField(max_length=40, null=True),
+        ),
+    ]

main/migrations/0013_sniperdata.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-06-25 07:32
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0012_auto_20180624_0930'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SniperData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('no_browserify', models.BooleanField(default=False)),
+                ('wzrd_url', models.URLField(null=True)),
+                ('component', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.Component')),
+            ],
+        ),
+    ]

main/models.py

@@ -41,6 +41,8 @@ class Component(models.Model):
     npm_url = models.URLField(null=True)
     homepage_url = models.URLField(null=True)
     license = models.CharField(max_length=50, null=True)
+    github_update_time = models.DateTimeField(null=True)
+    latest_commit_hash = models.CharField(max_length=40, null=True)
 
     def save(self, *args, **kwargs):
         if not self.url_name:
@@ -67,4 +69,34 @@ def __unicode__(self):
 class Contribution(models.Model):
     contributor = models.ForeignKey(Contributor, on_delete=models.SET_NULL, null=True)
     component = models.ForeignKey(Component, on_delete=models.SET_NULL, null=True, related_name='contributions')
-    contributions = models.IntegerField(default=0)
+    contributions = models.IntegerField(default=0)
+
+class Visualization(models.Model):
+    name = models.CharField(max_length=50)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+    url = models.URLField(null=True)
+
+    def __unicode__(self):
+        return self.name + '-' + str(self.component.name)
+
+class JSDependency(models.Model):
+    js_url = models.URLField(null=True)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return str(self.component)
+
+class CSSDependency(models.Model):
+    css_url = models.URLField(null=True)
+    component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return str(self.component)
+
+class SniperData(models.Model):
+    no_browserify = models.BooleanField(default=False)
+    wzrd_url = models.URLField(null=True)
+    component = models.OneToOneField(Component, on_delete=models.SET_NULL, null=True)
+
+    def __unicode__(self):
+        return str(self.component)