Skip to content

Commit dd1567e

Browse files
Megh-ThakkarMegh-Thakkar
authored
Merge branch 'master' into visualization-frontend
2 parents 8cb82aa + 8760053 commit dd1567e

File tree

4 files changed

+189
-113
lines changed

4 files changed

+189
-113
lines changed

‎main/management/commands/updatecomponents.py

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,52 @@ def update_visualizations(component, commit_hash, test=False):
5555
if dependency.startswith('https://'):
5656
dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
5757
elif dependency.startswith('/'):
58+
dependency_string = dependency
59+
if 'build/' in dependency_string:
60+
continue
5861
dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
59-
dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
62+
try:
63+
req_dependency = JSDependency.objects.get(component=component, sniper_data_value=dependency_string)
64+
req_dependency.js_url = dependency
65+
req_dependency.save()
66+
except:
67+
req_dependency = JSDependency.objects.create(component=component, sniper_data_value=dependency_string, js_url=dependency)
6068
else:
69+
dependency_string = dependency
70+
if 'build/' in dependency_string:
71+
continue
6172
dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
62-
dependency, created = JSDependency.objects.get_or_create(component=component, js_url=dependency)
73+
try:
74+
req_dependency = JSDependency.objects.get(component=component, sniper_data_value=dependency_string)
75+
req_dependency.js_url = dependency
76+
req_dependency.save()
77+
except:
78+
req_dependency = JSDependency.objects.create(component=component, sniper_data_value=dependency_string, js_url=dependency)
6379
for dependency in css_dependencies:
6480
if dependency.startswith('https://'):
6581
dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
6682
elif dependency.startswith('/'):
83+
dependency_string = dependency
84+
if 'build/' in dependency_string:
85+
continue
6786
dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
68-
dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
87+
try:
88+
req_dependency = CSSDependency.objects.get(component=component, sniper_data_value=dependency_string)
89+
req_dependency.css_url = dependency
90+
req_dependency.save()
91+
except:
92+
req_dependency = CSSDependency.objects.create(component=component, sniper_data_value=dependency_string, css_url=dependency)
6993
else:
70-
dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + "/" + dependency
71-
dependency, created = CSSDependency.objects.get_or_create(component=component, css_url=dependency)
94+
dependency_string = dependency
95+
if 'build/' in dependency_string:
96+
continue
97+
dependency = "https://cdn.rawgit.com/" + str(owner) + '/' + str(repo_name) + "/" + commit_hash + dependency
98+
try:
99+
req_dependency = CSSDependency.objects.get(component=component, sniper_data_value=dependency_string)
100+
req_dependency.css_url = dependency
101+
req_dependency.save()
102+
except:
103+
req_dependency = CSSDependency.objects.create(component=component, sniper_data_value=dependency_string, css_url=dependency)
72104
try:
73105
sniperData = SniperData.objects.get(component=component)
74106
except:

‎main/migrations/0016_auto_20180713_2103.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.11.13 on 2018-07-13 21:03
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('main', '0015_sniperdata_snippets_dir_name'),
12+
]
13+
14+
operations = [
15+
migrations.AddField(
16+
model_name='cssdependency',
17+
name='sniper_data_value',
18+
field=models.CharField(max_length=50, null=True),
19+
),
20+
migrations.AddField(
21+
model_name='jsdependency',
22+
name='sniper_data_value',
23+
field=models.CharField(max_length=50, null=True),
24+
),
25+
]

‎main/models.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,13 +82,15 @@ def __unicode__(self):
8282
class JSDependency(models.Model):
8383
js_url = models.URLField(null=True)
8484
component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
85+
sniper_data_value = models.CharField(max_length=50, null=True)
8586

8687
def __unicode__(self):
8788
return str(self.component)
8889

8990
class CSSDependency(models.Model):
9091
css_url = models.URLField(null=True)
9192
component = models.ForeignKey(Component, null=True, on_delete=models.SET_NULL)
93+
sniper_data_value = models.CharField(max_length=50, null=True)
9294

9395
def __unicode__(self):
9496
return str(self.component)

‎tests/test_views.py

Lines changed: 125 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@
1010
import json
1111
import ast
1212
from main.management.commands import updatecomponents
13+
try:
14+
from biojs.config import *
15+
except:
16+
GITHUB_CLIENT_ID = ''
17+
GITHUB_CLIENT_SECRET = ''
1318

1419
class IndexViewTest(TestCase):
1520

@@ -192,9 +197,9 @@ def setUpTestData(cls):
192197
except:
193198
pass
194199
try:
195-
github_url = component_data['links']['repository']
196-
url_list = github_url.split('/')
197-
_component.github_url = 'https://api.github.com/repos/' + str(url_list[3]) + '/' + str(url_list[4])
200+
github_url = component_data['links']['repository']
201+
url_list = github_url.split('/')
202+
_component.github_url = 'https://api.github.com/repos/' + str(url_list[3]) + '/' + str(url_list[4]) + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
198203
except:
199204
pass
200205
try:
@@ -208,9 +213,15 @@ def setUpTestData(cls):
208213
_component.save()
209214

210215
if _component.github_url:
216+
# print _component.github_url
211217
response = urllib.urlopen(_component.github_url)
212218
github_data = json.load(response)
213-
_component.stars = github_data['stargazers_count']
219+
try:
220+
_component.stars = github_data['stargazers_count']
221+
except:
222+
_component.github_url = None
223+
_component.save()
224+
continue
214225
_component.forks = github_data['forks']
215226
_component.watchers = github_data['watchers']
216227
_component.icon_url = github_data['owner']['avatar_url']
@@ -230,13 +241,13 @@ def setUpTestData(cls):
230241
req_date = datetime.strptime(str_date, "%Y-%m-%dT%H:%M:%SZ")
231242
aware_date = pytz.utc.localize(req_date)
232243
_component.github_update_time = aware_date
233-
commits_url = github_data['commits_url'].split('{')[0]
244+
commits_url = github_data['commits_url'].split('{')[0] + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET
234245
response = urllib.urlopen(commits_url)
235246
data = json.loads(response.read())[0]
236247
_component.latest_commit_hash = data['sha']
237248
_component.save()
238249
updatecomponents.update_visualizations(_component, _component.latest_commit_hash, True)
239-
contributors_data = json.load(urllib.urlopen(str(github_data['contributors_url'])))
250+
contributors_data = json.load(urllib.urlopen(str(github_data['contributors_url'] + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET)))
240251
commits = 0
241252
count = 0
242253
for contributor in contributors_data:
@@ -252,7 +263,7 @@ def setUpTestData(cls):
252263
_contribution = Contribution.objects.create(component=_component, contributor=_contributor, contributions=contributor["contributions"])
253264
commits += _contribution.contributions
254265
count +=1
255-
response = urllib.urlopen(github_data['downloads_url'])
266+
response = urllib.urlopen(github_data['downloads_url'] + '?client_id=' + GITHUB_CLIENT_ID + '&client_secret=' + GITHUB_CLIENT_SECRET)
256267
downloads = 0
257268
data = ast.literal_eval(response.read())
258269
for download in data:
@@ -281,6 +292,7 @@ def test_view_accessible_by_name(self):
281292
# Tests whether the relevant keys are present in the json response and length of response list is same as number of components in database
282293
def test_relevance_of_response(self):
283294
# call for biojs-vis-rohart-msc-test
295+
current_component = Component.objects.get(url_name="biojs-vis-rohart-msc-test")
284296
response_1 = self.client.get(reverse('main:component_details', kwargs={'url_name':'biojs-vis-rohart-msc-test'}))
285297
self.assertTrue('details' in response_1.json())
286298
objects = []
@@ -294,110 +306,115 @@ def test_relevance_of_response(self):
294306
self.assertTrue('details' in response_3.json())
295307
objects.append(response_3.json()['details'])
296308
# Test if all the required fields are present in the response
297-
for object in objects:
298-
self.assertTrue(
299-
'name' in object and
300-
'tags' in object and
301-
'stars' in object and
302-
'downloads' in object and
303-
'created_time' in object and
304-
'modified_time' in object and
305-
'icon_url' in object and
306-
'github_url' in object and
307-
'short_description' in object and
308-
'url_name' in object and
309-
'commits' in object and
310-
'forks' in object and
311-
'watchers' in object and
312-
'no_of_contributors' in object and
313-
'open_issues' in object and
314-
'version' in object and
315-
'author' in object and
316-
'license' in object and
317-
'github_update_time' in object and
318-
'latest_commit_hash' in object
319-
)
320-
# check if number of commits >= 50 for biojs-vis-rohart-msc-test
321-
# and >= 3757 for cytoscape, from the time the tests were initiated
322-
### As number of stars, watchers might go down in the future so they haven't been tested
323-
self.assertTrue(int(response_1.json()['details']['commits']) >= 50)
324-
self.assertTrue(int(response_2.json()['details']['commits']) >= 3757)
309+
if current_component.github_url: # Make sure data was returned by Github API. Ignore if no data returned
310+
for object in objects:
311+
self.assertTrue(
312+
'name' in object and
313+
'tags' in object and
314+
'stars' in object and
315+
'downloads' in object and
316+
'created_time' in object and
317+
'modified_time' in object and
318+
'icon_url' in object and
319+
'github_url' in object and
320+
'short_description' in object and
321+
'url_name' in object and
322+
'commits' in object and
323+
'forks' in object and
324+
'watchers' in object and
325+
'no_of_contributors' in object and
326+
'open_issues' in object and
327+
'version' in object and
328+
'author' in object and
329+
'license' in object and
330+
'github_update_time' in object and
331+
'latest_commit_hash' in object
332+
)
333+
# check if number of commits >= 50 for biojs-vis-rohart-msc-test
334+
# and >= 3757 for cytoscape, from the time the tests were initiated
335+
### As number of stars, watchers might go down in the future so they haven't been tested
336+
self.assertTrue(int(response_1.json()['details']['commits']) >= 50)
337+
self.assertTrue(int(response_2.json()['details']['commits']) >= 3757)
325338

326339
# Tests for benchmark component
327-
self.assertTrue(int(response_3.json()['details']['stars']) >= 3)
328-
self.assertTrue(int(response_2.json()['details']['commits']) >= 70)
340+
self.assertTrue(int(response_3.json()['details']['stars']) >= 3)
341+
self.assertTrue(int(response_2.json()['details']['commits']) >= 70)
329342

330343
# modified date should be after created date
331-
self.assertTrue(response_1.json()['details']['created_time'] <= response_1.json()['details']['github_update_time']) # for biojs-vis-rohart-msc-test
332-
self.assertTrue(response_2.json()['details']['created_time'] <= response_2.json()['details']['github_update_time']) # for cytoscape
333-
self.assertTrue(response_3.json()['details']['created_time'] <= response_3.json()['details']['github_update_time']) # for mplexviz-ngraph
334-
# check if number of contributors is same as contributors added
335-
self.assertEqual(response_1.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='biojs-vis-rohart-msc-test')).count())
336-
self.assertEqual(response_1.json()['details']['no_of_contributors'], len(response_1.json()['contributors']))
337-
self.assertEqual(response_2.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='cytoscape')).count())
338-
self.assertEqual(response_2.json()['details']['no_of_contributors'], len(response_2.json()['contributors']))
339-
self.assertEqual(response_3.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='mplexviz-ngraph')).count())
340-
self.assertEqual(response_3.json()['details']['no_of_contributors'], len(response_3.json()['contributors']))
341-
for object in response_1.json()['contributors'] + response_2.json()['contributors'] + response_3.json()['contributors']:
342-
self.assertTrue(
343-
'contributor' in object and
344-
'contributions' in object and
345-
'id' in object
346-
)
347-
contributor_details = object['contributor']
348-
self.assertTrue(
349-
'username' in contributor_details and
350-
'avatar_url' in contributor_details
351-
)
344+
self.assertTrue(response_1.json()['details']['created_time'] <= response_1.json()['details']['github_update_time']) # for biojs-vis-rohart-msc-test
345+
self.assertTrue(response_2.json()['details']['created_time'] <= response_2.json()['details']['github_update_time']) # for cytoscape
346+
self.assertTrue(response_3.json()['details']['created_time'] <= response_3.json()['details']['github_update_time']) # for mplexviz-ngraph
347+
# check if number of contributors is same as contributors added
348+
self.assertEqual(response_1.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='biojs-vis-rohart-msc-test')).count())
349+
self.assertEqual(response_1.json()['details']['no_of_contributors'], len(response_1.json()['contributors']))
350+
self.assertEqual(response_2.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='cytoscape')).count())
351+
self.assertEqual(response_2.json()['details']['no_of_contributors'], len(response_2.json()['contributors']))
352+
self.assertEqual(response_3.json()['details']['no_of_contributors'], Contribution.objects.filter(component=Component.objects.get(name='mplexviz-ngraph')).count())
353+
self.assertEqual(response_3.json()['details']['no_of_contributors'], len(response_3.json()['contributors']))
354+
for object in response_1.json()['contributors'] + response_2.json()['contributors'] + response_3.json()['contributors']:
355+
self.assertTrue(
356+
'contributor' in object and
357+
'contributions' in object and
358+
'id' in object
359+
)
360+
contributor_details = object['contributor']
361+
self.assertTrue(
362+
'username' in contributor_details and
363+
'avatar_url' in contributor_details
364+
)
352365

353-
# tests for visualizations
354-
# Test for JS and CSS dependencies as well as snippets names.
355-
# Cytoscape
356-
if 'js_dependencies' in response_2.json():
357-
for js_dependency in response_2.json()['js_dependencies']:
358-
url = js_dependency['js_url']
359-
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
360-
# Verify that cdn URL is configured correctly
361-
if('cdn.rawgit.com' in url):
362-
self.assertTrue(latest_commit_hash in url)
363-
if 'css_dependencies' in response_2.json():
364-
for css_dependency in response_2.json()['css_dependencies']:
365-
url = css_dependency['css_url']
366-
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
367-
# Verify that cdn URL is configured correctly
368-
if('cdn.rawgit.com' in url):
369-
self.assertTrue(latest_commit_hash in url)
370-
if 'snippets' in response_2.json():
371-
snippets_list = ['animated-bfs', 'images', 'performance-tuning', 'visual']
372-
for snippet in response_2.json()['snippets']:
373-
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
374-
url = snippet['url']
375-
name = snippet['name']
376-
if('cdn.rawgit.com' in url):
377-
self.assertTrue(latest_commit_hash in url)
378-
self.assertTrue(name in snippets_list)
366+
# tests for visualizations
367+
# Test for JS and CSS dependencies as well as snippets names.
368+
# Cytoscape
369+
if 'js_dependencies' in response_2.json():
370+
for js_dependency in response_2.json()['js_dependencies']:
371+
url = js_dependency['js_url']
372+
print url
373+
# verify absence of 'build/' dependencies
374+
self.assertTrue('build/' not in url)
375+
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
376+
# Verify that cdn URL is configured correctly
377+
if('cdn.rawgit.com' in url):
378+
self.assertTrue(latest_commit_hash in url)
379+
if 'css_dependencies' in response_2.json():
380+
for css_dependency in response_2.json()['css_dependencies']:
381+
url = css_dependency['css_url']
382+
self.assertTrue('build/' not in url)
383+
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
384+
# Verify that cdn URL is configured correctly
385+
if('cdn.rawgit.com' in url):
386+
self.assertTrue(latest_commit_hash in url)
387+
if 'snippets' in response_2.json():
388+
snippets_list = ['animated-bfs', 'images', 'performance-tuning', 'visual']
389+
for snippet in response_2.json()['snippets']:
390+
latest_commit_hash = response_2.json()['details']['latest_commit_hash']
391+
url = snippet['url']
392+
name = snippet['name']
393+
if('cdn.rawgit.com' in url):
394+
self.assertTrue(latest_commit_hash in url)
395+
self.assertTrue(name in snippets_list)
379396

380-
# mplexviz-ngraph
381-
if 'js_dependencies' in response_3.json():
382-
for js_dependency in response_3.json()['js_dependencies']:
383-
url = js_dependency['js_url']
384-
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
385-
# Verify that cdn URL is configured correctly
386-
if('cdn.rawgit.com' in url):
387-
self.assertTrue(latest_commit_hash in url)
388-
if 'css_dependencies' in response_3.json():
389-
for css_dependency in response_3.json()['css_dependencies']:
390-
url = css_dependency['css_url']
391-
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
392-
# Verify that cdn URL is configured correctly
393-
if('cdn.rawgit.com' in url):
394-
self.assertTrue(latest_commit_hash in url)
395-
if 'snippets' in response_3.json():
396-
snippets_list = ['one', 'two', 'three']
397-
for snippet in response_3.json()['snippets']:
398-
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
399-
url = snippet['url']
400-
name = snippet['name']
401-
if('cdn.rawgit.com' in url):
402-
self.assertTrue(latest_commit_hash in url)
403-
self.assertTrue(name in snippets_list)
397+
# mplexviz-ngraph
398+
if 'js_dependencies' in response_3.json():
399+
for js_dependency in response_3.json()['js_dependencies']:
400+
url = js_dependency['js_url']
401+
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
402+
# Verify that cdn URL is configured correctly
403+
if('cdn.rawgit.com' in url):
404+
self.assertTrue(latest_commit_hash in url)
405+
if 'css_dependencies' in response_3.json():
406+
for css_dependency in response_3.json()['css_dependencies']:
407+
url = css_dependency['css_url']
408+
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
409+
# Verify that cdn URL is configured correctly
410+
if('cdn.rawgit.com' in url):
411+
self.assertTrue(latest_commit_hash in url)
412+
if 'snippets' in response_3.json():
413+
snippets_list = ['one', 'two', 'three']
414+
for snippet in response_3.json()['snippets']:
415+
latest_commit_hash = response_3.json()['details']['latest_commit_hash']
416+
url = snippet['url']
417+
name = snippet['name']
418+
if('cdn.rawgit.com' in url):
419+
self.assertTrue(latest_commit_hash in url)
420+
self.assertTrue(name in snippets_list)

0 commit comments

Comments
 (0)