[F] New paginated components route (#48)

* Added Docker setup for local development.

* Re-add port to localhost CORS settings

* Update settings and docker-compose command.

* Clean up updatecomponents.py

* Re-wrote update_visualizations function.

* Mark components that don't need browserify

* Update deps for security vulnerabilities and remove Django warning.

* Prevent duplicate Snippets.

* Update visualizations template to work for built components

* Provide all components with pagination.

* Fix requirements that breaks builds.

* Add pagination.

* Remove offending package.

* Store docker settings outside of git.

* Turn debug off.

* Remove unused logger

* Update readme with updatecomponents command.

* Add dev.biojs.net to CORS whitelist.

* Add docs for github credentials.

* Add debug docs.

* Setup automated deployments.

* Use newly encrypted env vars.

* limit deployments to relevant branches

* Add GITHUB creds to deployment.

* Import github creds from ansible config

* Log GH ID when it fails.

* One more try with settings.

Author: Dennis Schwartz

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
 *.sqlite3
 media/
-config.py
+config.py
+creds.txt

.travis.yml

@@ -1,11 +1,51 @@
 language: python
-
 python:
-  - 2.7
-
-install:
-  - pip install -r requirements.txt
-
-script:
-  - python manage.py makemigrations
-  - python manage.py migrate
+- 2.7
+services:
+- docker
+jobs:
+  include:
+  - stage: test
+    install:
+    - pip install -r requirements.txt
+    script:
+    - python manage.py makemigrations
+    - python manage.py migrate
+  - stage: deploy-dev
+    if: branch = master
+    before_script:
+    - docker pull philm/ansible_playbook
+    - git clone https://github.com/biojs/biojs-backend-ansible.git
+    - openssl aes-256-cbc -K $encrypted_cf08b333dbdc_key -iv $encrypted_cf08b333dbdc_iv
+      -in deployment-key.enc -out ~/.ssh/id_rsa -d
+    script:
+    - docker run -it -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v "$(pwd)/biojs-backend-ansible":/ansible/playbooks
+      -e DB_USER=$DB_USER
+      -e DB_PASSWORD=$DB_PASSWORD
+      -e GITHUB_CLIENT_SECRET=$GITHUB_CLIENT_SECRET
+      -e GITHUB_CLIENT_ID=$GITHUB_CLIENT_ID
+      philm/ansible_playbook dev-deploy.yml
+      --private-key=~/.ssh/id_rsa -u ubuntu -i dev_hosts
+  - stage: deploy-production
+    if: branch = production
+    before_script:
+    - docker pull philm/ansible_playbook
+    - git clone https://github.com/biojs/biojs-backend-ansible.git
+    - openssl aes-256-cbc -K $encrypted_cf08b333dbdc_key -iv $encrypted_cf08b333dbdc_iv
+      -in deployment-key.enc -out ~/.ssh/id_rsa -d
+    script:
+    - docker run -it -v ~/.ssh/id_rsa:/root/.ssh/id_rsa -v "$(pwd)/biojs-backend-ansible":/ansible/playbooks
+      -e DB_USER=$DB_USER
+      -e DB_PASSWORD=$DB_PASSWORD
+      -e GITHUB_CLIENT_SECRET=$GITHUB_CLIENT_SECRET
+      -e GITHUB_CLIENT_ID=$GITHUB_CLIENT_ID
+      philm/ansible_playbook dev-production.yml
+      --private-key=~/.ssh/id_rsa -u ubuntu -i production_hosts
+notifications:
+  email: true
+env:
+  global:
+  - secure: FipxVvaQcL1ZG5eT7AgLil7ofDw7M1q6rxbwha6qO3hx1LiySZuaGK7fJW0iz9aPcF+SjtNCtE2o5zxANXgTAncSrqI/yTYhoPTW6WHfb/m/vustkkoGjtOMZDNRL+UJ2eYB3kKIXFdXnVkWKi8RuKjS38yioLxJs2aJdH/49yuMeY/+wIqGiVBYqG8b5VGl56gcbw9BgntTg9+OVDdz1Yflm8CdLtu2aqS7MIU66OU4H5GNgks+XuBGNfRiPAeyxR/jLkpQXWE0q67n3SKXQEzJEJlYuJNmRl14tcD3wigrm5BLpQwBVuKfS+XYTGYvb4FDh/oyTydGOvm/wFLN439en89YFtpHYVJZnon1G6BMNHbW07X8gnRLHw8jhosEwDHRDYyfCHaxvAz9v4tS00Q7ev6vhHKcpIBzi3vWbO6HXMw4QY8xhRAs84aDX1ZlHeQTJd5n+QJo1yPyumjTBWVWQg1o4k9zOOGU4JyG4IgiNCc8f1zDhw06h7YPhoOumLVhyd1rte1uvLW3Ai4dePUkJSu75YesfSQy4QYFPqao0HqshivpLhqq3zfNMpgqFuRK+27Yuck5FmaOIqDi3QDNGvql5WT/rIIwDKf9SZQHe6OJN+9wjHsta9oSR4QTmmpQm/iSbaUt7hPuU9tC5NzBWGukCTuWpzG8JeajnQs=
+  - secure: W2cTLqSeKqKnF6HaMcZTi6zAlfHHBrX+n5/OPqokUml5pBV3sGaoiBCIL1XYTb34zzYXQ1y8oq1DDW7K4H/+NdOHlLRpooBsiiSbaSv8+BllEfiDT1DtVQRyM5ys8rC1RGDKsMXZJw19ssGppKHaYAF9Beh7+Am415j38cgxZ7AG9vuKq8HlObbe55jrRRSchzEHv+Ttawd8pKvLXstyGagguxlGBpHJAZ01Abhcq0NMYkTCzXvNF9xUOa5262EX/MMJRFp2SWFJR+CnH32SFsHFDBcKIyrXhE6d0srIjB5FFuRzp7TrItk0oLDXhFPTw99t58cRArQsinTBkXwUGCf0XIgyIfu9r49tvy4A5ls12Xsb/vuYXOp8zvR85G75sVFIEmZSHn32qflONjna46/h848UO1z9F5+vq/RkzjPWScHhzFBswcYNeiImYbv9688cvdaD4GSHcPMSPOvQ0Vp2xU+G5uzUnO+ma0CGdG671RSpA5FCoLABOJ2iFUIdu8ProkXMzimz6Z/SYjFqTHGcOYyTPVU5w1NwbxGk6W3HkdeIlLswzwe267fx0GFy3hgchntn+zNevv4tWxHku/EKYghzL3x/4v/vw6x3Cudu4NU8HWwTCetq4zzCC4G+HkamlfUgP6G81cXRLkcNA8kGGEvqvh9lKvFZtfj4rWM=
+  - secure: E6y/1hOwG+hO84C8MualWLE17/QZg1czbrLG+0VeJHnvzBrNwJc0RwBBZipT7cUbxTM3JSqeKbK+YiVKZ5WWfxFNtO7X6aTjg1BJJfpa6ko4pe5p1F5BfkUDAlRP8FcY+POXdPEh8/han17U/6MH9l2+HumporYHH0gMLbwwkusKOyMgkzKD+UIZZFr3GampIng9jfedZqixJ3TvSvPG2b/Yzspqa4qztybVwcV7cU/xmT5rxB1SD8N9pjiQkacW5BILaGCXc4810reL1V6IaUBPbqYN4XpYj0T3DJlsE3TQ3m1kK4lvJJ8DUAlOgcXYpFbfLcJT3NvJ7Ft/CIM8Fbub16NF4k5f6DDlMzJc5v5t1VpWODfv5gWEXG9Q2zMTs12eBTMSUi1CzPEGJa7tZfQ5gwPvZUSuly8z2Kpw2o+jakoBJ3ZC9pznxLRkpd+jrOQuPuF4FIOyu/py1bVfZn/ss814LYaaPpfjWMMXK9hDJsBqbQncbZnfCWO9irgv7QGkuzzmZ85otCOCOSizLFAY0qYljWtCZeKnbHAJlPBuC/PT/JqjjRy1zhf2YIK6BHyAV7UxQq2xv3rtDyJbhuGQq9XbPV236ivJvwCzMt7zbp82nkBMkwXEa7cYTB1j4GjRdXIl3CGvd8BXYp6HJJc6iZQ1PWwdclY2Nj+CUGs=
+  - secure: Sknv+U5suurv3TTY32niO2n4vHp5LZZyHmJPu5t3i78k28AcFdA9wpGxuAKxBzPwnCNGs8wh90x+mZLhRdiRirvHKdPps+YXrRtwgVwmCvNxjjZshAP1FNfDhMv6n9uH6lMAO34oQ4DzomrkYtWGsiAyRJ/IDqXX9j7j8XfuKkcMYeFCyQuToQqHt5/hhc4rxwr1Lw2SIC4cti1jz7WjKKpRTOZRXwcQusacBgt0qz/KzdHSU9Y+dpLhHuaD5qOTSGv2QGyQVBkW1L6c/cio8Jkw38geHwH1nRKStDvusMTPd6BFhNRYg9H8rOUPQgxXfA917JNbo9fQGLjqvYCwmSKHw1wBOEkuqOyzO3Bd9DjyZp2jOazvN2PPxoL1OG/IRaCqKLnrAVQRuw+mcuBV9NFDmtFSSMqs00OaUfI8e3csnpXnI2p+HTg1gObAkcQjdMh/4REqqxrjggqq+EoaSF6boev/WjAsvnsy2xQvLc8BKaB4eea3oY1rLZjNAwWO3Lx6uJYh6VZYYIJ5ggeR3Is+EqoH2cM/GqzY78Z263Qp+epDpOrXWbRftQoJZSi2Twn5ge+XC04BBXEYr3EbO8ePCBrRnS0dmhCRaEZ35kQ5STAqDZfKJjeRYMmuWy1cWPhuDSEG5qaMTCWDpm/hhkPaaT81Cegnf/ditrLyJ5A=

README.md

@@ -29,8 +29,21 @@ $ pip install -r requirements.txt
 # migrate the database
 $ python manage.py migrate
 
+# Create a personal Github access token in settings and use github username for CLIENT_ID
+# export Github credentials to bypass rate limit
+$ export GITHUB_CLIENT_ID=<username> && export GITHUB_CLIENT_SECRET=<token>
+
 # start the server
 $ python manage.py runserver
+
+# Collect components
+$ python manage.py updatecomponents
 ```
 
 Navigate to [127.0.0.1:8000](http://127.0.0.1:8000/).
+
+## Settings
+
+### Debugging
+
+Enable debug logs by editing the DEBUG=False setting in `biojs/settings.py`

biojs/settings.py

@@ -22,15 +22,19 @@
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = '$&&+$xz)50khd6q+aq&95r1$2urmmdv37=-*lu3v-wwh46829t'
 
-GITHUB_CLIENT_ID = os.environ.get('GITHUB_CLIENT_ID') or ''
-GITHUB_CLIENT_SECRET = os.environ.get('GITHUB_CLIENT_SECRET') or ''
+try:
+    from config import *
+    GITHUB_CLIENT_ID = GITHUB_CLIENT_ID
+    GITHUB_CLIENT_SECRET = GITHUB_CLIENT_SECRET
+except:
+    GITHUB_CLIENT_ID = ''
+    GITHUB_CLIENT_SECRET = ''
 
 # SECURITY WARNING: don't run with debug turned on in production!
 try:
-    from config import *
-    DEBUG = DEBUG
+    from config import DEBUG
 except:
-    DEGUB = True
+    DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 
@@ -161,8 +165,25 @@
 # CORS settings
 
 CORS_ORIGIN_WHITELIST = (
+    'dev.biojs.net',
     'biojs.net',
     'biojs.io',
     'localhost:8080',
     '127.0.0.1:9000'
 )
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['console'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
+        },
+    },
+}

biojs/urls.py

@@ -13,5 +13,5 @@
 urlpatterns += staticfiles_urlpatterns()
 
 if settings.DEBUG:
-	urlpatterns += static(settings.STATIC_URL, serve, document_root=settings.STATIC_ROOT)
-	urlpatterns += static(settings.MEDIA_URL, serve, document_root=settings.MEDIA_ROOT)
+    urlpatterns += static(settings.STATIC_URL, serve, document_root=settings.STATIC_ROOT)
+    urlpatterns += static(settings.MEDIA_URL, serve, document_root=settings.MEDIA_ROOT)

deployment-key.enc

@@ -5,8 +5,8 @@ services:
     image: biojs-backend-base
     working_dir: /opt
     environment:
-      GITHUB_CLIENT_SECRET: 289807644cc6b4afe50b4bb2c5afcd7056047de0
-      GITHUB_CLIENT_ID: DennisSchwartz
+      GITHUB_CLIENT_SECRET: <token>
+      GITHUB_CLIENT_ID: <user>
     volumes:
       - .:/opt
     ports:

docker-compose.yml

@@ -176,6 +176,7 @@ def handle(self, *args, **options):
                 except urllib2.HTTPError as e:
                     print('Error getting github data!')
                     print(e)
+                    print GITHUB_CLIENT_ID
                     continue
                 except Exception as e:
                     print('Unexpected error accessing Github!')

main/management/commands/updatecomponents.py

@@ -5,6 +5,7 @@
 
 urlpatterns = [
 		url(r'^$', views.index, name="index"),
+		url(r'^components/?$', views.components, name="components"),
 		url(r'^updatedata/$', views.update_data, name="update_data"),
 		url(r'^all/$', views.all_components, name="all_components"),
         url(r'^top/$', views.top_components, name="top_components"),

main/urls.py

@@ -9,6 +9,7 @@
 from django.core.management import call_command
 from django.core.serializers import serialize
 from django.views.decorators.clickjacking import xframe_options_exempt
+from django.core.paginator import Paginator, EmptyPage
 import numpy as np
 
 def index(request):
@@ -37,6 +38,19 @@ def top_components(request):
         'top_components':top_components.data,
         })
 
+def components(request):
+    size = request.GET.get('size', 20)
+    page = request.GET.get('page', 1)
+    components_list = Component.objects.all().order_by('-modified_time')
+    paginator = Paginator(components_list, size)
+    try:
+        currentPage = paginator.page(page)
+        serialised = TopComponentSerializer(currentPage, many=True)
+        components = serialised.data
+    except EmptyPage:
+        components = []
+    return JsonResponse({ 'components': components })
+
 def component_details(request, url_name):
     component = Component.objects.get(url_name=url_name)
     details = DetailComponentSerializer(component, context={'request':request})