Merge branch 'master' into master

Author: aaronweaver

.travis.yml

@@ -3,6 +3,7 @@ language: python
 install: true
 
 env:
+  - TEST=smoke-test
   - TEST=integration-test
   - TEST=unit-test
   - TEST=sourceclear
@@ -29,6 +30,11 @@ script:
   - |
     echo "Running test=$TEST"
     case "$TEST" in
+      smoke-test)
+        travis_fold start "smoke-test"
+        bash ./scripts/travis-smoke-test.sh || exit 1
+        travis_fold end "smoke-test"
+        ;;
       unit-test)
         travis_fold start "unit-test"
         bash ./scripts/travis-unit-test.sh || exit 1

ansible/legacy/roles/webserver/tasks/app.yml

@@ -44,12 +44,7 @@
   sudo_user: '{{ dd_user }}'
 
 - name: Install Pillow
-  pip: name=Pillow version=2.3.0 virtualenv={{ venv_dir }}
-  sudo: yes
-  sudo_user: '{{ dd_user }}'
-
-- name: Install django-secure
-  pip: name=django-secure virtualenv={{ venv_dir }}
+  pip: name=Pillow version=5.0.0 virtualenv={{ venv_dir }}
   sudo: yes
   sudo_user: '{{ dd_user }}'
 
@@ -153,11 +148,6 @@
   sudo: yes
   sudo_user: '{{ dd_user }}'
 
-- name: Install sqlalchemy
-  pip: name=sqlalchemy virtualenv={{ venv_dir }}
-  sudo: yes
-  sudo_user: '{{ dd_user }}'
-
 - name: install pdfkit
   pip: name=pdfkit virtualenv={{ venv_dir }}
   sudo: yes

ansible/prod-install/files/requirements/production.txt

@@ -1,6 +1,5 @@
 Django==1.11.2
-Pillow==4.1.1
-django-secure>1.0
+Pillow==5.0.0
 django-tastypie>=0.12.2
 django-tastypie-swagger
 gunicorn>=19.1.1
@@ -17,8 +16,6 @@ vobject
 html2text
 django-watson==1.3.1
 celery==3.1.24
-kombu==3.0.37
-sqlalchemy
 django-polymorphic==1.2
 pdfkit==0.5.0
 django-overextends

components/package.json

@@ -28,7 +28,6 @@
     "@yarn_components/mocha": "mochajs/mocha#~1.17.1",
     "@yarn_components/moment": "moment/moment#^2.9.0",
     "@yarn_components/morrisjs": "morrisjs/morris.js#~0.5.1",
-    "@yarn_components/raphael": "DmitryBaranovskiy/raphael#~2.1.4",
     "@yarn_components/startbootstrap-sb-admin-2": "BlackrockDigital/startbootstrap-sb-admin-2#*"
   },
   "engines": {

docs/features.rst

@@ -586,10 +586,6 @@ you can run: ::
 
     pip install celery
 
-You will also need to install `sqlalchemy`: ::
-
-    pip install sqlalchemy
-
 If you are using virtual environments make sure your environment is activated.  You can also follow the `installation
 instructions`_ from the Celery documentation.
 

docs/getting-started.rst

@@ -84,8 +84,7 @@ And the `python` packages are (listed in `setup.py` as well):
 
 * 'Django==1.8',
 * 'MySQL-python==1.2.3',
-* 'Pillow==2.3.0',
-* 'django-secure==1.0',
+* 'Pillow==5.0.0',
 * 'django-tastypie==0.12.1',
 * 'django-tastypie-swagger',
 * 'gunicorn==19.1.1',

docs/running-in-production.rst

@@ -3,15 +3,27 @@ Running in Production
 
 This guide will walk you through how to setup DefectDojo for running in production using Ubuntu 16.04, nginx, and uwsgi.
 
-*Install, Setup, and Activate Virtualenv*
+**Install, Setup, and Activate Virtualenv**
+
+Assumes running as root or using sudo command for the below.
 
 .. code-block:: console
 
   pip install virtualenv
 
+  cd /opt
+  
   virtualenv dojo
+  
+  cd /opt/dojo
+  
+  git clone https://github.com/DefectDojo/django-DefectDojo.git
+  
+  useradd -m dojo
+  
+  chown -R dojo /opt/dojo
 
-  source dojo/bin/activate
+  source ./bin/activate
 
 **Install Dojo**
 
@@ -66,7 +78,7 @@ However, for a quick setup you can use the following to run both in the backgrou
 
   celery beat -A dojo -l info &
 
-*Start Uwsgi*
+**Start Uwsgi**
 
 From inside the django-DefectDojo/ directory execute:
 
@@ -80,6 +92,45 @@ It is recommended that you use an Upstart job or a @restart cron job to launch u
 
   uwsgi --socket :8001 --wsgi-file wsgi.py --workers 7 &
 
+**Making Defect Dojo start on boot**
+
+Below we configure service files for systemd.  The commands follow, the config files are below the Nginx in the next section.
+
+.. code-block:: console
+
+  cd /etc/systemd/system/
+  
+  vi dojo.service
+    [contents below]
+  
+  systemctl enable dojo
+  
+  systemctl start dojo
+  
+  systemctl status dojo
+    [ensure it launched OK]
+  
+  vi celery-worker.service
+    [contents below]
+  
+  systemctl enable celery-worker
+  
+  systemctl start celery-worker
+  
+  systemctl status celery-worker
+    [ensure it launched OK]
+  
+  vi celery-beat.service
+    [contents below]
+  
+  systemctl enable celery-beat
+  
+  systemctl start celery-beat
+  
+  systemctl status celery-beat
+    [ensure it launched OK]
+  
+
 *NGINX Configuration*
 
 Everyone feels a little differently about nginx settings, so here are the barebones to add your to your nginx configuration to proxy uwsgi. Make sure to modify the filesystem paths if needed:
@@ -122,4 +173,68 @@ Everyone feels a little differently about nginx settings, so here are the barebo
     }
   }
 
+*Systemd Configuration Files*
+
+dojo.service
+
+.. code-block:: console
+
+  [Unit]
+  Description=uWSGI instance to serve DefectDojo
+  Requires=nginx.service mysql.service
+  Before=nginx.service
+  After=mysql.service
+  
+  [Service]
+  ExecStart=/bin/bash -c 'su - dojo -c "cd /opt/dojo/django-DefectDojo && source ../bin/activate && uwsgi --socket :8001 --wsgi-file wsgi.py --workers 7"'
+  Restart=always
+  RestartSec=3
+  #StandardOutput=syslog 
+  #StandardError=syslog  
+  SyslogIdentifier=dojo
+  
+  [Install]
+  WantedBy=multi-user.target
+
+celery-worker.service
+
+.. code-block:: console
+
+  [Unit]
+  Description=celery workers for DefectDojo
+  Requires=dojo.service
+  After=dojo.service
+  
+  [Service]
+  ExecStart=/bin/bash -c 'su - dojo -c "cd /opt/dojo/django-DefectDojo && source ../bin/activate && celery -A dojo worker -l info --concurrency 3"'
+  Restart=always
+  RestartSec=3
+  #StandardOutput=syslog 
+  #StandardError=syslog  
+  SyslogIdentifier=celeryworker
+  
+  [Install]
+  WantedBy=multi-user.target
+
+celery-beat.service
+
+.. code-block:: console
+
+  [Unit]
+  Description=celery beat for DefectDojo
+  Requires=dojo.service
+  After=dojo.service
+  
+  [Service]
+  ExecStart=/bin/bash -c 'su - dojo -c "cd /opt/dojo/django-DefectDojo && source ../bin/activate && celery beat -A dojo -l info"'
+  Restart=always
+  RestartSec=3
+  #StandardOutput=syslog 
+  #StandardError=syslog  
+  SyslogIdentifier=celerybeat
+  
+  [Install]
+  WantedBy=multi-user.target
+
+
 *That's it!*

dojo/api.py

@@ -1321,7 +1321,7 @@ def obj_create(self, bundle, **kwargs):
                 item.last_reviewed_by = bundle.request.user
                 item.active = bundle.data['active']
                 item.verified = bundle.data['verified']
-                item.save()
+                item.save(dedupe_option=False)
 
                 if hasattr(item, 'unsaved_req_resp') and len(item.unsaved_req_resp) > 0:
                     for req_resp in item.unsaved_req_resp:
@@ -1349,6 +1349,7 @@ def obj_create(self, bundle, **kwargs):
                                                                  product=t.engagement.product)
 
                     item.endpoints.add(ep)
+                item.save()
 
                 if item.unsaved_tags is not None:
                     item.tags = item.unsaved_tags

dojo/endpoint/views.py

@@ -58,6 +58,7 @@ def vulnerable_endpoints(request):
 
 def all_endpoints(request):
     endpoints = Endpoint.objects.all()
+    show_uri = get_system_setting('display_endpoint_uri')
     # are they authorized
     if request.user.is_staff:
         pass
@@ -75,14 +76,18 @@ def all_endpoints(request):
             product = get_object_or_404(Product, id=p[0])
 
     ids = get_endpoint_ids(EndpointFilter(request.GET, queryset=endpoints, user=request.user).qs)
-    endpoints = EndpointFilter(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user)
-    paged_endpoints = get_page_items(request, endpoints.qs, 25)
+    if show_uri:
+        paged_endpoints = get_page_items(request, endpoints, 25)
+    else:
+        endpoints = EndpointFilter(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user)
+        paged_endpoints = get_page_items(request, endpoints.qs, 25)
     add_breadcrumb(title="All Endpoints", top_level=not len(request.GET), request=request)
     return render(request,
                   'dojo/endpoints.html',
                   {"endpoints": paged_endpoints,
                    "filtered": endpoints,
                    "name": "All Endpoints",
+                   "show_uri": show_uri
                    })
 
 
@@ -148,6 +153,11 @@ def view_endpoint(request, eid):
 
     paged_findings = get_page_items(request, active_findings, 25)
 
+    vulnerable = False
+
+    if active_findings.count() != 0:
+        vulnerable = True
+
     add_breadcrumb(parent=endpoint, top_level=False, request=request)
     return render(request,
                   "dojo/view_endpoint.html",
@@ -157,6 +167,7 @@ def view_endpoint(request, eid):
                    'all_findings': all_findings,
                    'opened_per_month': monthly_counts['opened_per_period'],
                    'endpoint_metadata': endpoint_metadata,
+                   'vulnerable': vulnerable,
                    })
 
 

dojo/engagement/views.py

@@ -194,11 +194,16 @@ def delete_engagement(request, eid):
                    })
 
 
-@user_passes_test(lambda u: u.is_staff)
 def view_engagement(request, eid):
     eng = Engagement.objects.get(id=eid)
     tests = Test.objects.filter(engagement=eng)
+    prod = eng.product
+    auth = request.user.is_staff or request.user in prod.authorized_users.all()
     risks_accepted = eng.risk_acceptance.all()
+    if not auth:
+        # will render 403
+        raise PermissionDenied
+
     try:
         jissue = JIRA_Issue.objects.get(engagement=eng)
     except:
@@ -220,7 +225,7 @@ def view_engagement(request, eid):
         check = None
         pass
     form = DoneForm()
-    if request.method == 'POST':
+    if request.method == 'POST' and request.user.is_staff:
         eng.progress = 'check_list'
         eng.save()