Merge branch 'dev' into fix-docker-with-override

Author: ptrovatelli

Dockerfile.django

@@ -1,6 +1,10 @@
 
 # code: language=Dockerfile
-FROM python:2
+
+# The code for the build image should be idendical with the code in
+# Dockerfile.nginx to use the caching mechanism of Docker.
+
+FROM python:2 as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -13,7 +17,36 @@ RUN \
   rm -rf /var/lib/apt/lists && \
   true
 COPY requirements.txt ./
-RUN pip install -r ./requirements.txt
+RUN pip wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
+
+FROM python:2-slim
+WORKDIR /app
+RUN \
+  apt-get -y update && \
+  # ugly fix to install postgresql-client without errors
+  mkdir -p /usr/share/man/man1 /usr/share/man/man7 && \
+  apt-get -y install --no-install-recommends \
+    # libopenjp2-7 libjpeg62 libtiff5 are required by the pillow package
+    libopenjp2-7 \
+    libjpeg62 \
+    libtiff5 \
+    dnsutils \
+    mysql-client \
+    libmariadbclient18 \
+    # only required for the dbshell (used by the initializer job)
+    postgresql-client \
+    && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists && \
+  true
+RUN pip install --no-cache-dir --upgrade pip
+COPY --from=build /tmp/wheels /tmp/wheels
+COPY requirements.txt ./
+RUN pip install \
+	--no-cache-dir \
+	--no-index \
+  --find-links=/tmp/wheels \
+  -r ./requirements.txt
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
@@ -39,7 +72,7 @@ USER 1001
 ENV \
   DD_ADMIN_USER=admin \
   [email protected] \
-  DD_ADMIN_PASSWORD= \
+  DD_ADMIN_PASSWORD='' \
   DD_ADMIN_FIRST_NAME=Administrator \
   DD_ADMIN_LAST_NAME=User \
   DD_ALLOWED_HOSTS="*" \

Dockerfile.nginx

@@ -1,5 +1,24 @@
 # code: language=Dockerfile
-FROM defectdojo/defectdojo-django:latest AS build
+
+# The code for the build image should be idendical with the code in
+# Dockerfile.django to use the caching mechanism of Docker.
+
+FROM python:2 as build
+WORKDIR /app
+RUN \
+  apt-get -y update && \
+  apt-get -y install \
+    dnsutils \
+    mysql-client \
+    postgresql-client \
+    && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists && \
+  true
+COPY requirements.txt ./
+RUN pip wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
+
+FROM build AS collectstatic
 
 USER root
 RUN \
@@ -15,7 +34,18 @@ RUN \
   apt-get clean && \
   rm -rf /var/lib/apt/lists && \
   true
+
+RUN pip install \
+	--no-cache-dir \
+	--no-index \
+  --find-links=/tmp/wheels \
+	-r ./requirements.txt
+
 COPY components/ ./components/
+COPY manage.py ./
+COPY dojo/ ./dojo/
+RUN \
+  cp dojo/settings/settings.dist.py dojo/settings/settings.py
 RUN \
   cd components && \
   yarn && \
@@ -24,7 +54,7 @@ RUN \
   true
 
 FROM nginx
-COPY --from=build /app/static/ /usr/share/nginx/html/static/
+COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/
 COPY wsgi_params nginx/nginx.conf /etc/nginx/
 COPY docker/entrypoint-nginx.sh /
 RUN \

README.md

@@ -69,7 +69,7 @@ should give you an idea of how to use DefectDojo for your own team.
 ![Slack](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/slack_rgb.png)
 
 Realtime discussion is done in the OWASP Slack Channel, #defectdojo.
-[Get Access.](http://owaspslack.com/)
+[Get Access.](https://join.slack.com/t/owasp/shared_invite/enQtNjExMTc3MTg0MzU4LTViMDg1MmJiMzMwZGUxZjgxZWQ1MTE0NTBlOTBhNjhhZDIzZTZiNmEwOTJlYjdkMzAxMGVhNDkwNDNiNjZiOWQ)
 
 ![Twitter](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/Twitter_Logo.png)
 
@@ -136,6 +136,9 @@ Proceeds are used for testing, infrastructure, etc.
 [![Xing](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/XING_logo.png)](https://corporate.xing.com/en/about-xing/security/)
 [![10Security](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/10Security-logo.png)](https://10security.com/services-by-technology/defectdojo-commercial-support/)
 [![GCSecurity](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/gc_logo_2018.png)](https://gcsec.com.br/)
+[![Timo-Pagel](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/timo-pagel-logo.png )](https://pagel.pro/)
+[![SDA-SE](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/sda-se-logo.png)](https://sda-se.com/)
+[![Signal-Iduna](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/signal-iduna.png)](https://signal-iduna.de/)
 
 Interested in becoming a sponsor and having your logo displayed? Please review
 our [sponsorship information](SPONSORING.md) or email [email protected]

docker/entrypoint-unit-tests.sh

@@ -2,6 +2,26 @@
 # Run available unittests with a simple setup
 
 cd /app
-python manage.py makemigrations dojo
-python manage.py migrate
-exec python manage.py test dojo.unittests --keepdb
+
+./manage.py makemigrations --no-input --check --dry-run || {
+    cat <<-EOF
+
+********************************************************************************
+
+You made changes to the models without creating a DB migration for them.
+
+**NEVER** change existing migrations, create a new one.
+
+If you're not familiar with migrations in Django, please read the
+great documentation thoroughly:
+https://docs.djangoproject.com/en/1.11/topics/migrations/
+
+********************************************************************************
+
+EOF
+    exit 1
+}
+
+./manage.py migrate
+
+exec ./manage.py test dojo.unittests

dojo/api_test_files/test.json

@@ -2,5 +2,6 @@
 	"engagement": "/api/v1/engagements/96222/",
 	"target_start": "2014-06-07",
 	"target_end": "2014-07-07",
+    "title": "some title",
 	"test_type": 3
 }

dojo/api_v2/serializers.py

@@ -17,7 +17,7 @@
 import six
 from django.utils.translation import ugettext_lazy as _
 import json
-
+from tagging.models import Tag
 
 class TagList(list):
     def __init__(self, *args, **kwargs):
@@ -557,13 +557,16 @@ def save(self):
             for item in parser.items:
                 if skip_duplicates:
                     hash_code = item.compute_hash_code()
+
                     if ((test.engagement.deduplication_on_engagement and
-                        Finding.objects.filter(Q(active=True) | Q(false_p=True) | Q(duplicate=True),
-                                              test__engagement=test.engagement,
-                                              hash_code=hash_code).exists()) or
-                       (Finding.objects.filter(Q(active=True) | Q(false_p=True) | Q(duplicate=True),
-                                              test__engagement__product=test.engagement.product,
-                                              hash_code=hash_code).exists())):
+                             Finding.objects.filter(Q(active=True) | Q(false_p=True) | Q(duplicate=True),
+                             test__engagement=test.engagement,
+                             hash_code=hash_code).exists()) or
+                        (not test.engagement.deduplication_on_engagement and
+                            Finding.objects.filter(Q(active=True) | Q(false_p=True) | Q(duplicate=True),
+                                                    test__engagement__product=test.engagement.product,
+                                                    hash_code=hash_code).exists())):
+
                         skipped_hashcodes.append(hash_code)
                         continue
 
@@ -649,6 +652,7 @@ def save(self):
                 old_finding.notes.create(author=self.context['request'].user,
                                          entry="This finding has been automatically closed"
                                          " as it is not present anymore in recent scans.")
+                Tag.objects.add_tag(old_finding, 'stale')
                 old_finding.save()
                 title = 'An old finding has been closed for "{}".' \
                         .format(test.engagement.product.name)

dojo/api_v2/views.py

@@ -1,7 +1,11 @@
+from django.http import HttpResponse
+from django.shortcuts import get_object_or_404
 from rest_framework import viewsets, mixins
 from rest_framework.permissions import DjangoModelPermissions
+from rest_framework.decorators import detail_route
 from django_filters.rest_framework import DjangoFilterBackend
 
+from dojo.engagement.services import close_engagement, reopen_engagement
 from dojo.models import Product, Product_Type, Engagement, Test, Test_Type, Finding, \
     User, ScanSettings, Scan, Stub_Finding, Finding_Template, \
     JIRA_Issue, Tool_Product_Settings, Tool_Configuration, Tool_Type, \
@@ -36,6 +40,18 @@ class EngagementViewSet(mixins.ListModelMixin,
                      'updated', 'threat_model', 'api_test',
                      'pen_test', 'status', 'product')
 
+    @detail_route(methods=["post"])
+    def close(self, request, pk=None):
+        eng = get_object_or_404(Engagement.objects, id=pk)
+        close_engagement(eng)
+        return HttpResponse()
+
+    @detail_route(methods=["post"])
+    def reopen(self, request, pk=None):
+        eng = get_object_or_404(Engagement.objects, id=pk)
+        reopen_engagement(eng)
+        return HttpResponse()
+
 
 class FindingTemplatesViewSet(mixins.ListModelMixin,
                               mixins.RetrieveModelMixin,
@@ -236,8 +252,9 @@ class TestsViewSet(mixins.ListModelMixin,
     serializer_class = serializers.TestSerializer
     queryset = Test.objects.all()
     filter_backends = (DjangoFilterBackend,)
-    filter_fields = ('id', 'test_type', 'target_start', 'target_end', 'notes',
-                     'percent_complete', 'actual_time', 'engagement')
+    filter_fields = ('id', 'title', 'test_type', 'target_start',
+                     'target_end', 'notes', 'percent_complete',
+                     'actual_time', 'engagement')
 
     def get_serializer_class(self):
         if self.request.method == 'POST':

dojo/db_migrations/0002_auto_20190503_1817.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.16 on 2019-05-03 18:17
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='finding',
+            name='hash_code',
+            field=models.TextField(blank=True, editable=False, null=True),
+        ),
+    ]

dojo/db_migrations/0003_test_title.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-04-23 09:34
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0002_auto_20190503_1817'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='test',
+            name='title',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

dojo/db_migrations/0004_cve_field.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-06 21:54
+from __future__ import unicode_literals
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0003_test_title'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='finding',
+            name='cve',
+            field=models.TextField(max_length=20, null=True, validators=[django.core.validators.RegexValidator(message=b"CVE must be entered in the format: 'CVE-9999-9999'. ", regex=b'^CVE-\\d{4}-\\d{4,7}$')]),
+        ),
+        migrations.AddField(
+            model_name='finding_template',
+            name='cve',
+            field=models.TextField(max_length=20, null=True, validators=[django.core.validators.RegexValidator(message=b"CVE must be entered in the format: 'CVE-9999-9999'. ", regex=b'^CVE-\\d{4}-\\d{4,7}$')]),
+        ),
+        migrations.AlterField(
+            model_name='child_rule',
+            name='match_field',
+            field=models.CharField(choices=[('id', 'id'), (b'title', b'title'), (b'date', b'date'), (b'cwe', b'cwe'), (b'cve', b'cve'), (b'url', b'url'), (b'severity', b'severity'), (b'description', b'description'), (b'mitigation', b'mitigation'), (b'impact', b'impact'), (b'steps_to_reproduce', b'steps_to_reproduce'), (b'severity_justification', b'severity_justification'), (b'references', b'references'), (b'test', b'test'), (b'is_template', b'is_template'), (b'active', b'active'), (b'verified', b'verified'), (b'false_p', b'false_p'), (b'duplicate', b'duplicate'), (b'duplicate_finding', b'duplicate_finding'), (b'out_of_scope', b'out_of_scope'), (b'under_review', b'under_review'), (b'review_requested_by', b'review_requested_by'), (b'under_defect_review', b'under_defect_review'), (b'defect_review_requested_by', b'defect_review_requested_by'), (b'thread_id', b'thread_id'), (b'mitigated', b'mitigated'), (b'mitigated_by', b'mitigated_by'), (b'reporter', b'reporter'), (b'numerical_severity', b'numerical_severity'), (b'last_reviewed', b'last_reviewed'), (b'last_reviewed_by', b'last_reviewed_by'), (b'line_number', b'line_number'), (b'sourcefilepath', b'sourcefilepath'), (b'sourcefile', b'sourcefile'), (b'param', b'param'), (b'payload', b'payload'), (b'hash_code', b'hash_code'), (b'line', b'line'), (b'file_path', b'file_path'), (b'static_finding', b'static_finding'), (b'dynamic_finding', b'dynamic_finding'), (b'created', b'created'), (b'scanner_confidence', b'scanner_confidence')], max_length=200),
+        ),
+        migrations.AlterField(
+            model_name='rule',
+            name='applied_field',
+            field=models.CharField(choices=[('id', 'id'), (b'title', b'title'), (b'date', b'date'), (b'cwe', b'cwe'), (b'cve', b'cve'), (b'url', b'url'), (b'severity', b'severity'), (b'description', b'description'), (b'mitigation', b'mitigation'), (b'impact', b'impact'), (b'steps_to_reproduce', b'steps_to_reproduce'), (b'severity_justification', b'severity_justification'), (b'references', b'references'), (b'test', b'test'), (b'is_template', b'is_template'), (b'active', b'active'), (b'verified', b'verified'), (b'false_p', b'false_p'), (b'duplicate', b'duplicate'), (b'duplicate_finding', b'duplicate_finding'), (b'out_of_scope', b'out_of_scope'), (b'under_review', b'under_review'), (b'review_requested_by', b'review_requested_by'), (b'under_defect_review', b'under_defect_review'), (b'defect_review_requested_by', b'defect_review_requested_by'), (b'thread_id', b'thread_id'), (b'mitigated', b'mitigated'), (b'mitigated_by', b'mitigated_by'), (b'reporter', b'reporter'), (b'numerical_severity', b'numerical_severity'), (b'last_reviewed', b'last_reviewed'), (b'last_reviewed_by', b'last_reviewed_by'), (b'line_number', b'line_number'), (b'sourcefilepath', b'sourcefilepath'), (b'sourcefile', b'sourcefile'), (b'param', b'param'), (b'payload', b'payload'), (b'hash_code', b'hash_code'), (b'line', b'line'), (b'file_path', b'file_path'), (b'static_finding', b'static_finding'), (b'dynamic_finding', b'dynamic_finding'), (b'created', b'created'), (b'scanner_confidence', b'scanner_confidence')], max_length=200),
+        ),
+        migrations.AlterField(
+            model_name='rule',
+            name='match_field',
+            field=models.CharField(choices=[('id', 'id'), (b'title', b'title'), (b'date', b'date'), (b'cwe', b'cwe'), (b'cve', b'cve'), (b'url', b'url'), (b'severity', b'severity'), (b'description', b'description'), (b'mitigation', b'mitigation'), (b'impact', b'impact'), (b'steps_to_reproduce', b'steps_to_reproduce'), (b'severity_justification', b'severity_justification'), (b'references', b'references'), (b'test', b'test'), (b'is_template', b'is_template'), (b'active', b'active'), (b'verified', b'verified'), (b'false_p', b'false_p'), (b'duplicate', b'duplicate'), (b'duplicate_finding', b'duplicate_finding'), (b'out_of_scope', b'out_of_scope'), (b'under_review', b'under_review'), (b'review_requested_by', b'review_requested_by'), (b'under_defect_review', b'under_defect_review'), (b'defect_review_requested_by', b'defect_review_requested_by'), (b'thread_id', b'thread_id'), (b'mitigated', b'mitigated'), (b'mitigated_by', b'mitigated_by'), (b'reporter', b'reporter'), (b'numerical_severity', b'numerical_severity'), (b'last_reviewed', b'last_reviewed'), (b'last_reviewed_by', b'last_reviewed_by'), (b'line_number', b'line_number'), (b'sourcefilepath', b'sourcefilepath'), (b'sourcefile', b'sourcefile'), (b'param', b'param'), (b'payload', b'payload'), (b'hash_code', b'hash_code'), (b'line', b'line'), (b'file_path', b'file_path'), (b'static_finding', b'static_finding'), (b'dynamic_finding', b'dynamic_finding'), (b'created', b'created'), (b'scanner_confidence', b'scanner_confidence')], max_length=200),
+        ),
+    ]