Merge pull request #2931 from DefectDojo/release/1.8.0

Release 1.8.0

Author: valentijnscholten

.github/workflows/flake8-your-pr.yml

@@ -0,0 +1,15 @@
+name: Flake8 your PR
+# run on pull_request_target instead of just pull_request as we need write access to update the status check
+on: [pull_request_target]
+jobs:
+  flake8-your-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      # - uses: tayfun/flake8-your-pr@master
+      - uses: valentijnscholten/flake8-your-pr@master
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/integration-tests.yml

@@ -0,0 +1,46 @@
+name: Integration tests
+# run on pull_request_target instead of just pull_request as we need write access to update the status check
+on: [pull_request_target]
+jobs:
+  integration_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      
+      - name: Set integration-test mode
+        run: docker/setEnv.sh integration_tests
+      
+      # Pull the latest image to build, and avoid caching pull-only images.
+      # (docker pull is faster than caching in most cases.)
+      - run: docker-compose pull
+      # In this step, this action saves a list of existing images,
+      # the cache is created without them in the post run.
+      # It also restores the cache if it exists.
+      - uses: satackey/action-docker-layer-caching@master
+        # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
+
+      - name: Build the stack
+        run: docker-compose build
+
+      # phased startup so we can use the exit code from integrationtest container
+      - name: Stary MySQL
+        run: docker-compose up -d 
+
+      - name: Initialize
+        run: docker-compose up --exit-code-from initializer initializer
+
+      - name: Start Dojo
+        # implicity starts uwsgi and rabbitmq
+        run: docker-compose up -d nginx celerybeat celeryworker
+
+      - name: Integration tests
+        run: docker-compose up --exit-code-from integrationtest integrationtest
+
+      - name: Logs
+        if: failure()
+        run: docker-compose logs --tail="2500" uwsgi
+
+      - name: Shutdown
+        if: always()
+        run: docker-compose down

.gitignore

@@ -76,6 +76,7 @@ dojo/uploads/risk/*
 dojo/uploads/reports/*
 dojo/scans/scan*
 dojo/uploads/threat/*
+dojo/fixtures/initial_surveys.json
 .idea
 *.sqlite
 *.db
@@ -113,3 +114,10 @@ quick.bash
 Pipfile
 Pipfile*
 
+
+#ignore locally added certs
+certs/
+
+# Helm dependencies
+helm/defectdojo/charts
+

.travis.yml

@@ -4,9 +4,9 @@ services:
   - docker
 env:
   global:
-    - K8S_VERSION=v1.13.4
-    - MINIKUBE_VERSION=v0.35.0
-    - HELM_VERSION=v2.13.0
+    - K8S_VERSION=v1.19.0
+    - MINIKUBE_VERSION=v1.12.3
+    - HELM_VERSION=v3.3.2
     - CHANGE_MINIKUBE_NONE_USER=true
   matrix:
     - TEST=flake8
@@ -15,12 +15,14 @@ env:
     - BROKER=rabbitmq DATABASE=postgresql
     - BROKER=redis DATABASE=mysql
     - BROKER=redis DATABASE=postgresql
+    - BROKER=rabbitmq DATABASE=postgresql REPLICATION=enabled
+    - BROKER=rabbitmq DATABASE=postgresql EXTRAVAL=enabled
     - TEST=snyk
-matrix: 
+matrix:
   allow_failures:
     - env: TEST=snyk
 jobs:
-  include:  
+  include:
     - stage: deploy
       env: TEST=deploy
 before_install: ['./travis/before-install.sh']

BRANCHING-MODEL.md

@@ -3,20 +3,16 @@ This section describes
 - how branches are handled
 - defectdojo release cycle
 
+Please be careful to submit your pull requests to the correct branch: 
+- bugfix: latest  "release/a.b.x" branch (+ merge using a separate PR against the dev branch)
+- evolutions: dev branch
+
+If in doubt please use dev branch.
+
 # Release and hotfix model
 ![Schemas](doc/branching_model.png)
-## Releasing
-- Start a release/x.y.0 release branch off dev branch
-- Commit only bug fixes from `dev` branch onto this branch
-- Dev branch keeps living with further evolutions
-- Every 4-8 weeks, merge the release branch to master and tag x.y.0: this is when the new release is out: x.(y+1).0
-
-# Issuing a hotfix
-- In case of major issue found after releasing, and fixed in `dev`:
-  - Issue a hotfix branch (first is x.y.1) holding this fix
-  - Merge to `master` and the next release branch
 
 
-Diagrams created with https://www.planttext.com
+Diagrams created with https://www.planttext.com/
 
-This model is close to gitflow https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow, https://nvie.com/posts/a-successful-git-branching-model/ with the feature branch being made in each contributor repository.
+This model is inspired by https://nvie.com/posts/a-successful-git-branching-model/ with the feature branch being made in each contributor repository.

Dockerfile.django

@@ -5,7 +5,7 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.6.11-slim-buster@sha256:9111ff37d96bdcd84bcac261951ac410ee276144eb8a02f06e5907ff4ff2ffea as build
+FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -24,8 +24,11 @@ RUN \
 COPY requirements.txt ./
 RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM python:3.6.11-slim-buster@sha256:9111ff37d96bdcd84bcac261951ac410ee276144eb8a02f06e5907ff4ff2ffea
+FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c
 WORKDIR /app
+ARG uid=1001
+ARG appuser=defectdojo
+ENV appuser ${appuser}
 RUN \
   apt-get -y update && \
   # ugly fix to install postgresql-client without errors
@@ -64,6 +67,7 @@ COPY \
   docker/entrypoint-unit-tests.sh \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
+  certs/* \
   /
 COPY wsgi.py manage.py tests/unit-tests.sh ./
 COPY dojo/ ./dojo/
@@ -74,11 +78,17 @@ COPY tests/ ./tests/
 RUN \
   mkdir -p dojo/migrations && \
   chmod g=u dojo/migrations && \
-  chmod g=u /var/run && \
   true
 USER root
-RUN chmod -R 0777 /app
-USER 1001
+RUN \
+    adduser --system --no-create-home --disabled-password --gecos '' \
+        --uid ${uid} ${appuser} && \
+    chown -R ${appuser} /app && \
+    chmod 0700 /app && \
+    chmod 0750 -R /app/* && \
+    mkdir /var/run/${appuser} && \
+    chown ${appuser} /var/run/${appuser}
+USER ${appuser}
 ENV \
   DD_ADMIN_USER=admin \
   [email protected] \
@@ -94,6 +104,14 @@ ENV \
   DD_CELERY_BROKER_PORT="5672" \
   DD_CELERY_BROKER_PATH="//" \
   DD_CELERY_LOG_LEVEL="INFO" \
+  DD_CELERY_WORKER_POOL_TYPE="solo" \
+  # Enable prefork and options below to ramp-up celeryworker performance. Presets should work fine for a machine with 8GB of RAM, while still leaving room.
+  # See https://docs.celeryproject.org/en/stable/userguide/workers.html#id12 for more details
+  # DD_CELERY_WORKER_POOL_TYPE="prefork" \
+  # DD_CELERY_WORKER_AUTOSCALE_MIN="2" \
+  # DD_CELERY_WORKER_AUTOSCALE_MAX="8" \
+  # DD_CELERY_WORKER_CONCURRENCY="8" \
+  # DD_CELERY_WORKER_PREFETCH_MULTIPLIER="128" \
   DD_DATABASE_ENGINE="django.db.backends.mysql" \
   DD_DATABASE_HOST="mysql" \
   DD_DATABASE_NAME="defectdojo" \
@@ -103,6 +121,8 @@ ENV \
   DD_INITIALIZE=true \
   DD_UWSGI_MODE="socket" \
   DD_UWSGI_ENDPOINT="0.0.0.0:3031" \
+  DD_UWSGI_NUM_OF_PROCESSES="2" \
+  DD_UWSGI_NUM_OF_THREADS="2" \
   DD_DJANGO_ADMIN_ENABLED="True" \
   DD_TRACK_MIGRATIONS="True" \
   DD_DJANGO_METRICS_ENABLED="False"

Dockerfile.integration-tests

@@ -1,7 +1,7 @@
 
 # code: language=Dockerfile
 
-FROM python:3.6.11-slim-buster@sha256:9111ff37d96bdcd84bcac261951ac410ee276144eb8a02f06e5907ff4ff2ffea as build
+FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c as build
 WORKDIR /app
 RUN \
   apt-get -y update && \

Dockerfile.nginx

@@ -3,7 +3,7 @@
 # The code for the build image should be idendical with the code in
 # Dockerfile.django to use the caching mechanism of Docker.
 
-FROM python:3.6.11-slim-buster@sha256:9111ff37d96bdcd84bcac261951ac410ee276144eb8a02f06e5907ff4ff2ffea as build
+FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -64,14 +64,17 @@ RUN \
   python3 manage.py collectstatic --noinput && \
   true
 
-FROM nginx:1.19.1-alpine@sha256:966f134cf5ddeb12a56ede0f40fff754c0c0a749182295125f01a83957391d84
+FROM nginx:1.19.2-alpine@sha256:4635b632d2aaf8c37c8a1cf76a1f96d11b899f74caa2c6946ea56d0a5af02c0c
+ARG uid=1001
+ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/
 COPY wsgi_params nginx/nginx.conf nginx/nginx_TLS.conf /etc/nginx/
 COPY docker/entrypoint-nginx.sh /
 RUN \
   apk add --no-cache openssl && \
   chmod -R g=u /var/cache/nginx && \
-  chmod -R g=u /var/run && \
+  mkdir /var/run/defectdojo && \
+  chmod -R g=u /var/run/defectdojo && \
   mkdir -p /etc/nginx/ssl && \
   chmod -R g=u /etc/nginx && \
   true
@@ -84,6 +87,6 @@ ENV \
   NGINX_METRICS_ENABLED="false" \
   METRICS_HTTP_AUTH_USER="" \
   METRICS_HTTP_AUTH_PASSWORD=""
-USER 1001
+USER ${uid}
 EXPOSE 8080
 ENTRYPOINT ["/entrypoint-nginx.sh"]

PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,16 @@
 This template is for your information. Please clear everything when submitting your pull request.
 
-**Note: DefectDojo is now on Python3.6 and Django 2.2.x. Please submit your pull requests to the 'dev' branch as the 'legacy-python2.7' branch is only for bug fixes. Any new features submitted to the legacy branch will be ignored and closed.**
+**Note: DefectDojo is now on Python3.6 and Django 2.2.x.**
+
+**Please submit your pull requests to :**
+
+**- evolutions: dev branch**
+
+**- bugfix: latest "release/x.y.z" branch**
+
+**- hotfixes: master branch**
+
+If in doubt you can use dev branch, it will just roll-out later. 
 
 When submitting a pull request, please make sure you have completed the following checklist:
 

components/package.json

@@ -5,11 +5,16 @@
     "bootstrap-social": "^4.0.0",
     "bootstrap-wysiwyg": "^2.0.0",
     "bootswatch": "3.4.1",
-    "chosen": "harvesthq/bower-chosen#~1.4.0",
-    "chosen-bootstrap": "dbtek/chosen-bootstrap#~1.1.0",
+    "chosen-bootstrap": "https://github.com/dbtek/chosen-bootstrap",
+    "chosen-js": "^1.8.7",
     "clipboard": "^2.0.6",
     "components-jqueryui": "^1.0.0",
-    "datatables": "1.10.18",
+    "datatables.net": "^1.10.22",
+    "datatables.net-bs": "^1.10.22",
+    "datatables.net-buttons-bs": "^1.6.4",
+    "datatables.net-buttons-dt": "^1.6.4",
+    "datatables.net-colreorder": "^1.5.2",
+    "datatables.net-dt": "^1.10.22",
     "drmonty-datatables-plugins": "^1.0.0",
     "drmonty-datatables-responsive": "^1.0.0",
     "easymde": "^2.11.0",
@@ -18,15 +23,17 @@
     "font-awesome": "^4.0.0",
     "fullcalendar": "^3.10.2",
     "google-code-prettify": "^1.0.0",
-    "jquery": "^3.4.0",
+    "jquery": "^3.5.1",
     "jquery-highlight": "3.5.0",
     "jquery.cookie": "1.4.1",
     "jquery.flot.tooltip": "^0.9.0",
     "jquery.hotkeys": "jeresig/jquery.hotkeys#master",
-    "justgage": "^1.3.0",
+    "jszip": "^3.5.0",
+    "justgage": "^1.4.0",
     "metismenu": "~3.0.6",
-    "moment": "^2.27.0",
+    "moment": "^2.29.0",
     "morris.js": "morrisjs/morris.js",
+    "pdfmake": "^0.1.68",
     "startbootstrap-sb-admin-2": "1.0.7"
   },
   "engines": {