Improve Google Sheets Sync feature (#1831)

Author: piyarathnalakmali

dojo/db_migrations/0035_system_settings_email_address.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.12 on 2020-04-08 16:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0035_push_all_issues_help_text_rename_gh_fields'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='system_settings',
+            name='email_address',
+            field=models.EmailField(blank=True, max_length=100),
+        ),
+    ]

dojo/forms.py

@@ -2006,6 +2006,10 @@ class GoogleSheetFieldsForm(forms.Form):
         required=True,
         label="Google Drive folder ID",
         help_text="Extract the Drive folder ID from the URL and provide it here")
+    email_address = forms.EmailField(
+        required=True,
+        label="Email Address",
+        help_text="Enter the same email Address used to create the Service Account")
     enable_service = forms.BooleanField(
         initial=False,
         required=False,

dojo/google_sheet/views.py

@@ -16,7 +16,7 @@
 from django.core.exceptions import PermissionDenied
 from django.contrib.auth.decorators import user_passes_test
 
-from dojo.models import Finding, System_Settings, Test, Dojo_User, Note_Type, NoteHistory, Notes
+from dojo.models import Finding, System_Settings, Test, Dojo_User, Note_Type, NoteHistory, Notes, Sonarqube_Issue
 from dojo.forms import GoogleSheetFieldsForm
 from dojo.utils import add_breadcrumb, Product_Tab
 
@@ -39,6 +39,7 @@ def configure_google_sheets(request):
             else:
                 initial['Protect ' + field.name] = True
         initial['drive_folder_ID'] = system_settings.drive_folder_ID
+        initial['email_address'] = system_settings.email_address
         initial['enable_service'] = system_settings.enable_google_sheets
         form = GoogleSheetFieldsForm(all_fields=fields, initial=initial, credentials_required=False)
     else:
@@ -53,6 +54,7 @@ def configure_google_sheets(request):
             system_settings.column_widths = ""
             system_settings.credentials = ""
             system_settings.drive_folder_ID = ""
+            system_settings.email_address = ""
             system_settings.enable_google_sheets = False
             system_settings.save()
             messages.add_message(
@@ -92,6 +94,7 @@ def configure_google_sheets(request):
                     system_settings.column_widths = column_widths
                     system_settings.credentials = cred_str
                     system_settings.drive_folder_ID = drive_folder_ID
+                    system_settings.email_address = form.cleaned_data['email_address']
                     system_settings.enable_google_sheets = form.cleaned_data['enable_service']
                     system_settings.save()
                     if initial:
@@ -261,9 +264,15 @@ def export_to_sheet(request, tid):
     except googleapiclient.errors.HttpError as error:
         error_message = 'There is a problem with the Google Sheets Sync Configuration. Contact your system admin to solve the issue.'
         return render(request, 'google_sheet_error.html', {'error_message': error_message})
+    except Exception as e:
+        error_message = e
+        return render(request, 'google_sheet_error.html', {'error_message': error_message})
 
 
 def create_googlesheet(request, tid):
+    user_email = request.user.email
+    if not user_email:
+        raise Exception('User must have an email address to use this feature.')
     test = Test.objects.get(id=tid)
     system_settings = get_object_or_404(System_Settings, id=1)
     service_account_info = json.loads(system_settings.credentials)
@@ -289,7 +298,7 @@ def create_googlesheet(request, tid):
                                         addParents=folder_id,
                                         removeParents=previous_parents,
                                         fields='id, parents').execute()
-    user_email = request.user.email
+    # Share created Spreadsheet with current user
     drive_service.permissions().create(body={'type': 'user', 'role': 'writer', 'emailAddress': user_email}, fileId=spreadsheetId).execute()
     populate_sheet(tid, spreadsheetId)
 
@@ -311,7 +320,10 @@ def sync_findings(request, tid, spreadsheetId):
             sheet_names.append(date)
         except:
             pass
-    sheet_title = str(max(sheet_names))
+    try:
+        sheet_title = str(max(sheet_names))
+    except:
+        raise Exception('Existing Google Spreadsheet has errors. Delete the speadsheet and export again.')
     res['sheet_title'] = sheet_title
 
     result = sheets_service.spreadsheets().values().get(spreadsheetId=spreadsheetId, range=sheet_title).execute()
@@ -443,6 +455,7 @@ def populate_sheet(tid, spreadsheetId):
     system_settings = get_object_or_404(System_Settings, id=1)
     service_account_info = json.loads(system_settings.credentials)
     service_account_email = service_account_info['client_email']
+    email_address = system_settings.email_address
     SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
     credentials = service_account.Credentials.from_service_account_info(service_account_info, scopes=SCOPES)
     sheets_service = googleapiclient.discovery.build('sheets', 'v4', credentials=credentials, cache_discovery=False)
@@ -548,6 +561,7 @@ def populate_sheet(tid, spreadsheetId):
                                   "editors": {
                                         "users": [
                                             service_account_email,
+                                            email_address
                                         ]
                                   },
                                   # "description": "Protecting total row",
@@ -618,6 +632,7 @@ def populate_sheet(tid, spreadsheetId):
                                   "editors": {
                                         "users": [
                                             service_account_email,
+                                            email_address
                                         ]
                                   },
                                   "warningOnly": False
@@ -745,13 +760,14 @@ def populate_sheet(tid, spreadsheetId):
                               "editors": {
                                     "users": [
                                         service_account_email,
+                                        email_address
                                     ]
                               },
                               "warningOnly": False
                         }
                   }
             })
-        elif column_name[:6] == '[note]':
+        elif column_name[:6] == '[note]' or column_name[:11] == '[duplicate]':
             body["requests"].append({
                   "autoResizeDimensions": {
                         "dimensions": {
@@ -767,6 +783,7 @@ def populate_sheet(tid, spreadsheetId):
 
 def get_findings_list(tid):
     test = Test.objects.get(id=tid)
+    system_settings = get_object_or_404(System_Settings, id=1)
     findings = Finding.objects.filter(test=test).order_by('numerical_severity')
     active_note_types = Note_Type.objects.filter(is_active=True).order_by('id')
     note_type_activation = active_note_types.count()
@@ -783,11 +800,15 @@ def get_findings_list(tid):
     for finding in findings:
         finding_details = []
         for field in fields:
-            value = eval("finding." + field.name)
+            value = getattr(finding, field.name)
             if type(value) == datetime.date or type(value) == Test or type(value) == datetime.datetime:
-                var = str(eval("finding." + field.name))
+                var = str(value)
             elif type(value) == User or type(value) == Dojo_User:
                 var = value.username
+            elif type(value) == Finding:
+                var = value.id
+            elif type(value) == Sonarqube_Issue:
+                var = value.key
             else:
                 var = value
             finding_details.append(var)
@@ -856,6 +877,29 @@ def get_findings_list(tid):
             for i in range(missing_notes_count):
                 findings_list[f + 1].append('')
                 findings_list[f + 1].append('')
+
+    if system_settings.enable_deduplication:
+        if note_type_activation:
+            for note_type in active_note_types:
+                findings_list[0].append('[duplicate] ' + note_type.name)
+            for f in range(findings.count()):
+                original_finding = findings[f].duplicate_finding
+                for note_type in active_note_types:
+                    try:
+                        note = original_finding.notes.filter(note_type=note_type).latest('date')
+                        findings_list[f + 1].append(note.entry)
+                    except:
+                        findings_list[f + 1].append('')
+        else:
+            findings_list[0].append('[duplicate] note')
+            for f in range(findings.count()):
+                original_finding = findings[f].duplicate_finding
+                try:
+                    note = original_finding.notes.latest('date')
+                    findings_list[f + 1].append(note.entry)
+                except:
+                    findings_list[f + 1].append('')
+
     findings_list[0].append('Last column')
     for f in range(findings.count()):
         findings_list[f + 1].append('-')

dojo/models.py

@@ -279,6 +279,7 @@ class System_Settings(models.Model):
     column_widths = models.CharField(max_length=1500, blank=True)
     drive_folder_ID = models.CharField(max_length=100, blank=True)
     enable_google_sheets = models.BooleanField(default=False, null=True, blank=True)
+    email_address = models.EmailField(max_length=100, blank=True)
 
     objects = SystemSettingsManager()