diff --git a/dejacode_toolkit/__init__.py b/dejacode_toolkit/__init__.py index c0e497a1..a75562d7 100644 --- a/dejacode_toolkit/__init__.py +++ b/dejacode_toolkit/__init__.py @@ -10,7 +10,6 @@ from os import getenv from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist import requests @@ -53,11 +52,7 @@ def __init__(self, dataspace): self.basic_auth_user = None self.basic_auth_password = None - try: - dataspace_configuration = dataspace.configuration - except ObjectDoesNotExist: - dataspace_configuration = None - + dataspace_configuration = dataspace.get_configuration() # Take the integration settings from the Dataspace when defined if dataspace_configuration: self.service_url = getattr(dataspace_configuration, self.url_field_name) diff --git a/dje/admin.py b/dje/admin.py index 54c07e44..cedac12b 100644 --- a/dje/admin.py +++ b/dje/admin.py @@ -1056,6 +1056,7 @@ class DataspaceConfigurationForm(forms.ModelForm): "scancodeio_api_key", "vulnerablecode_api_key", "purldb_api_key", + "github_token", ] def __init__(self, *args, **kwargs): @@ -1077,15 +1078,33 @@ class DataspaceConfigurationInline(DataspacedFKMixin, admin.StackedInline): form = DataspaceConfigurationForm verbose_name_plural = _("Configuration") verbose_name = _("Dataspace configuration") - fields = [ - "homepage_layout", - "scancodeio_url", - "scancodeio_api_key", - "vulnerablecode_url", - "vulnerablecode_api_key", - "vulnerabilities_risk_threshold", - "purldb_url", - "purldb_api_key", + fieldsets = [ + ( + "", + {"fields": ("homepage_layout",)}, + ), + ( + "AboutCode Integrations", + { + "fields": ( + "scancodeio_url", + "scancodeio_api_key", + "vulnerablecode_url", + "vulnerablecode_api_key", + "vulnerabilities_risk_threshold", + "purldb_url", + "purldb_api_key", + ) + }, + ), + ( + "GitHub Integration", + { + "fields": [ + "github_token", + ] + }, + ), ] can_delete = False diff --git a/dje/migrations/0008_dataspaceconfiguration_github_token.py b/dje/migrations/0008_dataspaceconfiguration_github_token.py new file mode 100644 index 00000000..7ba4c5a4 --- /dev/null +++ b/dje/migrations/0008_dataspaceconfiguration_github_token.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.4 on 2025-07-28 13:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dje', '0007_dejacodeuser_timezone'), + ] + + operations = [ + migrations.AddField( + model_name='dataspaceconfiguration', + name='github_token', + field=models.CharField(blank=True, help_text='Personal access token (PAT) or GitHub App token used to authenticate API requests for this integration. Keep this token secure.', max_length=255, verbose_name='GitHub token'), + ), + ] diff --git a/dje/models.py b/dje/models.py index 696012de..3fff6d15 100644 --- a/dje/models.py +++ b/dje/models.py @@ -518,6 +518,16 @@ class DataspaceConfiguration(models.Model): ), ) + github_token = models.CharField( + _("GitHub token"), + max_length=255, + blank=True, + help_text=_( + "Personal access token (PAT) or GitHub App token used to authenticate " + "API requests for this integration. Keep this token secure." + ), + ) + def __str__(self): return f"Configuration for {self.dataspace}" diff --git a/dje/tests/testfiles/test_dataset_workflow.json b/dje/tests/testfiles/test_dataset_workflow.json index a17cfd64..0f5e36a2 100644 --- a/dje/tests/testfiles/test_dataset_workflow.json +++ b/dje/tests/testfiles/test_dataset_workflow.json @@ -16,7 +16,8 @@ ], "is_active": true, "include_applies_to": true, - "include_product": true + "include_product": true, + "issue_tracker_id": "" } }, { diff --git a/docs/index.rst b/docs/index.rst index e8ed616b..0db5acb0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,12 @@ Welcome to the very start of your DejaCode journey! reference-2 reference-3-cravex +.. toctree:: + :maxdepth: 1 + :caption: Integrations + + integrations-github + .. toctree:: :maxdepth: 1 :caption: Miscellaneous diff --git a/docs/integrations-github.rst b/docs/integrations-github.rst new file mode 100644 index 00000000..7fd9e946 --- /dev/null +++ b/docs/integrations-github.rst @@ -0,0 +1,75 @@ +.. _integrations_github: + +GitHub Integration +================== + +DejaCode's integration with GitHub allows you to automatically forward +**Workflow Requests** to GitHub repository **Issues**. +This behavior can be selectively applied to any **Request Template** of your choice. + +GitHub Account and Personal Access Token +---------------------------------------- + +To enable integration, you need a GitHub **fine-grained personal access token (PAT)**. + +1. **Access GitHub Developer Settings**: + + - Go to: https://github.com/settings/personal-access-tokens + - Click **"Generate new token"** under *Fine-grained personal access tokens* + +2. **Configure the Token**: + + - **Name**: Give it a clear name (e.g., ``DejaCode Integration``) + - **Expiration**: Set an expiration date (recommended) + - **Resource owner**: Choose your personal GitHub account or organization + +.. note:: + + It is recommended to **create a dedicated GitHub user** with a clear, descriptive + name such as ``dejacode-integration``. This ensures that all GitHub issues managed by + the integration are clearly attributed to that user, improving traceability and + auditability. + +3. **Repository Access**: + + - Under *Repository access*, select **Only select repositories** + - Choose the repository where you want issues to be created and updated + +4. **Permissions**: + + - Under *Repository permissions*, enable:: + + Issues: Read and write + +5. **Save and Copy the Token**: + + - Click **Generate token** + - Copy the token and store it securely — you’ll need it for the next step + +DejaCode Dataspace Configuration +-------------------------------- + +To use your GitHub token in DejaCode: + +1. Go to the **Administration dashboard** +2. Navigate to **Dataspaces**, and select your Dataspace +3. Scroll to the **GitHub Integration** section under **Configuration** +4. Paste your GitHub token in the **GitHub token** field +5. Save the form + +Activate GitHub Integration on Request Templates +------------------------------------------------ + +1. Go to the **Administration dashboard** +2. Navigate to **Workflow** > **Request templates** +3. Create or edit a Request Template in your Dataspace +4. Set the **Issue Tracker ID** field to your GitHub repository URL, e.g.:: + + https://github.com/org/repo_name + +Once the integration is configured: + +- New **Requests** using this template will be automatically pushed to GitHub +- Field updates (like title or priority) and **status changes** (e.g. closed) will be + synced +- New **Comments** on a DejaCode Request will be propagated to the GitHub Issue. diff --git a/workflow/integrations/__init__.py b/workflow/integrations/__init__.py new file mode 100644 index 00000000..f61c4aae --- /dev/null +++ b/workflow/integrations/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/aboutcode-org/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# diff --git a/workflow/integrations/github.py b/workflow/integrations/github.py new file mode 100644 index 00000000..92aee9ea --- /dev/null +++ b/workflow/integrations/github.py @@ -0,0 +1,176 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# DejaCode is a trademark of nexB Inc. +# SPDX-License-Identifier: AGPL-3.0-only +# See https://github.com/aboutcode-org/dejacode for support or download. +# See https://aboutcode.org for more information about AboutCode FOSS projects. +# + +from urllib.parse import urlparse + +from django.conf import settings + +import requests + +GITHUB_API_URL = "https://api.github.com" +DEJACODE_SITE_URL = settings.SITE_URL.rstrip("/") + + +class GitHubIntegration: + """ + A class for managing GitHub issue creation, updates, and comments + from DejaCode requests. + """ + + api_url = GITHUB_API_URL + default_timeout = 10 + + def __init__(self, dataspace): + if not dataspace: + raise ValueError("Dataspace must be provided.") + self.dataspace = dataspace + self.session = self.get_session() + + def get_session(self): + session = requests.Session() + session.headers.update(self.get_headers()) + return session + + def get_headers(self): + github_token = self.dataspace.get_configuration(field_name="github_token") + if not github_token: + raise ValueError("The github_token is not set on the Dataspace.") + return {"Authorization": f"token {github_token}"} + + def sync(self, request): + """Sync the given request with GitHub by creating or updating an issue.""" + try: + repo_id = self.extract_github_repo_path(request.request_template.issue_tracker_id) + except ValueError as error: + raise ValueError(f"Invalid GitHub repository URL: {error}") + + labels = [] + if request.priority: + labels.append(str(request.priority)) + + external_issue = request.external_issue + if external_issue: + self.update_issue( + repo_id=repo_id, + issue_id=external_issue.issue_id, + title=self.make_issue_title(request), + body=self.make_issue_body(request), + state="closed" if request.is_closed else "open", + ) + else: + issue = self.create_issue( + repo_id=repo_id, + title=self.make_issue_title(request), + body=self.make_issue_body(request), + labels=labels, + ) + request.link_external_issue( + platform="github", + repo=repo_id, + issue_id=issue["number"], + ) + + def create_issue(self, repo_id, title, body="", labels=None): + """Create a new GitHub issue.""" + url = f"{self.api_url}/repos/{repo_id}/issues" + data = { + "title": title, + "body": body, + } + if labels: + data["labels"] = labels + + response = self.session.post( + url, + json=data, + timeout=self.default_timeout, + ) + response.raise_for_status() + return response.json() + + def update_issue(self, repo_id, issue_id, title=None, body=None, state=None): + """Update an existing GitHub issue.""" + url = f"{self.api_url}/repos/{repo_id}/issues/{issue_id}" + data = {} + if title: + data["title"] = title + if body: + data["body"] = body + if state: + data["state"] = state + + response = self.session.patch( + url, + json=data, + timeout=self.default_timeout, + ) + response.raise_for_status() + return response.json() + + def post_comment(self, repo_id, issue_id, comment_body): + """Post a comment on an existing GitHub issue.""" + url = f"{self.api_url}/repos/{repo_id}/issues/{issue_id}/comments" + data = {"body": comment_body} + + response = self.session.post( + url, + json=data, + timeout=self.default_timeout, + ) + response.raise_for_status() + return response.json() + + @staticmethod + def extract_github_repo_path(url): + """Extract 'username/repo-name' from a GitHub URL.""" + parsed = urlparse(url) + if "github.com" not in parsed.netloc: + raise ValueError("URL does not point to GitHub.") + + path_parts = [part for part in parsed.path.split("/") if part] + if len(path_parts) < 2: + raise ValueError("Incomplete GitHub repository path.") + + return f"{path_parts[0]}/{path_parts[1]}" + + @staticmethod + def make_issue_title(request): + return f"[DEJACODE] {request.title}" + + @staticmethod + def make_issue_body(request): + request_url = f"{DEJACODE_SITE_URL}{request.get_absolute_url()}" + label_fields = [ + ("📝 Request Template", request.request_template), + ("📦 Product Context", request.product_context), + ("📌 Applies To", request.content_object), + ("🙋 Submitted By", request.requester), + ("👤 Assigned To", request.assignee), + ("🚨 Priority", request.priority), + ("🗒️ Notes", request.notes), + ("🔗️ DejaCode URL", request_url), + ] + + lines = [] + for label, value in label_fields: + if value: + lines.append(f"### {label}\n{value}") + + lines.append("----") + + for question in request.get_serialized_data_as_list(): + label = question.get("label") + value = question.get("value") + input_type = question.get("input_type") + + if input_type == "BooleanField": + value = "Yes" if str(value).lower() in ("1", "true", "yes") else "No" + + lines.append(f"### {label}\n{value}") + + return "\n\n".join(lines) diff --git a/workflow/migrations/0002_requesttemplate_issue_tracker_id_externalissuelink_and_more.py b/workflow/migrations/0002_requesttemplate_issue_tracker_id_externalissuelink_and_more.py new file mode 100644 index 00000000..a7e97487 --- /dev/null +++ b/workflow/migrations/0002_requesttemplate_issue_tracker_id_externalissuelink_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 5.2.4 on 2025-07-30 07:04 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dje', '0008_dataspaceconfiguration_github_token'), + ('workflow', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='requesttemplate', + name='issue_tracker_id', + field=models.CharField(blank=True, help_text='Link to associated issue in a tracking application, provided by the integration when the issue is created.', max_length=1000, verbose_name='Issue Tracker ID'), + ), + migrations.CreateModel( + name='ExternalIssueLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, verbose_name='UUID')), + ('platform', models.CharField(choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('jira', 'Jira'), ('sourcehut', 'SourceHut')], help_text='External issue tracking platform.', max_length=20)), + ('repo', models.CharField(help_text="Repository or project identifier (e.g., 'user/repo-name').", max_length=100)), + ('issue_id', models.CharField(help_text='ID or key of the issue on the external platform.', max_length=100)), + ('dataspace', models.ForeignKey(editable=False, help_text='A Dataspace is an independent, exclusive set of DejaCode data, which can be either nexB master reference data or installation-specific data.', on_delete=django.db.models.deletion.PROTECT, to='dje.dataspace')), + ], + options={ + 'unique_together': {('dataspace', 'platform', 'repo', 'issue_id'), ('dataspace', 'uuid')}, + }, + ), + migrations.AddField( + model_name='request', + name='external_issue', + field=models.ForeignKey(blank=True, help_text='Link to external issue (GitHub, GitLab, Jira, etc.)', null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow.externalissuelink'), + ), + ] diff --git a/workflow/models.py b/workflow/models.py index 6c46c114..1be8da51 100644 --- a/workflow/models.py +++ b/workflow/models.py @@ -39,6 +39,7 @@ from dje.models import HistoryDateFieldsMixin from dje.models import HistoryFieldsMixin from dje.models import get_unsecured_manager +from workflow.integrations.github import GitHubIntegration from workflow.notification import request_comment_slack_payload from workflow.notification import request_slack_payload @@ -88,6 +89,44 @@ def __str__(self): return self.label +class ExternalIssueLink(DataspacedModel): + class Platform(models.TextChoices): + GITHUB = "github", _("GitHub") + GITLAB = "gitlab", _("GitLab") + JIRA = "jira", _("Jira") + SOURCEHUT = "sourcehut", _("SourceHut") + + platform = models.CharField( + max_length=20, choices=Platform.choices, help_text="External issue tracking platform." + ) + + repo = models.CharField( + max_length=100, help_text="Repository or project identifier (e.g., 'user/repo-name')." + ) + + issue_id = models.CharField( + max_length=100, help_text="ID or key of the issue on the external platform." + ) + + class Meta: + unique_together = ( + ("dataspace", "platform", "repo", "issue_id"), + ("dataspace", "uuid"), + ) + + def __str__(self): + return f"{self.get_platform_display()}:{self.repo}#{self.issue_id}" + + @property + def issue_url(self): + if self.platform == self.Platform.GITHUB: + return f"https://github.com/{self.repo}/issues/{self.issue_id}" + elif self.platform == self.Platform.GITLAB: + return f"https://gitlab.com/{self.repo}/-/issues/{self.issue_id}" + elif self.platform == self.Platform.JIRA: + return f"https://{self.repo}/browse/{self.issue_id}" + + class RequestQuerySet(DataspacedQuerySet): BASE_SELECT_RELATED = [ "request_template", @@ -95,6 +134,7 @@ class RequestQuerySet(DataspacedQuerySet): "assignee", "priority", "product_context", + "external_issue", "last_modified_by", ] @@ -348,6 +388,14 @@ class Status(models.TextChoices): ), ) + external_issue = models.ForeignKey( + to="workflow.ExternalIssueLink", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text=_("Link to external issue (GitHub, GitLab, Jira, etc.)"), + ) + last_modified_by = LastModifiedByField() objects = DataspacedManager.from_queryset(RequestQuerySet)() @@ -378,8 +426,8 @@ def save(self, *args, **kwargs): # `previous_object_id` logic is only required on edition. previous_object_id = None - is_addition = self.pk - if is_addition: + is_change = self.pk + if is_change: previous_object_id = self.__class__.objects.get(pk=self.pk).object_id super().save(*args, **kwargs) @@ -397,6 +445,8 @@ def save(self, *args, **kwargs): return previous_object.update_request_count() + self.handle_integrations() + def get_absolute_url(self): return reverse("workflow:request_details", args=[self.uuid]) @@ -520,6 +570,45 @@ def serialize_hook(self, hook): "data": serializer.data, } + def close(self, user, reason): + """ + Set the Request status to CLOSED. + A RequestEvent is created and returned. + """ + self.status = self.Status.CLOSED + self.last_modified_by = user + self.save() + event_instance = self.events.create( + user=user, + text=reason, + event_type=RequestEvent.CLOSED, + dataspace=self.dataspace, + ) + return event_instance + + def link_external_issue(self, platform, repo, issue_id): + """Create or return an ExternalIssueLink associated with this Request.""" + if self.external_issue: + return self.external_issue + + external_issue = ExternalIssueLink.objects.create( + dataspace=self.dataspace, + platform=platform, + repo=repo, + issue_id=str(issue_id), + ) + self.update(external_issue=external_issue) + + return external_issue + + def handle_integrations(self): + issue_tracker_id = self.request_template.issue_tracker_id + if not issue_tracker_id: + return + + if "github.com" in issue_tracker_id: + GitHubIntegration(dataspace=self.dataspace).sync(request=self) + @receiver(models.signals.post_delete, sender=Request) def update_request_count_on_delete(sender, instance=None, **kwargs): @@ -545,6 +634,16 @@ class AbstractRequestEvent(HistoryDateFieldsMixin, DataspacedModel): class Meta: abstract = True + def save(self, *args, **kwargs): + """Call the handle_integrations method on save, only for addition.""" + is_addition = not self.pk + super().save(*args, **kwargs) + if is_addition: + self.handle_integrations() + + def handle_integrations(self): + pass + class RequestEvent(AbstractRequestEvent): request = models.ForeignKey( @@ -574,6 +673,21 @@ class Meta: def __str__(self): return f"{self.get_event_type_display()} by {self.user.username}" + def handle_integrations(self): + external_issue = self.request.external_issue + if not external_issue: + return + + if not self.event_type == self.CLOSED: + return + + if external_issue.platform == ExternalIssueLink.Platform.GITHUB: + GitHubIntegration(dataspace=self.dataspace).post_comment( + repo_id=external_issue.repo, + issue_id=external_issue.issue_id, + comment_body=self.text, + ) + class RequestComment(AbstractRequestEvent): request = models.ForeignKey( @@ -644,6 +758,18 @@ def serialize_hook(self, hook): "data": data, } + def handle_integrations(self): + external_issue = self.request.external_issue + if not external_issue: + return + + if external_issue.platform == ExternalIssueLink.Platform.GITHUB: + GitHubIntegration(dataspace=self.dataspace).post_comment( + repo_id=external_issue.repo, + issue_id=external_issue.issue_id, + comment_body=self.text, + ) + class RequestTemplateQuerySet(DataspacedQuerySet): def actives(self): @@ -723,6 +849,16 @@ class RequestTemplate(HistoryFieldsMixin, DataspacedModel): ), ) + issue_tracker_id = models.CharField( + verbose_name=_("Issue Tracker ID"), + max_length=1000, + blank=True, + help_text=_( + "Link to associated issue in a tracking application, " + "provided by the integration when the issue is created." + ), + ) + objects = DataspacedManager.from_queryset(RequestTemplateQuerySet)() class Meta: diff --git a/workflow/templates/workflow/includes/request_list_table.html b/workflow/templates/workflow/includes/request_list_table.html index 2f1765ca..b346985a 100644 --- a/workflow/templates/workflow/includes/request_list_table.html +++ b/workflow/templates/workflow/includes/request_list_table.html @@ -46,6 +46,15 @@ #{{ req.id }} {{ req.title }} + {% if req.external_issue %} +
+ {% endif %}