GitHub integration

Makefile

@@ -9,7 +9,7 @@ SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
 SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
-EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key
+EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key github_integration_app_private_key github_integration_app_webhook_secret
 GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
 
 ENVREPLACE := deploy/scripts/envreplace

base.yml

@@ -67,6 +67,8 @@ services:
       - django_secret_key
       - github_client_secret
       - orcid_client_secret
+      - github_integration_app_private_key
+      - github_integration_app_webhook_secret
       - hcaptcha_secret
       - mail_api_key
       - youtube_api_key
@@ -99,6 +101,10 @@ secrets:
     file: ./build/secrets/django_secret_key
   github_client_secret:
     file: ./build/secrets/github_client_secret
+  github_integration_app_private_key:
+    file: ./build/secrets/github_integration_app_private_key
+  github_integration_app_webhook_secret:
+    file: ./build/secrets/github_integration_app_webhook_secret
   hcaptcha_secret:
     file: ./build/secrets/hcaptcha_secret
   mail_api_key:

deploy/conf/.env.template

@@ -45,6 +45,12 @@ DATACITE_DRY_RUN="true" # allowed values: "true" or "false"
 # youtube api settings
 YOUTUBE_CHANNEL_ID=
 
+# github integration app
+GITHUB_INTEGRATION_APP_ID=
+GITHUB_INTEGRATION_APP_NAME=
+GITHUB_INTEGRATION_APP_INSTALLATION_ID=
+GITHUB_MODEL_LIBRARY_ORG_NAME=
+
 # test
 TEST_USER_ID=10000000
 TEST_USERNAME=__test_user__

django/core/models.py

@@ -466,6 +466,14 @@ def github_url(self):
         """
         return self.get_social_account_profile_url("github")
 
+    @property
+    def github_username(self):
+        github_account = self.get_social_account("github")
+        if github_account:
+            return github_account.extra_data.get("login")
+        else:
+            return None
+
     def get_social_account_profile_url(self, provider_name):
         social_acct = self.get_social_account(provider_name)
         if social_acct:

django/core/settings/defaults.py

@@ -546,6 +546,20 @@ def set_environment(env: Environment):
 GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
 GITHUB_CLIENT_SECRET = read_secret("github_client_secret")
 
+GITHUB_INTEGRATION_APP_ID = int(os.getenv("GITHUB_INTEGRATION_APP_ID") or 0)
+GITHUB_INTEGRATION_APP_NAME = os.getenv("GITHUB_INTEGRATION_APP_NAME", "")
+GITHUB_INTEGRATION_APP_PRIVATE_KEY = read_secret("github_integration_app_private_key")
+GITHUB_INTEGRATION_APP_INSTALLATION_ID = int(
+    os.getenv("GITHUB_INTEGRATION_APP_INSTALLATION_ID") or 0
+)
+GITHUB_INTEGRATION_APP_WEBHOOK_SECRET = read_secret(
+    "github_integration_app_webhook_secret"
+)
+GITHUB_MODEL_LIBRARY_ORG_NAME = os.getenv("GITHUB_MODEL_LIBRARY_ORG_NAME", "")
+GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT = int(
+    os.getenv("GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT") or 100 * 1024 * 1024
+)
+
 TEST_BASIC_AUTH_PASSWORD = os.getenv("TEST_BASIC_AUTH_PASSWORD", "test password")
 TEST_USER_ID = os.getenv("TEST_USER_ID", 1000000)
 TEST_USERNAME = os.getenv("TEST_USERNAME", "__test_user__")

django/core/tests/base.py

@@ -188,5 +188,13 @@ def initialize_test_shared_folders():
     )
 
 
+def clear_test_shared_folder(dir=settings.REPOSITORY_ROOT):
+    for fs in os.scandir(dir):
+        if fs.is_dir():
+            shutil.rmtree(os.path.join(dir, fs.name), ignore_errors=True)
+        elif fs.is_file():
+            os.remove(os.path.join(dir, fs.name))
+
+
 def destroy_test_shared_folders():
     shutil.rmtree(settings.SHARE_DIR, ignore_errors=True)

django/curator/fs.py

@@ -8,6 +8,8 @@
 def fsck(queryset):
     results = OrderedDict()
     for release in queryset:
+        if release.is_imported:
+            continue
         rfsc = CodebaseReleaseFileConsistencyChecker(release)
         errors = rfsc.check()
         if errors:

django/curator/invoke_tasks/borg.py

@@ -11,6 +11,7 @@
 
 DEFAULT_LIBRARY_BASENAME = os.path.basename(settings.LIBRARY_ROOT)
 DEFAULT_MEDIA_BASENAME = os.path.basename(settings.MEDIA_ROOT)
+DEFAULT_REPOSITORY_BASENAME = os.path.basename(settings.REPOSITORY_ROOT)
 
 
 @task(aliases=["init"])
@@ -48,12 +49,14 @@ def backup(ctx):
     archive = "{utcnow}"
     library = os.path.relpath(settings.LIBRARY_ROOT, share)
     media = os.path.relpath(settings.MEDIA_ROOT, share)
+    repository = os.path.relpath(settings.REPOSITORY_ROOT, share)
     database = os.path.relpath(os.path.join(settings.BACKUP_ROOT, "latest"), share)
 
     error_msgs = []
     for p in (
         settings.LIBRARY_ROOT,
         settings.MEDIA_ROOT,
+        settings.REPOSITORY_ROOT,
         os.path.join(settings.BACKUP_ROOT, "latest"),
     ):
         if not os.path.exists(p):
@@ -63,44 +66,57 @@ def backup(ctx):
 
     with ctx.cd(share):
         ctx.run(
-            f'borg create --stats --compression lz4 {repo}::"{archive}" {library} {media} {database}',
+            f'borg create --stats --compression lz4 {repo}::"{archive}" {library} {media} {repository} {database}',
             echo=True,
             env=environment(),
         )
 
 
 def delete_latest_uncompressed_backup(
-    src_library=DEFAULT_LIBRARY_BASENAME, src_media=DEFAULT_MEDIA_BASENAME
+    src_library=DEFAULT_LIBRARY_BASENAME,
+    src_media=DEFAULT_MEDIA_BASENAME,
+    src_repository=DEFAULT_REPOSITORY_BASENAME,
 ):
     latest_dest_library = os.path.join(settings.PREVIOUS_SHARE_ROOT, src_library)
     latest_dest_media = os.path.join(settings.PREVIOUS_SHARE_ROOT, src_media)
+    latest_dest_repository = os.path.join(settings.PREVIOUS_SHARE_ROOT, src_repository)
 
     shutil.rmtree(latest_dest_library, ignore_errors=True)
     shutil.rmtree(latest_dest_media, ignore_errors=True)
+    shutil.rmtree(latest_dest_repository, ignore_errors=True)
 
 
 def rotate_library_and_media_files(
     working_directory,
     src_library=DEFAULT_LIBRARY_BASENAME,
     src_media=DEFAULT_MEDIA_BASENAME,
+    src_repository=DEFAULT_REPOSITORY_BASENAME,
 ):
     """
-    Rotate the current library and media files
+    Rotate the current library, media, and repository files
 
-    Current library and media files are moved to the '.latest' folder in case of problems during the restore process.
+    Current library, media, and repository files are moved to the '.latest' folder in case of problems during the restore process.
     Files in .latest can be deleted if the restore was successful
     """
-    print("rotating library and media files")
-    delete_latest_uncompressed_backup(src_library=src_library, src_media=src_media)
+    print("rotating library, media, and repository files")
+    delete_latest_uncompressed_backup(
+        src_library=src_library, src_media=src_media, src_repository=src_repository
+    )
 
     os.makedirs(settings.PREVIOUS_SHARE_ROOT, exist_ok=True)
     if os.path.exists(settings.LIBRARY_ROOT):
         shutil.move(settings.LIBRARY_ROOT, settings.PREVIOUS_SHARE_ROOT)
     if os.path.exists(settings.MEDIA_ROOT):
         shutil.move(settings.MEDIA_ROOT, settings.PREVIOUS_SHARE_ROOT)
+    if os.path.exists(settings.REPOSITORY_ROOT):
+        shutil.move(settings.REPOSITORY_ROOT, settings.PREVIOUS_SHARE_ROOT)
 
     shutil.move(os.path.join(working_directory, src_library), settings.SHARE_DIR)
     shutil.move(os.path.join(working_directory, src_media), settings.SHARE_DIR)
+    # repository dir may not exist in older backups
+    src_repository_path = os.path.join(working_directory, src_repository)
+    if os.path.exists(src_repository_path):
+        shutil.move(src_repository_path, settings.SHARE_DIR)
 
 
 def environment():

django/curator/tests/test_dump_restore.py

@@ -19,7 +19,7 @@
 from core.tests.base import EventFactory, JobFactory
 from library.fs import import_archive
 from library.models import Codebase
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def setUp(self):
         fs_api = self.release.get_fs_api()
         import_archive(
             codebase_release=self.release,
-            nested_code_folder_name="library/tests/archives/nestedcode",
+            nested_code_folder_name=TEST_SAMPLES_DIR / "archives" / "nestedcode",
             fs_api=fs_api,
         )