transplants: replace s3 uploads with Revision model (bug 1753728) (#296)

- remove S3 functionality + tests
- add Revision model
- add support for storing patches in new model

Author: zzzeid

landoapi/api/transplants.py

@@ -14,9 +14,8 @@
 from landoapi import auth
 from landoapi.commit_message import format_commit_message
 from landoapi.decorators import require_phabricator_api_key
-from landoapi.hgexports import build_patch_for_revision
 from landoapi.models.landing_job import LandingJob, LandingJobStatus
-from landoapi.patches import upload
+from landoapi.models.revisions import Revision
 from landoapi.phabricator import PhabricatorClient
 from landoapi.projects import (
     CHECKIN_PROJ_SLUG,
@@ -309,7 +308,6 @@ def post(phab: PhabricatorClient, data: dict):
     }
 
     # Build the patches to land.
-    patch_urls = []
     for revision, diff in to_land:
         reviewers = get_collated_reviewers(revision)
         accepted_reviewers = reviewers_for_commit_message(
@@ -341,23 +339,28 @@ def post(phab: PhabricatorClient, data: dict):
         author_name, author_email = select_diff_author(diff)
         timestamp = int(datetime.now().timestamp())
 
-        # Construct the patch that will be sent to transplant.
-        raw_diff = phab.call_conduit("differential.getrawdiff", diffID=diff["id"])
-        patch = build_patch_for_revision(
-            raw_diff, author_name, author_email, commit_message, timestamp
-        )
+        # Construct the patch that will be transplanted.
+        revision_id = revision["id"]
+        diff_id = diff["id"]
 
-        # Upload the patch to S3
-        patch_url = upload(
-            revision["id"],
-            diff["id"],
-            patch,
-            current_app.config["PATCH_BUCKET_NAME"],
-            aws_access_key=current_app.config["AWS_ACCESS_KEY"],
-            aws_secret_key=current_app.config["AWS_SECRET_KEY"],
-            endpoint_url=current_app.config["S3_ENDPOINT_URL"],
-        )
-        patch_urls.append(patch_url)
+        lando_revision = Revision.get_from_revision_id(revision_id)
+        if not lando_revision:
+            lando_revision = Revision(revision_id=revision_id)
+            db.session.add(lando_revision)
+
+        lando_revision.diff_id = diff_id
+        db.session.commit()
+
+        patch_data = {
+            "author_name": author_name,
+            "author_email": author_email,
+            "commit_message": commit_message,
+            "timestamp": timestamp,
+        }
+
+        raw_diff = phab.call_conduit("differential.getrawdiff", diffID=diff["id"])
+        lando_revision.set_patch(raw_diff, patch_data)
+        db.session.commit()
 
     ldap_username = g.auth0_user.email
     revision_to_diff_id = {str(r["id"]): d["id"] for r, d in to_land}

landoapi/app.py

@@ -18,7 +18,6 @@
 from landoapi.dockerflow import dockerflow
 from landoapi.hooks import initialize_hooks
 from landoapi.logging import logging_subsystem
-from landoapi.patches import patches_s3_subsystem
 from landoapi.phabricator import phabricator_subsystem
 from landoapi.repos import repo_clone_subsystem
 from landoapi.sentry import sentry_subsystem
@@ -41,7 +40,6 @@
     celery_subsystem,
     db_subsystem,
     lando_ui_subsystem,
-    patches_s3_subsystem,
     phabricator_subsystem,
     smtp_subsystem,
     treestatus_subsystem,
@@ -90,7 +88,6 @@ def load_config() -> dict[str, Any]:
         "PHABRICATOR_URL",
         "REPO_CLONES_PATH",
         "REPOS_TO_LAND",
-        "S3_ENDPOINT_URL",
         "SENTRY_DSN",
         "TRANSPLANT_PASSWORD",
         "TRANSPLANT_API_KEY",

landoapi/cli.py

@@ -13,9 +13,6 @@
 import connexion
 from flask.cli import FlaskGroup
 
-from landoapi import (
-    patches,
-)
 from landoapi.models.configuration import (
     ConfigurationVariable,
     ConfigurationKey,
@@ -57,18 +54,6 @@ def cli():
     """Lando API cli."""
 
 
-@cli.command()
-def init_s3():
-    """Initialize fake S3 bucket for development purposes."""
-    # Create a fake S3 bucket, ie for moto.
-    s3 = patches.create_s3(
-        aws_access_key=os.environ["AWS_ACCESS_KEY"],
-        aws_secret_key=os.environ["AWS_SECRET_KEY"],
-        endpoint_url=os.environ["S3_ENDPOINT_URL"],
-    )
-    s3.create_bucket(Bucket=os.environ["PATCH_BUCKET_NAME"])
-
-
 @cli.command(context_settings=dict(ignore_unknown_options=True))
 @click.argument("celery_arguments", nargs=-1, type=click.UNPROCESSED)
 def worker(celery_arguments):

landoapi/models/revisions.py

@@ -9,11 +9,14 @@
 Phabricator diff that is associated with a particular revision.
 """
 
+from __future__ import annotations
+
 import enum
 import logging
 
 from sqlalchemy.dialects.postgresql.json import JSONB
 
+from landoapi.hgexports import build_patch_for_revision
 from landoapi.models.base import Base
 from landoapi.storage import db
 
@@ -32,6 +35,40 @@ class DiffWarningGroup(enum.Enum):
     LINT = "LINT"
 
 
+class Revision(Base):
+    """
+    A representation of a revision in the database referencing a Phabricator revision.
+    """
+
+    # revision_id and diff_id map to Phabricator IDs (integers).
+    revision_id = db.Column(db.Integer, nullable=False, unique=True)
+    diff_id = db.Column(db.Integer, nullable=False)
+
+    # The actual patch.
+    patch_bytes = db.Column(db.LargeBinary, nullable=False, default=b"")
+
+    # Patch metadata, such as author, timestamp, etc...
+    patch_data = db.Column(JSONB, nullable=False, default=dict)
+
+    def __repr__(self):
+        """Return a human-readable representation of the instance."""
+        return (
+            f"<{self.__class__.__name__}: {self.id} "
+            f"[D{self.revision_id}-{self.diff_id}]>"
+        )
+
+    @classmethod
+    def get_from_revision_id(cls, revision_id: int) -> "Revision" | None:
+        """Return a Revision object from a given ID."""
+        return cls.query.filter(Revision.revision_id == revision_id).one_or_none()
+
+    def set_patch(self, raw_diff: bytes, patch_data: dict[str, str]):
+        """Given a raw_diff and patch data, build the patch and store it."""
+        self.patch_data = patch_data
+        patch = build_patch_for_revision(raw_diff, **self.patch_data)
+        self.patch_bytes = patch.encode("utf-8")
+
+
 class DiffWarning(Base):
     """Represents a warning message associated with a particular diff and revision."""