revisions: add revision worker and mots integration (bug 1740107)

This is a work in progress, do not merge!

- add new command (lando-cli revision-worker) to start new worker
- add new `revision-worker` that pre-processes revisions
- add method to parse diff and list affected files (should move to mots)
- store module output in revision model
- add mots to requirements file
- include revision info via API endpoint
- create abstract Worker class, use asyncio (bug 1744327)
- add phab integrations and proper loop/process functionality
- run pre/post mots query
- check mots hashes
- refactored revision worker and landing worker
- implement stack hashes
- functional multi edge search
- working mots implementation
- remove s3/boto/etc. dependencies
- add patch caching
- add many to many fields + association to revisions/landing jobs
- fix or temporarily mark tests as xfail (see TODOs)
- create /patches volume in Dockerfiles

Author: zzzeid

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 88
 select = C,E,F,W,B,B9
-ignore = E203, E501, W503, B006
+ignore = E203, E501, W503, B006, E712, E711
 exclude =
     .hg,
     .git,

.gitignore

@@ -6,3 +6,4 @@ __pycache__
 *.pyc
 /.db/
 /.vscode
+notes

.hgrc

@@ -0,0 +1,2 @@
+[ui]
+username=lando<lando@lando.test>

Dockerfile

@@ -52,6 +52,13 @@ RUN chown -R app:app /app
 
 # Create repos directory for transplanting in landing-worker
 RUN mkdir /repos
+RUN chown -R app:app /repos
+
+RUN mkdir /patches
+RUN chown -R app:app /patches
+
+RUN echo "[ui]" > /root/.hgrc
+RUN echo "username = revision worker <revision_worker@lando.test>" >> /root/.hgrc
 
 # Run as a non-privileged user
 USER app

Dockerfile-dev

@@ -11,6 +11,7 @@ ENV PYTHONUNBUFFERED=1
 ENV FLASK_RUN_PORT=9000
 ENV FLASK_RUN_HOST=0.0.0.0
 ENV FLASK_DEBUG=1
+ENV HTTP_ALLOWED=1
 
 ENTRYPOINT ["lando-cli"]
 CMD ["run"]
@@ -42,7 +43,15 @@ RUN chown -R app:app /app
 RUN mkdir /repos
 RUN chown -R app:app /repos
 
+RUN mkdir /patches
+RUN chown -R app:app /patches
+
+RUN echo "[ui]" > /root/.hgrc
+RUN echo "username = revision worker <revision_worker@lando.test>" >> /root/.hgrc
+
 # Run as a non-privileged user
 USER app
 
+# RUN 'echo -e "api-lefsv24henzsbzpw337bhizawuyh\n" | moz-phab install-certificate'
+
 WORKDIR /app

landoapi/api/landing_jobs.py

@@ -10,6 +10,7 @@
 
 from landoapi import auth
 from landoapi.models.landing_job import LandingJob, LandingJobStatus, LandingJobAction
+from landoapi.models.revisions import RevisionStatus
 from landoapi.storage import db
 
 logger = logging.getLogger(__name__)
@@ -63,6 +64,8 @@ def put(landing_job_id, data):
 
     if landing_job.status in (LandingJobStatus.SUBMITTED, LandingJobStatus.DEFERRED):
         landing_job.transition_status(LandingJobAction.CANCEL)
+        for revision in landing_job.get_revisions():
+            revision.status = RevisionStatus.STALE
         db.session.commit()
         return {"id": landing_job.id}, 200
     else:

landoapi/api/revisions.py

@@ -10,6 +10,7 @@
 from landoapi import auth
 from landoapi.decorators import require_phabricator_api_key
 from landoapi.models import SecApprovalRequest
+from landoapi.models.revisions import Revision, RevisionStatus
 from landoapi.projects import get_secure_project_phid
 from landoapi.revisions import revision_is_secure
 from landoapi.secapproval import send_sanitized_commit_message_for_review
@@ -90,3 +91,40 @@ def request_sec_approval(data=None):
     db.session.commit()
 
     return {}, 200
+
+
+@require_phabricator_api_key(optional=True)
+def process(revision_id, diff_id, target_phid):
+    revision = Revision.query.filter(
+        Revision.revision_id == revision_id,
+    ).one_or_none()
+    if revision:
+        # Update with target phid and mark revision as stale.
+        if revision.diff_id != diff_id:
+            logger.info(
+                f"diff_id changed for revision "
+                f"{revision.revision_id} {revision.diff_id} -> {diff_id}"
+            )
+            revision.diff_id = diff_id
+        revision.target = target_phid
+        revision.status = RevisionStatus.STALE
+    else:
+        # Create a new revision and add all relevant info.
+        revision = Revision(
+            revision_id=revision_id,
+            diff_id=diff_id,
+            target=target_phid,
+            status=RevisionStatus.NEW,
+        )
+        db.session.add(revision)
+        logger.info(
+            f"Revision {revision.revision_id} added with diff_id {revision.diff_id}"
+        )
+    db.session.commit()
+
+
+def get_stack_hashes(revision_id):
+    revision = Revision.query.filter(Revision.id == revision_id).one_or_none()
+    if revision:
+        return revision.stack_hashes, 200
+    return {}, 404

landoapi/api/stacks.py

@@ -8,6 +8,7 @@
 from flask import current_app, g
 from landoapi.commit_message import format_commit_message
 from landoapi.decorators import require_phabricator_api_key
+from landoapi.models.revisions import Revision
 from landoapi.phabricator import PhabricatorClient, PhabricatorAPIException
 from landoapi.projects import (
     get_sec_approval_project_phid,
@@ -112,6 +113,9 @@ def get(revision_id):
 
     revisions_response = []
     for _phid, revision in stack_data.revisions.items():
+        lando_revision = Revision.query.filter(
+            Revision.revision_id == revision["id"]
+        ).one_or_none()
         revision_phid = PhabricatorClient.expect(revision, "phid")
         fields = PhabricatorClient.expect(revision, "fields")
         diff_phid = PhabricatorClient.expect(fields, "diffPHID")
@@ -179,6 +183,9 @@ def get(revision_id):
                 "reviewers": serialize_reviewers(reviewers, users, projects, diff_phid),
                 "is_secure": secure,
                 "is_using_secure_commit_message": commit_description.sanitized,
+                "lando_revision": lando_revision.serialize()
+                if lando_revision
+                else None,
             }
         )