Migrate reviews when migrating repository from github

Author: lunny

models/migrations/migrations.go

@@ -304,6 +304,8 @@ var migrations = []Migration{
 	NewMigration("Add original informations for reactions", addReactionOriginals),
 	// v124 -> v125
 	NewMigration("Add columns to user and repository", addUserRepoMissingColumns),
+	// v125 -> v126
+	NewMigration("Add some columns on review for migration", addReviewMigrateInfo),
 }
 
 // Migrate database to current version

models/migrations/v125.go

@@ -0,0 +1,23 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+func addReviewMigrateInfo(x *xorm.Engine) error {
+	type Review struct {
+		OriginalAuthor   string
+		OriginalAuthorID int64
+	}
+
+	if err := x.Sync2(new(Review)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

models/review.go

@@ -45,13 +45,15 @@ func (rt ReviewType) Icon() string {
 
 // Review represents collection of code comments giving feedback for a PR
 type Review struct {
-	ID         int64 `xorm:"pk autoincr"`
-	Type       ReviewType
-	Reviewer   *User  `xorm:"-"`
-	ReviewerID int64  `xorm:"index"`
-	Issue      *Issue `xorm:"-"`
-	IssueID    int64  `xorm:"index"`
-	Content    string `xorm:"TEXT"`
+	ID               int64 `xorm:"pk autoincr"`
+	Type             ReviewType
+	Reviewer         *User `xorm:"-"`
+	ReviewerID       int64 `xorm:"index"`
+	OriginalAuthor   string
+	OriginalAuthorID int64
+	Issue            *Issue `xorm:"-"`
+	IssueID          int64  `xorm:"index"`
+	Content          string `xorm:"TEXT"`
 	// Official is a review made by an assigned approver (counts towards approval)
 	Official bool   `xorm:"NOT NULL DEFAULT false"`
 	CommitID string `xorm:"VARCHAR(40)"`
@@ -62,6 +64,8 @@ type Review struct {
 
 	// CodeComments are the initial code comments of the review
 	CodeComments CodeComments `xorm:"-"`
+
+	Comments []*Comment `xorm:"-"`
 }
 
 func (r *Review) loadCodeComments(e Engine) (err error) {
@@ -398,3 +402,36 @@ func MarkReviewsAsNotStale(issueID int64, commitID string) (err error) {
 
 	return
 }
+
+// InsertReviews inserts review and review comments
+func InsertReviews(reviews []*Review) error {
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	for _, review := range reviews {
+		if _, err := sess.NoAutoTime().Insert(review); err != nil {
+			return err
+		}
+
+		for _, c := range review.Comments {
+			c.ReviewID = review.ID
+		}
+	}
+
+	if err := sess.Commit(); err != nil {
+		return err
+	}
+	sess.Close()
+
+	for _, review := range reviews {
+		if _, err := x.NoAutoTime().Insert(review.Comments); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

modules/migrations/base/downloader.go

@@ -23,6 +23,7 @@ type Downloader interface {
 	GetIssues(page, perPage int) ([]*Issue, bool, error)
 	GetComments(issueNumber int64) ([]*Comment, error)
 	GetPullRequests(page, perPage int) ([]*PullRequest, error)
+	GetReviews(pullRequestNumber int64) ([]*Review, error)
 }
 
 // DownloaderFactory defines an interface to match a downloader implementation and create a downloader

modules/migrations/base/review.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Review is a standard review information
+type Review struct {
+	ID           int64
+	IssueIndex   int64
+	ReviewerID   int64
+	ReviewerName string
+	Official     bool
+	CommitID     string
+	Content      string
+	CreatedAt    time.Time
+	State        string // PENDING, APPROVE, REQUEST_CHANGES, or COMMENT
+	Comments     []*ReviewComment
+}
+
+// ReviewComment represents a review comment
+type ReviewComment struct {
+	ID        int64
+	InReplyTo int64
+	Content   string
+	TreePath  string
+	Position  int
+	CommitID  string
+	PosterID  int64
+	Reactions *Reactions
+	CreatedAt time.Time
+	UpdatedAt time.Time
+}

modules/migrations/base/uploader.go

@@ -17,6 +17,7 @@ type Uploader interface {
 	CreateIssues(issues ...*Issue) error
 	CreateComments(comments ...*Comment) error
 	CreatePullRequests(prs ...*PullRequest) error
+	CreateReviews(reviews ...*Review) error
 	Rollback() error
 	Close()
 }

modules/migrations/gitea.go

@@ -6,6 +6,7 @@
 package migrations
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -27,6 +28,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/services/gitdiff"
 
 	gouuid "github.com/satori/go.uuid"
 )
@@ -706,6 +708,115 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 	return &pullRequest, nil
 }
 
+func convertReviewState(state string) models.ReviewType {
+	switch state {
+	case "PENDING":
+		return models.ReviewTypePending
+	case "APPROVE":
+		return models.ReviewTypeApprove
+	case "REQUEST_CHANGES":
+		return models.ReviewTypeReject
+	case "COMMENT":
+		return models.ReviewTypeComment
+	default:
+		return models.ReviewTypePending
+	}
+}
+
+// CreateReviews create pull request reviews
+func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
+	var cms = make([]*models.Review, 0, len(reviews))
+	for _, review := range reviews {
+		var issueID int64
+		if issueIDStr, ok := g.issues.Load(review.IssueIndex); !ok {
+			issue, err := models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
+			if err != nil {
+				return err
+			}
+			issueID = issue.ID
+			g.issues.Store(review.IssueIndex, issueID)
+		} else {
+			issueID = issueIDStr.(int64)
+		}
+
+		userid, ok := g.userMap[review.ReviewerID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[review.ReviewerID] = userid
+			}
+		}
+
+		var cm = models.Review{
+			Type:        convertReviewState(review.State),
+			IssueID:     issueID,
+			Content:     review.Content,
+			Official:    review.Official,
+			CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
+			UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
+		}
+
+		if userid > 0 {
+			cm.ReviewerID = userid
+		} else {
+			cm.ReviewerID = g.doer.ID
+			cm.OriginalAuthor = review.ReviewerName
+			cm.OriginalAuthorID = review.ReviewerID
+		}
+
+		// TODO: cache pr
+		pr, err := models.GetPullRequestByID(issueID)
+		if err != nil {
+			return err
+		}
+
+		for _, comment := range review.Comments {
+			headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
+			if err != nil {
+				return fmt.Errorf("GetRefCommitID[%s]: %v", pr.GetGitRefName(), err)
+			}
+			patchBuf := new(bytes.Buffer)
+			if err := gitdiff.GetRawDiffForFile(g.gitRepo.Path, pr.MergeBase, headCommitID, gitdiff.RawDiffNormal, comment.TreePath, patchBuf); err != nil {
+				return fmt.Errorf("GetRawDiffForLine[%s, %s, %s, %s]: %v", err, g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath)
+			}
+			line := int64(comment.Position)
+			patch := gitdiff.CutDiffAroundLine(patchBuf, int64((&models.Comment{Line: line}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
+
+			var c = models.Comment{
+				Type:      models.CommentTypeCode,
+				PosterID:  comment.PosterID,
+				IssueID:   issueID,
+				Content:   comment.Content,
+				Line:      line,
+				TreePath:  comment.TreePath,
+				CommitSHA: comment.CommitID,
+				Patch:     patch,
+			}
+
+			if userid > 0 {
+				c.PosterID = userid
+			} else {
+				c.PosterID = g.doer.ID
+				c.OriginalAuthor = review.ReviewerName
+				c.OriginalAuthorID = review.ReviewerID
+			}
+
+			cm.Comments = append(cm.Comments, &c)
+		}
+
+		cms = append(cms, &cm)
+
+		// TODO: Reactions
+	}
+
+	return models.InsertReviews(cms)
+}
+
 // Rollback when migrating failed, this will rollback all the changes.
 func (g *GiteaLocalUploader) Rollback() error {
 	if g.repo != nil && g.repo.ID > 0 {

modules/migrations/github.go

@@ -614,3 +614,77 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 
 	return allPRs, nil
 }
+
+func convertGithubReview(r *github.PullRequestReview) *base.Review {
+	return &base.Review{
+		ID:           r.GetID(),
+		ReviewerID:   r.GetUser().GetID(),
+		ReviewerName: r.GetUser().GetLogin(),
+		CommitID:     r.GetCommitID(),
+		Content:      r.GetBody(),
+		CreatedAt:    r.GetSubmittedAt(),
+		State:        r.GetState(),
+	}
+}
+
+func convertGithubReviewComments(cs []*github.PullRequestComment) []*base.ReviewComment {
+	var rcs = make([]*base.ReviewComment, 0, len(cs))
+	for _, c := range cs {
+		rcs = append(rcs, &base.ReviewComment{
+			ID:        c.GetID(),
+			InReplyTo: c.GetInReplyTo(),
+			Content:   c.GetBody(),
+			TreePath:  c.GetPath(),
+			Position:  c.GetPosition(),
+			CommitID:  c.GetCommitID(),
+			PosterID:  c.GetUser().GetID(),
+			Reactions: convertGithubReactions(c.Reactions),
+			CreatedAt: c.GetCreatedAt(),
+			UpdatedAt: c.GetUpdatedAt(),
+		})
+	}
+	return rcs
+}
+
+// GetReviews returns pull requests review
+func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
+	var allReviews = make([]*base.Review, 0, 100)
+	opt := &github.ListOptions{
+		PerPage: 100,
+	}
+	for {
+		g.sleep()
+		reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt)
+		if err != nil {
+			return nil, fmt.Errorf("error while listing repos: %v", err)
+		}
+		g.rate = &resp.Rate
+		for _, review := range reviews {
+			r := convertGithubReview(review)
+			r.IssueIndex = pullRequestNumber
+			// retrieve all review comments
+			opt2 := &github.ListOptions{
+				PerPage: 100,
+			}
+			for {
+				g.sleep()
+				reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2)
+				if err != nil {
+					return nil, fmt.Errorf("error while listing repos: %v", err)
+				}
+				g.rate = &resp.Rate
+				r.Comments = append(r.Comments, convertGithubReviewComments(reviewComments)...)
+				if resp.NextPage == 0 {
+					break
+				}
+				opt2.Page = resp.NextPage
+			}
+			allReviews = append(allReviews, r)
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	return allReviews, nil
+}