Merge pull request #61 from elementsinteractive/archive

refactor(#52): download repository archive instead of using git clone for GitLab

Author: scastlara

.DS_Store

@@ -44,7 +44,7 @@ jobs:
         run: |
           docker run --rm elementsinteractive/lgtm-ai \
           review \
-          --pr-url "https://github.com/${{ github.repository }}/pull/${{ github.event.issue.number }}" \
           --git-api-key "${{ secrets.GITHUB_TOKEN }}" \
           --ai-api-key "${{ secrets.AI_API_TOKEN }}" \
-          -vv
+          -vv \
+          "https://github.com/${{ github.repository }}/pull/${{ github.event.issue.number }}"

.github/workflows/lgtm.yml

@@ -198,7 +198,7 @@ func (s *sheriffService) scanProject(project repository.Project) (report *scanne
 
 	// Clone the project
 	log.Info().Str("project", project.Path).Str("dir", dir).Str("url", project.RepoUrl).Msg("Cloning project")
-	if err := s.repoService.Provide(project.Repository).Clone(project.RepoUrl, dir); err != nil {
+	if err := s.repoService.Provide(project.Repository).Clone(project, dir); err != nil {
 		return nil, errors.Join(fmt.Errorf("failed to clone project %v", project.Path), err)
 	}
 

archive.tar.gz

@@ -239,8 +239,8 @@ func (c *mockClient) OpenVulnerabilityIssue(project repository.Project, report s
 	return args.Get(0).(*repository.Issue), args.Error(1)
 }
 
-func (c *mockClient) Clone(url string, dir string) error {
-	args := c.Called(url, dir)
+func (c *mockClient) Clone(project repository.Project, dir string) error {
+	args := c.Called(project.RepoUrl, dir)
 	return args.Error(0)
 }
 

internal/patrol/patrol.go

@@ -242,7 +242,7 @@ func (c *mockGitlabService) OpenVulnerabilityIssue(project repository.Project, r
 	return args.Get(0).(*repository.Issue), args.Error(1)
 }
 
-func (c *mockGitlabService) Clone(url string, dir string) error {
-	args := c.Called(url, dir)
+func (c *mockGitlabService) Clone(project repository.Project, dir string) error {
+	args := c.Called(project.RepoUrl, dir)
 	return args.Error(0)
 }

internal/patrol/patrol_test.go

@@ -67,9 +67,9 @@ func (s githubService) OpenVulnerabilityIssue(project repository.Project, report
 	return nil, errors.New("OpenVulnerabilityIssue not yet implemented") // TODO #9 Add github support
 }
 
-func (s githubService) Clone(url string, dir string) (err error) {
+func (s githubService) Clone(project repository.Project, dir string) (err error) {
 	_, err = git.PlainClone(dir, false, &git.CloneOptions{
-		URL: url,
+		URL: project.RepoUrl,
 		Auth: &http.BasicAuth{
 			Username: "N/A",
 			Password: s.token,

internal/publish/to_issue_test.go

@@ -1,14 +1,19 @@
 package gitlab
 
 import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
 	"errors"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
 	"sheriff/internal/repository"
+	"strings"
 	"sync"
 
 	"github.com/elliotchance/pie/v2"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/rs/zerolog/log"
 	gitlab "gitlab.com/gitlab-org/api/client-go"
 )
@@ -115,17 +120,19 @@ func (s gitlabService) OpenVulnerabilityIssue(project repository.Project, report
 	return
 }
 
-func (s gitlabService) Clone(url string, dir string) (err error) {
-	_, err = git.PlainClone(dir, false, &git.CloneOptions{
-		URL: url,
-		Auth: &http.BasicAuth{
-			Username: "N/A",
-			Password: s.token,
-		},
-		Depth: 1,
-	})
+func (s gitlabService) Clone(project repository.Project, dir string) (err error) {
+	archiveData, _, err := s.client.Archive(project.ID, &gitlab.ArchiveOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to download archive: %w", err)
+	}
+
+	// Create directory if it doesn't exist
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create directory: %w", err)
+	}
 
-	return err
+	// Extract archive to directory
+	return s.extractTarGz(bytes.NewReader(archiveData), dir)
 }
 
 // This function receives a list of paths which can be gitlab projects or groups
@@ -344,3 +351,65 @@ func mapIssuePtr(i *gitlab.Issue) *repository.Issue {
 
 	return &issue
 }
+
+// extractTarGz extracts a tar.gz archive
+func (s gitlabService) extractTarGz(reader io.Reader, destDir string) error {
+	// TODO(#52): Move to a separate package when implementing GitHub "clone" through downloading archives
+	gzReader, err := gzip.NewReader(reader)
+	if err != nil {
+		return fmt.Errorf("failed to create gzip reader: %w", err)
+	}
+	defer gzReader.Close()
+
+	tarReader := tar.NewReader(gzReader)
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed to read tar header: %w", err)
+		}
+
+		// Skip the root directory (GitLab archives have a root folder)
+		pathParts := strings.Split(header.Name, "/")
+		if len(pathParts) <= 1 {
+			continue
+		}
+
+		// Remove the first directory component (the root folder)
+		relativePath := strings.Join(pathParts[1:], "/")
+		if relativePath == "" {
+			continue
+		}
+
+		targetPath := filepath.Join(destDir, relativePath)
+		if !strings.HasPrefix(targetPath, filepath.Clean(destDir)+string(os.PathSeparator)) {
+			return fmt.Errorf("content of tar file is trying to write outside of destination directory: %s", relativePath)
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
+				return fmt.Errorf("failed to create directory %s: %w", targetPath, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
+				return fmt.Errorf("failed to create parent directory for %s: %w", targetPath, err)
+			}
+
+			file, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(header.Mode))
+			if err != nil {
+				return fmt.Errorf("failed to create file %s: %w", targetPath, err)
+			}
+			defer file.Close()
+
+			if _, err := io.Copy(file, tarReader); err != nil {
+				return fmt.Errorf("failed to write file %s: %w", targetPath, err)
+			}
+		}
+	}
+
+	return nil
+}

internal/repository/github/github.go

@@ -15,6 +15,7 @@ type iclient interface {
 	ListProjectIssues(projectId interface{}, opt *gitlab.ListProjectIssuesOptions, options ...gitlab.RequestOptionFunc) ([]*gitlab.Issue, *gitlab.Response, error)
 	CreateIssue(projectId interface{}, opt *gitlab.CreateIssueOptions, options ...gitlab.RequestOptionFunc) (*gitlab.Issue, *gitlab.Response, error)
 	UpdateIssue(projectId interface{}, issueId int, opt *gitlab.UpdateIssueOptions, options ...gitlab.RequestOptionFunc) (*gitlab.Issue, *gitlab.Response, error)
+	Archive(pid interface{}, opt *gitlab.ArchiveOptions, options ...gitlab.RequestOptionFunc) ([]byte, *gitlab.Response, error)
 }
 
 type client struct {
@@ -40,3 +41,7 @@ func (c *client) CreateIssue(projectId interface{}, opt *gitlab.CreateIssueOptio
 func (c *client) UpdateIssue(projectId interface{}, issueId int, opt *gitlab.UpdateIssueOptions, options ...gitlab.RequestOptionFunc) (*gitlab.Issue, *gitlab.Response, error) {
 	return c.client.Issues.UpdateIssue(projectId, issueId, opt, options...)
 }
+
+func (c *client) Archive(pid interface{}, opt *gitlab.ArchiveOptions, options ...gitlab.RequestOptionFunc) ([]byte, *gitlab.Response, error) {
+	return c.client.Repositories.Archive(pid, opt, options...)
+}

internal/repository/gitlab/gitlab.go

@@ -2,11 +2,14 @@ package gitlab
 
 import (
 	"errors"
+	"os"
+	"path/filepath"
 	"sheriff/internal/repository"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 	gitlab "gitlab.com/gitlab-org/api/client-go"
 )
 
@@ -199,6 +202,46 @@ func TestDereferenceProjectsPointers(t *testing.T) {
 	assert.Equal(t, 2, dereferencedProjects[1].ID)
 	assert.Equal(t, 2, errCount)
 }
+func TestCloneCompleteFlow(t *testing.T) {
+	// Create temporary directory for testing
+	tempDir, err := os.MkdirTemp("", "sheriff-clone-test-")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	stubArchive, err := os.ReadFile("testdata/sample-archive.tar.gz")
+	require.NoError(t, err)
+
+	mockClient := mockClient{}
+	mockClient.On("Archive", 123, mock.Anything, mock.Anything).Return(stubArchive, &gitlab.Response{}, nil)
+
+	svc := gitlabService{client: &mockClient}
+
+	// Create a test project
+	testProject := repository.Project{
+		ID:   123,
+		Name: "test-project",
+		Path: "group/project",
+	}
+
+	err = svc.Clone(testProject, tempDir)
+
+	// Verify no errors
+	assert.NoError(t, err)
+	mockClient.AssertExpectations(t)
+
+	// Verify files were extracted correctly
+	readmeContent, err := os.ReadFile(filepath.Join(tempDir, "README.md"))
+	assert.NoError(t, err)
+	assert.Equal(t, "# Test Project\n\nThis is a test project for testing GitLab archive extraction.", string(readmeContent))
+
+	srcContent, err := os.ReadFile(filepath.Join(tempDir, "src", "main.go"))
+	assert.NoError(t, err)
+	assert.Equal(t, "package main\n\nimport \"fmt\"\n\nfunc main() {\n\tfmt.Println(\"Hello from test project!\")\n}", string(srcContent))
+
+	// Verify directory structure
+	_, err = os.Stat(filepath.Join(tempDir, "src"))
+	assert.NoError(t, err, "src directory should exist")
+}
 
 type mockClient struct {
 	mock.Mock
@@ -248,3 +291,12 @@ func (c *mockClient) UpdateIssue(projectId interface{}, issueId int, opt *gitlab
 	}
 	return args.Get(0).(*gitlab.Issue), r, args.Error(2)
 }
+
+func (c *mockClient) Archive(pid interface{}, opt *gitlab.ArchiveOptions, options ...gitlab.RequestOptionFunc) ([]byte, *gitlab.Response, error) {
+	args := c.Called(pid, opt, options)
+	var r *gitlab.Response
+	if resp := args.Get(1); resp != nil {
+		r = args.Get(1).(*gitlab.Response)
+	}
+	return args.Get(0).([]byte), r, args.Error(2)
+}