diff --git a/backup.go b/backup.go index 7144ab3..ba6cdf9 100644 --- a/backup.go +++ b/backup.go @@ -63,29 +63,40 @@ func cloneNewRepo(repoDir string, repo *Repository, bare bool) ([]byte, error) { log.Printf("Cloning %s\n", repo.Name) log.Printf("%#v\n", repo) - if repo.Private && ignorePrivate != nil && *ignorePrivate { + if shouldSkipPrivateRepo(repo) { log.Printf("Skipping %s as it is a private repo.\n", repo.Name) return nil, nil } - cloneURL := repo.CloneURL - if useHTTPSClone != nil && *useHTTPSClone { - // Add username and token to the clone URL - // https://gitlab.com/amitsaha/testproject1 => https://amitsaha:token@gitlab.com/amitsaha/testproject1 - u, err := url.Parse(repo.CloneURL) - if err != nil { - log.Fatalf("Invalid clone URL: %v\n", err) - } - cloneURL = u.Scheme + "://" + gitHostUsername + ":" + gitHostToken + "@" + u.Host + u.Path + cloneURL := buildAuthenticatedCloneURL(repo.CloneURL) + cmd := buildGitCloneCommand(cloneURL, repoDir, bare) + return cmd.CombinedOutput() +} + +// shouldSkipPrivateRepo checks if a private repo should be skipped +func shouldSkipPrivateRepo(repo *Repository) bool { + return repo.Private && ignorePrivate != nil && *ignorePrivate +} + +// buildAuthenticatedCloneURL adds authentication to the clone URL if using HTTPS +func buildAuthenticatedCloneURL(originalURL string) string { + if useHTTPSClone == nil || !*useHTTPSClone { + return originalURL } - var cmd *exec.Cmd + u, err := url.Parse(originalURL) + if err != nil { + log.Fatalf("Invalid clone URL: %v\n", err) + } + return u.Scheme + "://" + gitHostUsername + ":" + gitHostToken + "@" + u.Host + u.Path +} + +// buildGitCloneCommand creates the appropriate git clone command +func buildGitCloneCommand(cloneURL, repoDir string, bare bool) *exec.Cmd { if bare { - cmd = execCommand(gitCommand, "clone", "--mirror", cloneURL, repoDir) - } else { - cmd = execCommand(gitCommand, "clone", cloneURL, repoDir) + return execCommand(gitCommand, "clone", "--mirror", cloneURL, repoDir) } - return cmd.CombinedOutput() + return execCommand(gitCommand, "clone", cloneURL, repoDir) } // setupBackupDir determines and creates the backup directory path diff --git a/bitbucket.go b/bitbucket.go index d9e458b..d0109ab 100644 --- a/bitbucket.go +++ b/bitbucket.go @@ -18,38 +18,60 @@ func getBitbucketRepositories( log.Fatalf("Couldn't acquire a client to talk to %s", service) } - var repositories []*Repository - - resp, err := client.(*bitbucket.Client).Workspaces.List() + workspaces, err := client.(*bitbucket.Client).Workspaces.List() if err != nil { return nil, err } - for _, workspace := range resp.Workspaces { - options := &bitbucket.RepositoriesOptions{Owner: workspace.Slug} + return fetchBitbucketRepositoriesFromWorkspaces(client.(*bitbucket.Client), workspaces.Workspaces) +} + +// fetchBitbucketRepositoriesFromWorkspaces retrieves repositories from all workspaces +func fetchBitbucketRepositoriesFromWorkspaces(client *bitbucket.Client, workspaces []bitbucket.Workspace) ([]*Repository, error) { + var repositories []*Repository - resp, err := client.(*bitbucket.Client).Repositories.ListForAccount(options) + for _, workspace := range workspaces { + workspaceRepos, err := fetchBitbucketWorkspaceRepositories(client, workspace.Slug) if err != nil { return nil, err } + repositories = append(repositories, workspaceRepos...) + } - for _, repo := range resp.Items { - namespace := strings.Split(repo.Full_name, "/")[0] + return repositories, nil +} - httpsURL, sshURL := extractBitbucketCloneURLs(repo.Links) - cloneURL := getCloneURL(httpsURL, sshURL) +// fetchBitbucketWorkspaceRepositories retrieves all repositories from a single workspace +func fetchBitbucketWorkspaceRepositories(client *bitbucket.Client, workspaceSlug string) ([]*Repository, error) { + options := &bitbucket.RepositoriesOptions{Owner: workspaceSlug} + + resp, err := client.Repositories.ListForAccount(options) + if err != nil { + return nil, err + } - repositories = append(repositories, &Repository{ - CloneURL: cloneURL, - Name: repo.Slug, - Namespace: namespace, - Private: repo.Is_private, - }) - } + var repositories []*Repository + for _, repo := range resp.Items { + repositories = append(repositories, buildBitbucketRepository(repo)) } + return repositories, nil } +// buildBitbucketRepository converts a Bitbucket repository to our Repository type +func buildBitbucketRepository(repo bitbucket.Repository) *Repository { + namespace := strings.Split(repo.Full_name, "/")[0] + httpsURL, sshURL := extractBitbucketCloneURLs(repo.Links) + cloneURL := getCloneURL(httpsURL, sshURL) + + return &Repository{ + CloneURL: cloneURL, + Name: repo.Slug, + Namespace: namespace, + Private: repo.Is_private, + } +} + func extractBitbucketCloneURLs(links map[string]interface{}) (httpsURL, sshURL string) { linkmaps, ok := links["clone"].([]interface{}) if !ok { diff --git a/github.go b/github.go index d8a2b37..78e1788 100644 --- a/github.go +++ b/github.go @@ -19,46 +19,30 @@ func getGithubRepositories( log.Fatalf("Couldn't acquire a client to talk to %s", service) } - var repositories []*Repository - ctx := context.Background() if githubRepoType == "starred" { return getGithubStarredRepositories(ctx, client.(*github.Client), ignoreFork) } - options := github.RepositoryListOptions{Type: githubRepoType} + return getGithubUserRepositories(ctx, client.(*github.Client), githubRepoType, githubNamespaceWhitelist, ignoreFork) +} + +// getGithubUserRepositories retrieves user repositories (not starred) from GitHub +func getGithubUserRepositories(ctx context.Context, client *github.Client, repoType string, namespaceWhitelist []string, ignoreFork bool) ([]*Repository, error) { + var repositories []*Repository + options := github.RepositoryListOptions{Type: repoType} for { - repos, resp, err := client.(*github.Client).Repositories.List(ctx, "", &options) + repos, resp, err := client.Repositories.List(ctx, "", &options) if err != nil { return nil, err } for _, repo := range repos { - if *repo.Fork && ignoreFork { + if shouldSkipGithubRepo(repo, namespaceWhitelist, ignoreFork) { continue } - namespace := strings.Split(*repo.FullName, "/")[0] - - if len(githubNamespaceWhitelist) > 0 && !contains(githubNamespaceWhitelist, namespace) { - continue - } - - var httpsCloneURL, sshCloneURL string - if repo.CloneURL != nil { - httpsCloneURL = *repo.CloneURL - } - if repo.SSHURL != nil { - sshCloneURL = *repo.SSHURL - } - - cloneURL := getCloneURL(httpsCloneURL, sshCloneURL) - repositories = append(repositories, &Repository{ - CloneURL: cloneURL, - Name: *repo.Name, - Namespace: namespace, - Private: *repo.Private, - }) + repositories = append(repositories, buildGithubRepository(repo)) } if resp.NextPage == 0 { break @@ -68,6 +52,35 @@ func getGithubRepositories( return repositories, nil } +// shouldSkipGithubRepo determines if a repository should be skipped based on filters +func shouldSkipGithubRepo(repo *github.Repository, namespaceWhitelist []string, ignoreFork bool) bool { + if *repo.Fork && ignoreFork { + return true + } + namespace := strings.Split(*repo.FullName, "/")[0] + return len(namespaceWhitelist) > 0 && !contains(namespaceWhitelist, namespace) +} + +// buildGithubRepository converts a GitHub repository to our Repository type +func buildGithubRepository(repo *github.Repository) *Repository { + namespace := strings.Split(*repo.FullName, "/")[0] + + var httpsCloneURL, sshCloneURL string + if repo.CloneURL != nil { + httpsCloneURL = *repo.CloneURL + } + if repo.SSHURL != nil { + sshCloneURL = *repo.SSHURL + } + + return &Repository{ + CloneURL: getCloneURL(httpsCloneURL, sshCloneURL), + Name: *repo.Name, + Namespace: namespace, + Private: *repo.Private, + } +} + func getGithubStarredRepositories(ctx context.Context, client *github.Client, ignoreFork bool) ([]*Repository, error) { var repositories []*Repository options := github.ActivityListStarredOptions{} @@ -81,23 +94,7 @@ func getGithubStarredRepositories(ctx context.Context, client *github.Client, ig if *star.Repository.Fork && ignoreFork { continue } - namespace := strings.Split(*star.Repository.FullName, "/")[0] - - var httpsCloneURL, sshCloneURL string - if star.Repository.CloneURL != nil { - httpsCloneURL = *star.Repository.CloneURL - } - if star.Repository.SSHURL != nil { - sshCloneURL = *star.Repository.SSHURL - } - - cloneURL := getCloneURL(httpsCloneURL, sshCloneURL) - repositories = append(repositories, &Repository{ - CloneURL: cloneURL, - Name: *star.Repository.Name, - Namespace: namespace, - Private: *star.Repository.Private, - }) + repositories = append(repositories, buildGithubRepository(star.Repository)) } if resp.NextPage == 0 { break diff --git a/github_create_user_migration.go b/github_create_user_migration.go index 94012c8..3b3edad 100644 --- a/github_create_user_migration.go +++ b/github_create_user_migration.go @@ -4,8 +4,13 @@ import ( "context" "log" "time" + + "github.com/google/go-github/v34/github" ) +// defaultMigrationPollingInterval is the default time to wait between migration status checks +const defaultMigrationPollingInterval = 60 * time.Second + func handleGithubCreateUserMigration(client interface{}, c *appConfig) { repos, err := getRepositories( client, @@ -21,6 +26,12 @@ func handleGithubCreateUserMigration(client interface{}, c *appConfig) { log.Fatalf("Error getting list of repositories: %v", err) } + createUserMigration(client, c, repos) + processOrganizationMigrations(client, c) +} + +// createUserMigration creates and optionally downloads a user migration +func createUserMigration(client interface{}, c *appConfig, repos []*Repository) { log.Printf("Creating a user migration for %d repos", len(repos)) m, err := createGithubUserMigration( context.Background(), @@ -33,47 +44,56 @@ func handleGithubCreateUserMigration(client interface{}, c *appConfig) { } if c.githubWaitForMigrationComplete { - migrationStatePollingDuration := 60 * time.Second err = downloadGithubUserMigrationData( context.Background(), client, c.backupDir, m.ID, - migrationStatePollingDuration, + defaultMigrationPollingInterval, ) if err != nil { log.Fatalf("Error querying/downloading migration: %v", err) } } +} +// processOrganizationMigrations creates migrations for all user-owned organizations +func processOrganizationMigrations(client interface{}, c *appConfig) { orgs, err := getGithubUserOwnedOrgs(context.Background(), client) if err != nil { log.Fatal("Error getting user organizations", err) } + for _, o := range orgs { - orgRepos, err := getGithubOrgRepositories(context.Background(), client, o) - if err != nil { - log.Fatal("Error getting org repos", err) - } - if len(orgRepos) == 0 { - log.Printf("No repos found in %s", *o.Login) - continue - } - log.Printf("Creating a org migration (%s) for %d repos", *o.Login, len(orgRepos)) - oMigration, err := createGithubOrgMigration(context.Background(), client, *o.Login, orgRepos) - if err != nil { - log.Fatalf("Error creating migration: %v", err) - } - if c.githubWaitForMigrationComplete { - migrationStatePollingDuration := 60 * time.Second - downloadGithubOrgMigrationData( - context.Background(), - client, - *o.Login, - c.backupDir, - oMigration.ID, - migrationStatePollingDuration, - ) - } + createOrganizationMigration(client, c, o) } +} +// createOrganizationMigration creates and optionally downloads a migration for a single organization +func createOrganizationMigration(client interface{}, c *appConfig, org *github.Organization) { + orgRepos, err := getGithubOrgRepositories(context.Background(), client, org) + if err != nil { + log.Fatal("Error getting org repos", err) + } + + if len(orgRepos) == 0 { + log.Printf("No repos found in %s", *org.Login) + return + } + + log.Printf("Creating a org migration (%s) for %d repos", *org.Login, len(orgRepos)) + oMigration, err := createGithubOrgMigration(context.Background(), client, *org.Login, orgRepos) + if err != nil { + log.Fatalf("Error creating migration: %v", err) + } + + if c.githubWaitForMigrationComplete { + downloadGithubOrgMigrationData( + context.Background(), + client, + *org.Login, + c.backupDir, + oMigration.ID, + defaultMigrationPollingInterval, + ) + } } diff --git a/gitlab.go b/gitlab.go index eb92f60..3c94344 100644 --- a/gitlab.go +++ b/gitlab.go @@ -18,59 +18,82 @@ func getGitlabRepositories( log.Fatalf("Couldn't acquire a client to talk to %s", service) } - var repositories []*Repository + gitlabListOptions := buildGitlabListOptions(gitlabProjectMembershipType, gitlabProjectVisibility) - var visibility gitlab.VisibilityValue - var boolTrue bool = true + return fetchGitlabProjects(client.(*gitlab.Client), gitlabListOptions) +} - gitlabListOptions := gitlab.ListProjectsOptions{} +// buildGitlabListOptions constructs the list options for GitLab project queries +func buildGitlabListOptions(membershipType, visibility string) gitlab.ListProjectsOptions { + var boolTrue = true + options := gitlab.ListProjectsOptions{} - switch gitlabProjectMembershipType { + // Set membership type filters + switch membershipType { case "owner": - gitlabListOptions.Owned = &boolTrue + options.Owned = &boolTrue case "member": - gitlabListOptions.Membership = &boolTrue + options.Membership = &boolTrue case "starred": - gitlabListOptions.Starred = &boolTrue + options.Starred = &boolTrue case "all": - gitlabListOptions.Owned = &boolTrue - gitlabListOptions.Membership = &boolTrue - gitlabListOptions.Starred = &boolTrue + options.Owned = &boolTrue + options.Membership = &boolTrue + options.Starred = &boolTrue } - if gitlabProjectVisibility != "all" { - switch gitlabProjectVisibility { - case "public": - visibility = gitlab.PublicVisibility - case "private": - visibility = gitlab.PrivateVisibility - case "internal": - fallthrough - case "default": - visibility = gitlab.InternalVisibility - } - gitlabListOptions.Visibility = &visibility + // Set visibility filter + if visibility != "all" { + visibilityValue := getGitlabVisibility(visibility) + options.Visibility = &visibilityValue } + return options +} + +// getGitlabVisibility converts a visibility string to GitLab's VisibilityValue type +func getGitlabVisibility(visibility string) gitlab.VisibilityValue { + switch visibility { + case "public": + return gitlab.PublicVisibility + case "private": + return gitlab.PrivateVisibility + case "internal", "default": + return gitlab.InternalVisibility + default: + return gitlab.InternalVisibility + } +} + +// fetchGitlabProjects retrieves all projects from GitLab with pagination +func fetchGitlabProjects(client *gitlab.Client, options gitlab.ListProjectsOptions) ([]*Repository, error) { + var repositories []*Repository + for { - repos, resp, err := client.(*gitlab.Client).Projects.ListProjects(&gitlabListOptions) + repos, resp, err := client.Projects.ListProjects(&options) if err != nil { return nil, err } for _, repo := range repos { - namespace := strings.Split(repo.PathWithNamespace, "/")[0] - cloneURL := getCloneURL(repo.WebURL, repo.SSHURLToRepo) - repositories = append(repositories, &Repository{ - CloneURL: cloneURL, - Name: repo.Name, - Namespace: namespace, - Private: repo.Visibility == "private", - }) + repositories = append(repositories, buildGitlabRepository(repo)) } if resp.NextPage == 0 { break } - gitlabListOptions.ListOptions.Page = resp.NextPage + options.ListOptions.Page = resp.NextPage } return repositories, nil } + +// buildGitlabRepository converts a GitLab project to our Repository type +func buildGitlabRepository(repo *gitlab.Project) *Repository { + namespace := strings.Split(repo.PathWithNamespace, "/")[0] + cloneURL := getCloneURL(repo.WebURL, repo.SSHURLToRepo) + + return &Repository{ + CloneURL: cloneURL, + Name: repo.Name, + Namespace: namespace, + Private: repo.Visibility == "private", + } +} diff --git a/helpers.go b/helpers.go index 11896ca..9230587 100644 --- a/helpers.go +++ b/helpers.go @@ -11,37 +11,48 @@ import ( // getUsername retrieves the username for the authenticated user from the git service func getUsername(client interface{}, service string) string { - if client == nil { log.Fatalf("Couldn't acquire a client to talk to %s", service) } - if service == "github" { - ctx := context.Background() - user, _, err := client.(*github.Client).Users.Get(ctx, "") - if err != nil { - log.Fatal("Error retrieving username", err.Error()) - } - return *user.Login + switch service { + case "github": + return getGithubUsername(client.(*github.Client)) + case "gitlab": + return getGitlabUsername(client.(*gitlab.Client)) + case "bitbucket": + return getBitbucketUsername(client.(*bitbucket.Client)) + default: + return "" } +} - if service == "gitlab" { - user, _, err := client.(*gitlab.Client).Users.CurrentUser() - if err != nil { - log.Fatal("Error retrieving username", err.Error()) - } - return user.Username +// getGithubUsername retrieves the GitHub username +func getGithubUsername(client *github.Client) string { + ctx := context.Background() + user, _, err := client.Users.Get(ctx, "") + if err != nil { + log.Fatal("Error retrieving username", err.Error()) } + return *user.Login +} - if service == "bitbucket" { - user, err := client.(*bitbucket.Client).User.Profile() - if err != nil { - log.Fatal("Error retrieving username", err.Error()) - } - return user.Username +// getGitlabUsername retrieves the GitLab username +func getGitlabUsername(client *gitlab.Client) string { + user, _, err := client.Users.CurrentUser() + if err != nil { + log.Fatal("Error retrieving username", err.Error()) } + return user.Username +} - return "" +// getBitbucketUsername retrieves the Bitbucket username +func getBitbucketUsername(client *bitbucket.Client) string { + user, err := client.User.Profile() + if err != nil { + log.Fatal("Error retrieving username", err.Error()) + } + return user.Username } // validGitlabProjectMembership checks if the given membership type is valid diff --git a/user_data.go b/user_data.go index 6177606..a343996 100644 --- a/user_data.go +++ b/user_data.go @@ -96,20 +96,24 @@ func createGithubOrgMigration(ctx context.Context, client interface{}, org strin func downloadGithubUserMigrationData( ctx context.Context, client interface{}, backupDir string, id *int64, migrationStatePollingDuration time.Duration, ) error { - - var ms *github.UserMigration - - ms, _, err := client.(*github.Client).Migrations.UserMigrationStatus(ctx, *id) + ghClient := client.(*github.Client) + ms, _, err := ghClient.Migrations.UserMigrationStatus(ctx, *id) if err != nil { return err } + return pollUserMigrationStatus(ctx, ghClient, backupDir, ms, migrationStatePollingDuration) +} + +// pollUserMigrationStatus polls for migration completion and downloads when ready +func pollUserMigrationStatus(ctx context.Context, client *github.Client, backupDir string, ms *github.UserMigration, pollDuration time.Duration) error { + var err error for { switch *ms.State { case migrationStateFailed: return errors.New("migration failed") case migrationStateExported: - archiveURL, err := client.(*github.Client).Migrations.UserMigrationArchiveURL(ctx, *ms.ID) + archiveURL, err := client.Migrations.UserMigrationArchiveURL(ctx, *ms.ID) if err != nil { return err } @@ -117,9 +121,9 @@ func downloadGithubUserMigrationData( return downloadMigrationArchive(archiveURL, archiveFilepath) default: log.Printf("Waiting for migration state to be exported: %s\n", *ms.State) - time.Sleep(migrationStatePollingDuration) + time.Sleep(pollDuration) - ms, _, err = client.(*github.Client).Migrations.UserMigrationStatus(ctx, *ms.ID) + ms, _, err = client.Migrations.UserMigrationStatus(ctx, *ms.ID) if err != nil { return err } @@ -130,18 +134,24 @@ func downloadGithubUserMigrationData( func downloadGithubOrgMigrationData( ctx context.Context, client interface{}, org string, backupDir string, id *int64, migrationStatePollingDuration time.Duration, ) error { - var ms *github.Migration - ms, _, err := client.(*github.Client).Migrations.MigrationStatus(ctx, org, *id) + ghClient := client.(*github.Client) + ms, _, err := ghClient.Migrations.MigrationStatus(ctx, org, *id) if err != nil { return err } + return pollOrgMigrationStatus(ctx, ghClient, org, backupDir, ms, migrationStatePollingDuration) +} + +// pollOrgMigrationStatus polls for organization migration completion and downloads when ready +func pollOrgMigrationStatus(ctx context.Context, client *github.Client, org, backupDir string, ms *github.Migration, pollDuration time.Duration) error { + var err error for { switch *ms.State { case migrationStateFailed: return errors.New("org migration failed") case migrationStateExported: - archiveURL, err := client.(*github.Client).Migrations.MigrationArchiveURL(ctx, org, *ms.ID) + archiveURL, err := client.Migrations.MigrationArchiveURL(ctx, org, *ms.ID) if err != nil { return err } @@ -150,8 +160,8 @@ func downloadGithubOrgMigrationData( return downloadMigrationArchive(archiveURL, archiveFilepath) default: log.Printf("Waiting for migration state to be exported: %s\n", *ms.State) - time.Sleep(migrationStatePollingDuration) - ms, _, err = client.(*github.Client).Migrations.MigrationStatus(ctx, org, *ms.ID) + time.Sleep(pollDuration) + ms, _, err = client.Migrations.MigrationStatus(ctx, org, *ms.ID) if err != nil { return err }