diff --git a/stepman/library.go b/stepman/library.go index dfe01c48..ca439034 100644 --- a/stepman/library.go +++ b/stepman/library.go @@ -2,18 +2,24 @@ package stepman import ( "fmt" + "io" + "net/http" "os" + "path/filepath" "strings" "time" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/command/git" - "github.com/bitrise-io/go-utils/pathutil" "github.com/bitrise-io/go-utils/retry" "github.com/bitrise-io/stepman/models" ) -const filePathPrefix = "file://" +const ( + filePathPrefix = "file://" + defaultStepLib = "https://github.com/bitrise-io/bitrise-steplib.git" + defaultStepLibSpecJSONURL = "http://bitrise-steplib-collection.s3.amazonaws.com/spec.json" +) // Logger ... type Logger interface { @@ -23,7 +29,6 @@ type Logger interface { Infof(format string, v ...interface{}) } -// SetupLibrary ... func SetupLibrary(libraryURI string, log Logger) error { if exist, err := RootExistForLibrary(libraryURI); err != nil { return fmt.Errorf("failed to check if routing exist for library (%s), error: %s", libraryURI, err) @@ -37,60 +42,46 @@ func SetupLibrary(libraryURI string, log Logger) error { FolderAlias: alias, } - // Cleanup - isSuccess := false - defer func() { - if !isSuccess { + isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + if isLocalLibrary { + if err := setupWithLocalStepLib(libraryURI, route); err != nil { + // TODO: is cleanup needed? if err := CleanupRoute(route); err != nil { log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) } + + return err } - }() - // Setup - isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) + return nil + } - pth := GetLibraryBaseDirPath(route) - if !isLocalLibrary { - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - repo, err := git.New(pth) - if err != nil { - return err + isCustomStepLibrary := libraryURI != defaultStepLib + if isCustomStepLibrary { + if err := setupWithStepLibRepo(libraryURI, route); err != nil { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) } - return repo.Clone(libraryURI).Run() - }); err != nil { - return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) - } - } else { - // Local spec path - if err := os.MkdirAll(pth, 0777); err != nil { - return fmt.Errorf("failed to create library dir (%s), error: %s", pth, err) - } - - libraryFilePath := libraryURI - if strings.HasPrefix(libraryURI, filePathPrefix) { - libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) - } - if err := command.CopyDir(libraryFilePath, pth, true); err != nil { - return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, pth, err) + return err } + return nil } - if err := ReGenerateLibrarySpec(route); err != nil { - return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) - } + if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil { + if err := setupWithStepLibRepo(libraryURI, route); err != nil { + if err := CleanupRoute(route); err != nil { + log.Warnf("Failed to cleanup routing for library (%s), error: %s", libraryURI, err) + } - if err := AddRoute(route); err != nil { - return fmt.Errorf("failed to add routing, error: %s", err) + return err + } + return nil } - isSuccess = true - return nil } -// UpdateLibrary ... func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, error) { route, found := ReadRoute(libraryURI) if !found { @@ -101,37 +92,164 @@ func UpdateLibrary(libraryURI string, log Logger) (models.StepCollectionModel, e } isLocalLibrary := strings.HasPrefix(libraryURI, filePathPrefix) - if isLocalLibrary { if err := CleanupRoute(route); err != nil { return models.StepCollectionModel{}, fmt.Errorf("failed to cleanup route for library (%s), error: %s", libraryURI, err) } - if err := SetupLibrary(libraryURI, log); err != nil { + if err := setupWithLocalStepLib(libraryURI, route); err != nil { return models.StepCollectionModel{}, fmt.Errorf("failed to setup library (%s), error: %s", libraryURI, err) } - } else { - pth := GetLibraryBaseDirPath(route) - if exists, err := pathutil.IsPathExists(pth); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to check if library (%s) directory (%s) exist, error: %s", libraryURI, pth, err) - } else if !exists { - return models.StepCollectionModel{}, fmt.Errorf("library (%s) not initialized", libraryURI) - } - if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { - repo, err := git.New(pth) - if err != nil { - return err - } - return repo.Pull().Run() - }); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) + return ReadStepSpec(libraryURI) + } + + isCustomStepLibrary := libraryURI != defaultStepLib + if isCustomStepLibrary { + if err := updateWithStepLibRepo(libraryURI, route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err) } - if err := ReGenerateLibrarySpec(route); err != nil { - return models.StepCollectionModel{}, fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) + return ReadStepSpec(libraryURI) + } + + if err := setupWithStepLibSpecURL(libraryURI, defaultStepLibSpecJSONURL, route); err != nil { + if err := updateWithStepLibRepo(libraryURI, route); err != nil { + return models.StepCollectionModel{}, fmt.Errorf("failed to update library (%s), error: %s", libraryURI, err) } + + return ReadStepSpec(libraryURI) } return ReadStepSpec(libraryURI) } + +func setupWithLocalStepLib(libraryURI string, route SteplibRoute) error { + libraryBaseDir := GetLibraryBaseDirPath(route) + + if err := os.MkdirAll(libraryBaseDir, 0777); err != nil { + return fmt.Errorf("failed to create library dir (%s), error: %s", libraryBaseDir, err) + } + + libraryFilePath := libraryURI + if strings.HasPrefix(libraryFilePath, filePathPrefix) { + libraryFilePath = strings.TrimPrefix(libraryURI, filePathPrefix) + } + + if err := command.CopyDir(libraryFilePath, libraryBaseDir, true); err != nil { + return fmt.Errorf("failed to copy dir (%s) to (%s), error: %s", libraryFilePath, libraryBaseDir, err) + } + + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) + } + + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } + return nil +} + +func setupWithStepLibRepo(libraryURI string, route SteplibRoute) error { + libraryBaseDir := GetLibraryBaseDirPath(route) + + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + repo, err := git.New(libraryBaseDir) + if err != nil { + return err + } + return repo.Clone(libraryURI).Run() + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) + } + + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to re-generate library (%s), error: %s", libraryURI, err) + } + + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } + + return nil +} + +func updateWithStepLibRepo(libraryURI string, route SteplibRoute) error { + pth := GetLibraryBaseDirPath(route) + + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + repo, err := git.New(pth) + if err != nil { + return err + } + cmd := repo.Pull() + return cmd.Run() + }); err != nil { + return fmt.Errorf("failed to pull library (%s), error: %s", libraryURI, err) + } + + if err := ReGenerateLibrarySpec(route); err != nil { + return fmt.Errorf("failed to generate spec for library (%s), error: %s", libraryURI, err) + } + + return nil +} + +func setupWithStepLibSpecURL(libraryURI string, specURL string, route SteplibRoute) error { + specPath := GetStepSpecPath(route) + + if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { + return err + } + + if err := retry.Times(2).Wait(3 * time.Second).Try(func(attempt uint) error { + specPath := GetStepSpecPath(route) + + if err := os.MkdirAll(filepath.Dir(specPath), 0755); err != nil { + return err + } + + if err := downloadSpec(specPath, specURL); err != nil { + return err + } + + return nil + }); err != nil { + return fmt.Errorf("failed to clone library (%s), error: %s", libraryURI, err) + } + + if err := AddRoute(route); err != nil { + return fmt.Errorf("failed to add routing, error: %s", err) + } + + return nil +} + +func downloadSpec(filepath string, url string) error { + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Get the data + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check server response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + // Writer the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} diff --git a/stepman/library_test.go b/stepman/library_test.go new file mode 100644 index 00000000..81eb3cf2 --- /dev/null +++ b/stepman/library_test.go @@ -0,0 +1,105 @@ +package stepman + +import ( + "errors" + "fmt" + "testing" + + "github.com/bitrise-io/go-utils/command" +) + +func BenchmarkStepLibSetup(b *testing.B) { + for _, mode := range []string{"repo-based", "spec-json-based"} { + b.Run(fmt.Sprintf("Benchmarking UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + var steplib string + if mode == "repo-based" { + // spec.json based update is only supported for the default StepLib URI ( "https://github.com/bitrise-io/bitrise-steplib.git") + steplib = "https://github.com/bitrise-io/bitrise-steplib" + } else { + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + + b.StopTimer() + route, found := ReadRoute(steplib) + if found { + if err := CleanupRoute(route); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + + if err := SetupLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkStepLibUpdate(b *testing.B) { + oldCommit := "bf150ba4c10a05b9dfb063178746cb76286d04f1" // Commits on Apr 14, 2022 + + for _, mode := range []string{"repo-based", "spec-json-based"} { + b.Run(fmt.Sprintf("Benchmarking UpdateLibrary=%s", mode), func(b *testing.B) { + for i := 0; i < b.N; i++ { + var steplib string + if mode == "repo-based" { + // spec.json based update is only supported for the default StepLib URI ( "https://github.com/bitrise-io/bitrise-steplib.git") + steplib = "https://github.com/bitrise-io/bitrise-steplib" + + b.StopTimer() + err := setupStepLib(steplib, oldCommit) + if err != nil { + b.Fatal(err) + } + b.StartTimer() + } else { + steplib = "https://github.com/bitrise-io/bitrise-steplib.git" + } + + if _, err := UpdateLibrary(steplib, NopeLogger{}); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func setupStepLib(uri, commit string) error { + route, found := ReadRoute(uri) + if found { + if err := CleanupRoute(route); err != nil { + return err + } + } + + if err := SetupLibrary(uri, NopeLogger{}); err != nil { + return err + } + + route, found = ReadRoute(uri) + if !found { + return errors.New("no rout found") + } + + pth := GetLibraryBaseDirPath(route) + cmd := command.New("git", "reset", "--hard", commit) + cmd.SetDir(pth) + if err := cmd.Run(); err != nil { + return err + } + + return nil +} + +type NopeLogger struct { +} + +func (l NopeLogger) Warnf(format string, v ...interface{}) {} + +func (l NopeLogger) Debugf(format string, v ...interface{}) {} + +func (l NopeLogger) Errorf(format string, v ...interface{}) {} + +func (l NopeLogger) Infof(format string, v ...interface{}) {}