feature/tas yml segregation (#223)

* added tasconfigdownloader

* refactor tasconfigdownloader

* added YMLParsingMessage struct

* modify YMLParsingMessage struct

* refactor code

* added message type for yml parsing

* fix issue with github download

* fix bitbucket download url

* fix gitlab download url

* resolve go lint error

* resolve PR comments

* fix panic in test case

Author: VikrantKS

cmd/synapse/bin.go

@@ -20,7 +20,8 @@ import (
 	"github.com/LambdaTest/test-at-scale/pkg/proxyserver"
 	"github.com/LambdaTest/test-at-scale/pkg/runner/docker"
 	"github.com/LambdaTest/test-at-scale/pkg/secrets"
-	"github.com/LambdaTest/test-at-scale/pkg/synapse"
+	synapsepkg "github.com/LambdaTest/test-at-scale/pkg/synapse"
+	"github.com/LambdaTest/test-at-scale/pkg/tasconfigdownloader"
 	"github.com/LambdaTest/test-at-scale/pkg/utils"
 	"github.com/joho/godotenv"
 	"github.com/spf13/cobra"
@@ -88,8 +89,8 @@ func run(cmd *cobra.Command, args []string) {
 	if err != nil {
 		logger.Fatalf("could not instantiate k8s runner %v", err)
 	}
-
-	synapse := synapse.New(runner, logger, secretsManager)
+	tasConfigDownloader := tasconfigdownloader.New(logger)
+	synapse := synapsepkg.New(runner, logger, secretsManager, tasConfigDownloader)
 
 	proxyHandler, err := proxyserver.NewProxyHandler(logger)
 	if err != nil {

pkg/core/interfaces.go

@@ -20,12 +20,17 @@ type TASConfigManager interface {
 
 	// GetVersion returns TAS yml version
 	GetVersion(path string) (int, error)
+
+	// GetTasConfigFilePath returns file path of tas config
+	GetTasConfigFilePath(payload *Payload) (string, error)
 }
 
 // GitManager manages the cloning of git repositories
 type GitManager interface {
 	// Clone repository from TAS config
 	Clone(ctx context.Context, payload *Payload, oauth *Oauth) error
+	// DownloadFileByCommit download file from repo for given commit
+	DownloadFileByCommit(ctx context.Context, gitProvider, repoSlug, commitID, filePath string, oauth *Oauth) (string, error)
 }
 
 // DiffManager manages the diff findings for the given payload
@@ -173,5 +178,5 @@ type LogWriterStrategy interface {
 // Builder builds the driver for given tas yml version
 type Builder interface {
 	// GetDriver returns driver for use
-	GetDriver(version int) (Driver, error)
+	GetDriver(version int, ymlFilePath string) (Driver, error)
 }

pkg/core/lifecycle.go

@@ -136,16 +136,19 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 			return err
 		}
 	}
-	// load tas yaml file
-	version, err := pl.TASConfigManager.GetVersion(payload.TasFileName)
+	filePath, err := pl.TASConfigManager.GetTasConfigFilePath(pl.Payload)
+	if err != nil {
+		return err
+	}
+	version, err := pl.TASConfigManager.GetVersion(filePath)
 	if err != nil {
 		pl.Logger.Errorf("Unable to load tas yaml file, error: %v", err)
 		err = &errs.StatusFailed{Remark: err.Error()}
 		return err
 	}
 	pl.Logger.Infof("TAS Version %f", version)
 	pl.setEnv(payload, coverageDir)
-	newDriver, err := pl.Builder.GetDriver(version)
+	newDriver, err := pl.Builder.GetDriver(version, filePath)
 	if err != nil {
 		pl.Logger.Errorf("error crearing driver, error %v", err)
 		return err

pkg/core/models.go

@@ -152,9 +152,7 @@ type DiscoveryResult struct {
 	TaskID          string             `json:"taskID"`
 	OrgID           string             `json:"orgID"`
 	Branch          string             `json:"branch"`
-	Tier            Tier               `json:"tier"`
 	SubModule       string             `json:"subModule"`
-	ContainerImage  string             `json:"containerImage"`
 }
 
 // ExecutionResult represents the request body for test and test suite execution
@@ -456,3 +454,29 @@ type TestExecutionArgs struct {
 	FrameWorkVersion  int
 	CWD               string
 }
+
+// YMLParsingRequestMessage defines yml parsing request received from TAS server
+type YMLParsingRequestMessage struct {
+	GitProvider string    `json:"gitProvider"`
+	CommitID    string    `json:"commitID"`
+	Event       EventType `json:"eventType"`
+	RepoSlug    string    `json:"repoSlug"`
+	TasFileName string    `json:"tasFilePath"`
+	LicenseTier Tier      `json:"license_tier"`
+	OrgID       string    `json:"orgID"`
+	BuildID     string    `json:"buildID"`
+}
+
+// TASConfigDownloaderOutput repersent output return by tasconfig downloader
+type TASConfigDownloaderOutput struct {
+	Version   int         `json:"version"`
+	TASConfig interface{} `json:"tasConfig"`
+}
+
+// YMLParsingResultMessage repersent message sent to TAS server in response of yml parsing request
+type YMLParsingResultMessage struct {
+	ErrorMsg  string                    `json:"ErrorMsg"`
+	OrgID     string                    `json:"orgID"`
+	BuildID   string                    `json:"buildID"`
+	YMLOutput TASConfigDownloaderOutput `json:"ymlOutput"`
+}

pkg/core/secrets.go

@@ -20,6 +20,8 @@ type SecretsManager interface {
 
 	// GetSynapseName returns synapse name mentioned in config
 	GetSynapseName() string
+	// GetOauthToken returns oauth token
+	GetOauthToken() *Oauth
 
 	// GetGitSecretBytes get git secrets in bytes
 	GetGitSecretBytes() ([]byte, error)

pkg/core/wsproto.go

@@ -11,14 +11,16 @@ type StatType string
 
 // types of messages
 const (
-	MsgLogin         MessageType = "login"
-	MsgLogout        MessageType = "logout"
-	MsgTask          MessageType = "task"
-	MsgInfo          MessageType = "info"
-	MsgError         MessageType = "error"
-	MsgResourceStats MessageType = "resourcestats"
-	MsgJobInfo       MessageType = "jobinfo"
-	MsgBuildAbort    MessageType = "build_abort"
+	MsgLogin             MessageType = "login"
+	MsgLogout            MessageType = "logout"
+	MsgTask              MessageType = "task"
+	MsgInfo              MessageType = "info"
+	MsgError             MessageType = "error"
+	MsgResourceStats     MessageType = "resourcestats"
+	MsgJobInfo           MessageType = "jobinfo"
+	MsgBuildAbort        MessageType = "build_abort"
+	MsgYMLParsingRequest MessageType = "yml_parsing_request"
+	MsgYMLParsingResult  MessageType = "yml_parsing_result"
 )
 
 // JobInfo types

pkg/driver/builder.go

@@ -34,7 +34,7 @@ type (
 	}
 )
 
-func (b *Builder) GetDriver(version int) (core.Driver, error) {
+func (b *Builder) GetDriver(version int, filePath string) (core.Driver, error) {
 	switch version {
 	case firstVersion:
 		return &driverV1{
@@ -49,6 +49,7 @@ func (b *Builder) GetDriver(version int) (core.Driver, error) {
 			DiffManager:          b.DiffManager,
 			ListSubModuleService: b.ListSubModuleService,
 			TASVersion:           firstVersion,
+			TASFilePath:          filePath,
 			nodeInstaller: NodeInstaller{
 				logger:           b.Logger,
 				ExecutionManager: b.ExecutionManager,
@@ -67,6 +68,7 @@ func (b *Builder) GetDriver(version int) (core.Driver, error) {
 			DiffManager:          b.DiffManager,
 			ListSubModuleService: b.ListSubModuleService,
 			TASVersion:           secondVersion,
+			TASFilePath:          filePath,
 			nodeInstaller: NodeInstaller{
 				logger:           b.Logger,
 				ExecutionManager: b.ExecutionManager,

pkg/driver/builder_test.go

@@ -8,7 +8,7 @@ import (
 func Test_driver(t *testing.T) {
 	b := Builder{}
 	invalidVersion := 4
-	_, err := b.GetDriver(invalidVersion)
+	_, err := b.GetDriver(invalidVersion, "")
 	wantErr := fmt.Sprintf("invalid version ( %d )  mentioned in yml file", invalidVersion)
 	if err.Error() != wantErr {
 		t.Errorf("want %s , got %s", err.Error(), wantErr)

pkg/driver/driver_v1.go

@@ -6,13 +6,15 @@ package driver
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 
 	"github.com/LambdaTest/test-at-scale/pkg/core"
 	"github.com/LambdaTest/test-at-scale/pkg/errs"
 	"github.com/LambdaTest/test-at-scale/pkg/global"
 	"github.com/LambdaTest/test-at-scale/pkg/logwriter"
 	"github.com/LambdaTest/test-at-scale/pkg/lumber"
+	"github.com/LambdaTest/test-at-scale/pkg/utils"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -32,6 +34,7 @@ type (
 		DiffManager          core.DiffManager
 		ListSubModuleService core.ListSubModuleService
 		TASVersion           int
+		TASFilePath          string
 	}
 
 	setUpResultV1 struct {
@@ -43,7 +46,7 @@ type (
 
 func (d *driverV1) RunDiscovery(ctx context.Context, payload *core.Payload,
 	taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
-	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
 	if err != nil {
 		d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
 		err = &errs.StatusFailed{Remark: err.Error()}
@@ -117,13 +120,16 @@ func (d *driverV1) RunDiscovery(ctx context.Context, payload *core.Payload,
 
 func (d *driverV1) RunExecution(ctx context.Context, payload *core.Payload,
 	taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
-	tas, err := d.TASConfigManager.LoadAndValidate(ctx, 1, payload.TasFileName, payload.EventType, payload.LicenseTier)
+	tas, err := d.TASConfigManager.LoadAndValidate(ctx, 1, d.TASFilePath, payload.EventType, payload.LicenseTier)
 	if err != nil {
 		d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
 		err = &errs.StatusFailed{Remark: err.Error()}
 		return err
 	}
 	tasConfig := tas.(*core.TASConfig)
+	if cachErr := d.setCache(tasConfig); cachErr != nil {
+		return cachErr
+	}
 	if errG := d.BlockTestService.GetBlockTests(ctx, tasConfig.Blocklist, payload.BranchName); errG != nil {
 		d.logger.Errorf("Unable to fetch blocklisted tests: %v", errG)
 		errG = errs.New(errs.GenericErrRemark.Error())
@@ -164,7 +170,9 @@ func (d *driverV1) RunExecution(ctx context.Context, payload *core.Payload,
 func (d *driverV1) setUp(ctx context.Context, payload *core.Payload,
 	tasConfig *core.TASConfig, oauth *core.Oauth, language string) (*setUpResultV1, error) {
 	d.logger.Infof("Tas yaml: %+v", tasConfig)
-
+	if err := d.setCache(tasConfig); err != nil {
+		return nil, err
+	}
 	cacheKey := ""
 	if language == languageJs {
 		cacheKey = tasConfig.Cache.Key
@@ -273,6 +281,20 @@ func (d *driverV1) getEnvAndPattern(payload *core.Payload, tasConfig *core.TASCo
 func populateDiscovery(testDiscoveryResult *core.DiscoveryResult, tasConfig *core.TASConfig) {
 	testDiscoveryResult.Parallelism = tasConfig.Parallelism
 	testDiscoveryResult.SplitMode = tasConfig.SplitMode
-	testDiscoveryResult.ContainerImage = tasConfig.ContainerImage
-	testDiscoveryResult.Tier = tasConfig.Tier
+}
+
+func (d *driverV1) setCache(tasConfig *core.TASConfig) error {
+	language := global.FrameworkLanguageMap[tasConfig.Framework]
+	if tasConfig.Cache == nil && language == "javascript" {
+		checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
+		if err != nil {
+			d.logger.Errorf("Error while computing checksum, error %v", err)
+			return err
+		}
+		tasConfig.Cache = &core.Cache{
+			Key:   checksum,
+			Paths: []string{},
+		}
+	}
+	return nil
 }

pkg/driver/driver_v2.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path"
 	"strings"
@@ -17,6 +18,7 @@ import (
 	"github.com/LambdaTest/test-at-scale/pkg/global"
 	"github.com/LambdaTest/test-at-scale/pkg/logwriter"
 	"github.com/LambdaTest/test-at-scale/pkg/lumber"
+	"github.com/LambdaTest/test-at-scale/pkg/utils"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -36,6 +38,7 @@ type (
 		nodeInstaller        NodeInstaller
 		TestDiscoveryService core.TestDiscoveryService
 		TASVersion           int
+		TASFilePath          string
 	}
 
 	setUpResultV2 struct {
@@ -49,7 +52,7 @@ func (d *driverV2) RunDiscovery(ctx context.Context, payload *core.Payload,
 	taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
 	// do something
 	d.logger.Debugf("Running in %d version", d.TASVersion)
-	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
 	if err != nil {
 		d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
 		err = &errs.StatusFailed{Remark: err.Error()}
@@ -94,14 +97,18 @@ func (d *driverV2) RunDiscovery(ctx context.Context, payload *core.Payload,
 
 func (d *driverV2) RunExecution(ctx context.Context, payload *core.Payload,
 	taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
-	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+	tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
 	if err != nil {
 		d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
 		err = &errs.StatusFailed{Remark: err.Error()}
 		return err
 	}
+
 	subModuleName := os.Getenv(global.SubModuleName)
 	tasConfig := tas.(*core.TASConfigV2)
+	if cachErr := d.setCache(tasConfig); cachErr != nil {
+		return cachErr
+	}
 	subModule, err := d.findSubmodule(tasConfig, payload, subModuleName)
 	if err != nil {
 		d.logger.Errorf("Error finding sub module %s in tas config file", subModuleName)
@@ -355,6 +362,9 @@ func (d *driverV2) setUpDiscovery(ctx context.Context,
 	payload *core.Payload,
 	tasConfig *core.TASConfigV2,
 	oauth *core.Oauth) (*setUpResultV2, error) {
+	if err := d.setCache(tasConfig); err != nil {
+		return nil, err
+	}
 	cacheKey := tasConfig.Cache.Key
 
 	g, errCtx := errgroup.WithContext(ctx)
@@ -445,8 +455,6 @@ func populateTestDiscoveryV2(testDiscoveryResult *core.DiscoveryResult, subModul
 	testDiscoveryResult.Parallelism = subModule.Parallelism
 	testDiscoveryResult.SplitMode = tasConfig.SplitMode
 	testDiscoveryResult.SubModule = subModule.Name
-	testDiscoveryResult.Tier = tasConfig.Tier
-	testDiscoveryResult.ContainerImage = tasConfig.ContainerImage
 }
 
 func (d *driverV2) findSubmodule(tasConfig *core.TASConfigV2, payload *core.Payload, subModuleName string) (*core.SubModule, error) {
@@ -503,3 +511,18 @@ func GetSubmoduleBasedDiff(diff map[string]int, subModulePath string) map[string
 	}
 	return newDiff
 }
+
+func (d *driverV2) setCache(tasConfig *core.TASConfigV2) error {
+	if tasConfig.Cache == nil {
+		checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
+		if err != nil {
+			d.logger.Errorf("Error while computing checksum, error %v", err)
+			return err
+		}
+		tasConfig.Cache = &core.Cache{
+			Key:   checksum,
+			Paths: []string{},
+		}
+	}
+	return nil
+}