refactor multi config nucleus code (#211)

* move list sumodule logic out of dicovery

* refactor submodue

* bug fix panic in test discovery

* fix process working directory for submodules

* added comments

* remove deadcode

* fix runner install location for flag runPreRunEveryTime in exec mode

* fix tests

* rename files driverV1 -> driver_v1 and driverV2 -> driver_v2

* resolve PR comments

* fix typo

* remove unused mock file

Author: VikrantKS

cmd/nucleus/bin.go

@@ -21,8 +21,10 @@ import (
 	"github.com/LambdaTest/test-at-scale/pkg/command"
 	"github.com/LambdaTest/test-at-scale/pkg/core"
 	"github.com/LambdaTest/test-at-scale/pkg/diffmanager"
+	"github.com/LambdaTest/test-at-scale/pkg/driver"
 	"github.com/LambdaTest/test-at-scale/pkg/gitmanager"
 	"github.com/LambdaTest/test-at-scale/pkg/global"
+	"github.com/LambdaTest/test-at-scale/pkg/listsubmoduleservice"
 	"github.com/LambdaTest/test-at-scale/pkg/lumber"
 	"github.com/LambdaTest/test-at-scale/pkg/payloadmanager"
 	"github.com/LambdaTest/test-at-scale/pkg/requestutils"
@@ -145,6 +147,20 @@ func run(cmd *cobra.Command, args []string) {
 	if err != nil {
 		logger.Fatalf("failed to initialize coverage service: %v", err)
 	}
+	listsubmodule := listsubmoduleservice.New(defaultRequests, logger)
+
+	builder := driver.Builder{
+		Logger:               logger,
+		TestExecutionService: tes,
+		TestDiscoveryService: tds,
+		AzureClient:          azureClient,
+		BlockTestService:     tbs,
+		ExecutionManager:     execManager,
+		TASConfigManager:     tcm,
+		CacheStore:           cache,
+		DiffManager:          dm,
+		ListSubModuleService: listsubmodule,
+	}
 
 	pl.PayloadManager = pm
 	pl.TASConfigManager = tcm
@@ -159,6 +175,7 @@ func run(cmd *cobra.Command, args []string) {
 	pl.Task = t
 	pl.CacheStore = cache
 	pl.SecretParser = secretParser
+	pl.Builder = &builder
 
 	logger.Infof("LambdaTest Nucleus version: %s", global.NucleusBinaryVersion)
 

pkg/blocktestservice/setup.go

@@ -163,11 +163,3 @@ func (tbs *TestBlockTestService) populateBlockList(blocktestSource string, block
 		}
 	}
 }
-
-func (tbs *TestBlockTestService) GetBlocklistYMLV1(tasConfig *core.TASConfig) []string {
-	return tasConfig.Blocklist
-}
-
-func (tbs *TestBlockTestService) GetBlocklistYMLV2(subModule *core.SubModule) []string {
-	return subModule.Blocklist
-}

pkg/command/run.go

@@ -1,7 +1,6 @@
 package command
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -35,6 +34,7 @@ func (m *manager) ExecuteUserCommands(ctx context.Context,
 	payload *core.Payload,
 	runConfig *core.Run,
 	secretData map[string]string,
+	logwriter core.LogWriterStrategy,
 	cwd string) error {
 	script, err := m.createScript(runConfig.Commands, secretData)
 	if err != nil {
@@ -44,12 +44,10 @@ func (m *manager) ExecuteUserCommands(ctx context.Context,
 	if err != nil {
 		return err
 	}
-
 	azureReader, azureWriter := io.Pipe()
 	defer azureWriter.Close()
 
-	blobPath := fmt.Sprintf("%s/%s/%s/%s.log", payload.OrgID, payload.BuildID, os.Getenv("TASK_ID"), commandType)
-	errChan := m.StoreCommandLogs(ctx, blobPath, azureReader)
+	errChan := logwriter.Write(ctx, azureReader)
 	defer m.closeAndWriteLog(azureWriter, errChan, commandType)
 	logWriter := lumber.NewWriter(m.logger)
 	defer logWriter.Close()
@@ -78,56 +76,6 @@ func (m *manager) ExecuteUserCommands(ctx context.Context,
 	return nil
 }
 
-// ExecuteUserCommandsV2 executes user commands for version 2
-func (m *manager) ExecuteUserCommandsV2(ctx context.Context,
-	commandType core.CommandType,
-	payload *core.Payload,
-	runConfig *core.Run,
-	secretData map[string]string,
-	cwd, subModule string,
-	buffer *bytes.Buffer) error {
-	script, err := m.createScript(runConfig.Commands, secretData)
-	if err != nil {
-		return err
-	}
-	envVars, err := m.GetEnvVariables(runConfig.EnvMap, secretData)
-	if err != nil {
-		return err
-	}
-
-	reader, writer := io.Pipe()
-
-	errChan := m.writeCommandLogsToBuffer(subModule, buffer, reader)
-	defer func() {
-		writer.Close()
-		if uploadErr := <-errChan; uploadErr != nil {
-			// not returning error here as upload logs should not fail the task
-			m.logger.Errorf("failed to upload logs for command %s, error: %v", commandType, uploadErr)
-		}
-	}()
-	logWriter := lumber.NewWriter(m.logger)
-	defer logWriter.Close()
-	multiWriter := io.MultiWriter(logWriter, writer)
-	maskWriter := logstream.NewMasker(multiWriter, secretData)
-
-	cmd := exec.CommandContext(ctx, "/bin/bash", "-c", script)
-	cmd.Dir = cwd
-	cmd.Env = envVars
-	cmd.Stdout = maskWriter
-	cmd.Stderr = maskWriter
-
-	if startErr := cmd.Start(); startErr != nil {
-		m.logger.Errorf("failed to start command: %s, error: %v", commandType, startErr)
-		return startErr
-	}
-	if execErr := cmd.Wait(); execErr != nil {
-		m.logger.Errorf("command %s, exited with error: %v", commandType, execErr)
-		return execErr
-	}
-
-	return nil
-}
-
 // ExecuteInternalCommands executes internal commands
 func (m *manager) ExecuteInternalCommands(ctx context.Context,
 	commandType core.CommandType,
@@ -164,46 +112,6 @@ func (m *manager) GetEnvVariables(envMap, secretData map[string]string) ([]strin
 	return envVars, nil
 }
 
-// StoreCommandLogs stores the command logs to blob
-func (m *manager) StoreCommandLogs(ctx context.Context, blobPath string, reader io.Reader) <-chan error {
-	errChan := make(chan error, 1)
-	go func() {
-		sasURL, err := m.azureClient.GetSASURL(ctx, blobPath, core.LogsContainer)
-		if err != nil {
-			m.logger.Errorf("failed to genereate SAS URL for path %s, error: %v", blobPath, err)
-			errChan <- err
-			return
-		}
-		blobPath, err := m.azureClient.CreateUsingSASURL(ctx, sasURL, reader, "text/plain")
-		if err != nil {
-			m.logger.Errorf("failed to create SAS URL for path %s, error: %v", blobPath, err)
-			errChan <- err
-			return
-		}
-		close(errChan)
-		m.logger.Debugf("created blob path %s", blobPath)
-	}()
-	return errChan
-}
-
-func (m *manager) writeCommandLogsToBuffer(submodule string, buffer *bytes.Buffer, reader io.Reader) <-chan error {
-	errChan := make(chan error, 1)
-	go func() {
-		if _, err := fmt.Fprintf(buffer, "<------ PRE RUN for submodule %s  ------> \n", submodule); err != nil {
-			m.logger.Debugf("Error writing the logs separator for submodule %s, error %v", submodule, err)
-			errChan <- err
-			return
-		}
-		if _, err := buffer.ReadFrom(reader); err != nil {
-			m.logger.Debugf("Error writing the logs to buffer for submodule %s, error %v", submodule, err)
-			errChan <- err
-			return
-		}
-		close(errChan)
-		m.logger.Debugf("written logs for sub module %s to buffer", submodule)
-	}()
-	return errChan
-}
 func (m *manager) closeAndWriteLog(azureWriter *io.PipeWriter, errChan <-chan error, commandType core.CommandType) {
 	azureWriter.Close()
 	if uploadErr := <-errChan; uploadErr != nil {

pkg/command/run_test.go

@@ -1,21 +1,17 @@
 package command
 
 import (
-	"context"
 	"fmt"
-	"io"
 	"os"
 	"reflect"
 	"sort"
 	"testing"
 
 	"github.com/LambdaTest/test-at-scale/pkg/core"
-	"github.com/LambdaTest/test-at-scale/pkg/errs"
 	"github.com/LambdaTest/test-at-scale/pkg/lumber"
 	"github.com/LambdaTest/test-at-scale/pkg/secret"
 	"github.com/LambdaTest/test-at-scale/testutils"
 	"github.com/LambdaTest/test-at-scale/testutils/mocks"
-	"github.com/stretchr/testify/mock"
 )
 
 func TestNewExecutionManager(t *testing.T) {
@@ -119,131 +115,3 @@ func Test_manager_GetEnvVariables(t *testing.T) {
 		})
 	}
 }
-
-func Test_manager_StoreCommandLogs(t *testing.T) {
-	logger, err := testutils.GetLogger()
-	if err != nil {
-		t.Errorf("Couldn't initialize logger, error: %v", err)
-	}
-	secretParser := new(mocks.SecretParser)
-	azureClientGetSASURL := new(mocks.AzureClient)
-	mockUtil(azureClientGetSASURL, "getSASURL", "createUsingSASURL", "error in GetSASURL", "error in CreateUsingSASURL", true, false)
-
-	azureClientCreateSASURL := new(mocks.AzureClient)
-	mockUtil(azureClientCreateSASURL, "getSASURL", "createUsingSASURL", "error in GetSASURL", "error in CreateUsingSASURL", false, true)
-
-	azureClientSuccess := new(mocks.AzureClient)
-	mockUtil(azureClientSuccess, "getSASURL", "createUsingSASURL", "error in GetSASURL", "error in CreateUsingSASURL", false, false)
-
-	errGetSASURL := make(chan error, 1)
-	defer func() { close(errGetSASURL) }()
-
-	errCreateUsingSASURL := make(chan error, 1)
-	defer func() { close(errCreateUsingSASURL) }()
-
-	errSuccess := make(chan error, 1)
-	defer func() { close(errSuccess) }()
-
-	type fields struct {
-		azureClient core.AzureClient
-	}
-	type args struct {
-		ctx      context.Context
-		blobPath string
-		reader   io.Reader
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		args    args
-		want    <-chan error
-		wantErr bool
-	}{
-		{"Test StoreCommandLogs for getSASURL error",
-			fields{
-				azureClient: azureClientGetSASURL,
-			},
-			args{
-				ctx:      context.TODO(),
-				blobPath: "blobpath",
-				reader:   &mocks.Reader{},
-			},
-			errGetSASURL,
-			true,
-		},
-		{"Test StoreCommandLogs for CreateUsingSASURL error",
-			fields{
-				azureClient: azureClientCreateSASURL,
-			},
-			args{
-				ctx:      context.TODO(),
-				blobPath: "blobpath",
-				reader:   &mocks.Reader{},
-			},
-			errCreateUsingSASURL,
-			true,
-		},
-		{"Test StoreCommandLogs for success",
-			fields{
-				azureClient: azureClientSuccess,
-			},
-			args{
-				ctx:      context.TODO(),
-				blobPath: "blobpath",
-				reader:   &mocks.Reader{},
-			},
-			errSuccess,
-			false,
-		},
-	}
-	errGetSASURL <- errs.New("error in GetSASURL")
-	errCreateUsingSASURL <- errs.New("error in CreateUsingSASURL")
-	errSuccess <- errs.New("")
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			m := &manager{
-				logger:       logger,
-				secretParser: secretParser,
-				azureClient:  tt.fields.azureClient,
-			}
-			got := m.StoreCommandLogs(tt.args.ctx, tt.args.blobPath, tt.args.reader)
-			if !tt.wantErr {
-				if len(got) != 0 {
-					t.Errorf("Expected channel to be empty, received: %v", <-got)
-				}
-				return
-			}
-			received := <-got
-			want := <-tt.want
-			if received.Error() != want.Error() {
-				t.Errorf("manager.StoreCommandLogs() = %+v, want %+v", received, want)
-			}
-		})
-	}
-}
-
-func mockUtil(azureClient *mocks.AzureClient, msgGet, msgCreate, errGet, errCreate string, wantErrGet, wantErrCreate bool) {
-	azureClient.On("GetSASURL", mock.AnythingOfType("*context.emptyCtx"),
-		mock.AnythingOfType("string"), mock.AnythingOfType("core.ContainerType")).Return(
-		func(ctx context.Context, containerPath string, containerType core.ContainerType) string {
-			return msgGet
-		},
-		func(ctx context.Context, containerPath string, containerType core.ContainerType) error {
-			if !wantErrGet {
-				return nil
-			}
-			return errs.New(errGet)
-		})
-
-	azureClient.On("CreateUsingSASURL", mock.AnythingOfType("*context.emptyCtx"),
-		mock.AnythingOfType("string"), mock.AnythingOfType("*mocks.Reader"), "text/plain").Return(
-		func(ctx context.Context, sasURL string, reader io.Reader, mimeType string) string {
-			return msgCreate
-		},
-		func(ctx context.Context, sasURL string, reader io.Reader, mimeType string) error {
-			if !wantErrCreate {
-				return nil
-			}
-			return errs.New(errCreate)
-		})
-}