Phx 3545 - Nucleus changes for go, java runner. (#201)

* golang runner integration

* java integration

* dockerfile changes

* refactor constants

* java api test execution

* java api test execution

* fixes and refactoring

* framework version in constants

* java 18

* fix comments

* fix conflicts

* fix conflicts

* updated docker file

* java jar name update

* fix cyclomatic complexity

* revert

* fix git checks

* dockerfile changes

* fix blocklisting

* resolve conflicts

* fix dockerfile

* refactor arguements

* fix dockerfile

* maven local repo path update

* refactor dockerfile

Co-authored-by: Shirish <[email protected]>
Co-authored-by: Sachin Pandey <[email protected]>
Co-authored-by: nevilm-lt <[email protected]>

Author: 4 people

build/nucleus/Dockerfile

@@ -18,7 +18,9 @@ ARG VERSION
 ENV VERSION=$VERSION
 
 # Installing chromium so that all linux libs get automatically installed for running puppeteer tests
-RUN apt update && apt install -y git zstd chromium curl
+RUN apt update && apt install -y git zstd chromium curl unzip zip xmlstarlet build-essential
+RUN curl -LJO https://go.dev/dl/go1.18.3.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzf go1.18.3.linux-amd64.tar.gz
 
 COPY bundle /usr/local/bin/bundle
 RUN chmod +x /usr/local/bin/bundle
@@ -39,11 +41,18 @@ RUN npm i -g nyc@^15.1.0
 
 RUN tar -zcf /custom-runners/custom-runners.tgz node_modules
 RUN rm -rf /tmp/custom-runners
-
 RUN mkdir /home/nucleus
 RUN mkdir /home/nucleus/.nvm
 ENV NVM_DIR=/home/nucleus/.nvm
 
+ENV GOROOT /usr/local/go
+ENV GOPATH /home/nucleus
+ENV PATH /usr/local/go/bin:/home/nucleus/bin:$PATH
+
+COPY ./build/nucleus/golang/server /home/nucleus
+
+RUN chmod 744 /home/nucleus/server
+
 # install nvm for nucleus user
 RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | /bin/bash
 
@@ -52,4 +61,43 @@ WORKDIR /home/nucleus
 COPY --from=builder /nucleus/nucleus /usr/local/bin/
 # run the binary
 COPY ./build/nucleus/entrypoint.sh /
-ENTRYPOINT  ["/bin/sh", "/entrypoint.sh"]
+
+RUN apt update -y && apt upgrade -y
+
+RUN  curl -s https://get.sdkman.io | bash
+RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh;sdk install java 18.0.1-oracle"
+
+ENV JAVA_HOME="/root/.sdkman/candidates/java/current"
+ENV PATH=$JAVA_HOME:$PATH
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+ARG MAVEN_VERSION=3.6.3
+
+# Define a constant with the working directory
+ARG USER_HOME_DIR="/root"
+# Define the URL where maven can be downloaded from
+ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+
+# Create the directories, download maven, validate the download, install it, remove downloaded file and set links
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+  && echo "Downlaoding maven" \
+  && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
+  \
+  && echo "Unziping maven" \
+  && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
+  \
+  && echo "Cleaning and setting links" \
+  && rm -f /tmp/apache-maven.tar.gz \
+  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+
+# Define environmental variables required by Maven, like Maven_Home directory and where the maven repo is located
+ENV MAVEN_HOME /usr/share/maven
+RUN mkdir -p /home/nucleus/.m2
+
+#update settings.xml file for new maven local repo location
+RUN xmlstarlet ed -O --inplace -N a='http://maven.apache.org/SETTINGS/1.0.0' -s /a:settings --type elem --name "localRepository" -v /home/nucleus/.m2/repository /usr/share/maven/conf/settings.xml
+
+COPY ./build/nucleus/java/test-at-scale-java.jar /
+RUN curl -o /home/nucleus/junit-platform-console-standalone-1.8.2.jar https://repo1.maven.org/maven2/org/junit/platform/junit-platform-console-standalone/1.8.2/junit-platform-console-standalone-1.8.2.jar
+COPY ./build/nucleus/entrypoint.sh /
+ENTRYPOINT  ["/bin/sh", "/entrypoint.sh"]

build/nucleus/golang/server

@@ -23,6 +23,7 @@ import (
 const (
 	endpointPostTestResults = "http://localhost:9876/results"
 	endpointPostTestList    = "http://localhost:9876/test-list"
+	languageJs              = "javascript"
 )
 
 // NewPipeline creates and returns a new Pipeline instance
@@ -56,7 +57,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 	pl.Logger.Debugf("Payload for current task: %+v \n", *payload)
 
 	if pl.Cfg.CoverageMode {
-		if err := pl.CoverageService.MergeAndUpload(ctx, payload); err != nil {
+		if err = pl.CoverageService.MergeAndUpload(ctx, payload); err != nil {
 			pl.Logger.Fatalf("error while merge and upload coverage files %v", err)
 		}
 		os.Exit(0)
@@ -211,11 +212,19 @@ func (pl *Pipeline) runOldVersion(ctx context.Context,
 		return err
 	}
 
-	cacheKey := fmt.Sprintf("%s/%s/%s/%s", tasConfig.Cache.Version, payload.OrgID, payload.RepoID, tasConfig.Cache.Key)
+	cacheKey := ""
+
+	language := global.FrameworkLanguageMap[tasConfig.Framework]
+
+	if language == languageJs {
+		cacheKey = fmt.Sprintf("%s/%s/%s/%s", tasConfig.Cache.Version, payload.OrgID, payload.RepoID, tasConfig.Cache.Key)
+	}
+
 	pl.Logger.Infof("Tas yaml: %+v", tasConfig)
+
 	os.Setenv("REPO_CACHE_DIR", global.RepoCacheDir)
 
-	if tasConfig.NodeVersion != "" {
+	if tasConfig.NodeVersion != "" && language == languageJs {
 		nodeVersion := tasConfig.NodeVersion
 		if nodeErr := pl.installNodeVersion(ctx, nodeVersion); nodeErr != nil {
 			return nodeErr
@@ -250,6 +259,13 @@ func (pl *Pipeline) runTestExecutionV1(ctx context.Context,
 	coverageDir string,
 	secretMap map[string]string,
 	taskPayload *TaskPayload, payload *Payload) error {
+	blYml := pl.BlockTestService.GetBlocklistYMLV1(tasConfig)
+	if errG := pl.BlockTestService.GetBlockTests(ctx, blYml, payload.BranchName); errG != nil {
+		pl.Logger.Errorf("Unable to fetch blocklisted tests: %v", errG)
+		errG = errs.New(errs.GenericErrRemark.Error())
+		return errG
+	}
+
 	executionResults, err := pl.TestExecutionService.RunV1(ctx, tasConfig, pl.Payload, coverageDir, secretMap)
 	if err != nil {
 		pl.Logger.Infof("Unable to perform test execution: %v", err)
@@ -291,20 +307,31 @@ func (pl *Pipeline) runDiscoveryV1(ctx context.Context,
 	// discover test cases
 	// mark status as passed
 	// Upload cache once for other builds
+	language := global.FrameworkLanguageMap[tasConfig.Framework]
+
 	if postErr := pl.TestDiscoveryService.UpdateSubmoduleList(ctx, payload.BuildID, 1); postErr != nil {
 		return postErr
 	}
 
 	g, errCtx := errgroup.WithContext(ctx)
 
-	g.Go(func() error {
-		if errG := pl.CacheStore.Download(errCtx, cacheKey); errG != nil {
-			pl.Logger.Errorf("Unable to download cache: %v", errG)
-			errG = errs.New(errs.GenericErrRemark.Error())
-			return errG
+	if language == languageJs {
+		g.Go(func() error {
+			if errG := pl.CacheStore.Download(errCtx, cacheKey); errG != nil {
+				pl.Logger.Errorf("Unable to download cache: %v", errG)
+				errG = errs.New(errs.GenericErrRemark.Error())
+				return errG
+			}
+			return nil
+		})
+
+		err := pl.ExecutionManager.ExecuteInternalCommands(ctx, InstallRunners, global.InstallRunnerCmds, global.RepoDir, nil, nil)
+		if err != nil {
+			pl.Logger.Errorf("Unable to install custom runners %v", err)
+			err = errs.New(errs.GenericErrRemark.Error())
+			return err
 		}
-		return nil
-	})
+	}
 
 	pl.Logger.Infof("Identifying changed files ...")
 	diffExists := true
@@ -338,12 +365,6 @@ func (pl *Pipeline) runDiscoveryV1(ctx context.Context,
 			return err
 		}
 	}
-	err = pl.ExecutionManager.ExecuteInternalCommands(ctx, InstallRunners, global.InstallRunnerCmds, global.RepoDir, nil, nil)
-	if err != nil {
-		pl.Logger.Errorf("Unable to install custom runners %v", err)
-		err = errs.New(errs.GenericErrRemark.Error())
-		return err
-	}
 
 	pl.Logger.Debugf("Caching workspace")
 
@@ -359,10 +380,12 @@ func (pl *Pipeline) runDiscoveryV1(ctx context.Context,
 		err = &errs.StatusFailed{Remark: "Failed in discovering tests"}
 		return err
 	}
-	if err = pl.CacheStore.Upload(ctx, cacheKey, tasConfig.Cache.Paths...); err != nil {
-		pl.Logger.Errorf("Unable to upload cache: %v", err)
-		err = errs.New(errs.GenericErrRemark.Error())
-		return err
+	if language == languageJs {
+		if err = pl.CacheStore.Upload(ctx, cacheKey, tasConfig.Cache.Paths...); err != nil {
+			pl.Logger.Errorf("Unable to upload cache: %v", err)
+			err = errs.New(errs.GenericErrRemark.Error())
+			return err
+		}
 	}
 	taskPayload.Status = Passed
 	pl.Logger.Debugf("Cache uploaded successfully")

build/nucleus/java/test-at-scale-java.jar

@@ -302,7 +302,7 @@ type Oauth struct {
 // TASConfig represents the .tas.yml file
 type TASConfig struct {
 	SmartRun          bool               `yaml:"smartRun"`
-	Framework         string             `yaml:"framework" validate:"required,oneof=jest mocha jasmine"`
+	Framework         string             `yaml:"framework" validate:"required,oneof=jest mocha jasmine golang junit"`
 	Blocklist         []string           `yaml:"blocklist"`
 	Postmerge         *Merge             `yaml:"postMerge" validate:"omitempty"`
 	Premerge          *Merge             `yaml:"preMerge" validate:"omitempty"`
@@ -317,6 +317,7 @@ type TASConfig struct {
 	Tier              Tier               `yaml:"tier" validate:"oneof=xsmall small medium large xlarge"`
 	NodeVersion       string             `yaml:"nodeVersion" validate:"omitempty,semver"`
 	ContainerImage    string             `yaml:"containerImage"`
+	FrameworkVersion  int                `yaml:"frameworkVersion" validate:"omitempty"`
 	Version           string             `yaml:"version" validate:"required"`
 }
 

pkg/core/lifecycle.go

@@ -31,6 +31,12 @@ const (
 	ModulePath               = "MODULE_PATH"
 	CacheVersion             = "v1"
 	PackageJSON              = "package.json"
+	ArgPattern               = "--pattern"
+	ArgConfig                = "--config"
+	ArgDiff                  = "--diff"
+	ArgCommand               = "--command"
+	ArgLocator               = "--locator-file"
+	ArgFrameworVersion       = "--frameworkVersion"
 	DefaultTASVersion        = "1.0.0"
 )
 
@@ -39,6 +45,8 @@ var FrameworkRunnerMap = map[string]string{
 	"jasmine": "./node_modules/.bin/jasmine-runner",
 	"mocha":   "./node_modules/.bin/mocha-runner",
 	"jest":    "./node_modules/.bin/jest-runner",
+	"golang":  "/home/nucleus/server",
+	"junit":   "java",
 }
 
 // APIHostURLMap is map of git provider with there api url
@@ -58,3 +66,11 @@ var NeuronHost string
 func SetNeuronHost(host string) {
 	NeuronHost = host
 }
+
+var FrameworkLanguageMap = map[string]string{
+	"jasmine": "javascript",
+	"mocha":   "javascript",
+	"jest":    "javascript",
+	"golang":  "golang",
+	"junit":   "java",
+}

pkg/core/models.go

@@ -69,7 +69,7 @@ func New(execManager core.ExecutionManager,
 
 }
 
-//mergeCodeCoverageFiles merge all the coverage.json into single entity
+// mergeCodeCoverageFiles merge all the coverage.json into single entity
 func (c *codeCoverageService) mergeCodeCoverageFiles(ctx context.Context, commitDir, coverageManifestPath string, threshold bool) error {
 	if _, err := os.Lstat(commitDir); os.IsNotExist(err) {
 		c.logger.Errorf("coverage files not found, skipping merge")
@@ -78,7 +78,7 @@ func (c *codeCoverageService) mergeCodeCoverageFiles(ctx context.Context, commit
 
 	coverageFiles := make([]string, 0)
 	if err := filepath.WalkDir(commitDir, func(path string, d fs.DirEntry, err error) error {
-		//add all individual coverage json files
+		// add all individual coverage json files
 		if d.Name() == coverageJSONFileName {
 			coverageFiles = append(coverageFiles, path)
 		}

pkg/global/nucleusconstants.go

@@ -81,7 +81,8 @@ func (tc *tasConfigManager) validateYMLV1(ctx context.Context,
 		return nil, err
 	}
 
-	if tasConfig.Cache == nil {
+	language := global.FrameworkLanguageMap[tasConfig.Framework]
+	if tasConfig.Cache == nil && language == "javascript" {
 		checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
 		if err != nil {
 			tc.logger.Errorf("Error while computing checksum, error %v", err)

pkg/service/coverage/coverage.go

@@ -55,27 +55,22 @@ func (tds *testDiscoveryService) Discover(ctx context.Context,
 		return err
 	}
 	impactAll := tds.shouldImpactAll(tasConfig, configFilePath, diff)
-	args := []string{"--command", "discover"}
+
+	args := utils.GetArgs("discover", tasConfig, target)
+
 	if !impactAll {
 		if len(diff) == 0 && diffExists {
 			// empty diff; in PR, a commit added and then reverted to cause an overall empty PR diff
-			args = append(args, "--diff")
+			args = append(args, global.ArgDiff)
 		} else {
 			for k, v := range diff {
 				// in changed files we only have added or modified files.
 				if v != core.FileRemoved {
-					args = append(args, "--diff", k)
+					args = append(args, global.ArgDiff, k)
 				}
 			}
 		}
 	}
-	if tasConfig.ConfigFile != "" {
-		args = append(args, "--config", tasConfig.ConfigFile)
-	}
-
-	for _, pattern := range target {
-		args = append(args, "--pattern", pattern)
-	}
 	tds.logger.Debugf("Discovering tests at paths %+v", target)
 
 	cmd := exec.CommandContext(ctx, global.FrameworkRunnerMap[tasConfig.Framework], args...) //nolint:gosec
@@ -177,26 +172,26 @@ func (tds *testDiscoveryService) DiscoverV2(ctx context.Context,
 	// discover all tests if tas.yml modified or smart run feature is set to false
 	discoverAll := tasYmlModified || !tasConfig.SmartRun
 
-	args := []string{"--command", "discover"}
+	args := []string{global.ArgCommand, "discover"}
 	if !discoverAll {
 		if len(diff) == 0 && diffExists {
 			// empty diff; in PR, a commit added and then reverted to cause an overall empty PR diff
-			args = append(args, "--diff")
+			args = append(args, global.ArgDiff)
 		} else {
 			for k, v := range diff {
 				// in changed files we only have added or modified files.
 				if v != core.FileRemoved {
-					args = append(args, "--diff", k)
+					args = append(args, global.ArgDiff, k)
 				}
 			}
 		}
 	}
 	if subModule.ConfigFile != "" {
-		args = append(args, "--config", subModule.ConfigFile)
+		args = append(args, global.ArgConfig, subModule.ConfigFile)
 	}
 
 	for _, pattern := range target {
-		args = append(args, "--pattern", pattern)
+		args = append(args, global.ArgPattern, pattern)
 	}
 	tds.logger.Debugf("Discovering tests at paths %+v", target)
 

pkg/tasconfigmanager/setup.go

@@ -308,13 +308,9 @@ func (tes *testExecutionService) buildCmdArgsV1(ctx context.Context,
 	tasConfig *core.TASConfig,
 	payload *core.Payload,
 	target []string) ([]string, error) {
-	args := []string{global.FrameworkRunnerMap[tasConfig.Framework], "--command", "execute"}
-	if tasConfig.ConfigFile != "" {
-		args = append(args, "--config", tasConfig.ConfigFile)
-	}
-	for _, pattern := range target {
-		args = append(args, "--pattern", pattern)
-	}
+	args := []string{global.FrameworkRunnerMap[tasConfig.Framework]}
+
+	args = append(args, utils.GetArgs("execute", tasConfig, target)...)
 
 	if payload.LocatorAddress != "" {
 		locatorFile, err := tes.getLocatorsFile(ctx, payload.LocatorAddress)
@@ -323,21 +319,23 @@ func (tes *testExecutionService) buildCmdArgsV1(ctx context.Context,
 			tes.logger.Errorf("failed to get locator file, error: %v", err)
 			return nil, err
 		}
-		args = append(args, "--locator-file", locatorFile)
+
+		args = append(args, global.ArgLocator, locatorFile)
 	}
+
 	return args, nil
 }
 
 func (tes *testExecutionService) buildCmdArgsV2(ctx context.Context,
 	subModule *core.SubModule,
 	payload *core.Payload,
 	target []string) ([]string, error) {
-	args := []string{global.FrameworkRunnerMap[subModule.Framework], "--command", "execute"}
+	args := []string{global.FrameworkRunnerMap[subModule.Framework], global.ArgCommand, "execute"}
 	if subModule.ConfigFile != "" {
-		args = append(args, "--config", subModule.ConfigFile)
+		args = append(args, global.ArgConfig, subModule.ConfigFile)
 	}
 	for _, pattern := range target {
-		args = append(args, "--pattern", pattern)
+		args = append(args, global.ArgPattern, pattern)
 	}
 
 	if payload.LocatorAddress != "" {
@@ -347,7 +345,7 @@ func (tes *testExecutionService) buildCmdArgsV2(ctx context.Context,
 			tes.logger.Errorf("failed to get locator file, error: %v", err)
 			return nil, err
 		}
-		args = append(args, "--locator-file", locatorFile)
+		args = append(args, global.ArgLocator, locatorFile)
 	}
 	return args, nil
 }