fix: support both sarif and json file output for native code implementation (#375)

Co-authored-by: Peter Schäfer <[email protected]>

Author: CatalinSnyk

pkg/local_workflows/output_workflow.go

@@ -31,9 +31,6 @@ func InitOutputWorkflow(engine workflow.Engine) error {
 	outputConfig.Bool(configuration.FLAG_INCLUDE_IGNORES, false, "Include ignored findings in the output")
 	outputConfig.String(configuration.FLAG_SEVERITY_THRESHOLD, "low", "Severity threshold for findings to be included in the output")
 
-	// set default values for configuration values
-	engine.GetConfiguration().AddDefaultValue(output_workflow.OUTPUT_CONFIG_WRITE_EMPTY_FILE, configuration.StandardDefaultValueFunction(true))
-
 	entry, err := engine.Register(WORKFLOWID_OUTPUT_WORKFLOW, workflow.ConfigurationOptionsFromFlagset(outputConfig), outputWorkflowEntryPointImpl)
 	entry.SetVisibility(false)
 

pkg/local_workflows/output_workflow/constants.go

@@ -1,11 +1,19 @@
 package output_workflow
 
+import "github.com/snyk/go-application-framework/internal/presenters"
+
 const (
 	OUTPUT_CONFIG_KEY_JSON         = "json"
 	OUTPUT_CONFIG_KEY_JSON_FILE    = "json-file-output"
 	OUTPUT_CONFIG_KEY_SARIF        = "sarif"
 	OUTPUT_CONFIG_KEY_SARIF_FILE   = "sarif-file-output"
 	OUTPUT_CONFIG_TEMPLATE_FILE    = "internal_template_file"
-	OUTPUT_CONFIG_WRITE_EMPTY_FILE = "internal_write_empty_file"
+	OUTPUT_CONFIG_KEY_FILE_WRITERS = "internal_output_file_writers"
 	DEFAULT_WRITER                 = "default"
+	DEFAULT_MIME_TYPE              = presenters.DefaultMimeType
+	SARIF_MIME_TYPE                = presenters.ApplicationSarifMimeType
 )
+
+// DefaultTemplateFiles is an instance of TemplatePathsStruct with the template paths.
+var DefaultTemplateFiles = presenters.DefaultTemplateFiles
+var ApplicationSarifTemplates = presenters.ApplicationSarifTemplates

pkg/local_workflows/output_workflow/delayedfileopenwritecloser.go

@@ -0,0 +1,39 @@
+package output_workflow
+
+import (
+	"io"
+	"os"
+
+	iUtils "github.com/snyk/go-application-framework/internal/utils"
+)
+
+type delayedFileOpenWriteCloser struct {
+	Filename string
+	file     io.WriteCloser
+}
+
+func (wc *delayedFileOpenWriteCloser) Write(p []byte) (n int, err error) {
+	// lazy open file if not exists
+	if wc.file == nil {
+		pathError := iUtils.CreateFilePath(wc.Filename)
+		if pathError != nil {
+			return 0, pathError
+		}
+
+		file, fileErr := os.OpenFile(wc.Filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, iUtils.FILEPERM_666)
+		if fileErr != nil {
+			return 0, fileErr
+		}
+
+		wc.file = file
+	}
+
+	return wc.file.Write(p)
+}
+
+func (wc *delayedFileOpenWriteCloser) Close() error {
+	if wc.file != nil {
+		return wc.file.Close()
+	}
+	return nil
+}

pkg/local_workflows/output_workflow/findings_model_tools.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"strings"
 
 	"github.com/rs/zerolog"
@@ -19,11 +18,20 @@ import (
 	"github.com/snyk/go-application-framework/pkg/workflow"
 )
 
+// WriterEntry is an internal structure to handle all template based writers
 type WriterEntry struct {
-	writer    io.Writer
-	mimeType  string
-	templates []string
-	closer    func() error
+	writer          io.WriteCloser
+	mimeType        string
+	templates       []string
+	renderEmptyData bool
+}
+
+// FileWriter is a public structure used to configure file based rendering
+type FileWriter struct {
+	NameConfigKey     string   // defines the configuration key to look up the filename under
+	MimeType          string   // defines the final mime type of the rendering
+	TemplateFiles     []string // defines the set of template files to use for rendering
+	WriteEmptyContent bool     // specifies if anything should be written at all if the input data is empty
 }
 
 func getListOfFindings(input []workflow.Data, debugLogger *zerolog.Logger) (findings []*local_models.LocalFinding, remainingData []workflow.Data) {
@@ -65,83 +73,89 @@ func getTotalNumberOfFindings(findings []*local_models.LocalFinding) uint32 {
 	return count
 }
 
-func getWritersToUse(config configuration.Configuration, outputDestination iUtils.OutputDestination, findings []*local_models.LocalFinding) (map[string]*WriterEntry, error) {
+func getWritersToUse(config configuration.Configuration, outputDestination iUtils.OutputDestination) map[string]*WriterEntry {
+	// resulting map of writers and their templates
 	writerMap := map[string]*WriterEntry{
-		DEFAULT_WRITER: {
-			writer:    outputDestination.GetWriter(),
-			mimeType:  presenters.DefaultMimeType,
-			templates: presenters.DefaultTemplateFiles,
-			closer: func() error {
-				_, err := fmt.Fprintln(outputDestination.GetWriter(), "")
-				return err
-			},
-		},
+		DEFAULT_WRITER: getDefaultWriter(config, outputDestination),
 	}
 
-	sarifWriter, err := getSarifFileRenderer(config, findings)
-	if err != nil {
-		return writerMap, err
+	// default file writers
+	fileWriters := []FileWriter{
+		{
+			OUTPUT_CONFIG_KEY_SARIF_FILE,
+			SARIF_MIME_TYPE,
+			ApplicationSarifTemplates,
+			true,
+		},
+		/*
+			skipping support for json file output by default, since there is no supporting rendering yet.
+			{
+				OUTPUT_CONFIG_KEY_JSON_FILE,
+				SARIF_MIME_TYPE,
+				ApplicationSarifTemplates,
+				true,
+			},*/
+	}
+
+	// use configured file writers if available
+	if tmp, ok := config.Get(OUTPUT_CONFIG_KEY_FILE_WRITERS).([]FileWriter); ok {
+		fileWriters = tmp
+	}
+
+	for _, fileWriter := range fileWriters {
+		if config.IsSet(fileWriter.NameConfigKey) {
+			writerMap[fileWriter.NameConfigKey] = &WriterEntry{
+				writer:          &delayedFileOpenWriteCloser{Filename: config.GetString(fileWriter.NameConfigKey)},
+				mimeType:        fileWriter.MimeType,
+				templates:       fileWriter.TemplateFiles,
+				renderEmptyData: fileWriter.WriteEmptyContent,
+			}
+		}
 	}
 
-	if sarifWriter != nil {
-		writerMap[OUTPUT_CONFIG_KEY_SARIF_FILE] = sarifWriter
+	return writerMap
+}
+
+func getDefaultWriter(config configuration.Configuration, outputDestination iUtils.OutputDestination) *WriterEntry {
+	writer := &WriterEntry{
+		writer: &newLineCloser{
+			writer: outputDestination.GetWriter(),
+		},
+		mimeType:        DEFAULT_MIME_TYPE,
+		templates:       DefaultTemplateFiles,
+		renderEmptyData: true,
 	}
 
 	if config.GetBool(OUTPUT_CONFIG_KEY_SARIF) {
-		writerMap[DEFAULT_WRITER].mimeType = presenters.ApplicationSarifMimeType
-		writerMap[DEFAULT_WRITER].templates = presenters.ApplicationSarifTemplates
+		writer.mimeType = SARIF_MIME_TYPE
+		writer.templates = ApplicationSarifTemplates
 	}
 
 	if config.IsSet(OUTPUT_CONFIG_TEMPLATE_FILE) {
-		writerMap[DEFAULT_WRITER].templates = []string{config.GetString(OUTPUT_CONFIG_TEMPLATE_FILE)}
+		writer.templates = []string{config.GetString(OUTPUT_CONFIG_TEMPLATE_FILE)}
 	}
 
-	return writerMap, nil
+	return writer
 }
 
-func getSarifFileRenderer(config configuration.Configuration, findings []*local_models.LocalFinding) (*WriterEntry, error) {
-	outputFileName := config.GetString(OUTPUT_CONFIG_KEY_SARIF_FILE)
-	if len(outputFileName) == 0 {
-		//nolint:nilnil // returning a nil writer is a valid case based on the configuration and is not an error case
-		return nil, nil
-	}
-
-	if !config.GetBool(OUTPUT_CONFIG_WRITE_EMPTY_FILE) && getTotalNumberOfFindings(findings) == 0 {
-		//nolint:nilnil // returning a nil writer is a valid case based on the configuration and is not an error case
-		return nil, nil
-	}
-
-	pathError := iUtils.CreateFilePath(outputFileName)
-	if pathError != nil {
-		return nil, pathError
-	}
-
-	file, fileErr := os.OpenFile(outputFileName, os.O_WRONLY|os.O_CREATE, 0644)
-	if fileErr != nil {
-		return nil, fileErr
-	}
+func useRendererWith(name string, wEntry *WriterEntry, findings []*local_models.LocalFinding, invocation workflow.InvocationContext) {
+	debugLogger := invocation.GetEnhancedLogger()
 
-	writer := &WriterEntry{
-		writer:    file,
-		mimeType:  presenters.ApplicationSarifMimeType,
-		templates: presenters.ApplicationSarifTemplates,
-		closer:    func() error { return file.Close() },
+	if !wEntry.renderEmptyData && getTotalNumberOfFindings(findings) == 0 {
+		debugLogger.Info().Msgf("[%s] The input is empty, skipping rendering!", name)
+		return
 	}
-	return writer, nil
-}
 
-func useRendererWith(name string, wEntry *WriterEntry, debugLogger *zerolog.Logger, findings []*local_models.LocalFinding, config configuration.Configuration, invocation workflow.InvocationContext) {
 	debugLogger.Info().Msgf("[%s] Creating findings model renderer", name)
 
-	if wEntry.closer != nil {
-		defer func() {
-			closeErr := wEntry.closer()
-			if closeErr != nil {
-				debugLogger.Err(closeErr).Msgf("[%s] Error while closing writer.", name)
-			}
-		}()
-	}
+	defer func() {
+		closeErr := wEntry.writer.Close()
+		if closeErr != nil {
+			debugLogger.Err(closeErr).Msgf("[%s] Error while closing writer.", name)
+		}
+	}()
 
+	config := invocation.GetConfiguration()
 	renderer := presenters.NewLocalFindingsRenderer(
 		findings,
 		config,
@@ -160,6 +174,7 @@ func useRendererWith(name string, wEntry *WriterEntry, debugLogger *zerolog.Logg
 }
 
 func HandleContentTypeFindingsModel(input []workflow.Data, invocation workflow.InvocationContext, outputDestination iUtils.OutputDestination) ([]workflow.Data, error) {
+	var err error
 	debugLogger := invocation.GetEnhancedLogger()
 	config := invocation.GetConfiguration()
 
@@ -172,24 +187,22 @@ func HandleContentTypeFindingsModel(input []workflow.Data, invocation workflow.I
 	threadCount := max(int64(config.GetInt(configuration.MAX_THREADS)), 1)
 	debugLogger.Info().Msgf("Thread count: %d", threadCount)
 
-	writerMap, err := getWritersToUse(config, outputDestination, findings)
-	if err != nil {
-		debugLogger.Err(err).Msg("Failed to initialize all required writers")
-	}
+	writerMap := getWritersToUse(config, outputDestination)
 
 	// iterate over all writers and render for each of them
 	ctx := context.Background()
 	availableThreads := semaphore.NewWeighted(threadCount)
 	for k, v := range writerMap {
 		err = availableThreads.Acquire(ctx, 1)
 		if err != nil {
-			debugLogger.Err(err).Msgf("[%s] Failed to acquire threading lock.", k)
+			debugLogger.Err(err).Msgf("[%s] Failed to acquire threading lock. Cancel rendering.", k)
+			break
 		}
 
-		go func() {
+		go func(name string, writer *WriterEntry) {
 			defer availableThreads.Release(1)
-			useRendererWith(k, v, debugLogger, findings, config, invocation)
-		}()
+			useRendererWith(name, writer, findings, invocation)
+		}(k, v)
 	}
 
 	err = availableThreads.Acquire(ctx, threadCount)