implemented multipart upload

Author: cx-sumit-morchhale

internal/commands/scan.go

@@ -129,6 +129,8 @@ const (
 	sbomScanTypeErrMsg       = "The --sbom-only flag can only be used when the scan type is sca"
 	BranchPrimaryPrefix      = "--branch-primary="
 	OverridePolicyManagement = "override-policy-management"
+	maxSizeGB                = 5                              // 5 GB
+	maxSizeBytes             = maxSizeGB * 1024 * 1024 * 1024 // 5 GB in bytes
 )
 
 var (
@@ -2082,7 +2084,26 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip
 	var zipFilePathErr error
 	// Send a request to uploads service
 	var preSignedURL *string
-	preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper)
+
+	// calculate file size and compare with 5GB limit
+	fileInfo, err := os.Stat(zipFilePath)
+	if err != nil {
+		return "", zipFilePath, errors.Wrapf(err, "Failed to stat %s", zipFilePath)
+	}
+	logger.PrintIfVerbose(fmt.Sprintf("Zip size before upload:  %.2fMB\n", float64(fileInfo.Size())/mbBytes))
+
+	// check for INCREASE_FILE_UPLOAD_LIMIT feature flag
+	flagResponse, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.IncreaseFileUploadLimit)
+
+	if flagResponse.Status && fileInfo.Size() > maxSizeBytes {
+		// File size >5GB, proceed with multipart upload
+		logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...")
+		preSignedURL, zipFilePathErr = uploadsWrapper.UploadFileInMultipart(zipFilePath, featureFlagsWrapper)
+	} else {
+		// File size is within <=5GB, proceed with upload in single part
+		logger.PrintIfVerbose("File size is within the limit and uploading file in a single part...")
+		preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper)
+	}
 	if zipFilePathErr != nil {
 		if unzip || !userProvidedZip {
 			return "", zipFilePath, errors.Wrapf(zipFilePathErr, "%s: Failed to upload sources file\n", failedCreating)

internal/params/binds.go

@@ -80,4 +80,8 @@ var EnvVarsBinds = []struct {
 	{RiskManagementPathKey, RiskManagementPathEnv, "api/risk-management/projects/%s/results?scanID=%s"},
 	{ConfigFilePathKey, ConfigFilePathEnv, ""},
 	{RealtimeScannerPathKey, RealtimeScannerPathEnv, "api/realtime-scanner"},
+	{StartMultiPartUploadPathKey, StartMultiPartUploadPathEnv, "api/uploads/start-multipart-upload"},
+	{MultipartPresignedPathKey, MultipartPresignedPathEnv, "api/uploads/multipart-presigned"},
+	{CompleteMultiPartUploadPathKey, CompleteMultipartUploadPathEnv, "api/uploads/complete-multipart-upload"},
+	{MultipartFileSize, MultipartFileSizeEnv, "2"},
 }

internal/params/envs.go

@@ -80,4 +80,8 @@ const (
 	RiskManagementPathEnv               = "CX_RISK_MANAGEMENT_PATH"
 	ConfigFilePathEnv                   = "CX_CONFIG_FILE_PATH"
 	RealtimeScannerPathEnv              = "CX_REALTIME_SCANNER_PATH"
+	StartMultiPartUploadPathEnv         = "CX_START_MULTIPART_UPLOAD_PATH"
+	MultipartPresignedPathEnv           = "CX_MULTIPART_PRESIGNED_URL_PATH"
+	CompleteMultipartUploadPathEnv      = "CX_COMPLETE_MULTIPART_UPLOAD_PATH"
+	MultipartFileSizeEnv                = "MULTIPART_FILE_SIZE"
 )

internal/params/keys.go

@@ -79,4 +79,8 @@ var (
 	RiskManagementPathKey               = strings.ToLower(RiskManagementPathEnv)
 	ConfigFilePathKey                   = strings.ToLower(ConfigFilePathEnv)
 	RealtimeScannerPathKey              = strings.ToLower(RealtimeScannerPathEnv)
+	StartMultiPartUploadPathKey         = strings.ToLower(StartMultiPartUploadPathEnv)
+	MultipartPresignedPathKey           = strings.ToLower(MultipartPresignedPathEnv)
+	CompleteMultiPartUploadPathKey      = strings.ToLower(CompleteMultipartUploadPathEnv)
+	MultipartFileSize                   = strings.ToLower(MultipartFileSizeEnv)
 )

internal/wrappers/feature-flags.go

@@ -17,6 +17,7 @@ const OssRealtimeEnabled = "OSS_REALTIME_ENABLED"
 const ScsLicensingV2Enabled = "SSCS_NEW_LICENSING_ENABLED"
 const DirectAssociationEnabled = "DIRECT_APP_ASSOCIATION_ENABLED"
 const maxRetries = 3
+const IncreaseFileUploadLimit = "INCREASE_FILE_UPLOAD_LIMIT"
 
 var DefaultFFLoad bool = false