Add S3 URI handling to modules

Author: ericzbeard

cft/pkg/content.go

@@ -17,33 +17,14 @@ type ModuleContent struct {
 	BaseUri    string
 }
 
-// Helper function to handle zip file extraction with proper path resolution
-func handleZipFile(root string, location string, hash string, path string) ([]byte, error) {
-	// Resolve the path to the zip file if it's a local path
-	zipPath := location
-	if !strings.HasPrefix(zipPath, "http://") && !strings.HasPrefix(zipPath, "https://") {
-		// If it's a relative path, resolve it relative to the template's directory
-		if !filepath.IsAbs(zipPath) {
-			zipPath = filepath.Join(root, zipPath)
-		}
-	}
 
-	// Check if the zip file exists if it's a local file
-	if !strings.HasPrefix(zipPath, "http://") && !strings.HasPrefix(zipPath, "https://") {
-		_, err := os.Stat(zipPath)
-		if err != nil {
-			return nil, fmt.Errorf("error accessing zip file %s: %v", zipPath, err)
-		}
-	}
 
-	// Unzip, verify hash if there is one, and put the files in memory
-	content, err := DownloadFromZip(zipPath, hash, path)
-	if err != nil {
-		config.Debugf("ZIP: Error extracting from zip: %v", err)
-		return nil, err
-	}
+func isHttpsUrl(uri string) bool {
+	return strings.HasPrefix(uri, "https://")
+}
 
-	return content, nil
+func isS3URI(uri string) bool {
+	return strings.HasPrefix(uri, "s3://")
 }
 
 // Get the module's content from a local file, memory, or a remote uri
@@ -54,6 +35,8 @@ func getModuleContent(
 	baseUri string,
 	uri string) (*ModuleContent, error) {
 
+	config.Debugf("getModuleContent root: %s, uri: %s", root, uri)
+
 	var content []byte
 	var err error
 	var newRootDir string
@@ -71,7 +54,15 @@ func getModuleContent(
 
 					if strings.HasSuffix(packageAlias.Location, ".zip") {
 						isZip = true
-						content, err = handleZipFile(root, packageAlias.Location, packageAlias.Hash, path)
+						
+						// Use DownloadFromZip directly
+						zipLocation := packageAlias.Location
+						// For local files, resolve the path relative to the template's directory
+						if !isS3URI(zipLocation) && !isHttpsUrl(zipLocation) && !filepath.IsAbs(zipLocation) {
+							zipLocation = filepath.Join(root, zipLocation)
+						}
+						
+						content, err = DownloadFromZip(zipLocation, packageAlias.Hash, path)
 						if err != nil {
 							return nil, err
 						}
@@ -92,12 +83,25 @@ func getModuleContent(
 	// getModuleContent: root=cft/pkg/tmpl/awscli-modules, baseUri=, uri=package.zip/zip-module.yaml
 	if strings.Contains(uri, ".zip/") {
 		isZip = true
-		tokens := strings.Split(uri, "/")
-		location := tokens[0]
-		path := strings.Join(tokens[1:], "/")
-		content, err = handleZipFile(root, location, "", path)
-		if err != nil {
-			return nil, err
+		
+		// Extract the zip location and path within the zip
+		zipIndex := strings.Index(uri, ".zip/")
+		if zipIndex > 0 {
+			zipLocation := uri[:zipIndex+4]  // Include the .zip part
+			zipPath := uri[zipIndex+5:]      // Skip the .zip/ part
+			
+			// For local files, resolve the path relative to the template's directory
+			if !isS3URI(zipLocation) && !isHttpsUrl(zipLocation) && !filepath.IsAbs(zipLocation) {
+				zipLocation = filepath.Join(root, zipLocation)
+			}
+			
+			config.Debugf("Extracting from zip: %s, path: %s", zipLocation, zipPath)
+			
+			// Use DownloadFromZip directly - it can handle S3, HTTPS, and local files
+			content, err = DownloadFromZip(zipLocation, "", zipPath)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -109,7 +113,15 @@ func getModuleContent(
 			path := strings.Replace(uri, packageAlias.Alias+"/", "", 1)
 			if strings.HasSuffix(packageAlias.Location, ".zip") {
 				isZip = true
-				content, err = handleZipFile(root, packageAlias.Location, packageAlias.Hash, path)
+				
+				// Use DownloadFromZip directly
+				zipLocation := packageAlias.Location
+				// For local files, resolve the path relative to the template's directory
+				if !isS3URI(zipLocation) && !isHttpsUrl(zipLocation) && !filepath.IsAbs(zipLocation) {
+					zipLocation = filepath.Join(root, zipLocation)
+				}
+				
+				content, err = DownloadFromZip(zipLocation, packageAlias.Hash, path)
 				if err != nil {
 					return nil, err
 				}
@@ -122,7 +134,7 @@ func getModuleContent(
 	// Is this a local file or a URL or did we already unzip a package?
 	if isZip {
 		config.Debugf("Using content from a zipped module package (length: %d bytes)", len(content))
-	} else if strings.HasPrefix(uri, "https://") {
+	} else if isHttpsUrl(uri) || isS3URI(uri) {
 		config.Debugf("Downloading from URL: %s", uri)
 		content, err = downloadModule(uri)
 		if err != nil {
@@ -138,6 +150,7 @@ func getModuleContent(
 		baseUri = strings.Join(urlParts[:len(urlParts)-1], "/")
 
 	} else {
+		config.Debugf("Downloading from a local file, baseUri=%s, uri=%s", baseUri, uri)
 		if baseUri != "" {
 			// If we have a base URL, prepend it to the relative path
 			uri = baseUri + "/" + uri

cft/pkg/download.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/aws-cloudformation/rain/internal/aws/s3"
 	"github.com/aws-cloudformation/rain/internal/config"
 	"github.com/google/uuid"
 )
@@ -42,11 +43,25 @@ func downloadHash(uri string) (string, error) {
 
 // DownloadFromZip retrieves a single file from a zip file hosted on a URI
 func DownloadFromZip(uriString string, verifyHash string, path string) ([]byte, error) {
+
+	config.Debugf("DownloadFromZip uriString: %s, path: %s",
+		uriString, path)
+
 	var zipData []byte
 	var err error
-	
-	// Check if it's a URL or local file
-	if strings.HasPrefix(uriString, "http://") || strings.HasPrefix(uriString, "https://") {
+
+	isUrl := isHttpsUrl(uriString)
+	isS3 := isS3URI(uriString)
+
+	// Check if it's an S3 URI, HTTPS URL, or local file
+	if isS3 {
+		// Download from S3
+		config.Debugf("Downloading from S3: %s", uriString)
+		zipData, err = downloadS3(uriString)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download zip from S3: %v", err)
+		}
+	} else if isUrl {
 		// Download from URL
 		config.Debugf("Downloading %s", uriString)
 		resp, err := http.Get(uriString)
@@ -59,7 +74,7 @@ func DownloadFromZip(uriString string, verifyHash string, path string) ([]byte,
 				config.Debugf("Error closing body: %v", err)
 			}
 		}(resp.Body)
-		
+
 		zipData, err = io.ReadAll(resp.Body)
 		if err != nil {
 			return nil, err
@@ -112,7 +127,7 @@ func DownloadFromZip(uriString string, verifyHash string, path string) ([]byte,
 
 		// Download or read the hash
 		var originalHash string
-		if strings.HasPrefix(verifyHash, "http://") || strings.HasPrefix(verifyHash, "https://") {
+		if isUrl {
 			originalHash, err = downloadHash(verifyHash)
 			if err != nil {
 				return nil, err
@@ -215,9 +230,31 @@ func Unzip(f *os.File, dest string) error {
 	return nil
 }
 
+func downloadS3(uri string) ([]byte, error) {
+	// Parse the S3 URI
+	bucket, key, err := s3.ParseURI(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	// Download the file from S3
+	content, err := s3.GetObject(bucket, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return content, nil
+}
+
 // downloadModule downloads the file from the given URI and returns its content as a byte slice.
 func downloadModule(uri string) ([]byte, error) {
 	config.Debugf("Downloading %s", uri)
+
+	// If it's an S3 uri, use the s3 package to download the file
+	if strings.HasPrefix(uri, "s3://") {
+		return downloadS3(uri)
+	}
+
 	resp, err := http.Get(uri)
 	if err != nil {
 		return nil, err

cft/pkg/tmpl/awscli-modules/s3-template.yaml

@@ -0,0 +1,19 @@
+Packages:
+  abc:
+    Source: s3://ezbeard-awscli/modules/package.zip
+Constants:
+  ModuleSource: s3://ezbeard-awscli/modules
+Modules:
+  Content:
+    Source: !Sub ${Const::ModuleSource}/basic-module.yaml  
+    Properties:
+      Name: foo
+    Overrides:
+      Bucket:
+        Properties:
+          OverrideMe: def
+  TestPackage:
+    Source: $abc/zip-module.yaml
+Resources:
+  OtherResource:
+    Type: AWS::S3::Bucket

internal/aws/s3/s3.go

@@ -295,9 +295,52 @@ func RainBucket(forceCreation bool) string {
 	return bucketName
 }
 
+// ParseURI parses an S3 URI like s3://bucket/key
+// The object key name is a sequence of Unicode characters with UTF-8 encoding of up to 1,024 bytes long.
+// The bucket name must be a valid DNS name and follow S3 bucket naming rules.
+func ParseURI(uri string) (bucket string, key string, err error) {
+	// Check if the URI starts with s3:// prefix
+	if !strings.HasPrefix(uri, "s3://") {
+		err = fmt.Errorf("invalid s3 uri: %s, must start with s3://", uri)
+		return
+	}
+	
+	// Remove the s3:// prefix
+	uri = strings.TrimPrefix(uri, "s3://")
+	
+	// Split the remaining string by the first /
+	parts := strings.SplitN(uri, "/", 2)
+	if len(parts) == 0 || parts[0] == "" {
+		err = fmt.Errorf("invalid s3 uri: %s, bucket name missing", uri)
+		return
+	}
+	
+	bucket = parts[0]
+	
+	// Validate bucket name (basic validation)
+	if len(bucket) < 3 || len(bucket) > 63 {
+		err = fmt.Errorf("invalid bucket name: %s, length must be between 3 and 63 characters", bucket)
+		return
+	}
+	
+	if len(parts) == 1 {
+		key = ""
+	} else {
+		key = parts[1]
+		
+		// Validate key length (S3 limit is 1024 bytes)
+		if len(key) > 1024 {
+			err = fmt.Errorf("invalid key: %s, length exceeds 1024 bytes", key)
+			return
+		}
+	}
+	return
+}
+
 // GetObject gets an object by key from an S3 bucket
 func GetObject(bucketName string, key string) ([]byte, error) {
 
+	config.Debugf("GetObject bucket %s, key: %s", bucketName, key)
 	accountId, err := getAccountId()
 	if err != nil {
 		return nil, err