Adding support for OCI archives

Fixes #1

Author: clareliguori

README.md

@@ -19,7 +19,12 @@ If a layer is already published to Lambda (same layer name, SHA256 digest, and s
     + [Binaries](#binaries)
     + [From Source](#from-source)
 - [Permissions](#permissions)
-- [Example](#example)
+- [Examples](#examples)
+    + [Docker Example](#docker-example)
+    + [OCI Example](#oci-example)
+    + [Deploy Manually](#deploy-manually)
+    + [Deploy with AWS Serverless Application Model (SAM)](#deploy-with-aws-serverless-application-model-sam)
+    + [Deploy with Serverless Framework](#deploy-with-serverless-framework)
 - [License Summary](#license-summary)
 - [Security Disclosures](#security-disclosures)
 
@@ -32,7 +37,8 @@ USAGE:
    img2lambda [options]
 
 GLOBAL OPTIONS:
-   --image value, -i value                 Name of the source container image. For example, 'my-docker-image:latest'. The Docker image must be pulled locally already.
+   --image value, -i value                 Name or path of the source container image. For example, 'my-docker-image:latest' or './my-oci-image-archive'. The image must be pulled locally already.
+   --image-type value, -t value            Type of the source container image. Valid values: 'docker' (Docker image from the local Docker daemon), 'oci' (OCI image archive at the given path and optional tag) (default: "docker")
    --region value, -r value                AWS region (default: "us-east-1")
    --profile value, -p value               AWS credentials profile. Credentials will default to the same chain as the AWS CLI: environment variables, default profile, container credentials, EC2 instance credentials
    --output-directory value, -o value      Destination directory for command output (default: "./output")
@@ -114,7 +120,9 @@ For example:
 }
 ```
 
-## Example
+## Examples
+
+### Docker Example
 
 Build the example Docker image to create a PHP Lambda custom runtime:
 ```
@@ -135,6 +143,22 @@ Run the tool to create and publish Lambda layers that contain the PHP custom run
 ../bin/local/img2lambda -i lambda-php:latest -r us-east-1 -o ./output
 ```
 
+### OCI Example
+
+Create an OCI image from the example Dockerfile:
+```
+cd example
+
+podman build --format oci -t lambda-php .
+
+podman push lambda-php oci-archive:./lambda-php-oci
+```
+
+Run the tool to create and publish Lambda layers that contain the PHP custom runtime:
+```
+../bin/local/img2lambda -i ./lambda-php-oci -t oci -r us-east-1 -o ./output
+```
+
 ### Deploy Manually
 Create a PHP function that uses the layers:
 ```

img2lambda/cli/main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/awslabs/aws-lambda-container-image-converter/img2lambda/extract"
 	"github.com/awslabs/aws-lambda-container-image-converter/img2lambda/publish"
@@ -28,14 +29,20 @@ func createApp() (*cli.App, *types.CmdOptions) {
 		opts.CompatibleRuntimes = c.StringSlice("cr")
 
 		validateCliOptions(&opts, c)
-		return repackImageAction(&opts)
+		return repackImageAction(&opts, c)
 	}
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "image, i",
-			Usage:       "Name of the source container image. For example, 'my-docker-image:latest'. The Docker image must be pulled locally already.",
+			Usage:       "Name or path of the source container image. For example, 'my-docker-image:latest' or './my-oci-image-archive'. The image must be pulled locally already.",
 			Destination: &opts.Image,
 		},
+		cli.StringFlag{
+			Name:        "image-type, t",
+			Usage:       "Type of the source container image. Valid values: 'docker' (Docker image from the local Docker daemon), 'oci' (OCI image archive at the given path)",
+			Value:       "docker",
+			Destination: &opts.ImageType,
+		},
 		cli.StringFlag{
 			Name:        "region, r",
 			Usage:       "AWS region",
@@ -101,8 +108,24 @@ func validateCliOptions(opts *types.CmdOptions, context *cli.Context) {
 	}
 }
 
-func repackImageAction(opts *types.CmdOptions) error {
-	layers, err := extract.RepackImage("docker-daemon:"+opts.Image, opts.OutputDir)
+func repackImageAction(opts *types.CmdOptions, context *cli.Context) error {
+	var imageTransport string
+	switch opts.ImageType {
+	case "docker":
+		imageTransport = "docker-daemon:"
+	case "oci":
+		imageTransport = "oci-archive:"
+	default:
+		fmt.Println("ERROR: Image type must be one of the supported image types")
+		cli.ShowAppHelpAndExit(context, 1)
+	}
+
+	imageLocation := imageTransport + opts.Image
+	if opts.ImageType == "docker" && strings.Count(imageLocation, ":") == 1 {
+		imageLocation += ":latest"
+	}
+
+	layers, err := extract.RepackImage(imageLocation, opts.OutputDir)
 	if err != nil {
 		return err
 	}

img2lambda/extract/repack_image.go

@@ -4,6 +4,7 @@ package extract
 
 import (
 	"archive/tar"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -24,7 +25,7 @@ import (
 
 // Converts container image to Lambda layer archive files
 func RepackImage(imageName string, layerOutputDir string) (layers []types.LambdaLayer, retErr error) {
-	log.Printf("Parsing the docker image %s", imageName)
+	log.Printf("Parsing the image %s", imageName)
 
 	// Get image's layer data from image name
 	ref, err := alltransports.ParseImageName(imageName)
@@ -104,9 +105,21 @@ func repackImage(opts *repackOptions) (layers []types.LambdaLayer, retErr error)
 		}
 		defer layerStream.Close()
 
-		fileCreated, err := repackLayer(lambdaLayerFilename, layerStream)
+		fileCreated, err := repackLayer(lambdaLayerFilename, layerStream, false)
 		if err != nil {
-			return nil, err
+			tarErr := err
+
+			// tar extraction failed, try tar.gz
+			layerStream, _, err = opts.rawImageSource.GetBlob(opts.ctx, layerInfo, opts.cache)
+			if err != nil {
+				return nil, err
+			}
+			defer layerStream.Close()
+
+			fileCreated, err = repackLayer(lambdaLayerFilename, layerStream, true)
+			if err != nil {
+				return nil, fmt.Errorf("could not read layer with tar nor tar.gz: %v, %v", err, tarErr)
+			}
 		}
 
 		if fileCreated {
@@ -126,10 +139,21 @@ func repackImage(opts *repackOptions) (layers []types.LambdaLayer, retErr error)
 // Converts container image layer archive (tar) to Lambda layer archive (zip).
 // Filters files from the source and only writes a new archive if at least
 // one file in the source matches the filter (i.e. does not create empty archives).
-func repackLayer(outputFilename string, layerContents io.Reader) (created bool, retError error) {
+func repackLayer(outputFilename string, layerContents io.Reader, isGzip bool) (created bool, retError error) {
 	t := archiver.NewTar()
+	contentsReader := layerContents
+	var err error
+
+	if isGzip {
+		gzr, err := gzip.NewReader(layerContents)
+		if err != nil {
+			return false, fmt.Errorf("could not create gzip reader for layer: %v", err)
+		}
+		defer gzr.Close()
+		contentsReader = gzr
+	}
 
-	err := t.Open(layerContents, 0)
+	err = t.Open(contentsReader, 0)
 	if err != nil {
 		return false, fmt.Errorf("opening layer tar: %v", err)
 	}

img2lambda/extract/repack_image_test.go

@@ -6,6 +6,7 @@ import (
 	"archive/zip"
 	"bufio"
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
 	"os"
@@ -20,12 +21,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func createImageLayer(t *testing.T,
-	rawSource *mocks.MockImageSource,
+func CreateLayerData(t *testing.T,
 	filename string,
 	fileContents string,
-	digest string) *imgtypes.BlobInfo {
-
+	digest string) *bytes.Buffer {
 	tar := archiver.NewTar()
 
 	layerFile, err := ioutil.TempFile("", "")
@@ -60,6 +59,17 @@ func createImageLayer(t *testing.T,
 	err = os.Remove(layerFile.Name())
 	assert.Nil(t, err)
 
+	return &tarContents
+}
+
+func createImageLayer(t *testing.T,
+	rawSource *mocks.MockImageSource,
+	filename string,
+	fileContents string,
+	digest string) *imgtypes.BlobInfo {
+
+	tarContents := CreateLayerData(t, filename, fileContents, digest)
+
 	blobInfo := imgtypes.BlobInfo{Digest: godigest.Digest(digest)}
 
 	rawSource.EXPECT().GetBlob(gomock.Any(),
@@ -69,6 +79,40 @@ func createImageLayer(t *testing.T,
 	return &blobInfo
 }
 
+func createGzipImageLayer(t *testing.T,
+	rawSource *mocks.MockImageSource,
+	filename string,
+	fileContents string,
+	digest string) *imgtypes.BlobInfo {
+
+	tarContents := CreateLayerData(t, filename, fileContents, digest)
+
+	var targzContents bytes.Buffer
+	bufWriter := bufio.NewWriter(&targzContents)
+
+	gz := archiver.NewGz()
+	err := gz.Compress(bytes.NewReader(tarContents.Bytes()), bufWriter)
+	assert.Nil(t, err)
+	err = bufWriter.Flush()
+	assert.Nil(t, err)
+
+	blobInfo := imgtypes.BlobInfo{Digest: godigest.Digest(digest)}
+
+	contentsBytes := targzContents.Bytes()
+
+	rawSource.EXPECT().GetBlob(gomock.Any(),
+		blobInfo,
+		gomock.Any()).
+		DoAndReturn(
+			func(context context.Context, blobInfo imgtypes.BlobInfo, cache imgtypes.BlobInfoCache) (io.ReadCloser, int64, error) {
+				// checks whatever
+				return ioutil.NopCloser(bytes.NewReader(contentsBytes)), int64(0), nil
+			}).
+		Times(2)
+
+	return &blobInfo
+}
+
 func validateLambdaLayer(t *testing.T,
 	layer *types.LambdaLayer,
 	expectedFilename string,
@@ -126,7 +170,7 @@ func TestRepack(t *testing.T) {
 	blobInfos = append(blobInfos, *blobInfo1)
 
 	// Second matching file
-	blobInfo2 := createImageLayer(t, rawSource, "opt/hello/file2", "hello world 2", "digest2")
+	blobInfo2 := createGzipImageLayer(t, rawSource, "opt/hello/file2", "hello world 2", "digest2")
 	blobInfos = append(blobInfos, *blobInfo2)
 
 	// Irrelevant file
@@ -161,3 +205,44 @@ func TestRepack(t *testing.T) {
 	err = os.Remove(dir)
 	assert.Nil(t, err)
 }
+
+func TestRepackFailure(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	source := mocks.NewMockImageCloser(ctrl)
+	rawSource := mocks.NewMockImageSource(ctrl)
+
+	// Create layer tar files
+	var blobInfos []imgtypes.BlobInfo
+
+	// Add garbage data to layer
+	blobInfo := imgtypes.BlobInfo{Digest: godigest.Digest("digest1")}
+	rawSource.EXPECT().GetBlob(gomock.Any(),
+		blobInfo,
+		gomock.Any()).
+		Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), int64(0), nil).
+		Times(2)
+	blobInfos = append(blobInfos, blobInfo)
+
+	source.EXPECT().LayerInfos().Return(blobInfos)
+
+	dir, err := ioutil.TempDir("", "")
+	assert.Nil(t, err)
+
+	layers, err := repackImage(&repackOptions{
+		ctx:            nil,
+		cache:          nil,
+		imageSource:    source,
+		rawImageSource: rawSource,
+		imageName:      "test-image",
+		layerOutputDir: dir,
+	})
+
+	assert.Nil(t, layers)
+	assert.NotNil(t, err)
+	assert.Equal(t, err.Error(), "could not read layer with tar nor tar.gz: could not create gzip reader for layer: EOF, opening next file in layer tar: unexpected EOF")
+
+	err = os.Remove(dir)
+	assert.Nil(t, err)
+}

img2lambda/types/types.go

@@ -14,6 +14,7 @@ type LambdaLayer struct {
 
 type CmdOptions struct {
 	Image              string   // Name of the container image
+	ImageType          string   // Type of the container image
 	Region             string   // AWS region
 	Profile            string   // AWS credentials profile
 	OutputDir          string   // Output directory for the Lambda layers