Publish layers to Lambda

Author: clareliguori

README.md

@@ -13,8 +13,11 @@ docker run lambda-php goodbye '{"name": "World"}'
 ```
 
 TODO:
+* Support image types other than local Docker images, where the layer format is tar. For example, layers directly from a Docker registry will be .tar.gz-formatted. OCI images can be either tar or tar.gz, based on the layer's media type.
+* De-dupe Lambda layers before publishing them (compare local file's SHA256 to published layer versions with the same name)
+* Accept additional parameters for PublishLayerVersion API (license, description, etc)
+* Support Lambda compatible runtimes other than 'provided'
 * Utility for creating a function deployment package from a Docker image
-* Support image types other than local Docker images ('docker-daemon' transport), where the layer format is tar. For example, layers directly from a Docker registry will be .tar.gz-formatted. OCI images can be either tar or tar.gz, based on the layer's media type.
 
 ## License Summary
 

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"log"
 	"os"
 
@@ -28,7 +29,7 @@ func createApp() (*cli.App, *cmdOptions) {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "image, i",
-			Usage:       "Name of the source container image. Format is 'transport:details', such as 'docker-daemon:my-docker-image:latest",
+			Usage:       "Name of the source container image. For example, 'my-docker-image:latest'",
 			Destination: &opts.image,
 		},
 		cli.StringFlag{
@@ -59,7 +60,23 @@ func createApp() (*cli.App, *cmdOptions) {
 }
 
 func repackImageAction(opts *cmdOptions) error {
-	return RepackImage(opts.image, opts.outputDir)
+	layers, err := RepackImage("docker-daemon:"+opts.image, opts.outputDir)
+	if err != nil {
+		return err
+	}
+
+	if len(layers) == 0 {
+		return errors.New("No compatible layers found in the image (likely nothing found in /opt)")
+	}
+
+	if !opts.dryRun {
+		err := PublishLambdaLayers(opts.image, layers, opts.region, opts.layerNamespace, opts.outputDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func main() {

publish_layers.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/lambda"
+)
+
+func PublishLambdaLayers(sourceImageName string, layers []LambdaLayer, region string, layerPrefix string, resultsDir string) error {
+	layerArns := []string{}
+
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+
+	svc := lambda.New(sess, &aws.Config{Region: aws.String(region)})
+
+	for _, layer := range layers {
+		layerName := layerPrefix + "-" + strings.Replace(layer.Digest, ":", "-", -1)
+
+		layerContents, err := ioutil.ReadFile(layer.File)
+		if err != nil {
+			return err
+		}
+
+		publishArgs := &lambda.PublishLayerVersionInput{
+			CompatibleRuntimes: []*string{aws.String("provided")},
+			Content:            &lambda.LayerVersionContentInput{ZipFile: layerContents},
+			Description:        aws.String("created by img2lambda from image " + sourceImageName),
+			LayerName:          aws.String(layerName),
+		}
+
+		resp, err := svc.PublishLayerVersion(publishArgs)
+		if err != nil {
+			return err
+		}
+
+		layerArns = append(layerArns, *resp.LayerVersionArn)
+
+		err = os.Remove(layer.File)
+		if err != nil {
+			return err
+		}
+	}
+
+	jsonArns, err := json.MarshalIndent(layerArns, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	resultsPath := filepath.Join(resultsDir, "layers.json")
+	jsonFile, err := os.Create(resultsPath)
+	if err != nil {
+		return err
+	}
+	defer jsonFile.Close()
+
+	_, err = jsonFile.Write(jsonArns)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

repack_image.go

@@ -13,12 +13,17 @@ import (
 	"github.com/pkg/errors"
 )
 
+type LambdaLayer struct {
+	Digest string
+	File   string
+}
+
 // Converts container image to Lambda layer archive files
-func RepackImage(imageName string, layerOutputDir string) (retErr error) {
+func RepackImage(imageName string, layerOutputDir string) (layers []LambdaLayer, retErr error) {
 	// Get image's layer data from image name
 	ref, err := alltransports.ParseImageName(imageName)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	sys := &types.SystemContext{}
@@ -29,16 +34,16 @@ func RepackImage(imageName string, layerOutputDir string) (retErr error) {
 
 	rawSource, err := ref.NewImageSource(ctx, sys)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	src, err := image.FromSource(ctx, sys, rawSource)
 	if err != nil {
 		if closeErr := rawSource.Close(); closeErr != nil {
-			return errors.Wrapf(err, " (close error: %v)", closeErr)
+			return nil, errors.Wrapf(err, " (close error: %v)", closeErr)
 		}
 
-		return err
+		return nil, err
 	}
 	defer func() {
 		if err := src.Close(); err != nil {
@@ -50,7 +55,7 @@ func RepackImage(imageName string, layerOutputDir string) (retErr error) {
 
 	// Unpack and inspect each image layer, copy relevant files to new Lambda layer
 	if err := os.MkdirAll(layerOutputDir, 0777); err != nil {
-		return err
+		return nil, err
 	}
 
 	lambdaLayerNum := 1
@@ -60,19 +65,20 @@ func RepackImage(imageName string, layerOutputDir string) (retErr error) {
 
 		layerStream, _, err := rawSource.GetBlob(ctx, layerInfo, cache)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		defer layerStream.Close()
 
 		fileCreated, err := RepackLayer(lambdaLayerFilename, layerStream)
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		if fileCreated {
 			lambdaLayerNum++
+			layers = append(layers, LambdaLayer{Digest: string(layerInfo.Digest), File: lambdaLayerFilename})
 		}
 	}
 
-	return nil
+	return layers, nil
 }

repack_layer.go

@@ -26,8 +26,8 @@ func RepackLayer(outputFilename string, layerContents io.Reader) (created bool,
 	defer t.Close()
 
 	// Walk the files in the tar
-	z := (*archiver.Zip)(nil)
-	out := (*os.File)(nil)
+	var z *archiver.Zip
+	var out *os.File
 	defer func() {
 		if z != nil {
 			if err := z.Close(); err != nil {