Railpack CLI feature (#1301)

* Allow no physical Dockerfile and adjust for Normalization

* Added testdata Compose file for railpack

* Railpack logic

1: If there is a nontraditionally named Dockerfile in the tree and
one specified in the compose file, use it.
2: If there is no Dockerfile in the tree, but one is specified
which is named "Dockerfile" then use the pack builder.
3: If there is no Dockerfile in the tree, but one is specified
which is named something else that is not "Dockerfile" then we error.
4: If there is a Dockerfile in the tree, we use it if not specified
in the compose file.

* Expected testcases

* Handle case if there no dockerile at the desc location in compose

* simplfy railpack logic

* Railpack feature passing all cases

* update UploadModIgnore

* Revert test

* Move contextAwareWriter role to ContextAwareReader

* reverted testcases

* Apply suggestions from code review

* Collect dockerfile field modifcation to one area

---------

Co-authored-by: Lio李歐 <[email protected]>

Author: KevyVo

src/pkg/cli/compose/context.go

@@ -2,13 +2,15 @@ package compose
 
 import (
 	"archive/tar"
+	"archive/zip"
 	"bytes"
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -72,6 +74,94 @@ defang.exe
 .defang`
 )
 
+type ArchiveType string
+
+const ArchiveTypeZip ArchiveType = "application/zip"
+const ArchiveTypeGzip ArchiveType = "application/gzip"
+
+type WriterFactory interface {
+	CreateHeader(info fs.FileInfo, slashPath string) (io.Writer, error)
+	Close() error
+}
+
+type tarFactory struct {
+	*tar.Writer
+	gzipWriter io.WriteCloser
+}
+
+func (tw *tarFactory) CreateHeader(info fs.FileInfo, slashPath string) (io.Writer, error) {
+	// Convert zip header to tar header
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.Mode().IsRegular() {
+		return nil, nil
+	}
+
+	// Make reproducible; WalkDir walks files in lexical order.
+	header.ModTime = time.Unix(sourceDateEpoch, 0)
+	header.Gid = 0
+	header.Uid = 0
+	header.Name = slashPath
+	err = tw.WriteHeader(header)
+	return tw.Writer, err
+}
+
+func (tw *tarFactory) Close() error {
+	// Close the tar and gzip writers before returning the buffer
+	err := tw.Writer.Close()
+	if err != nil {
+		return err
+	}
+
+	err = tw.gzipWriter.Close()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type zipFactory struct {
+	*zip.Writer
+}
+
+func (zw *zipFactory) CreateHeader(info fs.FileInfo, slashPath string) (io.Writer, error) {
+	// Create a new zip file header
+	header := &zip.FileHeader{
+		Name:   slashPath,
+		Method: zip.Deflate,
+	}
+
+	// Make reproducible
+	header.Modified = time.Unix(sourceDateEpoch, 0)
+
+	if !info.Mode().IsRegular() {
+		if info.IsDir() {
+			header.Name = slashPath + "/" // Ensure directory paths end with slash
+			header.Method = zip.Store     // Directories are stored without compression
+			_, err := zw.Writer.CreateHeader(header)
+			return nil, err
+		}
+		return nil, nil
+	}
+
+	// Create file entry in zip
+	writer, err := zw.Writer.CreateHeader(header)
+	return writer, err
+}
+
+func (zw *zipFactory) Close() error {
+	// Close the zip writer before returning the buffer
+	err := zw.Writer.Close()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func parseContextLimit(limit string, def int64) int64 {
 	if size, err := units.RAMInBytes(limit); err == nil {
 		return size
@@ -89,17 +179,26 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec
 		return "", fmt.Errorf("invalid build context: %w", err) // already checked in ValidateProject
 	}
 
+	var archiveType ArchiveType
+	// If we have a Railpack build, we use a zip archive
+	if build.Dockerfile == RAILPACK {
+		archiveType = ArchiveTypeZip
+		// We use tar for all other builds
+	} else {
+		archiveType = ArchiveTypeGzip
+	}
+
 	switch upload {
 	case UploadModeIgnore:
-		// `compose config`, ie. dry-run: don't upload the tarball, just return the path as-is
+		// `compose config`, ie. dry-run: don't upload the archive, just return the path as-is
 		return root, nil
 	case UploadModeEstimate:
 		// For estimation, we don't bother packaging the files, we just return a placeholder URL
 		return fmt.Sprintf("s3://cd-preview/%v", time.Now().Unix()), nil
 	}
 
 	term.Info("Packaging the project files for", name, "at", root)
-	buffer, err := createTarball(ctx, build.Context, build.Dockerfile)
+	buffer, err := createArchive(ctx, build.Context, build.Dockerfile, archiveType)
 	if err != nil {
 		return "", err
 	}
@@ -121,19 +220,19 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec
 	}
 
 	term.Info("Uploading the project files for", name)
-	return uploadTarball(ctx, provider, project, buffer, digest)
+	return uploadArchive(ctx, provider, project, buffer, archiveType, digest)
 }
 
-func uploadTarball(ctx context.Context, provider client.Provider, project string, body io.Reader, digest string) (string, error) {
-	// Upload the tarball to the fabric controller storage;; TODO: use a streaming API
+func uploadArchive(ctx context.Context, provider client.Provider, project string, body io.Reader, contentType ArchiveType, digest string) (string, error) {
+	// Upload the archive to the fabric controller storage;; TODO: use a streaming API
 	ureq := &defangv1.UploadURLRequest{Digest: digest, Project: project}
 	res, err := provider.CreateUploadURL(ctx, ureq)
 	if err != nil {
 		return "", err
 	}
 
 	// Do an HTTP PUT to the generated URL
-	resp, err := http.Put(ctx, res.Url, "application/gzip", body)
+	resp, err := http.Put(ctx, res.Url, string(contentType), body)
 	if err != nil {
 		return "", err
 	}
@@ -150,17 +249,17 @@ func uploadTarball(ctx context.Context, provider client.Provider, project string
 	return url, nil
 }
 
-type contextAwareWriter struct {
+type contextAwareReader struct {
 	ctx context.Context
-	io.WriteCloser
+	io.ReadCloser
 }
 
-func (cw contextAwareWriter) Write(p []byte) (n int, err error) {
+func (cr contextAwareReader) Read(p []byte) (n int, err error) {
 	select {
-	case <-cw.ctx.Done(): // Detect context cancelation
-		return 0, cw.ctx.Err()
+	case <-cr.ctx.Done(): // Detect context cancelation
+		return 0, cr.ctx.Err()
 	default:
-		return cw.WriteCloser.Write(p)
+		return cr.ReadCloser.Read(p)
 	}
 }
 
@@ -229,7 +328,6 @@ func getDockerIgnorePatterns(root, dockerfile string) ([]string, string, error)
 }
 
 func WalkContextFolder(root, dockerfile string, fn func(path string, de os.DirEntry, slashPath string) error) error {
-	foundDockerfile := false
 	if dockerfile == "" {
 		dockerfile = "Dockerfile"
 	} else {
@@ -267,7 +365,6 @@ func WalkContextFolder(root, dockerfile string, fn func(path string, de os.DirEn
 
 		// we need the Dockerfile, even if it's in the .dockerignore file
 		if relPath == dockerfile {
-			foundDockerfile = true
 		} else if relPath == dockerignore {
 			// we need the .dockerignore file too: it might ignore itself and/or the Dockerfile, but is needed by the builder
 		} else {
@@ -291,19 +388,23 @@ func WalkContextFolder(root, dockerfile string, fn func(path string, de os.DirEn
 		return err
 	}
 
-	if !foundDockerfile {
-		return fmt.Errorf("the specified dockerfile could not be read: %q", dockerfile)
-	}
-
 	return nil
 }
 
-func createTarball(ctx context.Context, root, dockerfile string) (*bytes.Buffer, error) {
+func createArchive(ctx context.Context, root string, dockerfile string, contentType ArchiveType) (*bytes.Buffer, error) {
 	fileCount := 0
 	// TODO: use io.Pipe and do proper streaming (instead of buffering everything in memory)
+
 	buf := &bytes.Buffer{}
-	gzipWriter := &contextAwareWriter{ctx, gzip.NewWriter(buf)}
-	tarWriter := tar.NewWriter(gzipWriter)
+	var factory WriterFactory
+	if contentType == ArchiveTypeZip {
+		zipWriter := zip.NewWriter(buf)
+		factory = &zipFactory{zipWriter}
+	} else {
+		gzipWriter := gzip.NewWriter(buf)
+		tarWriter := tar.NewWriter(gzipWriter)
+		factory = &tarFactory{tarWriter, gzipWriter}
+	}
 
 	doProgress := term.StdoutCanColor() && term.IsTerminal()
 	err := WalkContextFolder(root, dockerfile, func(path string, de os.DirEntry, slashPath string) error {
@@ -319,38 +420,27 @@ func createTarball(ctx context.Context, root, dockerfile string) (*bytes.Buffer,
 			return err
 		}
 
-		header, err := tar.FileInfoHeader(info, info.Name())
-		if err != nil {
-			return err
-		}
-
-		// Make reproducible; WalkDir walks files in lexical order.
-		header.ModTime = time.Unix(sourceDateEpoch, 0)
-		header.Gid = 0
-		header.Uid = 0
-		header.Name = slashPath
-		err = tarWriter.WriteHeader(header)
-		if err != nil {
+		writer, err := factory.CreateHeader(info, slashPath)
+		if err != nil || writer == nil {
 			return err
 		}
 
-		if !info.Mode().IsRegular() {
-			return nil
-		}
-
 		file, err := os.Open(path)
 		if err != nil {
 			return err
 		}
 		defer file.Close()
 
+		// Wrap the file reader with context-aware reader
+		contextReader := &contextAwareReader{ctx, file}
+
 		fileCount++
 		if fileCount == ContextFileLimit+1 {
 			term.Warnf("the build context contains more than %d files; use --debug or create .dockerignore to exclude caches and build artifacts", ContextFileLimit)
 		}
 
 		bufLen := buf.Len()
-		_, err = io.Copy(tarWriter, file)
+		_, err = io.Copy(writer, contextReader)
 		if int64(buf.Len()) > ContextSizeHardLimit {
 			return fmt.Errorf("the build context is limited to %s; consider downloading large files in the Dockerfile or set the DEFANG_BUILD_CONTEXT_LIMIT environment variable", units.BytesSize(float64(ContextSizeHardLimit)))
 		}
@@ -364,12 +454,8 @@ func createTarball(ctx context.Context, root, dockerfile string) (*bytes.Buffer,
 		return nil, err
 	}
 
-	// Close the tar and gzip writers before returning the buffer
-	if err = tarWriter.Close(); err != nil {
-		return nil, err
-	}
-
-	if err = gzipWriter.Close(); err != nil {
+	err = factory.Close() // Close the tar or zip writer
+	if err != nil {
 		return nil, err
 	}
 

src/pkg/cli/compose/context_test.go

@@ -38,7 +38,7 @@ func Test_parseContextLimit(t *testing.T) {
 	})
 }
 
-func TestUploadTarball(t *testing.T) {
+func TestUploadArchive(t *testing.T) {
 	const path = "/upload/x/"
 	const digest = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
 
@@ -49,28 +49,39 @@ func TestUploadTarball(t *testing.T) {
 		if !strings.HasPrefix(r.URL.Path, path) {
 			t.Errorf("Expected prefix %v, got %v", path, r.URL.Path)
 		}
-		if r.Header.Get("Content-Type") != "application/gzip" {
-			t.Errorf("Expected Content-Type: application/gzip, got %v", r.Header.Get("Content-Type"))
+		if !(r.Header.Get("Content-Type") == string(ArchiveTypeGzip) || r.Header.Get("Content-Type") == string(ArchiveTypeZip)) {
+			t.Errorf("Expected Content-Type: application/gzip or application/zip, got %v", r.Header.Get("Content-Type"))
 		}
 		w.WriteHeader(200)
 	}))
 	defer server.Close()
 
 	t.Run("upload with digest", func(t *testing.T) {
-		url, err := uploadTarball(context.Background(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, digest)
+		url, err := uploadArchive(context.Background(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, digest)
 		if err != nil {
-			t.Fatalf("uploadTarball() failed: %v", err)
+			t.Fatalf("uploadArchive() failed: %v", err)
 		}
 		const expectedPath = path + digest
 		if url != server.URL+expectedPath {
 			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
+
+		t.Run("upload with zip", func(t *testing.T) {
+			url, err := uploadArchive(context.Background(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, "")
+			if err != nil {
+				t.Fatalf("uploadContent() failed: %v", err)
+			}
+			if url != server.URL+path {
+				t.Errorf("Expected %v, got %v", server.URL+path, url)
+			}
+		})
+
 	})
 
 	t.Run("force upload without digest", func(t *testing.T) {
-		url, err := uploadTarball(context.Background(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, "")
+		url, err := uploadArchive(context.Background(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, "")
 		if err != nil {
-			t.Fatalf("uploadTarball() failed: %v", err)
+			t.Fatalf("uploadArchive() failed: %v", err)
 		}
 		if url != server.URL+path {
 			t.Errorf("Expected %v, got %v", server.URL+path, url)
@@ -134,7 +145,7 @@ func TestWalkContextFolder(t *testing.T) {
 
 func TestCreateTarballReader(t *testing.T) {
 	t.Run("Default Dockerfile", func(t *testing.T) {
-		buffer, err := createTarball(context.Background(), "../../../testdata/testproj", "")
+		buffer, err := createArchive(context.Background(), "../../../testdata/testproj", "", ArchiveTypeGzip)
 		if err != nil {
 			t.Fatalf("createTarballReader() failed: %v", err)
 		}
@@ -171,14 +182,14 @@ func TestCreateTarballReader(t *testing.T) {
 	})
 
 	t.Run("Missing Dockerfile", func(t *testing.T) {
-		_, err := createTarball(context.Background(), "../../testdata", "Dockerfile.missing")
+		_, err := createArchive(context.Background(), "../../testdata", "Dockerfile.missing", ArchiveTypeGzip)
 		if err == nil {
 			t.Fatal("createTarballReader() should have failed")
 		}
 	})
 
 	t.Run("Missing Context", func(t *testing.T) {
-		_, err := createTarball(context.Background(), "asdfqwer", "")
+		_, err := createArchive(context.Background(), "asdfqwer", "", ArchiveTypeGzip)
 		if err == nil {
 			t.Fatal("createTarballReader() should have failed")
 		}