Merge pull request #126 from netlify/get_original_size

Read original size from lfs pointers.

Author: calavera

Makefile

@@ -1,9 +1,6 @@
 .PHONY: all build deps generate help test validate
 CHECK_FILES?=$$(go list ./... | grep -v /vendor/)
 
-
-
-
 help: ## Show this help.
 		@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 

README.md

@@ -8,7 +8,7 @@ It's still a work in progress and we welcome feedback and contributions.
 
 ## Usage
 
-The [`swagger.yml`](swagger.yml) file is the master copy of the Open API 2.0 definition.  Additional context on using the API can be found on our [docs site](https://www.netlify.com/docs/api/).
+The [`swagger.yml`](swagger.yml) file is the master copy of the Open API 2.0 definition. Additional context on using the API can be found on our [docs site](https://www.netlify.com/docs/api/).
 
 The spec is published and versioned for various ecosystems:
 

go/plumbing/operations/upload_deploy_file_parameters.go

@@ -2,6 +2,7 @@ package porcelain
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"crypto/sha1"
 	"crypto/sha256"
@@ -12,7 +13,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -33,15 +34,19 @@ const (
 	goRuntime = "go"
 
 	preProcessingTimeout = time.Minute * 5
-)
-
-type uploadType int
 
-const (
 	fileUpload uploadType = iota
 	functionUpload
+
+	lfsVersionString = "version https://git-lfs.github.com/spec/v1"
 )
 
+type uploadType int
+type pointerData struct {
+	SHA  string
+	Size int64
+}
+
 type DeployObserver interface {
 	OnSetupWalk() error
 	OnSuccessfulStep(*FileBundle) error
@@ -86,6 +91,7 @@ type FileBundle struct {
 	Name    string
 	Sum     string
 	Runtime string
+	Size    *int64 `json:"size,omitempty"`
 
 	// Path OR Buffer should be populated
 	Path   string
@@ -410,6 +416,9 @@ func (n *Netlify) uploadFile(ctx context.Context, d *models.Deploy, f *FileBundl
 			if operationError == nil {
 				defer body.Close()
 				params := operations.NewUploadDeployFileParams().WithDeployID(d.ID).WithPath(f.Name).WithFileBody(body)
+				if f.Size != nil {
+					params.WithSize(f.Size)
+				}
 				if timeout != 0 {
 					params.SetTimeout(timeout)
 				}
@@ -500,9 +509,19 @@ func walk(dir string, observer DeployObserver, useLargeMedia bool) (*deployFiles
 					return err
 				}
 				defer o.Close()
-				originalSha := getLFSSha(o)
-				if originalSha != "" {
-					file.Sum += ":" + string(originalSha)
+
+				data, err := readLFSData(o)
+				if err != nil {
+					return err
+				}
+
+				if data != nil {
+					if data.SHA != "" {
+						file.Sum += ":" + data.SHA
+					}
+					if data.Size > 0 {
+						file.Size = &data.Size
+					}
 				}
 			}
 
@@ -654,18 +673,44 @@ func createHeader(archive *zip.Writer, i os.FileInfo, runtime string) (io.Writer
 	return archive.Create(i.Name())
 }
 
-func getLFSSha(file io.Reader) string {
+func readLFSData(file io.Reader) (*pointerData, error) {
 	// currently this only supports certain type of git lfs pointer files
 	// version [version]\noid sha256:[oid]\nsize [size]
-	data := make([]byte, 150)
-	if count, err := file.Read(data); err == nil {
-		r, _ := regexp.Compile(`^version \S+\noid sha256:(\S+)\n`)
-		res := r.FindSubmatch(data[:count])
-		if len(res) == 2 {
-			if originalSha := res[1]; len(originalSha) == 64 {
-				return string(originalSha)
-			}
+	data := make([]byte, len(lfsVersionString))
+	count, err := file.Read(data)
+	if err != nil {
+		// ignore file if it's not an LFS pointer with the expected header
+		return nil, nil
+	}
+	if count != len(lfsVersionString) || string(data) != lfsVersionString {
+		// ignore file if it's not an LFS pointer with the expected header
+		return nil, nil
+	}
+
+	scanner := bufio.NewScanner(file)
+	values := map[string]string{}
+	for scanner.Scan() {
+		keyAndValue := bytes.SplitN(scanner.Bytes(), []byte(" "), 2)
+		if len(keyAndValue) > 1 {
+			values[string(keyAndValue[0])] = string(keyAndValue[1])
 		}
 	}
-	return ""
+
+	var sha string
+	oid, ok := values["oid"]
+	if !ok {
+		return nil, fmt.Errorf("missing LFS OID")
+	}
+
+	sha = strings.SplitN(oid, ":", 2)[1]
+
+	size, err := strconv.ParseInt(values["size"], 10, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pointerData{
+		SHA:  sha,
+		Size: size,
+	}, nil
 }

go/plumbing/operations/upload_deploy_function_parameters.go

@@ -6,23 +6,37 @@ import (
 )
 
 func TestGetLFSSha(t *testing.T) {
-	tests := []struct {
-		contents string
-		length   int
-	}{
-		{"Not a pointer file", 0},
-		{"version https://git-lfs.github.com/spec/v1\n" +
-			"oid sha256:7e56e498ccb4cbb9c672e1aed6710fb91b2fd314394a666c11c33b2059ea3d71\n" +
-			"size 1743570", 64},
-	}
+	t.Run("test with not a pointer file", func(t *testing.T) {
+		file := strings.NewReader("Not a pointer file")
+		data, err := readLFSData(file)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-	for _, test := range tests {
-		file := strings.NewReader(test.contents)
-		out := getLFSSha(file)
-		if len(out) != test.length {
-			t.Fatalf("expected `%d`, got `%d`", test.length, len(out))
+		if data != nil {
+			t.Fatal("expected data to be nil without proper formatting")
 		}
-	}
+	})
+
+	t.Run("test with v1 pointer", func(t *testing.T) {
+		content := `version https://git-lfs.github.com/spec/v1
+oid sha256:7e56e498ccb4cbb9c672e1aed6710fb91b2fd314394a666c11c33b2059ea3d71
+size 1743570
+`
+		file := strings.NewReader(content)
+		data, err := readLFSData(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if data.SHA != "7e56e498ccb4cbb9c672e1aed6710fb91b2fd314394a666c11c33b2059ea3d71" {
+			t.Fatalf("expected `7e56e498ccb4cbb9c672e1aed6710fb91b2fd314394a666c11c33b2059ea3d71`, got `%v`", data.SHA)
+		}
+
+		if data.Size != 1743570 {
+			t.Fatalf("expected `1743570`, got `%v`", data.Size)
+		}
+	})
 }
 
 func TestAddWithLargeMedia(t *testing.T) {

go/porcelain/deploy.go

@@ -757,6 +757,9 @@ paths:
           type: string
           in: path
           required: true
+        - name: size
+          type: integer
+          in: query
         - name: file_body
           in: body
           schema:
@@ -787,6 +790,9 @@ paths:
         - name: runtime
           type: string
           in: query
+        - name: size
+          type: integer
+          in: query
         - name: file_body
           in: body
           schema: