Output json file version manifest

Author: djfarrelly

README.md

@@ -1,3 +1,32 @@
 # buffer-static-upload
 
-A straightforward static asset uploader
+A straightforward static asset uploader which generates a json file for your
+application to read the uploaded file locations from.
+
+## Usage
+
+Ensure your AWS credentials environment variables are set (`AWS_ACCESS_KEY_ID`,
+`AWS_SECRET_ACCESS_KEY`). The cli has the following argument options:
+
+```
+$ buffer-static-upload -h
+Usage of buffer-static-upload:
+  -bucket string
+    	the s3 bucket to upload to (default "static.buffer.com")
+  -dir string
+    	required, the directory to upload files to in the bucket
+  -files string
+    	the path to the files you'd like to upload, ex. "public/**/.*js,public/style.css"
+  -o string
+    	the json file you'd like your generate (default "staticAssets.json")
+```
+
+For example, you can use glob patterns to match multiple sets of files:
+
+```
+buffer-static-upload -files "public/js/**/*.js,public/css/*.css"
+```
+
+*Note* - The default bucket is used by multiple teams, so if you use that you
+must specify a directory to use for your project as not to create unnecessary
+collisions.

main.go

@@ -1,30 +1,181 @@
 package main
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
 
-	"github.com/bufferapp/buffer-static-upload/utils"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 )
 
-func main() {
-	s3Bucket := "buffer-dan-test"
+var defaultS3Bucket string = "static.buffer.com"
+
+// GetFileMd5 returns a checksum for a given file
+func GetFileMd5(file *os.File) (string, error) {
+	var fileHash string
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return fileHash, err
+	}
+	hashInBytes := hash.Sum(nil)[:16]
+	fileHash = hex.EncodeToString(hashInBytes)
+	return fileHash, nil
+}
+
+// GetVersionedFilename returns a new filename with the version before the extension
+func GetVersionedFilename(filename string, version string) string {
+	ext := filepath.Ext(filename)
+	versionedExt := "." + version + ext
+	versionedFilename := strings.Replace(filename, ext, versionedExt, 1)
+	return versionedFilename
+}
+
+// GetFileMimeType returns the mime type of a file using it's extension
+func GetFileMimeType(filename string) string {
+	ext := filepath.Ext(filename)
+	return mime.TypeByExtension(ext)
+}
+
+// GetFilesFromGlobsList returns a list of files that match a list of
+// comma-deliniated file path globs
+func GetFilesFromGlobsList(globList string) ([]string, error) {
+	var files []string
+	globs := strings.Split(globList, ",")
+
+	for _, glob := range globs {
+		fileList, err := filepath.Glob(glob)
+		if err != nil {
+			return files, err
+		}
+		files = append(files, fileList...)
+	}
+	return files, nil
+}
+
+// GetS3Uploader returns a configured Uploader
+func GetS3Uploader() (*s3manager.Uploader, error) {
+	var uploader *s3manager.Uploader
+
+	awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
+	awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
+
+	creds := credentials.NewStaticCredentials(awsAccessKeyID, awsSecretAccessKey, "")
+
+	sess := session.Must(session.NewSession(&aws.Config{
+		Credentials: creds,
+		Region:      aws.String(endpoints.UsEast1RegionID),
+	}))
+
+	_, err := creds.Get()
+	if err != nil {
+		fmt.Printf("failed to load AWS credentials %s", err)
+		return uploader, err
+	}
+
+	uploader = s3manager.NewUploader(sess)
+	return uploader, nil
+}
 
-	filesArg := flag.String("files", "", "the path to the files you'd like to upload")
+// VersionAndUploadFiles will verion files and upload them to s3 and return
+// a map of filenames and their version hashes
+func VersionAndUploadFiles(
+	bucket string,
+	directory string,
+	filenames []string,
+) (map[string]string, error) {
+	fileVersions := map[string]string{}
+
+	uploader, err := GetS3Uploader()
+	if err != nil {
+		return fileVersions, err
+	}
+
+	for _, filename := range filenames {
+		file, err := os.Open(filename)
+		if err != nil {
+			return fileVersions, err
+		}
+		defer file.Close()
+
+		checksum, err := GetFileMd5(file)
+		if err != nil {
+			return fileVersions, err
+		}
+
+		versionedFilename := GetVersionedFilename(filename, checksum)
+		bucketFilename := path.Join(directory, versionedFilename)
+		mimeType := GetFileMimeType(filename)
+
+		result, err := uploader.Upload(&s3manager.UploadInput{
+			Bucket:       aws.String(bucket),
+			Key:          aws.String(bucketFilename),
+			ContentType:  aws.String(mimeType),
+			CacheControl: aws.String("public, max-age=31520626"),
+			Expires:      aws.Time(time.Now().AddDate(10, 0, 0)),
+			Body:         file,
+		})
+		if err != nil {
+			return fileVersions, err
+		}
+
+		// the static.buffer.com bucket has a domain alias
+		if bucket == defaultS3Bucket {
+			fileVersions[filename] = "https://" + path.Join(bucket, bucketFilename)
+		} else {
+			fileVersions[filename] = result.Location
+		}
+		fmt.Printf("Uploaded %s\n", fileVersions[filename])
+	}
+
+	return fileVersions, nil
+}
+
+func main() {
+	s3Bucket := flag.String("bucket", defaultS3Bucket, "the s3 bucket to upload to")
+	directory := flag.String("dir", "", "required, the directory to upload files to in the bucket")
+	filesArg := flag.String("files", "", "the path to the files you'd like to upload, ex. \"public/**/.*js,public/style.css\"")
+	outputFilename := flag.String("o", "staticAssets.json", "the json file you'd like your generate")
 	flag.Parse()
 
-	files, err := utils.GetFilesFromGlobsList(*filesArg)
+	if *directory == "" && *s3Bucket == defaultS3Bucket {
+		fmt.Println("To use the default bucket you need to specify an upload directory (-dir)")
+		os.Exit(1)
+	}
+
+	files, err := GetFilesFromGlobsList(*filesArg)
 	if err != nil {
-		fmt.Printf("err %s", err)
+		fmt.Printf("failed to get files %s", err)
 	}
-	fmt.Printf("Found %d files to upload and version\n", len(files))
+	fmt.Printf("Found %d files to upload and version:\n", len(files))
 
-	fileVersions, err := utils.VersionAndUploadFiles(s3Bucket, files)
+	fileVersions, err := VersionAndUploadFiles(*s3Bucket, *directory, files)
 	if err != nil {
-		fmt.Printf("Failed to upload files: %s", err)
+		fmt.Printf("failed to upload files %s", err)
 	}
 
-	for file, version := range fileVersions {
-		fmt.Printf("%s = %s\n", file, version)
+	output, err := json.MarshalIndent(fileVersions, "", "  ")
+	if err != nil {
+		fmt.Printf("failed to generate versions json file %s", err)
 	}
+
+	err = ioutil.WriteFile(*outputFilename, output, 0644)
+	if err != nil {
+		fmt.Printf("failed to write versions json file %s", err)
+	}
+
+	fmt.Printf("\nSuccessfully uploaded static assets and generated %s\n", *outputFilename)
 }