Merge pull request #444 from jamiehannaford/checksum

Add MD5 checksum check to file uploads

Author: jrperritt

openstack/objectstorage/v1/objects/fixtures.go

@@ -3,7 +3,9 @@
 package objects
 
 import (
+	"crypto/md5"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 
@@ -107,20 +109,26 @@ func HandleListObjectNamesSuccessfully(t *testing.T) {
 
 // HandleCreateTextObjectSuccessfully creates an HTTP handler at `/testContainer/testObject` on the test handler mux
 // that responds with a `Create` response. A Content-Type of "text/plain" is expected.
-func HandleCreateTextObjectSuccessfully(t *testing.T) {
+func HandleCreateTextObjectSuccessfully(t *testing.T, content string) {
 	th.Mux.HandleFunc("/testContainer/testObject", func(w http.ResponseWriter, r *http.Request) {
 		th.TestMethod(t, r, "PUT")
 		th.TestHeader(t, r, "X-Auth-Token", fake.TokenID)
 		th.TestHeader(t, r, "Content-Type", "text/plain")
 		th.TestHeader(t, r, "Accept", "application/json")
+
+		hash := md5.New()
+		io.WriteString(hash, content)
+		localChecksum := hash.Sum(nil)
+
+		w.Header().Set("ETag", fmt.Sprintf("%x", localChecksum))
 		w.WriteHeader(http.StatusCreated)
 	})
 }
 
 // HandleCreateTypelessObjectSuccessfully creates an HTTP handler at `/testContainer/testObject` on the test handler
 // mux that responds with a `Create` response. No Content-Type header may be present in the request, so that server-
 // side content-type detection will be triggered properly.
-func HandleCreateTypelessObjectSuccessfully(t *testing.T) {
+func HandleCreateTypelessObjectSuccessfully(t *testing.T, content string) {
 	th.Mux.HandleFunc("/testContainer/testObject", func(w http.ResponseWriter, r *http.Request) {
 		th.TestMethod(t, r, "PUT")
 		th.TestHeader(t, r, "X-Auth-Token", fake.TokenID)
@@ -130,6 +138,11 @@ func HandleCreateTypelessObjectSuccessfully(t *testing.T) {
 			t.Errorf("Expected Content-Type header to be omitted, but was %#v", contentType)
 		}
 
+		hash := md5.New()
+		io.WriteString(hash, content)
+		localChecksum := hash.Sum(nil)
+
+		w.Header().Set("ETag", fmt.Sprintf("%x", localChecksum))
 		w.WriteHeader(http.StatusCreated)
 	})
 }

openstack/objectstorage/v1/objects/requests.go

@@ -2,9 +2,11 @@ package objects
 
 import (
 	"crypto/hmac"
+	"crypto/md5"
 	"crypto/sha1"
 	"fmt"
 	"io"
+	"net/http"
 	"strings"
 	"time"
 
@@ -188,7 +190,8 @@ func (opts CreateOpts) ToObjectCreateParams() (map[string]string, string, error)
 	return h, q.String(), nil
 }
 
-// Create is a function that creates a new object or replaces an existing object.
+// Create is a function that creates a new object or replaces an existing object. If the returned response's ETag
+// header fails to match the local checksum, the failed request will automatically be retried up to a maximum of 3 times.
 func Create(c *gophercloud.ServiceClient, containerName, objectName string, content io.ReadSeeker, opts CreateOptsBuilder) CreateResult {
 	var res CreateResult
 
@@ -214,11 +217,30 @@ func Create(c *gophercloud.ServiceClient, containerName, objectName string, cont
 		MoreHeaders: h,
 	}
 
-	resp, err := c.Request("PUT", url, ropts)
-	if resp != nil {
-		res.Header = resp.Header
+	doUpload := func() (*http.Response, error) {
+		resp, err := c.Request("PUT", url, ropts)
+		if resp != nil {
+			res.Header = resp.Header
+		}
+		return resp, err
 	}
-	res.Err = err
+
+	hash := md5.New()
+	io.Copy(hash, content)
+	localChecksum := hash.Sum(nil)
+
+	for i := 1; i <= 3; i++ {
+		resp, err := doUpload()
+		if resp.Header.Get("ETag") == fmt.Sprintf("%x", localChecksum) {
+			res.Err = err
+			break
+		}
+		if i == 3 {
+			res.Err = fmt.Errorf("Local checksum does not match API ETag header")
+			return res
+		}
+	}
+
 	return res
 }
 

openstack/objectstorage/v1/objects/requests_test.go

@@ -2,7 +2,9 @@ package objects
 
 import (
 	"bytes"
+	"fmt"
 	"io"
+	"net/http"
 	"strings"
 	"testing"
 
@@ -84,22 +86,42 @@ func TestListObjectNames(t *testing.T) {
 func TestCreateObject(t *testing.T) {
 	th.SetupHTTP()
 	defer th.TeardownHTTP()
-	HandleCreateTextObjectSuccessfully(t)
 
-	content := strings.NewReader("Did gyre and gimble in the wabe")
+	content := "Did gyre and gimble in the wabe"
+
+	HandleCreateTextObjectSuccessfully(t, content)
+
 	options := &CreateOpts{ContentType: "text/plain"}
-	res := Create(fake.ServiceClient(), "testContainer", "testObject", content, options)
+	res := Create(fake.ServiceClient(), "testContainer", "testObject", strings.NewReader(content), options)
 	th.AssertNoErr(t, res.Err)
 }
 
 func TestCreateObjectWithoutContentType(t *testing.T) {
 	th.SetupHTTP()
 	defer th.TeardownHTTP()
-	HandleCreateTypelessObjectSuccessfully(t)
+
+	content := "The sky was the color of television, tuned to a dead channel."
+
+	HandleCreateTypelessObjectSuccessfully(t, content)
+
+	res := Create(fake.ServiceClient(), "testContainer", "testObject", strings.NewReader(content), &CreateOpts{})
+	th.AssertNoErr(t, res.Err)
+}
+
+func TestErrorIsRaisedForChecksumMismatch(t *testing.T) {
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/testContainer/testObject", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("ETag", "acbd18db4cc2f85cedef654fccc4a4d8")
+		w.WriteHeader(http.StatusCreated)
+	})
 
 	content := strings.NewReader("The sky was the color of television, tuned to a dead channel.")
 	res := Create(fake.ServiceClient(), "testContainer", "testObject", content, &CreateOpts{})
-	th.AssertNoErr(t, res.Err)
+
+	err := fmt.Errorf("Local checksum does not match API ETag header")
+	th.AssertDeepEquals(t, err, res.Err)
 }
 
 func TestCopyObject(t *testing.T) {

rackspace/objectstorage/v1/objects/delegate_test.go

@@ -66,21 +66,23 @@ func TestListObjectNames(t *testing.T) {
 func TestCreateObject(t *testing.T) {
 	th.SetupHTTP()
 	defer th.TeardownHTTP()
-	os.HandleCreateTextObjectSuccessfully(t)
+	
+        content := "Did gyre and gimble in the wabe"
+        os.HandleCreateTextObjectSuccessfully(t, content)
 
-	content := strings.NewReader("Did gyre and gimble in the wabe")
 	options := &os.CreateOpts{ContentType: "text/plain"}
-	res := Create(fake.ServiceClient(), "testContainer", "testObject", content, options)
+	res := Create(fake.ServiceClient(), "testContainer", "testObject", strings.NewReader(content), options)
 	th.AssertNoErr(t, res.Err)
 }
 
 func TestCreateObjectWithoutContentType(t *testing.T) {
 	th.SetupHTTP()
 	defer th.TeardownHTTP()
-	os.HandleCreateTypelessObjectSuccessfully(t)
 
-	content := strings.NewReader("The sky was the color of television, tuned to a dead channel.")
-	res := Create(fake.ServiceClient(), "testContainer", "testObject", content, &os.CreateOpts{})
+        content := "The sky was the color of television, tuned to a dead channel."
+	os.HandleCreateTypelessObjectSuccessfully(t, content)
+
+	res := Create(fake.ServiceClient(), "testContainer", "testObject", strings.NewReader(content), &os.CreateOpts{})
 	th.AssertNoErr(t, res.Err)
 }