Merge pull request #51 from deepsheth/master

Add POST functions

Author: anaskhan96

README.md

@@ -11,10 +11,12 @@ Exported variables and functions implemented till now :
 ```go
 var Headers map[string]string // Set headers as a map of key-value pairs, an alternative to calling Header() individually
 var Cookies map[string]string // Set cookies as a map of key-value  pairs, an alternative to calling Cookie() individually
-func Get(string) (string,error){} // Takes the url as an argument, returns HTML string
-func GetWithClient(string, *http.Client){} // Takes the url and a custom HTTP client as arguments, returns HTML string
-func Header(string, string){} // Takes key,value pair to set as headers for the HTTP request made in Get()
-func Cookie(string, string){} // Takes key, value pair to set as cookies to be sent with the HTTP request in Get()
+func Get(string) (string,error) {} // Takes the url as an argument, returns HTML string
+func GetWithClient(string, *http.Client) {} // Takes the url and a custom HTTP client as arguments, returns HTML string
+func Post(string, string, interface{}) (string, error) {} // Takes the url, bodyType, and payload as an argument, returns HTML string
+func PostForm(string, url.Values) {} // Takes the url and body. bodyType is set to "application/x-www-form-urlencoded"
+func Header(string, string) {} // Takes key,value pair to set as headers for the HTTP request made in Get()
+func Cookie(string, string) {} // Takes key, value pair to set as cookies to be sent with the HTTP request in Get()
 func HTMLParse(string) Root {} // Takes the HTML string as an argument, returns a pointer to the DOM constructed
 func Find([]string) Root {} // Element tag,(attribute key-value pair) as argument, pointer to first occurence returned
 func FindAll([]string) []Root {} // Same as Find(), but pointers to all occurrences returned

soup.go

@@ -6,9 +6,14 @@ package soup
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
+	"net/url"
+	netURL "net/url"
 	"regexp"
 	"strings"
 
@@ -35,6 +40,10 @@ const (
 	ErrCreatingGetRequest
 	// ErrInGetRequest will be returned when there was an error during the get request
 	ErrInGetRequest
+	// ErrCreatingPostRequest will be returned when the post request couldn't be created
+	ErrCreatingPostRequest
+	// ErrMarshallingPostRequest will be returned when the body of a post request couldn't be serialized
+	ErrMarshallingPostRequest
 	// ErrReadingResponse will be returned if there was an error reading the response to our get request
 	ErrReadingResponse
 )
@@ -99,10 +108,34 @@ func GetWithClient(url string, client *http.Client) (string, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		if debug {
-			panic("Couldn't perform GET request to " + url)
+			panic("Couldn't create GET request to " + url)
 		}
 		return "", newError(ErrCreatingGetRequest, "error creating get request to "+url)
 	}
+
+	setHeadersAndCookies(req)
+
+	// Perform request
+	resp, err := client.Do(req)
+	if err != nil {
+		if debug {
+			panic("Couldn't perform GET request to " + url)
+		}
+		return "", newError(ErrInGetRequest, "couldn't perform GET request to "+url)
+	}
+	defer resp.Body.Close()
+	bytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		if debug {
+			panic("Unable to read the response body")
+		}
+		return "", newError(ErrReadingResponse, "unable to read the response body")
+	}
+	return string(bytes), nil
+}
+
+// setHeadersAndCookies helps build a request
+func setHeadersAndCookies(req *http.Request) {
 	// Set headers
 	for hName, hValue := range Headers {
 		req.Header.Set(hName, hValue)
@@ -114,13 +147,66 @@ func GetWithClient(url string, client *http.Client) (string, error) {
 			Value: cValue,
 		})
 	}
+}
+
+// getBodyReader serializes the body for a network request. See the test file for examples
+func getBodyReader(rawBody interface{}) (io.Reader, error) {
+	var bodyReader io.Reader
+
+	if rawBody != nil {
+		switch body := rawBody.(type) {
+		case map[string]string:
+			jsonBody, err := json.Marshal(body)
+			if err != nil {
+				if debug {
+					panic("Unable to read the response body")
+				}
+				return nil, newError(ErrMarshallingPostRequest, "couldn't serialize map of strings to JSON.")
+			}
+			bodyReader = bytes.NewBuffer(jsonBody)
+		case netURL.Values:
+			bodyReader = strings.NewReader(body.Encode())
+		case []byte: //expects JSON format
+			bodyReader = bytes.NewBuffer(body)
+		case string: //expects JSON format
+			bodyReader = strings.NewReader(body)
+		default:
+			return nil, newError(ErrMarshallingPostRequest, fmt.Sprintf("Cannot handle body type %T", rawBody))
+		}
+	}
+
+	return bodyReader, nil
+}
+
+// PostWithClient returns the HTML returned by the url using a provided HTTP client
+// The type of the body must conform to one of the types listed in func getBodyReader()
+func PostWithClient(url string, bodyType string, body interface{}, client *http.Client) (string, error) {
+	bodyReader, err := getBodyReader(body)
+	if err != nil {
+		return "todo:", err
+	}
+
+	req, err := http.NewRequest("POST", url, bodyReader)
+	Header("Content-Type", bodyType)
+	setHeadersAndCookies(req)
+
+	if debug {
+		// Save a copy of this request for debugging.
+		requestDump, err := httputil.DumpRequest(req, true)
+		if err != nil {
+			fmt.Println(err)
+		}
+		fmt.Println(string(requestDump))
+	}
+
 	// Perform request
 	resp, err := client.Do(req)
+
 	if err != nil {
 		if debug {
-			panic("Couldn't perform GET request to " + url)
+			panic("Couldn't perform POST request to " + url)
 		}
-		return "", newError(ErrInGetRequest, "couldn't perform GET request to "+url)
+		return "", newError(ErrCreatingPostRequest, "couldn't perform POST request to"+url)
 	}
 	defer resp.Body.Close()
 	bytes, err := ioutil.ReadAll(resp.Body)
@@ -133,7 +219,17 @@ func GetWithClient(url string, client *http.Client) (string, error) {
 	return string(bytes), nil
 }
 
-// Get returns the HTML returned by the url in string using the default HTTP client
+// Post returns the HTML returned by the url as a string using the default HTTP client
+func Post(url string, bodyType string, body interface{}) (string, error) {
+	return PostWithClient(url, bodyType, body, defaultClient)
+}
+
+// PostForm is a convenience method for POST requests that
+func PostForm(url string, data url.Values) (string, error) {
+	return PostWithClient(url, "application/x-www-form-urlencoded", data, defaultClient)
+}
+
+// Get returns the HTML returned by the url as a string using the default HTTP client
 func Get(url string) (string, error) {
 	return GetWithClient(url, defaultClient)
 }

soup_test.go

@@ -1,6 +1,11 @@
 package soup
 
 import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"strconv"
 	"strings"
 	"testing"
@@ -177,7 +182,113 @@ func TestFindReturnsInspectableError(t *testing.T) {
 	assert.Equal(t, ErrElementNotFound, r.Error.(Error).Type)
 }
 
+// Similar test: https://github.com/hashicorp/go-retryablehttp/blob/master/client_test.go#L616
+func TestClient_Post(t *testing.T) {
+	// Mock server which always responds 200.
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatalf("bad method: %s", r.Method)
+		}
+		if r.RequestURI != "/foo/bar" {
+			t.Fatalf("bad uri: %s", r.RequestURI)
+		}
+		if ct := r.Header.Get("Content-Type"); ct != "application/json" {
+			t.Fatalf("bad content-type: %s", ct)
+		}
+
+		// Check the payload
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		expected := []byte(`{"hello":"world"}`)
+		if !bytes.Equal(body, expected) {
+			t.Fatalf("bad: %v", string(body))
+		}
+
+		w.WriteHeader(200)
+	}))
+	defer ts.Close()
+
+	// Make the request with JSON payload
+	_, err := Post(
+		ts.URL+"/foo/bar", "application/json", `{"hello":"world"}`)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Make the request with byte payload
+	_, err = Post(
+		ts.URL+"/foo/bar", "application/json", []byte(`{"hello":"world"}`))
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Make the request with string map payload
+	_, err = Post(
+		ts.URL+"/foo/bar",
+		"application/json",
+		map[string]string{
+			"hello": "world",
+		})
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+}
+
+// Similar test: https://github.com/hashicorp/go-retryablehttp/blob/add-circleci/client_test.go#L631
+func TestClient_PostForm(t *testing.T) {
+	// Mock server which always responds 200.
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatalf("bad method: %s", r.Method)
+		}
+		if r.RequestURI != "/foo/bar" {
+			t.Fatalf("bad uri: %s", r.RequestURI)
+		}
+		if ct := r.Header.Get("Content-Type"); ct != "application/x-www-form-urlencoded" {
+			t.Fatalf("bad content-type: %s", ct)
+		}
+
+		// Check the payload
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		expected := []byte(`hello=world`)
+		if !bytes.Equal(body, expected) {
+			t.Fatalf("bad: %v", string(body))
+		}
+
+		w.WriteHeader(200)
+	}))
+	defer ts.Close()
+
+	// Create the form data.
+	form1, err := url.ParseQuery("hello=world")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	form2 := url.Values{
+		"hello": []string{"world"},
+	}
+
+	// Make the request.
+	_, err = PostForm(ts.URL+"/foo/bar", form1)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Make the request.
+	_, err = PostForm(ts.URL+"/foo/bar", form2)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+}
+
 func TestHTML(t *testing.T) {
 	li := doc.Find("ul").Find("li")
 	assert.Equal(t, "<li>To a <a href=\"hello.jsp\">JSP page</a> right?</li>", li.HTML())
+
 }