Merge branch 'master' into dag/put#option

Author: postables

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "testenv"]
+	path = testenv
+	url = https://github.com/RTradeLtd/testenv.git

.travis.yml

@@ -1,42 +1,22 @@
-os:
-  - linux
-
 language: go
-
+sudo: required
 go:
-  - 1.11.x
-
+  - "1.12"
 services:
-  - docker
-
-env:
-  global:
-    - GOTFLAGS="-race"
-    - IPFS_PATH=/tmp/ipfs
-  matrix:
-    - BUILD_DEPTYPE=gx
-    - BUILD_DEPTYPE=gomod
+- docker
+env: GO111MODULE=on
 
-before_install:
-  - docker pull ipfs/go-ipfs:master
-  - mkdir /tmp/ipfs && chmod 0777 /tmp/ipfs
-  - docker run -d -v /tmp/ipfs:/data/ipfs -p 8080:8080 -p 4001:4001 -p 5001:5001 ipfs/go-ipfs:master --enable-pubsub-experiment
-
-
-# disable travis install
-install:
-  - true
+before_script:
+  - go vet ./...
+  - go build ./...
+  - go test -run xxxx ./...
+  - make testenv
 
 script:
-  - go get -d -t github.com/cheekybits/is/... # remove with gx
-  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
-
+ - go test -race -failfast -coverprofile=coverage.txt ./...
 
-cache:
-  directories:
-    - $GOPATH/src/gx
-    - $GOPATH/pkg/mod
-    - $HOME/.cache/go-build
+after_success:
+- bash <(curl -s https://codecov.io/bash)
 
 notifications:
   email: false

Makefile

@@ -1,8 +1,21 @@
-all: deps
-gx:
-	go get github.com/whyrusleeping/gx
-	go get github.com/whyrusleeping/gx-go
-deps: gx
-	gx --verbose install --global
-	gx-go rewrite
-.PHONY: all gx deps
+GO=env GO111MODULE=on go
+GONOMOD=env GO111MODULE=off go
+
+.PHONY: deps
+deps:
+	$(GO) mod vendor
+	$(GO) mod verify
+		
+# Set up test environment
+.PHONY: testenv
+WAIT=3
+testenv:
+	@echo "===================   preparing test env    ==================="
+	( cd testenv ; make testenv )
+	@echo "===================          done           ==================="
+
+.PHONY: clean
+clean:
+	@echo "===================   stopping test env    ==================="
+	( cd testenv ; make clean )
+	@echo "===================          done           ==================="

dag.go

@@ -7,8 +7,8 @@ import (
 	"io"
 	"strings"
 
-	"github.com/ipfs/go-ipfs-api/options"
-	"github.com/ipfs/go-ipfs-files"
+	"github.com/RTradeLtd/go-ipfs-api/options"
+	files "github.com/ipfs/go-ipfs-files"
 )
 
 func (s *Shell) DagGet(ref string, out interface{}) error {

go.mod

@@ -1,9 +1,8 @@
-module github.com/ipfs/go-ipfs-api
+module github.com/RTradeLtd/go-ipfs-api
 
 require (
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927
 	github.com/ipfs/go-ipfs-files v0.0.1
-	github.com/ipfs/go-ipfs-util v0.0.1
 	github.com/libp2p/go-libp2p-metrics v0.0.1
 	github.com/libp2p/go-libp2p-peer v0.0.1
 	github.com/mitchellh/go-homedir v1.1.0

go.sum

@@ -23,8 +23,6 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ipfs/go-ipfs-files v0.0.1 h1:OroTsI58plHGX70HPLKy6LQhPR3HZJ5ip61fYlo6POM=
 github.com/ipfs/go-ipfs-files v0.0.1/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
-github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
-github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

pubsub.go

@@ -3,7 +3,7 @@ package shell
 import (
 	"encoding/json"
 
-	"github.com/libp2p/go-libp2p-peer"
+	peer "github.com/libp2p/go-libp2p-peer"
 )
 
 // Message is a pubsub message.

request.go

@@ -29,7 +29,6 @@ func NewRequest(ctx context.Context, url, command string, args ...string) *Reque
 	if !strings.HasPrefix(url, "http") {
 		url = "http://" + url
 	}
-
 	opts := map[string]string{
 		"encoding":        "json",
 		"stream-channels": "true",
@@ -114,14 +113,13 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 	if err != nil {
 		return nil, err
 	}
-
+  
 	req = req.WithContext(r.Ctx)
 
 	// Add any headers that were supplied via the RequestBuilder.
 	for k, v := range r.Headers {
 		req.Header.Add(k, v)
 	}
-
 	if fr, ok := r.Body.(*files.MultiFileReader); ok {
 		req.Header.Set("Content-Type", "multipart/form-data; boundary="+fr.Boundary())
 		req.Header.Set("Content-Disposition", "form-data; name=\"files\"")
@@ -149,18 +147,18 @@ func (r *Request) Send(c *http.Client) (*Response, error) {
 		case contentType == "text/plain":
 			out, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) read error: %s\n", resp.StatusCode, err)
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response read error: %s\n", err)
 			}
 			e.Message = string(out)
 		case contentType == "application/json":
 			if err = json.NewDecoder(resp.Body).Decode(e); err != nil {
-				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response (%d) unmarshall error: %s\n", resp.StatusCode, err)
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response unmarshall error: %s\n", err)
 			}
 		default:
-			fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response (%d) encoding: %s", resp.StatusCode, contentType)
+			fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response encoding: %s", contentType)
 			out, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "ipfs-shell: response (%d) read error: %s\n", resp.StatusCode, err)
+				fmt.Fprintf(os.Stderr, "ipfs-shell: response read error: %s\n", err)
 			}
 			e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out)
 		}
@@ -187,3 +185,58 @@ func (r *Request) getURL() string {
 
 	return fmt.Sprintf("%s/%s?%s", r.ApiBase, r.Command, values.Encode())
 }
+
+func (r *Request) SendGET(c *http.Client) (*Response, error) {
+	url := r.getURLNoEncode()
+	req, err := http.NewRequest("GET", url, r.Body)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	contentType := resp.Header.Get("Content-Type")
+	parts := strings.Split(contentType, ";")
+	contentType = parts[0]
+	nresp := new(Response)
+	nresp.Output = resp.Body
+	if resp.StatusCode >= http.StatusBadRequest {
+		e := &Error{
+			Command: r.Command,
+		}
+		switch {
+		case resp.StatusCode == http.StatusNotFound:
+			e.Message = "command not found"
+		case contentType == "text/plain":
+			out, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response read error: %s\n", err)
+			}
+			e.Message = string(out)
+		case contentType == "application/json":
+			if err = json.NewDecoder(resp.Body).Decode(e); err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: warning! response unmarshall error: %s\n", err)
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "ipfs-shell: warning! unhandled response encoding: %s", contentType)
+			out, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "ipfs-shell: response read error: %s\n", err)
+			}
+			e.Message = fmt.Sprintf("unknown ipfs-shell error encoding: %q - %q", contentType, out)
+		}
+		nresp.Error = e
+		nresp.Output = nil
+
+		// drain body and close
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}
+
+	return nresp, nil
+}
+
+func (r *Request) getURLNoEncode() string {
+	return fmt.Sprintf("%s/%s/%s", r.ApiBase, r.Command, r.Args[0])
+}

shell.go

@@ -72,6 +72,25 @@ func NewShell(url string) *Shell {
 	return NewShellWithClient(url, c)
 }
 
+// NewDirectShell creates a new shell that directly uses the provided URL,
+// instead of attempting to parse it into a multiaddr.
+//
+// For Nexus-hosted IPFS nodes, for example, use:
+//
+//     shell := NewDirectShell(fmt.Sprintf("nexus.temporal.cloud/network/%s", networkName))
+//
+func NewDirectShell(url string) *Shell {
+	return &Shell{
+		url: url,
+		httpcli: gohttp.Client{
+			Transport: &gohttp.Transport{
+				Proxy:             gohttp.ProxyFromEnvironment,
+				DisableKeepAlives: true,
+			},
+		},
+	}
+}
+
 func NewShellWithClient(url string, c *gohttp.Client) *Shell {
 	if a, err := ma.NewMultiaddr(url); err == nil {
 		_, host, err := manet.DialArgs(a)
@@ -89,6 +108,22 @@ func NewShellWithClient(url string, c *gohttp.Client) *Shell {
 	return &sh
 }
 
+// WithAuthorization returns a Shell that sets the provided token to be used as
+// an Authorization header in API requests. For example:
+//
+//    resp, err := NewDirectShell(addr).
+//        WithAuthorization(token).
+//        Cat(hash)
+//
+func (s *Shell) WithAuthorization(token string) *Shell {
+	return &Shell{
+		url: s.url,
+		httpcli: gohttp.Client{
+			Transport: newAuthenticatedTransport(s.httpcli.Transport, token),
+		},
+	}
+}
+
 func (s *Shell) SetTimeout(d time.Duration) {
 	s.httpcli.Timeout = d
 }
@@ -178,6 +213,15 @@ func (s *Shell) Pin(path string) error {
 		Exec(context.Background(), nil)
 }
 
+// PinUpdate is used to update one pin path to another followed by unpinning
+func (s *Shell) PinUpdate(fromPath, toPath string) (map[string][]string, error) {
+	var out map[string][]string
+	if err := s.Request("pin/update", fromPath, toPath).Exec(context.Background(), &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Unpin the given path
 func (s *Shell) Unpin(path string) error {
 	return s.Request("pin/rm", path).
@@ -222,9 +266,10 @@ func (s *Shell) FindPeer(peer string) (*PeerInfo, error) {
 	return &peers.Responses[0], nil
 }
 
-func (s *Shell) Refs(hash string, recursive bool) (<-chan string, error) {
+func (s *Shell) Refs(hash string, recursive, unique bool) (<-chan string, error) {
 	resp, err := s.Request("refs", hash).
 		Option("recursive", recursive).
+		Option("unique", unique).
 		Send(context.Background())
 	if err != nil {
 		return nil, err

shell_test.go

@@ -12,13 +12,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/RTradeLtd/go-ipfs-api/options"
 	"github.com/cheekybits/is"
-	"github.com/ipfs/go-ipfs-api/options"
 )
 
 const (
 	examplesHash = "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"
-	shellUrl     = "localhost:5001"
+	shellUrl     = "192.168.1.101:5001"
 )
 
 func TestAdd(t *testing.T) {
@@ -112,16 +112,6 @@ func TestAddDir(t *testing.T) {
 	is.Equal(cid, "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")
 }
 
-func TestLocalShell(t *testing.T) {
-	is := is.New(t)
-	s := NewLocalShell()
-	is.NotNil(s)
-
-	mhash, err := s.Add(bytes.NewBufferString("Hello IPFS Shell tests"))
-	is.Nil(err)
-	is.Equal(mhash, "QmUfZ9rAdhV5ioBzXKdUTh2ZNsz9bzbkaLVyQ8uc8pj21F")
-}
-
 func TestCat(t *testing.T) {
 	is := is.New(t)
 	s := NewShell(shellUrl)
@@ -387,7 +377,7 @@ func TestRefs(t *testing.T) {
 	cid, err := s.AddDir("./testdata")
 	is.Nil(err)
 	is.Equal(cid, "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv")
-	refs, err := s.Refs(cid, false)
+	refs, err := s.Refs(cid, false, false)
 	is.Nil(err)
 	expected := []string{
 		"QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V",