Merge branch 'master' into issue-1703

Author: kortatu

.travis.yml

@@ -15,12 +15,25 @@ jobs:
     - stage: lint
       os: linux
       dist: trusty
-      go: 1.12.x
+      go: 1.13.x
       env:
         - lint
       script:
         - go run build/ci.go lint
 
+    # Go 1.12.x is needed because of the Ubuntu PPA builds
+    - stage: build
+      os: linux
+      dist: trusty
+      sudo: required
+      go: 1.12.x
+      script:
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
     # Go 1.11.x is needed because of the Ubuntu PPA builds
     - stage: build
       os: linux
@@ -39,7 +52,7 @@ jobs:
       os: linux
       dist: trusty
       sudo: required
-      go: 1.12.x
+      go: 1.13.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -49,7 +62,7 @@ jobs:
 
     - stage: build
       os: osx
-      go: 1.12.x
+      go: 1.13.x
       script:
         - echo "Increase the maximum number of open file descriptors on macOS"
         - NOFILE=20480
@@ -68,7 +81,7 @@ jobs:
       if: type = push
       os: linux
       dist: trusty
-      go: 1.12.x
+      go: 1.13.x
       env:
         - ubuntu-ppa
       addons:
@@ -90,7 +103,7 @@ jobs:
       os: linux
       dist: trusty
       sudo: required
-      go: 1.12.x
+      go: 1.13.x
       env:
         - azure-linux
       addons:
@@ -124,7 +137,7 @@ jobs:
       dist: trusty
       services:
         - docker
-      go: 1.12.x
+      go: 1.13.x
       env:
         - azure-linux-mips
       script:
@@ -148,7 +161,7 @@ jobs:
     - stage: deploy
       if: type = push
       os: osx
-      go: 1.12.x
+      go: 1.13.x
       env:
         - azure-osx
       script:
@@ -161,7 +174,7 @@ jobs:
       if: type = cron
       os: linux
       dist: trusty
-      go: 1.12.x
+      go: 1.13.x
       env:
         - azure-purge
       script:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12-alpine as builder
+FROM golang:1.13-alpine as builder
 RUN apk add --no-cache make gcc musl-dev linux-headers git
 ADD . /swarm
 WORKDIR /swarm

Dockerfile.alltools

@@ -1,4 +1,4 @@
-FROM golang:1.12-alpine as builder
+FROM golang:1.13-alpine as builder
 RUN apk add --no-cache make gcc musl-dev linux-headers git
 ADD . /swarm
 WORKDIR /swarm

api/api_test.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	crand "crypto/rand"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -34,15 +33,14 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethersphere/swarm/chunk"
+	chunktesting "github.com/ethersphere/swarm/chunk/testing"
 	"github.com/ethersphere/swarm/sctx"
 	"github.com/ethersphere/swarm/storage"
 	"github.com/ethersphere/swarm/testutil"
 )
 
 func init() {
-	loglevel := flag.Int("loglevel", 2, "loglevel")
-	flag.Parse()
-	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
+	testutil.Init()
 }
 
 func testAPI(t *testing.T, f func(*API, *chunk.Tags, bool)) {
@@ -143,7 +141,7 @@ func TestApiPut(t *testing.T) {
 		resp := testGet(t, api, addr.Hex(), "")
 		checkResponse(t, resp, exp)
 		tag := tags.All()[0]
-		testutil.CheckTag(t, tag, 2, 2, 0, 2) //1 chunk data, 1 chunk manifest
+		chunktesting.CheckTag(t, tag, 2, 2, 0, 2) //1 chunk data, 1 chunk manifest
 	})
 }
 
@@ -170,11 +168,11 @@ func TestApiTagLarge(t *testing.T) {
 		if toEncrypt {
 			tag := tags.All()[0]
 			expect := int64(4095 + 64 + 1)
-			testutil.CheckTag(t, tag, expect, expect, 0, expect)
+			chunktesting.CheckTag(t, tag, expect, expect, 0, expect)
 		} else {
 			tag := tags.All()[0]
 			expect := int64(4095 + 32 + 1)
-			testutil.CheckTag(t, tag, expect, expect, 0, expect)
+			chunktesting.CheckTag(t, tag, expect, expect, 0, expect)
 		}
 	})
 }

api/client/client_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethersphere/swarm/api"
 	swarmhttp "github.com/ethersphere/swarm/api/http"
+	chunktesting "github.com/ethersphere/swarm/chunk/testing"
 	"github.com/ethersphere/swarm/storage"
 	"github.com/ethersphere/swarm/storage/feed"
 	"github.com/ethersphere/swarm/storage/feed/lookup"
@@ -50,7 +51,7 @@ func TestClientUploadDownloadRaw(t *testing.T) {
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 1, 1, 0, 1)
+	chunktesting.CheckTag(t, tag, 1, 1, 0, 1)
 }
 
 func TestClientUploadDownloadRawEncrypted(t *testing.T) {
@@ -68,7 +69,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) {
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 1, 1, 0, 1)
+	chunktesting.CheckTag(t, tag, 1, 1, 0, 1)
 }
 
 func testClientUploadDownloadRaw(srv *swarmhttp.TestSwarmServer, toEncrypt bool, t *testing.T, data []byte, toPin bool) string {
@@ -227,7 +228,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 9, 9, 0, 9)
+	chunktesting.CheckTag(t, tag, 9, 9, 0, 9)
 
 	// check we can download the individual files
 	checkDownloadFile := func(path string, expected []byte) {
@@ -371,7 +372,7 @@ func TestClientMultipartUpload(t *testing.T) {
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 9, 9, 0, 9)
+	chunktesting.CheckTag(t, tag, 9, 9, 0, 9)
 
 	// check we can download the individual files
 	checkDownloadFile := func(path string) {

api/http/server_test.go

@@ -23,7 +23,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -44,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethersphere/swarm/api"
 	"github.com/ethersphere/swarm/chunk"
+	chunktesting "github.com/ethersphere/swarm/chunk/testing"
 	"github.com/ethersphere/swarm/storage"
 	"github.com/ethersphere/swarm/storage/feed"
 	"github.com/ethersphere/swarm/storage/feed/lookup"
@@ -52,9 +52,7 @@ import (
 )
 
 func init() {
-	loglevel := flag.Int("loglevel", 2, "loglevel")
-	flag.Parse()
-	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
+	testutil.Init()
 }
 
 func serverFunc(api *api.API, pinAPI *pin.API) TestServer {
@@ -947,7 +945,7 @@ func testBzzTar(encrypted bool, t *testing.T) {
 
 	// check that the tag was written correctly
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 4, 4, 0, 4)
+	chunktesting.CheckTag(t, tag, 4, 4, 0, 4)
 
 	swarmHash, err := ioutil.ReadAll(resp2.Body)
 	resp2.Body.Close()
@@ -1083,7 +1081,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) {
 				<-time.After(10 * time.Millisecond)
 			case 1:
 				tag := srv.Tags.All()[0]
-				testutil.CheckTag(t, tag, 0, 0, 0, v.expChunks)
+				chunktesting.CheckTag(t, tag, 0, 0, 0, v.expChunks)
 				srv.Tags.Delete(tag.Uid)
 				done = true
 			}

api/http/test_server.go

@@ -57,7 +57,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API, *pin.API) TestSe
 	}
 
 	tags := chunk.NewTags()
-	fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams(), tags)
+	fileStore := storage.NewFileStore(localStore, localStore, storage.NewFileStoreParams(), tags)
 
 	// Swarm feeds test setup
 	feedsDir, err := ioutil.TempDir("", "swarm-feeds-test")

api/inspector.go

@@ -18,24 +18,29 @@ package api
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethersphere/swarm/log"
 	"github.com/ethersphere/swarm/network"
+	stream "github.com/ethersphere/swarm/network/stream/v2"
 	"github.com/ethersphere/swarm/storage"
 )
 
+const InspectorIsPullSyncingTolerance = 15 * time.Second
+
 type Inspector struct {
 	api      *API
 	hive     *network.Hive
 	netStore *storage.NetStore
+	stream   *stream.Registry
 }
 
-func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore) *Inspector {
-	return &Inspector{api, hive, netStore}
+func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore, pullSyncer *stream.Registry) *Inspector {
+	return &Inspector{api, hive, netStore, pullSyncer}
 }
 
 // Hive prints the kademlia table
@@ -49,15 +54,12 @@ func (i *Inspector) KademliaInfo() network.KademliaInfo {
 }
 
 func (i *Inspector) IsPullSyncing() bool {
-	lastReceivedChunksMsg := metrics.GetOrRegisterGauge("network.stream.received_chunks", nil)
-
-	// last received chunks msg time
-	lrct := time.Unix(0, lastReceivedChunksMsg.Value())
+	t := i.stream.LastReceivedChunkTime()
 
 	// if last received chunks msg time is after now-15sec. (i.e. within the last 15sec.) then we say that the node is still syncing
 	// technically this is not correct, because this might have been a retrieve request, but for the time being it works for our purposes
 	// because we know we are not making retrieve requests on the node while checking this
-	return lrct.After(time.Now().Add(-15 * time.Second))
+	return t.After(time.Now().Add(-InspectorIsPullSyncingTolerance))
 }
 
 // DeliveriesPerPeer returns the sum of chunks we received from a given peer
@@ -96,3 +98,15 @@ func (i *Inspector) Has(chunkAddresses []storage.Address) string {
 
 	return strings.Join(hostChunks, "")
 }
+
+func (i *Inspector) PeerStreams() (string, error) {
+	peerInfo, err := i.stream.PeerInfo()
+	if err != nil {
+		return "", err
+	}
+	v, err := json.Marshal(peerInfo)
+	if err != nil {
+		return "", err
+	}
+	return string(v), nil
+}

api/inspector_test.go

@@ -0,0 +1,66 @@
+package api
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/ethersphere/swarm/network"
+	stream "github.com/ethersphere/swarm/network/stream/v2"
+	"github.com/ethersphere/swarm/storage"
+	"github.com/ethersphere/swarm/storage/localstore"
+
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethersphere/swarm/state"
+)
+
+// TestInspectorPeerStreams validates that response from RPC peerStream has at
+// least some data.
+func TestInspectorPeerStreams(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	baseKey := make([]byte, 32)
+	_, err = rand.Read(baseKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	localStore, err := localstore.New(dir, baseKey, &localstore.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	netStore := storage.NewNetStore(localStore, baseKey, enode.ID{})
+
+	i := NewInspector(nil, nil, netStore, stream.New(state.NewInmemoryStore(), baseKey, stream.NewSyncProvider(netStore, network.NewKademlia(
+		baseKey,
+		network.NewKadParams(),
+	), false, false)))
+
+	server := rpc.NewServer()
+	if err := server.RegisterName("inspector", i); err != nil {
+		t.Fatal(err)
+	}
+
+	client := rpc.DialInProc(server)
+
+	var peerInfo string
+
+	err = client.Call(&peerInfo, "inspector_peerStreams")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(peerInfo, `"base":"`+hex.EncodeToString(baseKey)[:16]+`"`) {
+		t.Error("missing base key in response")
+	}
+
+	t.Log(peerInfo)
+}