Merge remote-tracking branch 'origin/master' into ethereum-master

* origin/master: (23 commits)
  (#1919) Migrate to AddWatchedAddresses (#1920)
  Do not mark wallet address as watched + fix earlier txn errors
  Refactor zcash buildtx
  Import latest zcash changes
  Update make openbazaard command to not include tag
  Point Dockfiles at public Docker Hub repo; Add make help
  Bump version to 0.13.7
  Update zcash branchID
  Remove unused imports
  Fix linter issues
  Update transports
  Update Bitcoin Cash rates dialer
  Refactor Dockerfile.dev to derive from latest Dockerfile.qa
  Refactor Dockerfile.qa
  Add openbazaard and qa targets to Makefile; Fix docker build targets
  Add QA docker commands to Makefile
  QA dockerfile
  Add qa.sh script
  Fix transports for message retriever and images
  Fix formatted logging messages
  ...

 Conflicts:
        Makefile
        net/service/handlers.go
        qa/runtests.sh
        vendor/github.com/OpenBazaar/multiwallet/bitcoin/wallet.go
        vendor/github.com/OpenBazaar/multiwallet/bitcoincash/exchange_rates.go
        vendor/github.com/OpenBazaar/multiwallet/bitcoincash/wallet.go
        vendor/github.com/OpenBazaar/multiwallet/litecoin/wallet.go
        vendor/github.com/OpenBazaar/multiwallet/zcash/exchange_rates.go
        vendor/github.com/OpenBazaar/multiwallet/zcash/wallet.go

Author: placer14

Dockerfile.dev

@@ -1,28 +1,4 @@
-FROM golang:1.11
-VOLUME /var/lib/openbazaar
-
-RUN wget https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz && \
-    tar xvf Python-3.6.0.tgz && \
-    cd Python-3.6.0 && \
-    ./configure --enable-optimizations && \
-    make -j8
-RUN apt-get update && apt-get install -yq zlib1g-dev libssl-dev unzip
-RUN cd Python-3.6.0 && \
-    make altinstall && \
-    ln -s /usr/local/bin/python3.6 /usr/local/bin/python3
-
-COPY ./qa/requirements.txt ./requirements.txt
-
-RUN pip3.6 install --upgrade pip && \
-    pip3.6 install -r requirements.txt && \
-    wget https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz && \
-    tar -xvzf bitcoin-0.16.3-x86_64-linux-gnu.tar.gz -C /opt
-
-RUN wget https://github.com/google/protobuf/releases/download/v3.6.0/protoc-3.6.0-linux-x86_64.zip && \
-    unzip ./protoc-3.6.0-linux-x86_64.zip -x readme.txt && \
-    mv ./include/* /usr/local/include/ && \
-    mv ./bin/protoc /usr/local/bin/ && \
-    rm -rf ./include ./bin
+FROM openbazaar/server-qa:0.10
 
 RUN go get -u github.com/gogo/protobuf/proto \
               github.com/golang/protobuf/protoc-gen-go \

Dockerfile.qa

@@ -0,0 +1,34 @@
+FROM golang:1.11
+
+ENV BITCOIND_VERSION=0.16.3
+ENV BITCOIND_PATH=/opt/bitcoin-${BITCOIND_VERSION}
+
+# software installs, from most stable to most volatile
+RUN apt-get update -y
+RUN apt-get install -yq software-properties-common \
+                        zlib1g-dev \
+                        libssl-dev \
+                        unzip \
+                        python3 \
+                        python3-pip
+
+RUN wget https://github.com/google/protobuf/releases/download/v3.6.0/protoc-3.6.0-linux-x86_64.zip && \
+    unzip ./protoc-3.6.0-linux-x86_64.zip -x readme.txt && \
+    mv ./include/* /usr/local/include/ && \
+    mv ./bin/protoc /usr/local/bin/ && \
+    rm -rf ./include ./bin
+
+COPY ./qa/requirements.txt ./requirements.txt
+
+RUN python3 -m pip install --upgrade pip && \
+    pip install -r ./requirements.txt && \
+    wget https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz && \
+    tar -xvzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz -C /opt
+
+WORKDIR /go/src/github.com/OpenBazaar/openbazaar-go
+
+COPY ./Makefile ./Makefile
+
+VOLUME /go/src/github.com/OpenBazaar/openbazaar-go
+
+CMD make qa_test

Makefile

@@ -1,11 +1,31 @@
+.DEFAULT_GOAL := help
+
+##
+## Global ENV vars
+##
+
+GIT_SHA ?= $(shell git rev-parse --short=8 HEAD)
+GIT_TAG ?= $(shell git describe --tags --abbrev=0)
+
+##
+## Helpful Help
+##
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+
 ##
 ## Building
 ##
 
-ios_framework:
+.PHONY: ios_framework
+ios_framework: ## Build iOS Framework for mobile
 	gomobile bind -target=ios github.com/OpenBazaar/openbazaar-go/mobile
 
-android_framework:
+.PHONY: android_framework
+android_framework: ## Build Android Framework for mobile
 	gomobile bind -target=android github.com/OpenBazaar/openbazaar-go/mobile
 
 ##
@@ -17,32 +37,62 @@ P_ANY = Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any
 PKGMAP = $(P_TIMESTAMP),$(P_ANY)
 
 .PHONY: protos
-protos:
+protos: ## Build go files for proto definitions
 	cd pb/protos && PATH=$(PATH):$(GOPATH)/bin protoc --go_out=$(PKGMAP):.. *.proto
 
+
+##
+## Testing
+##
+OPENBAZAARD_NAME ?= openbazaard-$(GIT_SHA)
+BITCOIND_PATH ?= .
+
+.PHONY: openbazaard
+openbazaard: ## Build daemon
+	$(info "Building openbazaar daemon...")
+	go build -o ./$(OPENBAZAARD_NAME) .
+
+.PHONY: qa_test
+qa_test: openbazaard ## Run QA test suite against current working copy
+	$(info "Running QA... (openbazaard: ../$(OPENBAZAARD_NAME) bitcoind: $(BITCOIND_PATH)/bin/bitcoind)")
+	(cd qa && ./runtests.sh ../$(OPENBAZAARD_NAME) $(BITCOIND_PATH)/bin/bitcoind)
+
+.PHONY: qa_eth_test
+qa_eth_test: openbazaard ## Run ETH-based QA test suite against current working copy
+	$(info "Running ETH QA... (openbazaard: ../$(OPENBAZAARD_NAME))")
+	(cd qa && ./runtests_eth.sh ../$(OPENBAZAARD_NAME))
+
 ##
 ## Docker
 ##
-DOCKER_PROFILE ?= openbazaar
-DOCKER_VERSION ?= $(shell git describe --tags --abbrev=0)
-DOCKER_IMAGE_NAME ?= $(DOCKER_PROFILE)/server:$(DOCKER_VERSION)
-
-.PHONY: docker
-docker:
-	docker build -t $(DOCKER_IMAGE_NAME) .
-
-.PHONY: push_docker
-push_docker:
-	docker push $(DOCKER_IMAGE_NAME)
-
-.PHONY: qa
-qa:
-	go build -o ./openbazaar-qa ./openbazaard.go
-	(cd qa && ./runtests.sh ../openbazaar-qa /opt/bitcoin-0.16.3/bin/bitcoind)
-	rm ./openbazaar-qa
-
-.PHONY: qa_eth
-qa_eth:
-	go build -o ./openbazaar-qa ./openbazaard.go
-	(cd qa && ./runtests_eth.sh ../openbazaar-qa)
-	rm ./openbazaar-qa
+PUBLIC_DOCKER_REGISTRY ?= openbazaar
+QA_DEV_TAG ?= 0.10
+
+DOCKER_SERVER_IMAGE_NAME ?= $(PUBLIC_DOCKER_REGISTRY)/server:$(GIT_TAG)
+DOCKER_QA_IMAGE_NAME ?= $(PUBLIC_DOCKER_REGISTRY)/server-qa:$(QA_DEV_TAG)
+DOCKER_DEV_IMAGE_NAME ?= $(PUBLIC_DOCKER_REGISTRY)/server-dev:$(QA_DEV_TAG)
+
+
+.PHONY: docker_build
+docker_build: ## Build container for daemon
+	docker build -t $(DOCKER_SERVER_IMAGE_NAME) .
+
+.PHONY: docker_push
+docker_push: docker ## Push container for daemon
+	docker push $(DOCKER_SERVER_IMAGE_NAME)
+
+.PHONY: qa_docker_build
+qa_docker_build: ## Build container with QA test dependencies included
+	docker build -t $(DOCKER_QA_IMAGE_NAME) -f ./Dockerfile.qa .
+
+.PHONY: qa_docker_push
+qa_docker_push: qa_docker_build ## Push container for daemon QA test environment
+	docker push $(DOCKER_QA_IMAGE_NAME)
+
+.PHONY: dev_docker_build
+dev_docker: ## Build container with dev dependencies included
+	docker build -t $(DOCKER_DEV_IMAGE_NAME) -f ./Dockerfile.dev .
+
+.PHONY: dev_docker_push
+dev_docker_push: dev_docker_build ## Push container for daemon dev environment
+	docker push $(DOCKER_DEV_IMAGE_NAME)

cmd/start.go

@@ -61,11 +61,11 @@ import (
 )
 
 var stdoutLogFormat = logging.MustStringFormatter(
-	`%{color:reset}%{color}%{time:15:04:05.000} [%{level}] [%{module}/%{shortfunc}] %{message}`,
+	`%{color:reset}%{color}%{time:2006-01-02 15:04:05.000} [%{level}] [%{module}/%{shortfunc}] %{message}`,
 )
 
 var fileLogFormat = logging.MustStringFormatter(
-	`%{time:15:04:05.000} [%{level}] [%{module}/%{shortfunc}] %{message}`,
+	`%{time:2006-01-02 15:04:05.000} [%{level}] [%{module}/%{shortfunc}] %{message}`,
 )
 
 var ErrNoGateways = errors.New("no gateway addresses configured")

core/core.go

@@ -30,7 +30,7 @@ import (
 
 const (
 	// VERSION - current version
-	VERSION = "0.13.6"
+	VERSION = "0.13.7"
 	// USERAGENT - user-agent header string
 	USERAGENT = "/openbazaar-go:" + VERSION + "/"
 )

core/images.go

@@ -9,7 +9,6 @@ import (
 	_ "image/png"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	netUrl "net/url"
 	"os"
@@ -167,12 +166,14 @@ func (n *OpenBazaarNode) FetchImage(peerID string, imageType string, size string
 
 // GetBase64Image - fetch the image and return it as base64 encoded string
 func (n *OpenBazaarNode) GetBase64Image(url string) (base64ImageData, filename string, err error) {
-	dial := net.Dial
+	var client *http.Client
 	if n.TorDialer != nil {
-		dial = n.TorDialer.Dial
+		tbTransport := &http.Transport{Dial: n.TorDialer.Dial}
+		client = &http.Client{Transport: tbTransport, Timeout: time.Second * 30}
+	} else {
+		client = &http.Client{Timeout: time.Second * 30}
 	}
-	tbTransport := &http.Transport{Dial: dial}
-	client := &http.Client{Transport: tbTransport, Timeout: time.Second * 30}
+
 	resp, err := client.Get(url)
 	if err != nil {
 		return "", "", err

core/order.go

@@ -278,7 +278,7 @@ func prepareModeratedOrderContract(data *repo.PurchaseData, n *OpenBazaarNode, c
 	fee := wal.GetFeePerByte(wallet.NORMAL)
 	contract.BuyerOrder.BigRefundFee = fee.String()
 
-	err = wal.AddWatchedAddress(addr)
+	err = wal.AddWatchedAddresses(addr)
 	if err != nil {
 		return nil, err
 	}
@@ -315,7 +315,7 @@ func processOnlineDirectOrder(resp *pb.Message, n *OpenBazaarNode, wal wallet.Wa
 	if err != nil {
 		return "", "", *big.NewInt(0), false, err
 	}
-	err = wal.AddWatchedAddress(addr)
+	err = wal.AddWatchedAddresses(addr)
 	if err != nil {
 		return "", "", *big.NewInt(0), false, err
 	}
@@ -374,7 +374,7 @@ func processOfflineDirectOrder(n *OpenBazaarNode, wal wallet.Wallet, contract *p
 	payment.RedeemScript = hex.EncodeToString(redeemScript)
 	payment.Chaincode = hex.EncodeToString(chaincode)
 
-	err = wal.AddWatchedAddress(addr)
+	err = wal.AddWatchedAddresses(addr)
 	if err != nil {
 		return "", "", *big.NewInt(0), err
 	}

core/ulimit_unix.go

@@ -4,6 +4,7 @@ package core
 
 import (
 	"fmt"
+	"runtime"
 	"syscall"
 )
 
@@ -27,6 +28,16 @@ func CheckAndSetUlimit() error {
 		setting = true
 	}
 
+	// If we're on darwin, work around the fact that Getrlimit reports
+	// the wrong value. See https://github.com/golang/go/issues/30401
+	if runtime.GOOS == "darwin" && rLimit.Cur > 10240 {
+		// The max file limit is 10240, even though
+		// the max returned by Getrlimit is 1<<63-1.
+		// This is OPEN_MAX in sys/syslimits.h.
+		rLimit.Max = 10240
+		rLimit.Cur = 10240
+	}
+
 	// Try updating the limit. If it fails, try using the previous maximum instead
 	// of our new maximum. Not all users have permissions to increase the maximum.
 	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
@@ -44,6 +55,6 @@ func CheckAndSetUlimit() error {
 		return nil
 	}
 
-	log.Debug("Successfully raised file descriptor limit to", fileDescriptorLimit)
+	log.Debug("Successfully raised file descriptor limit to", rLimit.Cur)
 	return nil
 }

docs/install-linux.md

@@ -54,7 +54,7 @@ It will use git to checkout the source code into `$GOPATH/src/github.com/OpenBaz
 
 Checkout a release version:
 ```
-git checkout v0.13.6
+git checkout v0.13.7
 ```
 
 Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use in checkout.

docs/install-osx.md

@@ -56,7 +56,7 @@ It will use git to checkout the source code into `$GOPATH/src/github.com/OpenBaz
 
 Checkout a release version:
 ```
-git checkout v0.13.6
+git checkout v0.13.7
 ```
 
 Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use in checkout.