first commit

Author: ebuildy

.gitignore

@@ -0,0 +1,3 @@
+.idea
+_dist
+bin

.golangci.yml

@@ -0,0 +1,27 @@
+run:
+  timeout: 2m
+
+linters:
+  disable-all: true
+  enable:
+    - deadcode
+    - dupl
+    - gofmt
+    - goimports
+    - golint
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - structcheck
+    - unused
+    - varcheck
+
+linters-settings:
+  gofmt:
+    simplify: true
+  goimports:
+    local-prefixes: github.com/ebuildy/elastic-copy
+  dupl:
+    threshold: 400

LICENSE

@@ -0,0 +1,154 @@
+BINDIR     := $(CURDIR)/bin
+DIST_DIRS  := find * -type d -exec
+TARGETS    := darwin/amd64 linux/amd64#linux/386 linux/arm linux/arm64 linux/ppc64le windows/amd64
+BINNAME    ?= elasticcopy
+
+GOPATH        = $(shell go env GOPATH)
+DEP           = $(GOPATH)/bin/dep
+GOX           = $(GOPATH)/bin/gox
+GOIMPORTS     = $(GOPATH)/bin/goimports
+NAMESPACE     = github.com/ebuildy/elastic-copy
+
+ACCEPTANCE_DIR:=$(GOPATH)/src/helm.sh/acceptance-testing
+# To specify the subset of acceptance tests to run. '.' means all tests
+ACCEPTANCE_RUN_TESTS=.
+
+# go option
+PKG        := ./...
+TAGS       :=
+TESTS      := .
+TESTFLAGS  :=
+LDFLAGS    := -w -s
+GOFLAGS    :=
+SRC        := $(shell find . -type f -name '*.go' -print)
+
+# Required for globs to work correctly
+SHELL      = /bin/bash
+
+GIT_COMMIT = $(shell git rev-parse HEAD)
+GIT_SHA    = $(shell git rev-parse --short HEAD)
+GIT_TAG    = $(shell git describe --tags --abbrev=0 2>/dev/null)
+GIT_DIRTY  = $(shell test -n "`git status --porcelain`" && echo "dirty" || echo "clean")
+
+ifdef VERSION
+	BINARY_VERSION = $(VERSION)
+endif
+BINARY_VERSION ?= ${GIT_TAG}
+
+# Only set Version if building a tag or VERSION is set
+ifneq ($(BINARY_VERSION),)
+	LDFLAGS += -X /internal/version.version=${BINARY_VERSION}
+endif
+
+# Clear the "unreleased" string in BuildMetadata
+ifneq ($(GIT_TAG),)
+	LDFLAGS += -X ${NAMESPACE}/internal/version.metadata=
+endif
+LDFLAGS += -X ${NAMESPACE}/internal/version.gitCommit=${GIT_COMMIT}
+LDFLAGS += -X ${NAMESPACE}/internal/version.gitTreeState=${GIT_DIRTY}
+
+.PHONY: all
+all: build
+
+# ------------------------------------------------------------------------------
+#  build
+
+.PHONY: build
+build: $(BINDIR)/$(BINNAME)
+
+$(BINDIR)/$(BINNAME): $(SRC)
+	GO111MODULE=on go build $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) .
+
+# ------------------------------------------------------------------------------
+#  test
+
+.PHONY: test
+test: build
+test: TESTFLAGS += -race -v
+test: test-style
+test: test-unit
+
+.PHONY: test-unit
+test-unit:
+	@echo
+	@echo "==> Running unit tests <=="
+	GO111MODULE=on go test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS)
+
+.PHONY: test-coverage
+test-coverage:
+	@echo
+	@echo "==> Running unit tests with coverage <=="
+	@ ./.ci/scripts/coverage.sh
+
+.PHONY: test-acceptance
+test-acceptance: TARGETS = linux/amd64
+test-acceptance: build build-cross
+	@if [ -d "${ACCEPTANCE_DIR}" ]; then \
+		cd ${ACCEPTANCE_DIR} && \
+			ROBOT_RUN_TESTS=$(ACCEPTANCE_RUN_TESTS) ROBOT_HELM_PATH=$(BINDIR) make acceptance; \
+	else \
+		echo "You must clone the acceptance_testing repo under $(ACCEPTANCE_DIR)"; \
+		echo "You can find the acceptance_testing repo at https://github.com/helm/acceptance-testing"; \
+	fi
+
+.PHONY: test-acceptance-completion
+test-acceptance-completion: ACCEPTANCE_RUN_TESTS = shells.robot
+test-acceptance-completion: test-acceptance
+
+.PHONY: coverage
+coverage:
+	@scripts/coverage.sh
+
+.PHONY: format
+format: $(GOIMPORTS)
+	GO111MODULE=on go list -f '{{.Dir}}' ./... | xargs $(GOIMPORTS) -w -local ${NAMESPACE}
+
+# ------------------------------------------------------------------------------
+#  dependencies
+
+# If go get is run from inside the project directory it will add the dependencies
+# to the go.mod file. To avoid that we change to a directory without a go.mod file
+# when downloading the following dependencies
+
+$(GOX):
+	(cd /; GO111MODULE=on go get -u github.com/mitchellh/gox)
+
+$(GOIMPORTS):
+	(cd /; GO111MODULE=on go get -u golang.org/x/tools/cmd/goimports)
+
+# ------------------------------------------------------------------------------
+#  release
+
+.PHONY: build-cross
+build-cross: LDFLAGS += -extldflags "-static"
+build-cross: $(GOX)
+	GO111MODULE=on CGO_ENABLED=0 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' .
+
+.PHONY: dist
+dist:
+	( \
+		cd _dist && \
+		$(DIST_DIRS) cp ../LICENSE {} \; && \
+		$(DIST_DIRS) cp ../README.md {} \; && \
+		$(DIST_DIRS) tar -zcf ${BINNAME}-${BINARY_VERSION}-{}.tar.gz {} \; && \
+		$(DIST_DIRS) zip -r ${BINNAME}-${BINARY_VERSION}-{}.zip {} \; \
+	)
+
+.PHONY: checksum
+checksum:
+	for f in _dist/*.{gz,zip} ; do \
+		shasum -a 256 "$${f}"  | awk '{print $$1}' > "$${f}.sha256" ; \
+	done
+
+# ------------------------------------------------------------------------------
+
+.PHONY: clean
+clean:
+	@rm -rf $(BINDIR) ./_dist
+
+.PHONY: info
+info:
+	 @echo "Version:           ${BINARY_VERSION}"
+	 @echo "Git Tag:           ${GIT_TAG}"
+	 @echo "Git Commit:        ${GIT_COMMIT}"
+	 @echo "Git Tree State:    ${GIT_DIRTY}"

Makefile

@@ -0,0 +1,27 @@
+package commands
+
+import (
+	"github.com/ebuildy/elastic-copy/pkg/action"
+	"github.com/spf13/cobra"
+)
+
+func newCountCmd()  *cobra.Command {
+	action := action.NewCountAction(settings)
+
+	cmd := &cobra.Command{
+		Use:     "count",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			action.Run()
+
+			return nil
+		},
+	}
+
+	flags := cmd.PersistentFlags()
+
+	flags.StringVar(&action.Source, "source", "http://localhost:9200", "source URL")
+	flags.StringVar(&action.Query, "query", "", "")
+	flags.StringArrayVar(&action.Indices, "indices", nil, "indices to copy")
+
+	return cmd
+}

commands/count.go

@@ -0,0 +1,44 @@
+package commands
+
+import (
+	"fmt"
+	"github.com/ebuildy/elastic-copy/pkg/cli"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"io"
+)
+
+var (
+	settings = cli.New()
+)
+
+func NewRootCmd(out io.Writer, args []string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "elasticcopy",
+		Short:        "Copy data from & to elasticsearch",
+		SilenceUsage: true,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			l, err := logrus.ParseLevel(settings.LogLevel)
+
+			if err != nil {
+				fmt.Printf("log level \"%s\" is invalid!", settings.LogLevel)
+				l = logrus.InfoLevel
+			}
+
+			logrus.SetLevel(l)
+		},
+	}
+
+	cmd.AddCommand(
+		newRunCmd(),
+		newSamplerCmd(),
+		newCountCmd(),
+	)
+
+	flags := cmd.PersistentFlags()
+
+	flags.ParseErrorsWhitelist.UnknownFlags = true
+	flags.Parse(args)
+
+	return cmd
+}

commands/root.go

@@ -0,0 +1,34 @@
+package commands
+
+import (
+	"github.com/ebuildy/elastic-copy/pkg/action"
+	"github.com/ebuildy/elastic-copy/pkg/utils"
+	"github.com/spf13/cobra"
+)
+
+func newRunCmd()  *cobra.Command {
+	action := action.NewRunAction(settings)
+
+	cmd := &cobra.Command{
+		Use:     "run",
+		Aliases: []string{"gen"},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			action.Run()
+
+			return nil
+		},
+	}
+
+	flags := cmd.PersistentFlags()
+
+	flags.StringVar(&action.Source, "source", "http://localhost:9200", "source URL")
+	flags.StringVar(&action.Target, "target", utils.TARGET_STDOUT, "target URL")
+	flags.StringVar(&action.Query, "query", "", "")
+	flags.StringArrayVar(&action.Indices, "indices", nil, "indices to copy")
+	flags.Uint64Var(&action.Count, "count", 0, "0 => all, X => count")
+	flags.IntVar(&action.ReadBatchSize, "read_batch", 100, "how many documents to read in one scroll")
+	flags.IntVar(&action.WriteBatchSize, "write_batch", 20, "how many documents to send to writer in one batch")
+	flags.IntVar(&action.Threads, "threads", 5, "Number of threads in pool")
+
+	return cmd
+}

commands/run.go

@@ -0,0 +1,27 @@
+package commands
+
+import (
+	"github.com/ebuildy/elastic-copy/pkg/action"
+	"github.com/spf13/cobra"
+)
+
+func newSamplerCmd()  *cobra.Command {
+	action := action.NewSampleAction(settings)
+
+	cmd := &cobra.Command{
+		Use:     "sample",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			action.Run()
+
+			return nil
+		},
+	}
+
+	flags := cmd.PersistentFlags()
+
+	flags.StringVar(&action.Target, "target", "http://localhost:9200", "target URL")
+	flags.StringVar(&action.Index, "index", "", "")
+	flags.IntVar(&action.Count, "count", 100, "")
+
+	return cmd
+}

commands/sampler.go

@@ -0,0 +1,20 @@
+version: "3"
+
+services:
+  es2:
+    image: elasticsearch:2.4
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+    - 9200:9200
+
+  es7:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.4.2
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+    - 9201:9200

docker-compose.yml

@@ -0,0 +1,15 @@
+module github.com/ebuildy/elastic-copy
+
+go 1.13
+
+require (
+	github.com/cheggaaa/pb/v3 v3.0.4
+	github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20200210103642-33004104143c
+	github.com/panjf2000/ants/v2 v2.3.0
+	github.com/sirupsen/logrus v1.4.2
+	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.3
+	github.com/stretchr/testify v1.4.0
+	github.com/tidwall/gjson v1.5.0
+	github.com/tidwall/pretty v1.0.1 // indirect
+)