Blocks copy tool (#169)

* Blocks copy tool
Signed-off-by: Peter Štibraný <[email protected]>

* Make lint happy.
Signed-off-by: Peter Štibraný <[email protected]>

* Check if block was copied first.
Signed-off-by: Peter Štibraný <[email protected]>

* Add basic metrics and http server for scraping them.
Signed-off-by: Peter Štibraný <[email protected]>

* Dockerfile and Makefile updates.
Signed-off-by: Peter Štibraný <[email protected]>

* Update cmd/blockscopy/main.go

Co-authored-by: Marco Pracucci <[email protected]>

Co-authored-by: Marco Pracucci <[email protected]>

Author: pstibrany

Makefile

@@ -14,6 +14,7 @@ cortextool: cmd/cortextool/cortextool
 chunktool: cmd/chunktool/chunktool
 logtool: cmd/logtool/logtool
 e2ealerting: cmd/e2ealerting/e2ealerting
+blockscopy: cmd/blockscopy/blockscopy
 
 benchtool-image:
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/benchtool -f cmd/benchtool/Dockerfile .
@@ -37,6 +38,10 @@ e2ealerting-image:
 push-e2ealerting-image: e2ealerting-image
 	$(SUDO) docker push $(IMAGE_PREFIX)/e2ealerting:$(IMAGE_TAG)
 
+blockscopy-image:
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/blockscopy -f cmd/blockscopy/Dockerfile .
+	$(SUDO) docker tag $(IMAGE_PREFIX)/blockscopy $(IMAGE_PREFIX)/blockscopy:$(IMAGE_TAG)
+
 cmd/benchtool/benchtool: $(APP_GO_FILES) cmd/benchtool/main.go
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
 
@@ -55,6 +60,9 @@ cmd/e2ealerting/e2ealerting: $(APP_GO_FILES) cmd/e2ealerting/main.go
 cmd/rules-migrator/rules-migrator: $(APP_GO_FILES) cmd/rules-migrator/main.go
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
 
+cmd/blockscopy/blockscopy: $(APP_GO_FILES) cmd/blockscopy/main.go
+	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+
 lint:
 	golangci-lint run -v
 
@@ -73,3 +81,4 @@ clean:
 	rm -rf cmd/chunktool/chunktool
 	rm -rf cmd/logtool/logtool
 	rm -rf cmd/e2ealerting/e2ealerting
+	rm -rf cmd/blockscopy/blockscopy

cmd/blockscopy/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.16.3-stretch as build
+ARG GOARCH="amd64"
+COPY . /build_dir
+WORKDIR /build_dir
+ENV GOPROXY=https://proxy.golang.org
+RUN make clean && make chunktool
+
+FROM       alpine:3.13
+RUN        apk add --update --no-cache ca-certificates
+COPY       --from=build /build_dir/cmd/blockscopy/blockscopy /usr/bin/blockscopy
+EXPOSE     80
+ENTRYPOINT [ "/usr/bin/blockscopy" ]

cmd/blockscopy/README.md

@@ -0,0 +1,9 @@
+# Blocks copy
+
+This program can copy Cortex blocks between two GCS buckets. It is GCS specific and uses special calls to copy
+files between buckets directly, without download to a local system first.
+
+Blocks copy can run in one-time mode, or as a service and do regular copies.
+
+It can be configured with minimum block time range to avoid copying of blocks that are too small.
+It can also be configured with list of users to copy, or avoid copying blocks for.