Initial commit

Author: bluepartyhat

Dockerfile

@@ -0,0 +1,35 @@
+FROM alpine:edge AS mongodb-dumper
+
+RUN apk update
+RUN apk upgrade
+RUN apk add --update go=1.16.4-r0 gcc g++ vips-dev
+
+WORKDIR /bitclout/src
+
+COPY mongodb-dumper/go.mod mongodb-dumper/
+COPY mongodb-dumper/go.sum mongodb-dumper/
+COPY core/go.mod core/
+COPY core/go.sum core/
+COPY core/third_party/ core/third_party/
+
+WORKDIR /bitclout/src/mongodb-dumper
+
+RUN go mod download
+
+# include mongodb-dumper src
+COPY mongodb-dumper/cmd     cmd
+COPY mongodb-dumper/mongodb mongodb
+COPY mongodb-dumper/main.go .
+
+# include core src
+COPY core/clouthash ../core/clouthash
+COPY core/cmd       ../core/cmd
+COPY core/lib       ../core/lib
+
+# build mongodb-dumper
+RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/mongodb-dumper main.go
+
+# create tiny image
+FROM alpine:edge
+
+COPY --from=mongodb-dumper /bitclout/src/mongodb-dumper/bin/mongodb-dumper /bitclout/bin/mongodb-dumper

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 BitClout Developer Community
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,31 @@
+# mongodb-dumper
+
+`mongodb-dumper` runs a full BitClout node and dumps the chain data into a MongoDB database
+
+## Build
+
+Running the following commands will create a Docker image called `mongodb-dumper:latest`.
+
+1. Checkout `mongodb-dumper` and `core` in the same directory
+
+2. In the `mongodb-dumper` repo, run the following (you may need sudo):
+
+```
+docker build -t mongodb-dumper -f Dockerfile ..
+```
+
+### Run
+
+You may need sudo:
+
+```
+docker run -it mongodb-dumper /bitclout/bin/mongodb-dumper run
+```
+
+Configure the connection to mongodb:
+
+```
+   --mongodb-collection   string    MongoDB collection name  (default "data")
+   --mongodb-database     string    MongoDB database name    (default "bitclout")
+   --mongodb-uri          string    MongoDB connection URI   (default "mongodb://localhost:27017")
+```

cmd/config.go

@@ -0,0 +1,22 @@
+package cmd
+
+import "github.com/spf13/viper"
+
+type Mode string
+type Network string
+
+type Config struct {
+	MongoURI        string
+	MongoDatabase   string
+	MongoCollection string
+}
+
+func LoadConfig() *Config {
+	config := Config{}
+
+	config.MongoURI = viper.GetString("monog-uri")
+	config.MongoDatabase = viper.GetString("monog-database")
+	config.MongoCollection = viper.GetString("monog-collection")
+
+	return &config
+}

cmd/node.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	coreCmd "github.com/bitclout/core/cmd"
+	"github.com/bitclout/mongodb-dumper/mongodb"
+)
+
+type Node struct {
+	SyncingService *mongodb.SyncingService
+	Config         *Config
+
+	CoreNode       *coreCmd.Node
+}
+
+func NewNode(config *Config, coreNode *coreCmd.Node) *Node {
+	result := Node{}
+	result.Config = config
+	result.CoreNode = coreNode
+
+	return &result
+}
+
+func (node *Node) Start() {
+	node.SyncingService = mongodb.NewSyncingService(
+		node.CoreNode.Server.GetBlockchain().DB(),
+		node.Config.MongoURI,
+		node.Config.MongoDatabase,
+		node.Config.MongoCollection)
+
+	go func() {
+		node.SyncingService.ConnectToMongo()
+		node.SyncingService.Start()
+	}()
+}
+
+func (node *Node) Stop() {
+	node.SyncingService.DisconnectFromMongo()
+}

cmd/root.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/spf13/viper"
+)
+
+var cfgFile string
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "mongodb-dumper",
+	Short: "BitClout node with mongodb dumper",
+	Long: `...`,
+}
+
+func Execute() {
+	cobra.CheckErr(rootCmd.Execute())
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.bitclout/mongodb-dumper.yaml)")
+}
+
+func initConfig() {
+	if cfgFile != "" {
+		viper.SetConfigFile(cfgFile)
+	} else {
+		home, err := homedir.Expand("~/.bitclout")
+		cobra.CheckErr(err)
+
+		viper.AddConfigPath(home)
+		viper.SetConfigName("mongodb-dumper")
+	}
+
+	// Environment variable support
+	viper.AutomaticEnv()
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
+	}
+}

cmd/run.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+	"os"
+	"os/signal"
+	"syscall"
+
+	coreCmd "github.com/bitclout/core/cmd"
+	"github.com/golang/glog"
+)
+
+// runCmd represents the run command
+var runCmd = &cobra.Command{
+	Use:   "run",
+	Short: "Run the MongoDB dumper",
+	Long: `...`,
+	Run: Run,
+}
+
+func Run(cmd *cobra.Command, args []string) {
+	// Start the core node
+	coreConfig := coreCmd.LoadConfig()
+	coreNode := coreCmd.NewNode(coreConfig)
+	coreNode.Start()
+
+	// Start the mongo dumper
+	mongoConfig := LoadConfig()
+	mongoNode := NewNode(mongoConfig, coreNode)
+	mongoNode.Start()
+
+	shutdownListener := make(chan os.Signal)
+	signal.Notify(shutdownListener, syscall.SIGINT, syscall.SIGTERM)
+	defer func() {
+		coreNode.Stop()
+		mongoNode.Stop()
+		glog.Info("Shutdown complete")
+	}()
+
+	<-shutdownListener
+}
+
+func init() {
+	// Add all the core node flags
+	coreCmd.SetupRunFlags(runCmd)
+
+	// Add the mongo dumper flags
+	runCmd.PersistentFlags().String("mongo-uri", "mongodb://localhost:27017", "Mongo connection URI")
+	runCmd.PersistentFlags().String("mongo-database", "bitclout", "Mongo database name")
+	runCmd.PersistentFlags().String("mongo-collection", "data", "Mongo collection name")
+
+	runCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) {
+		viper.BindPFlag(flag.Name, flag)
+	})
+
+	rootCmd.AddCommand(runCmd)
+}

go.mod

@@ -0,0 +1,42 @@
+module github.com/bitclout/mongodb-dumper
+
+go 1.16
+
+replace github.com/bitclout/core => ../core/
+
+replace github.com/golang/glog => ../core/third_party/github.com/golang/glog
+
+replace github.com/laser/go-merkle-tree => ../core/third_party/github.com/laser/go-merkle-tree
+
+replace github.com/sasha-s/go-deadlock => ../core/third_party/github.com/sasha-s/go-deadlock
+
+require (
+	cloud.google.com/go/storage v1.15.0
+	github.com/DataDog/datadog-go v4.5.0+incompatible
+	github.com/bitclout/core v0.0.0-00010101000000-000000000000
+	github.com/btcsuite/btcd v0.21.0-beta
+	github.com/btcsuite/btcutil v1.0.2
+	github.com/davecgh/go-spew v1.1.1
+	github.com/dgraph-io/badger/v3 v3.2011.1
+	github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1
+	github.com/fatih/structs v1.1.0
+	github.com/golang/glog v0.0.0-20210429001901-424d2337a529
+	github.com/gorilla/mux v1.8.0
+	github.com/h2non/bimg v1.1.5
+	github.com/kevinburke/twilio-go v0.0.0-20210327194925-1623146bcf73
+	github.com/laser/go-merkle-tree v0.0.0-20180821204614-16c2f6ea4444
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/nyaruka/phonenumbers v1.0.69
+	github.com/pkg/errors v0.9.1
+	github.com/rollbar/rollbar-go v1.4.0
+	github.com/sasha-s/go-deadlock v0.2.0
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
+	github.com/tyler-smith/go-bip39 v1.1.0
+	go.mongodb.org/mongo-driver v1.4.5
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	google.golang.org/api v0.46.0
+	gopkg.in/DataDog/dd-trace-go.v1 v1.29.0
+)