Merge remote-tracking branch 'origin/master'

# Conflicts:
#	app/cmd/server_test.go

Author: umputun

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: set up go 1.14
+      - name: set up go 1.15
         uses: actions/setup-go@v1
         with:
           go-version: 1.15
@@ -20,7 +20,7 @@ jobs:
       - name: launch mongodb
         uses: wbari/start-mongoDB@v0.2
         with:
-          mongoDBVersion: "4.0"
+          mongoDBVersion: "4.2"
 
       - name: checkout
         uses: actions/checkout@v2
@@ -42,8 +42,10 @@ jobs:
 
       - name: run linters
         run: $GITHUB_WORKSPACE/golangci-lint run --out-format=github-actions ./...
+        working-directory: app
         env:
           GO111MODULE: on
+          GOFLAGS: "-mod=vendor"
 
       - name: submit coverage
         run: $(go env GOPATH)/bin/goveralls -service="github" -coverprofile=$GITHUB_WORKSPACE/profile.cov

.golangci.yml

@@ -72,6 +72,9 @@ issues:
     - text: "at least one file in a package should have a package comment"
       linters:
         - stylecheck
+    - text: "whyNoLint: include an explanation for nolint directive"
+      linters:
+        - gocritic
   exclude-use-default: false
 
 service:

app/agent/demo_mode.go

@@ -3,7 +3,7 @@ package agent
 import (
 	"context"
 	"fmt"
-	"math/rand"
+	"math/rand" //nolint:gosec
 	"time"
 
 	log "github.com/go-pkgz/lgr"
@@ -27,11 +27,11 @@ func (d *DemoEmitter) Logs(o docker.LogsOptions) error {
 			var line string
 			switch o.Container {
 			case "nginx":
-				line = nginxDemo[rand.Intn(len(nginxDemo)-1)]
+				line = nginxDemo[rand.Intn(len(nginxDemo)-1)] //nolint:gosec
 			case "rest":
-				line = restDemo[rand.Intn(len(restDemo)-1)]
+				line = restDemo[rand.Intn(len(restDemo)-1)] //nolint:gosec
 			case "mongo":
-				line = mongoDemo[rand.Intn(len(mongoDemo)-1)]
+				line = mongoDemo[rand.Intn(len(mongoDemo)-1)] //nolint:gosec
 
 			}
 

app/client/cli.go

@@ -39,7 +39,7 @@ type APIParams struct {
 // DisplayParams customizes how records will be showed
 type DisplayParams struct {
 	ShowPid    bool           // include pid
-	ShowTs     bool           // include time stamp as "2006-01-02 15:04:05.999999" in given TZ
+	ShowTS     bool           // include time stamp as "2006-01-02 15:04:05.999999" in given TZ
 	FollowMode bool           // follow mode, like -f in grep
 	TailMode   bool           // tail mode, like -t in grep
 	ShowSyslog bool           // show non-docker messages from syslog, off by default
@@ -133,8 +133,8 @@ func (c *CLI) makeOutLine(e core.LogEntry) (string, bool) {
 	}
 
 	ts := ""
-	if c.ShowTs {
-		ts = fmt.Sprintf(" - %s", e.Ts.In(c.TimeZone).Format("2006-01-02 15:04:05.999999"))
+	if c.ShowTS {
+		ts = fmt.Sprintf(" - %s", e.TS.In(c.TimeZone).Format("2006-01-02 15:04:05.999999"))
 	}
 	line := fmt.Sprintf("%s:%s%s%s - %s\n", red(e.Host), green(e.Container), yellow(ts), yellow(pid), white(e.Msg))
 	return line, true

app/client/cli_test.go

@@ -42,7 +42,7 @@ func TestCliWithPidAndTS(t *testing.T) {
 
 	out := bytes.Buffer{}
 
-	c := NewCLI(APIParams{API: ts.URL + "/v1", Client: &http.Client{}}, DisplayParams{Out: &out, ShowPid: true, ShowTs: true})
+	c := NewCLI(APIParams{API: ts.URL + "/v1", Client: &http.Client{}}, DisplayParams{Out: &out, ShowPid: true, ShowTS: true})
 	_, err := c.Activate(context.Background(), core.Request{})
 	require.NoError(t, err)
 	exp := "h1:c1 - 2019-05-24 20:54:30 [0] - msg1\nh1:c2 - 2019-05-24 20:54:31 [0] - msg2\n" +
@@ -60,7 +60,7 @@ func TestCliWithCustomTZ(t *testing.T) {
 
 	tz, err := time.LoadLocation("America/New_York")
 	require.NoError(t, err)
-	c := NewCLI(APIParams{API: ts.URL + "/v1", Client: &http.Client{}}, DisplayParams{Out: &out, TimeZone: tz, ShowTs: true})
+	c := NewCLI(APIParams{API: ts.URL + "/v1", Client: &http.Client{}}, DisplayParams{Out: &out, TimeZone: tz, ShowTS: true})
 	_, err = c.Activate(context.Background(), core.Request{})
 	require.NoError(t, err)
 	exp := "h1:c1 - 2019-05-24 21:54:30 - msg1\nh1:c2 - 2019-05-24 21:54:31 - msg2\nh2:c1 - 2019-05-24 21:54:32 - msg3\n" +
@@ -101,7 +101,7 @@ func TestLastID(t *testing.T) {
 			}
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			rec := core.LogEntry{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1",
-				Msg: "msg1", Ts: ts.Add(0 * time.Second)}
+				Msg: "msg1", TS: ts.Add(0 * time.Second)}
 
 			err := json.NewEncoder(w).Encode(&rec)
 			require.NoError(t, err)
@@ -141,12 +141,12 @@ func TestCliFindFailedAndRestored(t *testing.T) {
 		if r.URL.Path == "/v1/find" && r.Method == "POST" {
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			recs := []core.LogEntry{
-				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", Ts: ts.Add(0 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", Ts: ts.Add(1 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", Ts: ts.Add(2 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", Ts: ts.Add(3 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", Ts: ts.Add(4 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", Ts: ts.Add(5 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", TS: ts.Add(0 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", TS: ts.Add(1 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", TS: ts.Add(2 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", TS: ts.Add(3 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", TS: ts.Add(4 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", TS: ts.Add(5 * time.Second)},
 			}
 			err := json.NewEncoder(w).Encode(recs)
 			require.NoError(t, err)
@@ -198,7 +198,7 @@ func TestCliFindFollow(t *testing.T) {
 				{
 					ID:   fmt.Sprintf("5ce8718aef1d7346a5443a1%d", c),
 					Host: "h1", Container: "c1", Msg: fmt.Sprintf("msg%d", c),
-					Ts: ts.Add(time.Duration(c) * time.Second),
+					TS: ts.Add(time.Duration(c) * time.Second),
 				},
 			}
 			require.NoError(t, json.NewEncoder(w).Encode(recs))
@@ -237,7 +237,7 @@ func TestCliFindFollowWithDelay(t *testing.T) {
 				{
 					ID:   fmt.Sprintf("5ce8718aef1d7346a5443a1%d", c),
 					Host: "h1", Container: "c1", Msg: fmt.Sprintf("msg%d", c),
-					Ts: ts.Add(time.Duration(c) * time.Second),
+					TS: ts.Add(time.Duration(c) * time.Second),
 				},
 			}
 			require.NoError(t, json.NewEncoder(w).Encode(recs))
@@ -269,7 +269,7 @@ func TestCliFindTail(t *testing.T) {
 		if r.URL.Path == "/v1/last" && r.Method == "GET" {
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			rec := core.LogEntry{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1",
-				Msg: "msg1", Ts: ts.Add(5 * time.Second)}
+				Msg: "msg1", TS: ts.Add(5 * time.Second)}
 			err := json.NewEncoder(w).Encode(&rec)
 			require.NoError(t, err)
 		}
@@ -286,7 +286,7 @@ func TestCliFindTail(t *testing.T) {
 			require.Equal(t, "5ce8718aef1d7346a5443a1f", req.LastID)
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			recs := []core.LogEntry{{ID: "5ce8718aef1d7346a5443a12f", Host: "h1", Container: "c1",
-				Msg: "msg1", Ts: ts.Add(15 * time.Second)}}
+				Msg: "msg1", TS: ts.Add(15 * time.Second)}}
 			require.NoError(t, json.NewEncoder(w).Encode(recs))
 		}
 	}))
@@ -349,12 +349,12 @@ func prepTestServer(t *testing.T) *httptest.Server {
 
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			recs := []core.LogEntry{
-				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", Ts: ts.Add(0 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", Ts: ts.Add(1 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", Ts: ts.Add(2 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", Ts: ts.Add(3 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", Ts: ts.Add(4 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", Ts: ts.Add(5 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", TS: ts.Add(0 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", TS: ts.Add(1 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", TS: ts.Add(2 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", TS: ts.Add(3 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", TS: ts.Add(4 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", TS: ts.Add(5 * time.Second)},
 			}
 			err = json.NewEncoder(w).Encode(recs)
 			require.NoError(t, err)
@@ -364,7 +364,7 @@ func prepTestServer(t *testing.T) *httptest.Server {
 		if r.URL.Path == "/v1/last" && r.Method == "GET" {
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			rec := core.LogEntry{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1",
-				Msg: "msg1", Ts: ts.Add(5 * time.Second)}
+				Msg: "msg1", TS: ts.Add(5 * time.Second)}
 			err := json.NewEncoder(w).Encode(&rec)
 			require.NoError(t, err)
 		}

app/cmd/client.go

@@ -17,7 +17,7 @@ type ClientOpts struct {
 	Containers []string `short:"c" description:"show container(s) only"`
 	Hosts      []string `short:"h" description:"show host(s) only"`
 	Excludes   []string `short:"x" description:"exclude container(s)"`
-	ShowTs     bool     `short:"m" description:"show syslog timestamp"`
+	ShowTS     bool     `short:"m" description:"show syslog timestamp"`
 	ShowPid    bool     `short:"p" description:"show pid"`
 	ShowSyslog bool     `short:"s" description:"show syslog messages"`
 	FollowMode bool     `short:"f" description:"follow mode"`
@@ -56,7 +56,7 @@ func (c ClientCmd) Run(ctx context.Context) error {
 
 	display := client.DisplayParams{
 		ShowPid:    c.ShowPid,
-		ShowTs:     c.ShowTs,
+		ShowTS:     c.ShowTS,
 		FollowMode: c.FollowMode,
 		TailMode:   c.TailMode,
 		ShowSyslog: c.ShowSyslog,

app/cmd/client_test.go

@@ -25,7 +25,7 @@ func TestClient(t *testing.T) {
 	c := ClientCmd{ClientOpts{
 		API:      ts.URL + "/v1",
 		TimeZone: "America/New_York",
-		ShowTs:   true,
+		ShowTS:   true,
 	}}
 
 	lgr.Out(ioutil.Discard)
@@ -67,12 +67,12 @@ func prepTestServer(t *testing.T) *httptest.Server {
 
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			recs := []core.LogEntry{
-				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", Ts: ts.Add(0 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", Ts: ts.Add(1 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", Ts: ts.Add(2 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", Ts: ts.Add(3 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", Ts: ts.Add(4 * time.Second)},
-				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", Ts: ts.Add(5 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1", Msg: "msg1", TS: ts.Add(0 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a2f", Host: "h1", Container: "c2", Msg: "msg2", TS: ts.Add(1 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a3f", Host: "h2", Container: "c1", Msg: "msg3", TS: ts.Add(2 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a4f", Host: "h1", Container: "c1", Msg: "msg4", TS: ts.Add(3 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a5f", Host: "h1", Container: "c2", Msg: "msg5", TS: ts.Add(4 * time.Second)},
+				{ID: "5ce8718aef1d7346a5443a6f", Host: "h2", Container: "c2", Msg: "msg6", TS: ts.Add(5 * time.Second)},
 			}
 			err = json.NewEncoder(w).Encode(recs)
 			require.NoError(t, err)
@@ -82,7 +82,7 @@ func prepTestServer(t *testing.T) *httptest.Server {
 		if r.URL.Path == "/v1/last" && r.Method == "GET" {
 			ts := time.Date(2019, 5, 24, 20, 54, 30, 0, time.Local)
 			rec := core.LogEntry{ID: "5ce8718aef1d7346a5443a1f", Host: "h1", Container: "c1",
-				Msg: "msg1", Ts: ts.Add(5 * time.Second)}
+				Msg: "msg1", TS: ts.Add(5 * time.Second)}
 			err := json.NewEncoder(w).Encode(&rec)
 			require.NoError(t, err)
 		}

app/cmd/server.go

@@ -9,9 +9,12 @@ import (
 	"path"
 	"time"
 
-	"github.com/globalsign/mgo"
-	"gopkg.in/natefinch/lumberjack.v2"
 	log "github.com/go-pkgz/lgr"
+	"github.com/go-pkgz/mongo/v2"
+	"github.com/pkg/errors"
+	mdrv "go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/umputun/dkll/app/server"
 )
@@ -20,13 +23,8 @@ import (
 type ServerOpts struct {
 	Port               int           `long:"api-port" env:"API_PORT" default:"8080" description:"rest server port"`
 	SyslogPort         int           `long:"syslog-port" env:"SYSLOG_PORT" default:"5514" description:"syslog server port"`
-	Mongo              []string      `long:"mongo" env:"MONGO" required:"true" env-delim:"," description:"mongo host:port"`
-	MongoUser          string        `long:"mongo-user" env:"MONGO_USER" default:"admin" description:"mongo user for auth"`
-	MongoPasswd        string        `long:"mongo-passwd" env:"MONGO_PASSWD" default:"" description:"mongo password"`
-	MongoDelay         time.Duration `long:"mongo-delay" env:"MONGO_DELAY" default:"0s" description:"mongo initial delay"`
+	MongoURL           string        `long:"mongo" env:"MONGO" required:"true" env-delim:"," description:"mongo host:port"`
 	MongoTimeout       time.Duration `long:"mongo-timeout" env:"MONGO_TIMEOUT" default:"5s" description:"mongo timeout"`
-	MongoDB            string        `long:"mongo-db" env:"MONGO_DB" default:"dkll" description:"mongo database name"`
-	MongoColl          string        `long:"mongo-coll" env:"MONGO_COLL" default:"msgs" description:"mongo collection name"`
 	MongoMaxSize       int           `long:"mongo-size" env:"MONGO_SIZE" default:"10000000000" description:"max collection size"`
 	MongoMaxDocs       int           `long:"mongo-docs" env:"MONGO_DOCS" default:"50000000" description:"max docs in collection"`
 	FileBackupLocation string        `long:"backup" default:"" env:"BACK_LOG" description:"backup log files location"`
@@ -61,22 +59,13 @@ func (s ServerCmd) Run(ctx context.Context) error {
 		return err
 	}
 
-	dial := mgo.DialInfo{
-		Addrs:    s.Mongo,
-		AppName:  "dkll",
-		Timeout:  s.MongoTimeout,
-		Database: "admin",
-	}
-
-	if s.MongoPasswd != "" {
-		dial.Username = s.MongoUser
-		dial.Password = s.MongoPasswd
-		log.Printf("[INFO] mongo auth enforced with user %s", s.MongoUser)
+	mclient, ex, err := makeMongoClient(s.MongoURL, s.MongoTimeout)
+	if err != nil {
+		return errors.Wrap(err, "can't make mongo client")
 	}
-
-	mgParams := server.MongoParams{DBName: s.MongoDB, Collection: s.MongoColl, Delay: s.MongoDelay,
+	mgParams := server.MongoParams{DBName: ex["db"].(string), Collection: ex["collection"].(string),
 		MaxDocs: s.MongoMaxDocs, MaxCollectionSize: s.MongoMaxSize}
-	mg, err := server.NewMongo(dial, mgParams)
+	mg, err := server.NewMongo(mclient, mgParams)
 	if err != nil {
 		return err
 	}
@@ -105,6 +94,23 @@ func (s ServerCmd) Run(ctx context.Context) error {
 	return nil
 }
 
+func makeMongoClient(mongoURL string, timeout time.Duration) (*mdrv.Client, map[string]interface{}, error) {
+	log.Printf("[DEBUG] make mongo client for %q", mongoURL)
+	if mongoURL == "" {
+		return nil, nil, errors.New("no mongo URL provided")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	client, ex, err := mongo.Connect(ctx, options.Client().
+		SetAppName("dkll-server").SetConnectTimeout(timeout), mongoURL, "db", "collection")
+
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "can't connect to mongo %s", mongoURL)
+	}
+	return client, ex, nil
+}
+
 func (s ServerCmd) makeWriters() (wrf server.WritersFactory, mergeLogWriter io.Writer, err error) {
 
 	// default loggers empty

app/cmd/server_test.go

@@ -13,9 +13,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/globalsign/mgo"
 	log "github.com/go-pkgz/lgr"
-	"github.com/go-pkgz/mongo"
+	"github.com/go-pkgz/mongo/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -24,20 +23,21 @@ import (
 
 func TestServer(t *testing.T) {
 	log.Printf("start server test")
-	mg, err := mongo.NewServer(mgo.DialInfo{Addrs: []string{"127.0.0.1:27017"}, Database: "test"}, mongo.ServerParams{})
-	require.NoError(t, err)
-	mgConn := mongo.NewConnection(mg, "test", "msgs")
-	cleanupTestAssets(t, "/tmp/dkll-test", mgConn)
-	defer cleanupTestAssets(t, "/tmp/dkll-test", mgConn)
+
+	_, coll, teardown := mongo.MakeTestConnection(t)
+	defer teardown()
+	// m, err := server.NewMongo(mg, server.MongoParams{DBName: "test", Collection: coll.Name()})
+	// require.NoError(t, err)
+
+	os.RemoveAll("/tmp/dkll-test")
 
 	opts := ServerOpts{
 		Port:               8080,
 		SyslogPort:         15514,
-		Mongo:              []string{"127.0.0.1:27017"},
-		MongoDB:            "test",
-		MongoColl:          "msgs",
+		MongoURL:           getMongoURL(t) + "?db=test&collection=" + coll.Name(),
 		EnableMerged:       true,
 		FileBackupLocation: "/tmp/dkll-test",
+		MongoMaxDocs:       100,
 	}
 	s := ServerCmd{ServerOpts: opts}
 
@@ -114,9 +114,14 @@ func TestServer(t *testing.T) {
 	log.Printf("start wait completed")
 }
 
-func cleanupTestAssets(_ *testing.T, loc string, conn *mongo.Connection) {
-	_ = os.RemoveAll(loc)
-	_ = conn.WithCollection(func(coll *mgo.Collection) error {
-		return coll.DropCollection()
-	})
+func getMongoURL(t *testing.T) string {
+	mongoURL := os.Getenv("MONGO_TEST")
+	if mongoURL == "" {
+		mongoURL = "mongodb://mongo:27017"
+		t.Logf("no MONGO_TEST in env, defaulted to %s", mongoURL)
+	}
+	if mongoURL == "skip" {
+		t.Skip("skip mongo test")
+	}
+	return mongoURL
 }