feat(NSQ): update maxInFlight default value as 1 (#15)

appleboy · web-flow · commit 5cee079aaed4 · 2022-04-20T13:35:30.000+08:00
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        go: [1.13, 1.14, 1.15, 1.16, 1.17, 1.18]
+        go: [1.17, 1.18]
         include:
           - os: ubuntu-latest
             go-build: ~/.cache/go-build
diff --git a/_example/server-client/client/main.go b/_example/server-client/client/main.go
@@ -9,6 +9,8 @@ import (
 	"github.com/golang-queue/nsq"
 	"github.com/golang-queue/queue"
 	"github.com/golang-queue/queue/core"
+
+	"github.com/appleboy/graceful"
 )
 
 type job struct {
@@ -27,36 +29,54 @@ func main() {
 	taskN := 10000
 	rets := make(chan string, taskN)
 
+	m := graceful.NewManager()
+
 	// define the worker
 	w := nsq.NewWorker(
 		nsq.WithAddr("127.0.0.1:4150"),
 		nsq.WithTopic("example"),
 		nsq.WithChannel("foobar"),
-		// concurrent job number
-		nsq.WithMaxInFlight(10),
 		nsq.WithRunFunc(func(ctx context.Context, m core.QueuedMessage) error {
 			var v *job
 			if err := json.Unmarshal(m.Bytes(), &v); err != nil {
 				return err
 			}
 			rets <- v.Message
-			time.Sleep(6 * time.Second)
+			fmt.Println("got message:", v.Message)
+			fmt.Println("wait 10 seconds")
+			time.Sleep(10 * time.Second)
 			return nil
 		}),
 	)
 
 	// define the queue
 	q := queue.NewPool(
-		5,
+		1,
 		queue.WithWorker(w),
 	)
 
-	// wait until all tasks done
-	for i := 0; i < taskN; i++ {
-		fmt.Println("message:", <-rets)
-		time.Sleep(50 * time.Millisecond)
-	}
+	m.AddRunningJob(func(ctx context.Context) error {
+		for {
+			select {
+			case <-ctx.Done():
+				select {
+				case m := <-rets:
+					fmt.Println("message:", m)
+				default:
+				}
+				return nil
+			case m := <-rets:
+				fmt.Println("message:", m)
+				time.Sleep(50 * time.Millisecond)
+			}
+		}
+	})
+
+	m.AddShutdownJob(func() error {
+		// shutdown the service and notify all the worker
+		q.Release()
+		return nil
+	})
 
-	// shutdown the service and notify all the worker
-	q.Release()
+	<-m.Done()
 }
diff --git a/_example/server-client/go.mod b/_example/server-client/go.mod
@@ -3,8 +3,9 @@ module example
 go 1.18
 
 require (
+	github.com/appleboy/graceful v0.0.4
 	github.com/golang-queue/nsq v0.0.0-00010101000000-000000000000
-	github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00
+	github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732
 )
 
 require (
diff --git a/_example/server-client/go.sum b/_example/server-client/go.sum
@@ -1,6 +1,10 @@
+github.com/appleboy/graceful v0.0.4 h1:Q4LCeq4DFy59qiACLtuH+mSqDERtUzwkQbCWpRaWwvQ=
+github.com/appleboy/graceful v0.0.4/go.mod h1:Q2mVx0t+N0lCDZc5MJudbcpTm6cgGM/J2gZCZIqD9dc=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00 h1:EFiINOvAuGgxiE3MNu7PKY0O2Pvvc7r6YApdqAnhWnQ=
 github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00/go.mod h1:g1yxxDl8JMo4gUfxt11fjjU3SXU1ah61EvwshmDoSIs=
+github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732 h1:Wqs2Dsl+jUCcGSniLP5o1lmDcGrRvujRCTyqFwHDrvw=
+github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732/go.mod h1:g1yxxDl8JMo4gUfxt11fjjU3SXU1ah61EvwshmDoSIs=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
diff --git a/_example/server-client/server/main.go b/_example/server-client/server/main.go
@@ -23,7 +23,7 @@ func (j *job) Bytes() []byte {
 }
 
 func main() {
-	taskN := 100
+	taskN := 5
 
 	// define the worker
 	w := nsq.NewWorker(
diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/golang-queue/nsq
 go 1.18
 
 require (
-	github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00
+	github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732
 	github.com/nsqio/go-nsq v1.1.0
 	github.com/stretchr/testify v1.7.1
 	go.uber.org/goleak v1.1.12
diff --git a/go.sum b/go.sum
@@ -1,7 +1,7 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00 h1:EFiINOvAuGgxiE3MNu7PKY0O2Pvvc7r6YApdqAnhWnQ=
-github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00/go.mod h1:g1yxxDl8JMo4gUfxt11fjjU3SXU1ah61EvwshmDoSIs=
+github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732 h1:Wqs2Dsl+jUCcGSniLP5o1lmDcGrRvujRCTyqFwHDrvw=
+github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732/go.mod h1:g1yxxDl8JMo4gUfxt11fjjU3SXU1ah61EvwshmDoSIs=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
diff --git a/nsq.go b/nsq.go
@@ -32,7 +32,7 @@ func NewWorker(opts ...Option) *Worker {
 	w := &Worker{
 		opts:  newOptions(opts...),
 		stop:  make(chan struct{}),
-		tasks: make(chan *nsq.Message, 1),
+		tasks: make(chan *nsq.Message),
 	}
 
 	cfg := nsq.NewConfig()
@@ -76,13 +76,19 @@ func (w *Worker) startConsumer(cfg *nsq.Config) error {
 			return nil
 		}
 
-		select {
-		case w.tasks <- msg:
-		case <-w.stop:
-			if msg != nil {
-				// re-queue the job if worker has been shutdown.
-				w.opts.logger.Info("re-queue the old job")
-				msg.Requeue(-1)
+	loop:
+		for {
+			select {
+			case w.tasks <- msg:
+				break loop
+			case <-w.stop:
+				if msg != nil {
+					// re-queue the job if worker has been shutdown.
+					msg.Requeue(-1)
+				}
+				break loop
+			case <-time.After(2 * time.Second):
+				msg.Touch()
 			}
 		}
 
diff --git a/nsq_test.go b/nsq_test.go
@@ -391,13 +391,14 @@ func TestNSQStatsInWorker(t *testing.T) {
 	assert.NoError(t, w.Queue(m))
 	assert.NoError(t, w.Queue(m))
 	assert.Equal(t, int(1), w.Stats().Connections)
-	assert.Equal(t, int(1), len(w.tasks))
 
-	time.Sleep(300 * time.Millisecond)
+	time.Sleep(50 * time.Millisecond)
 
-	assert.Equal(t, uint64(3), w.Stats().MessagesReceived)
-	assert.Equal(t, uint64(1), w.Stats().MessagesFinished)
+	assert.Equal(t, uint64(1), w.Stats().MessagesReceived)
+	assert.Equal(t, uint64(0), w.Stats().MessagesFinished)
 	assert.Equal(t, uint64(0), w.Stats().MessagesRequeued)
 
 	_ = w.Shutdown()
+	time.Sleep(50 * time.Millisecond)
+	assert.Equal(t, uint64(1), w.Stats().MessagesRequeued)
 }
diff --git a/options.go b/options.go
@@ -2,7 +2,6 @@ package nsq
 
 import (
 	"context"
-	"runtime"
 
 	"github.com/golang-queue/queue"
 	"github.com/golang-queue/queue/core"
@@ -85,7 +84,7 @@ func newOptions(opts ...Option) Options {
 		addr:        "127.0.0.1:4150",
 		topic:       "gorush",
 		channel:     "ch",
-		maxInFlight: runtime.NumCPU(),
+		maxInFlight: 1,
 
 		logger: queue.NewLogger(),
 		runFunc: func(context.Context, core.QueuedMessage) error {

Original file line number	Diff line number	Diff line change
`@@ -3,8 +3,9 @@ module example`
`3`	`3`	`go 1.18`
`4`	`4`
`5`	`5`	`require (`
	`6`	`+ github.com/appleboy/graceful v0.0.4`
`6`	`7`	`github.com/golang-queue/nsq v0.0.0-00010101000000-000000000000`
`7`		`- github.com/golang-queue/queue v0.0.13-0.20220408035349-ed24fa14aa00`
	`8`	`+ github.com/golang-queue/queue v0.0.13-0.20220420024737-03d90b78b732`
`8`	`9`	`)`
`9`	`10`
`10`	`11`	`require (`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ func (j *job) Bytes() []byte {`
`23`	`23`	`}`
`24`	`24`
`25`	`25`	`func main() {`
`26`		`- taskN := 100`
	`26`	`+ taskN := 5`
`27`	`27`
`28`	`28`	`// define the worker`
`29`	`29`	`w := nsq.NewWorker(`