chore: support nats platform (#14)

Author: appleboy

.drone.yml

@@ -32,6 +32,9 @@ services:
   commands:
   - /nsqd
 
+- name: nats
+  image: nats
+
 volumes:
 - name: gopath
   temp: {}

go.mod

@@ -3,6 +3,10 @@ module github.com/appleboy/queue
 go 1.16
 
 require (
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/nats-io/nats-server/v2 v2.3.2 // indirect
+	github.com/nats-io/nats.go v1.11.1-0.20210623165838-4b75fc59ae30
 	github.com/nsqio/go-nsq v1.0.8
 	github.com/stretchr/testify v1.7.0
+	google.golang.org/protobuf v1.27.1 // indirect
 )

go.sum

@@ -1,14 +1,74 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk=
+github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
+github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
+github.com/nats-io/jwt v1.2.2 h1:w3GMTO969dFg+UOKTmmyuu7IGdusK+7Ytlt//OYH/uU=
+github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q=
+github.com/nats-io/jwt/v2 v2.0.2 h1:ejVCLO8gu6/4bOKIHQpmB5UhhUJfAQw55yvLWpfmKjI=
+github.com/nats-io/jwt/v2 v2.0.2/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY=
+github.com/nats-io/nats-server/v2 v2.3.2 h1:SGJLWrjBHsl0DsdY8PeTR3YKEfiUEYVVq2STw9d8MSY=
+github.com/nats-io/nats-server/v2 v2.3.2/go.mod h1:dUf7Cm5z5LbciFVwWx54owyCKm8x4/hL6p7rrljhLFY=
+github.com/nats-io/nats.go v1.11.1-0.20210623165838-4b75fc59ae30 h1:9GqilBhZaR3xYis0JgMlJjNw933WIobdjKhilXm+Vls=
+github.com/nats-io/nats.go v1.11.1-0.20210623165838-4b75fc59ae30/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
+github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
+github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nsqio/go-nsq v1.0.8 h1:3L2F8tNLlwXXlp2slDUrUWSBn2O3nMh8R1/KEDFTHPk=
 github.com/nsqio/go-nsq v1.0.8/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

nats/nats.go

@@ -0,0 +1,156 @@
+package nats
+
+import (
+	"sync"
+
+	"github.com/appleboy/queue"
+
+	"github.com/nats-io/nats.go"
+)
+
+var _ queue.Worker = (*Worker)(nil)
+
+// Option for queue system
+type Option func(*Worker)
+
+// Job with NSQ message
+type Job struct {
+	Body []byte
+}
+
+// Bytes get bytes format
+func (j *Job) Bytes() []byte {
+	return j.Body
+}
+
+// Worker for NSQ
+type Worker struct {
+	addr     string
+	subj     string
+	queue    string
+	client   *nats.Conn
+	stop     chan struct{}
+	stopOnce sync.Once
+	runFunc  func(queue.QueuedMessage, <-chan struct{}) error
+}
+
+// WithAddr setup the addr of NATS
+func WithAddr(addr string) Option {
+	return func(w *Worker) {
+		w.addr = "nats://" + addr
+	}
+}
+
+// WithSubj setup the subject of NATS
+func WithSubj(subj string) Option {
+	return func(w *Worker) {
+		w.subj = subj
+	}
+}
+
+// WithQueue setup the queue of NATS
+func WithQueue(queue string) Option {
+	return func(w *Worker) {
+		w.queue = queue
+	}
+}
+
+// WithRunFunc setup the run func of queue
+func WithRunFunc(fn func(queue.QueuedMessage, <-chan struct{}) error) Option {
+	return func(w *Worker) {
+		w.runFunc = fn
+	}
+}
+
+// NewWorker for struc
+func NewWorker(opts ...Option) *Worker {
+	var err error
+	w := &Worker{
+		addr:  "nats://127.0.0.1:4222",
+		subj:  "foobar",
+		queue: "foobar",
+		stop:  make(chan struct{}),
+		runFunc: func(queue.QueuedMessage, <-chan struct{}) error {
+			return nil
+		},
+	}
+
+	// Loop through each option
+	for _, opt := range opts {
+		// Call the option giving the instantiated
+		opt(w)
+	}
+
+	w.client, err = nats.Connect(w.addr)
+	if err != nil {
+		panic(err)
+	}
+
+	return w
+}
+
+// BeforeRun run script before start worker
+func (s *Worker) BeforeRun() error {
+	return nil
+}
+
+// AfterRun run script after start worker
+func (s *Worker) AfterRun() error {
+	return nil
+}
+
+// Run start the worker
+func (s *Worker) Run() error {
+	wg := &sync.WaitGroup{}
+
+	_, err := s.client.QueueSubscribe(s.subj, s.queue, func(m *nats.Msg) {
+		wg.Add(1)
+		defer wg.Done()
+		job := &Job{
+			Body: m.Data,
+		}
+
+		// run custom process function
+		_ = s.runFunc(job, s.stop)
+	})
+	if err != nil {
+		return err
+	}
+
+	// wait close signal
+	<-s.stop
+
+	// wait job completed
+	wg.Wait()
+
+	return nil
+}
+
+// Shutdown worker
+func (s *Worker) Shutdown() error {
+	s.stopOnce.Do(func() {
+		close(s.stop)
+		s.client.Close()
+	})
+	return nil
+}
+
+// Capacity for channel
+func (s *Worker) Capacity() int {
+	return 0
+}
+
+// Usage for count of channel usage
+func (s *Worker) Usage() int {
+	return 0
+}
+
+// Queue send notification to queue
+func (s *Worker) Queue(job queue.QueuedMessage) error {
+	err := s.client.Publish(s.subj, job.Bytes())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

nats/nats_test.go

@@ -0,0 +1,86 @@
+package nats
+
+import (
+	"log"
+	"testing"
+	"time"
+
+	"github.com/appleboy/queue"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var host = "nats"
+
+func TestNATSDefaultFlow(t *testing.T) {
+	m := &Job{
+		Body: []byte("foo"),
+	}
+	w := NewWorker(
+		WithAddr(host+":4222"),
+		WithSubj("test"),
+		WithQueue("test"),
+	)
+	q, err := queue.NewQueue(
+		queue.WithWorker(w),
+		queue.WithWorkerCount(2),
+	)
+	assert.NoError(t, err)
+	q.Start()
+	time.Sleep(100 * time.Millisecond)
+	assert.NoError(t, q.Queue(m))
+	m.Body = []byte("new message")
+	assert.NoError(t, q.Queue(m))
+	q.Shutdown()
+	q.Wait()
+}
+
+func TestNATSShutdown(t *testing.T) {
+	w := NewWorker(
+		WithAddr(host+":4222"),
+		WithSubj("test"),
+		WithQueue("test"),
+	)
+	q, err := queue.NewQueue(
+		queue.WithWorker(w),
+		queue.WithWorkerCount(2),
+	)
+	assert.NoError(t, err)
+	q.Start()
+	time.Sleep(1 * time.Second)
+	q.Shutdown()
+	// check shutdown once
+	q.Shutdown()
+	q.Wait()
+}
+
+func TestNATSCustomFuncAndWait(t *testing.T) {
+	m := &Job{
+		Body: []byte("foo"),
+	}
+	w := NewWorker(
+		WithAddr(host+":4222"),
+		WithSubj("test"),
+		WithQueue("test"),
+		WithRunFunc(func(msg queue.QueuedMessage, s <-chan struct{}) error {
+			log.Println("show message: " + string(msg.Bytes()))
+			time.Sleep(500 * time.Millisecond)
+			return nil
+		}),
+	)
+	q, err := queue.NewQueue(
+		queue.WithWorker(w),
+		queue.WithWorkerCount(2),
+	)
+	assert.NoError(t, err)
+	q.Start()
+	time.Sleep(100 * time.Millisecond)
+	assert.NoError(t, q.Queue(m))
+	assert.NoError(t, q.Queue(m))
+	assert.NoError(t, q.Queue(m))
+	assert.NoError(t, q.Queue(m))
+	time.Sleep(600 * time.Millisecond)
+	q.Shutdown()
+	q.Wait()
+	// you will see the execute time > 1000ms
+}

nsq/nsq_test.go

@@ -1,6 +1,7 @@
 package nsq
 
 import (
+	"log"
 	"testing"
 	"time"
 
@@ -11,14 +12,14 @@ import (
 
 var host = "nsq"
 
-func TestDefaultFlow(t *testing.T) {
+func TestNSQDefaultFlow(t *testing.T) {
 	m := &Job{
 		Body: []byte("foo"),
 	}
 	w := NewWorker(
 		WithAddr(host+":4150"),
-		WithTopic("test"),
-		WithChannel("test"),
+		WithTopic("test1"),
+		WithChannel("test1"),
 	)
 	q, err := queue.NewQueue(
 		queue.WithWorker(w),
@@ -34,10 +35,10 @@ func TestDefaultFlow(t *testing.T) {
 	q.Wait()
 }
 
-func TestShutdown(t *testing.T) {
+func TestNSQShutdown(t *testing.T) {
 	w := NewWorker(
 		WithAddr(host+":4150"),
-		WithTopic("test"),
+		WithTopic("test2"),
 	)
 	q, err := queue.NewQueue(
 		queue.WithWorker(w),
@@ -52,15 +53,16 @@ func TestShutdown(t *testing.T) {
 	q.Wait()
 }
 
-func TestCustomFuncAndWait(t *testing.T) {
+func TestNSQCustomFuncAndWait(t *testing.T) {
 	m := &Job{
 		Body: []byte("foo"),
 	}
 	w := NewWorker(
 		WithAddr(host+":4150"),
-		WithTopic("test"),
+		WithTopic("test3"),
 		WithMaxInFlight(2),
 		WithRunFunc(func(msg queue.QueuedMessage, s <-chan struct{}) error {
+			log.Println("show message: " + string(msg.Bytes()))
 			time.Sleep(500 * time.Millisecond)
 			return nil
 		}),