Skip to content

Commit 765a163

Browse files
author
Jian Shen
authored
better error handling on placement does not exist (#355)
* better error handling on placement does not exist * better error handling * more test coverage * handle table non exist error in job config handler
1 parent 42e7331 commit 765a163

File tree

5 files changed

+87
-19
lines changed

5 files changed

+87
-19
lines changed

‎controller/handlers/config.go‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,15 @@ func (h ConfigHandler) GetJob(w *utils.ResponseWriter, r *http.Request) {
114114

115115
table, err := h.schemaMutator.GetTable(req.Namespace, job.AresTableConfig.Name)
116116
if err != nil {
117-
w.WriteErrorWithCode(
118-
http.StatusInternalServerError,
119-
ErrFailedToFetchTableSchemaForJobConfig)
120-
return
117+
if !mutatorCom.IsNonExist(err) {
118+
w.WriteErrorWithCode(
119+
http.StatusInternalServerError,
120+
ErrFailedToFetchTableSchemaForJobConfig)
121+
return
122+
}
123+
} else {
124+
job.AresTableConfig.Table = table
121125
}
122-
job.AresTableConfig.Table = table
123126

124127
numShards, err := h.getNumShards(req.Namespace)
125128
if err != nil {
@@ -158,24 +161,20 @@ func (h ConfigHandler) GetJobs(w *utils.ResponseWriter, r *http.Request) {
158161
return
159162
}
160163

161-
numValidJobs := 0
162-
for _, job := range jobs {
164+
for idx, job := range jobs {
165+
jobs[idx].NumShards = numShards
163166
table, err := h.schemaMutator.GetTable(req.Namespace, job.AresTableConfig.Name)
164167
if err != nil {
165-
// ignore job with table not found
166-
if mutatorCom.IsNonExist(err) {
167-
continue
168+
// only return system error when error is not NonExist
169+
if !mutatorCom.IsNonExist(err) {
170+
w.WriteErrorWithCode(http.StatusInternalServerError, ErrFailedToFetchTableSchemaForJobConfig)
171+
return
168172
}
169-
err = ErrFailedToFetchTableSchemaForJobConfig
170-
w.WriteErrorWithCode(http.StatusInternalServerError, err)
171-
return
173+
} else {
174+
jobs[idx].AresTableConfig.Table = table
172175
}
173-
jobs[numValidJobs] = job
174-
jobs[numValidJobs].AresTableConfig.Table = table
175-
jobs[numValidJobs].NumShards = numShards
176-
numValidJobs++
177176
}
178-
w.WriteObject(jobs[:numValidJobs])
177+
w.WriteObject(jobs)
179178
}
180179

181180
// DeleteJob swagger:route DELETE /config/{namespace}/jobs/{job} deleteJob

‎controller/handlers/placement.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ func (h *PlacementHandler) Get(rw *utils.ResponseWriter, r *http.Request) {
9393
}
9494
plm, err := h.placementMutator.GetCurrentPlacement(req.Namespace)
9595
if err != nil {
96+
if mutatorCom.IsNonExist(err) {
97+
rw.WriteErrorWithCode(http.StatusNotFound, err)
98+
return
99+
}
96100
rw.WriteError(err)
97101
return
98102
}
@@ -117,6 +121,10 @@ func (h *PlacementHandler) Add(rw *utils.ResponseWriter, r *http.Request) {
117121
// validate all shards available in placement before adding instance
118122
plm, err := h.placementMutator.AddInstance(req.Namespace, instances)
119123
if err != nil {
124+
if mutatorCom.IsNonExist(err) {
125+
rw.WriteErrorWithCode(http.StatusNotFound, err)
126+
return
127+
}
120128
rw.WriteError(err)
121129
return
122130
}
@@ -141,6 +149,10 @@ func (h *PlacementHandler) Replace(rw *utils.ResponseWriter, r *http.Request) {
141149
// validate all shards are available before replace instance
142150
plm, err := h.placementMutator.ReplaceInstance(req.Namespace, req.Body.LeavingInstances, newInstances)
143151
if err != nil {
152+
if mutatorCom.IsNonExist(err) {
153+
rw.WriteErrorWithCode(http.StatusNotFound, err)
154+
return
155+
}
144156
rw.WriteError(err)
145157
return
146158
}
@@ -164,6 +176,10 @@ func (h *PlacementHandler) Remove(rw *utils.ResponseWriter, r *http.Request) {
164176
// validate all shards are available before replace instance
165177
plm, err := h.placementMutator.RemoveInstance(req.Namespace, req.Body.LeavingInstances)
166178
if err != nil {
179+
if mutatorCom.IsNonExist(err) {
180+
rw.WriteErrorWithCode(http.StatusNotFound, err)
181+
return
182+
}
167183
rw.WriteError(err)
168184
return
169185
}
@@ -181,6 +197,10 @@ func (h *PlacementHandler) MarkNamespaceAvailable(rw *utils.ResponseWriter, r *h
181197

182198
plm, err := h.placementMutator.MarkNamespaceAvailable(req.Namespace)
183199
if err != nil {
200+
if mutatorCom.IsNonExist(err) {
201+
rw.WriteErrorWithCode(http.StatusNotFound, err)
202+
return
203+
}
184204
rw.WriteError(err)
185205
return
186206
}
@@ -203,6 +223,10 @@ func (h *PlacementHandler) MarkInstanceAvailable(rw *utils.ResponseWriter, r *ht
203223
p, err = h.placementMutator.MarkShardsAvailable(req.Namespace, req.Instance, req.Body.Shards)
204224
}
205225
if err != nil {
226+
if mutatorCom.IsNonExist(err) {
227+
rw.WriteErrorWithCode(http.StatusNotFound, err)
228+
return
229+
}
206230
rw.WriteError(err)
207231
return
208232
}

‎controller/mutators/common/errors.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ var (
5252
ErrInstanceDoesNotExist = NotExist("Instance does not exist")
5353
// ErrSubscriberDoesNotExist indicates an subscriber does not exist
5454
ErrSubscriberDoesNotExist = NotExist("Subscriber does not exist")
55+
// ErrPlacementDoesNotExist indicates that placement does not exist
56+
ErrPlacementDoesNotExist = NotExist("placement does not exist")
5557

5658
// ErrMsgFailedToBuildInitialPlacement represents error message for initial placement failure
5759
ErrMsgFailedToBuildInitialPlacement = "failed build initial placement"

‎controller/mutators/etcd/placement_mutator.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package etcd
22

33
import (
44
"fmt"
5+
"github.com/m3db/m3/src/cluster/kv"
56
"github.com/m3db/m3/src/cluster/placement"
67
"github.com/m3db/m3/src/cluster/services"
78
"github.com/m3db/m3/src/x/instrument"
@@ -77,6 +78,9 @@ func (p *placementMutator) GetCurrentPlacement(namespace string) (placement.Plac
7778
}
7879
plm, err := placementSvc.Placement()
7980
if err != nil {
81+
if err == kv.ErrNotFound {
82+
return nil, common.ErrPlacementDoesNotExist
83+
}
8084
return nil, utils.StackError(err, common.ErrMsgFailedToGetCurrentPlacement)
8185
}
8286
return plm, nil
@@ -91,6 +95,9 @@ func (p *placementMutator) AddInstance(namespace string, instances []placement.I
9195
}
9296
plm, _, err := placementSvc.AddInstances(instances)
9397
if err != nil {
98+
if err == kv.ErrNotFound {
99+
return nil, common.ErrPlacementDoesNotExist
100+
}
94101
return nil, utils.StackError(err, common.ErrMsgFailedToAddInstance)
95102
}
96103
return plm, nil
@@ -105,6 +112,9 @@ func (p *placementMutator) ReplaceInstance(namespace string, leavingInstances []
105112
}
106113
plm, _, err := placementSvc.ReplaceInstances(leavingInstances, newInstances)
107114
if err != nil {
115+
if err == kv.ErrNotFound {
116+
return nil, common.ErrPlacementDoesNotExist
117+
}
108118
return nil, utils.StackError(err, common.ErrMsgFailedToReplaceInstance)
109119
}
110120
return plm, nil
@@ -119,6 +129,9 @@ func (p *placementMutator) RemoveInstance(namespace string, leavingInstances []s
119129
}
120130
plm, err := placementSvc.RemoveInstances(leavingInstances)
121131
if err != nil {
132+
if err == kv.ErrNotFound {
133+
return nil, common.ErrPlacementDoesNotExist
134+
}
122135
return nil, utils.StackError(err, common.ErrMsgFailedToRemoveInstance)
123136
}
124137
return plm, nil
@@ -132,6 +145,9 @@ func (p *placementMutator) MarkNamespaceAvailable(namespace string) (placement.P
132145
}
133146
plm, err := placementSvc.MarkAllShardsAvailable()
134147
if err != nil {
148+
if err == kv.ErrNotFound {
149+
return nil, common.ErrPlacementDoesNotExist
150+
}
135151
return nil, utils.StackError(err, common.ErrMsgFailedToMarkAvailable)
136152
}
137153
return plm, nil
@@ -145,6 +161,9 @@ func (p *placementMutator) MarkInstanceAvailable(namespace string, instance stri
145161
}
146162
plm, err := placementSvc.MarkInstanceAvailable(instance)
147163
if err != nil {
164+
if err == kv.ErrNotFound {
165+
return nil, common.ErrPlacementDoesNotExist
166+
}
148167
return nil, utils.StackError(err, common.ErrMsgFailedToMarkAvailable)
149168
}
150169
return plm, nil
@@ -158,6 +177,9 @@ func (p *placementMutator) MarkShardsAvailable(namespace string, instance string
158177
}
159178
plm, err := placementSvc.MarkShardsAvailable(instance, shards...)
160179
if err != nil {
180+
if err == kv.ErrNotFound {
181+
return nil, common.ErrPlacementDoesNotExist
182+
}
161183
return nil, utils.StackError(err, common.ErrMsgFailedToMarkAvailable)
162184
}
163185
return plm, nil

‎controller/mutators/etcd/placement_mutator_test.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/m3db/m3/src/cluster/shard"
1010
"github.com/stretchr/testify/assert"
1111
"github.com/uber/aresdb/cluster/kvstore"
12+
"github.com/uber/aresdb/controller/mutators/common"
1213
"testing"
1314
)
1415

@@ -101,8 +102,28 @@ func TestPlacementMutator(t *testing.T) {
101102
initInstances[0], _ = placement.NewInstanceFromProto(&instancepb0)
102103
initInstances[1], _ = placement.NewInstanceFromProto(&instancepb1)
103104

105+
// 0. all procedure should fail with non exist placement before initialization
106+
plmt, err := placementMutator.GetCurrentPlacement(testNamespace)
107+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
108+
assert.Nil(t, plmt)
109+
plmt, err = placementMutator.MarkNamespaceAvailable(testNamespace)
110+
assert.Nil(t, plmt)
111+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
112+
plmt, err = placementMutator.ReplaceInstance(testNamespace, []string{}, []placement.Instance{})
113+
assert.Nil(t, plmt)
114+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
115+
plmt, err = placementMutator.MarkInstanceAvailable(testNamespace, "0")
116+
assert.Nil(t, plmt)
117+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
118+
plmt, err = placementMutator.AddInstance(testNamespace, []placement.Instance{})
119+
assert.Nil(t, plmt)
120+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
121+
plmt, err = placementMutator.RemoveInstance(testNamespace, []string{"0"})
122+
assert.Nil(t, plmt)
123+
assert.EqualError(t, err, common.ErrPlacementDoesNotExist.Error())
124+
104125
// 1. initialize placement
105-
plmt, err := placementMutator.BuildInitialPlacement(testNamespace, 2, 2, initInstances)
126+
plmt, err = placementMutator.BuildInitialPlacement(testNamespace, 2, 2, initInstances)
106127
assert.NoError(t, err)
107128
initPlacement, _ := plmt.Proto()
108129

0 commit comments

Comments
 (0)