MB-51403 Handle etmpfail in CRUD endpoints

Rather than crash with a Server Error when memcached returns etmpfail to
a CRUD endpoint request we'll now bubble up the error and return http
error 503.

Change-Id: Ib3420d20c33cb9c190fabef89b3ead681dcebf0b
Reviewed-on: https://review.couchbase.org/c/ns_server/+/172224
Well-Formed: Build Bot <[email protected]>
Tested-by: Steve Watanabe <[email protected]>
Reviewed-by: Artem Stemkovski <[email protected]>

Author: stevewatanabe

src/capi_crud.erl

@@ -24,7 +24,6 @@
 construct_error_context(Context) ->
     mochijson2:encode({[{error, [{"context", Context}]}]}).
 
-%% TODO: handle tmp failures here. E.g. during warmup
 handle_mutation_rv(#mc_header{status = ?SUCCESS} = _Header, _Entry) ->
     ok;
 handle_mutation_rv(#mc_header{status = ?KEY_ENOENT} = _Header, _Entry) ->
@@ -33,6 +32,8 @@ handle_mutation_rv(#mc_header{status = ?EINVAL} = _Header, Entry) ->
     {error, Entry#mc_entry.data};
 handle_mutation_rv(#mc_header{status = ?UNKNOWN_COLLECTION}, Entry) ->
     {error, Entry#mc_entry.data};
+handle_mutation_rv(#mc_header{status = ?ETMPFAIL}, _Entry) ->
+    {retry_needed, etmpfail};
 handle_mutation_rv(#mc_header{status = ?NOT_MY_VBUCKET} = _Header, _Entry) ->
     throw(not_my_vbucket).
 
@@ -126,6 +127,8 @@ get_inner(Bucket, DocId, CollectionUid, VBucket, Options, Identity,
             throw(not_my_vbucket);
         ?UNKNOWN_COLLECTION ->
             {error, Entry#mc_entry.data};
+        ?ETMPFAIL ->
+            {retry_needed, etmpfail};
         ?EINVAL ->
             {error, Entry#mc_entry.data}
     end.

src/menelaus_web_crud.erl

@@ -255,6 +255,10 @@ construct_error_reply(Msg) ->
              end,
     {struct, [{error, <<"bad_request">>}, {reason, Reason}]}.
 
+construct_etmpfail_error_reply() ->
+    {struct, [{error, <<"retry_needed">>},
+              {reason, <<"etmpfail returned from memcached">>}]}.
+
 handle_get(BucketId, DocId, Req) ->
     CollectionUid = assert_default_collection_uid(BucketId),
     XAttrPerm = get_xattrs_permissions(BucketId, "_default", "_default", Req),
@@ -267,6 +271,8 @@ do_handle_get(BucketId, DocId, CollectionUid, XAttrPermissions, Req) ->
             menelaus_util:reply(Req, 404);
         {error, Msg} ->
             menelaus_util:reply_json(Req, construct_error_reply(Msg), 400);
+        {retry_needed, etmpfail} ->
+            menelaus_util:reply_json(Req,construct_etmpfail_error_reply(), 503);
         {ok, EJSON, {XAttrs}} ->
             {Json} = capi_utils:couch_doc_to_json(EJSON, unparsed),
             ns_audit:read_doc(Req, BucketId, DocId),
@@ -334,6 +340,9 @@ handle_post(BucketId, DocId, CollectionUid, Req) ->
     case mutate(Req, set, BucketId, DocId, CollectionUid, Value, Flags) of
         ok ->
             menelaus_util:reply_json(Req, []);
+        {retry_needed, etmpfail} ->
+            menelaus_util:reply_json(Req, construct_etmpfail_error_reply(),
+                                     503);
         {error, Msg} ->
             menelaus_util:reply_json(Req, construct_error_reply(Msg), 400)
     end.
@@ -347,6 +356,9 @@ handle_delete(BucketId, DocId, CollectionUid, Req) ->
                 undefined, undefined) of
         ok ->
             menelaus_util:reply_json(Req, []);
+        {retry_needed, etmpfail} ->
+            menelaus_util:reply_json(Req, construct_etmpfail_error_reply(),
+                                     503);
         {error, Msg} ->
             menelaus_util:reply_json(Req, construct_error_reply(Msg), 400)
     end.