CDRIVER-1056 use setversion to detect stale primary

Author: ajdavis

src/mongoc/mongoc-server-description-private.h

@@ -25,6 +25,8 @@
 #define MONGOC_DEFAULT_BSON_OBJ_SIZE 16 * 1024 * 1024
 #define MONGOC_DEFAULT_MAX_MSG_SIZE 48000000
 
+/* represent a server or topology with no replica set config version */
+#define MONGOC_NO_SET_VERSION -1
 
 typedef enum
    {
@@ -68,6 +70,7 @@ struct _mongoc_server_description_t
 
    bson_t                           tags;
    const char                      *current_primary;
+   int64_t                          set_version;
    bson_oid_t                       election_id;
 };
 
@@ -79,6 +82,10 @@ bool
 mongoc_server_description_has_rs_member (mongoc_server_description_t *description,
                                          const char                  *address);
 
+
+bool
+mongoc_server_description_has_set_version (mongoc_server_description_t *description);
+
 bool
 mongoc_server_description_has_election_id (mongoc_server_description_t *description);
 
@@ -92,6 +99,9 @@ void
 mongoc_server_description_set_state (mongoc_server_description_t     *description,
                                      mongoc_server_description_type_t type);
 void
+mongoc_server_description_set_set_version (mongoc_server_description_t *description,
+                                           int64_t                      set_version);
+void
 mongoc_server_description_set_election_id (mongoc_server_description_t *description,
                                            const bson_oid_t            *election_id);
 void

src/mongoc/mongoc-server-description.c

@@ -96,6 +96,7 @@ mongoc_server_description_init (mongoc_server_description_t *sd,
    sd->round_trip_time = -1;
 
    sd->set_name = NULL;
+   sd->set_version = MONGOC_NO_SET_VERSION;
    sd->current_primary = NULL;
 
    if (!_mongoc_host_list_from_string(&sd->host, address)) {
@@ -193,6 +194,25 @@ mongoc_server_description_has_rs_member(mongoc_server_description_t *server,
    return false;
 }
 
+/*
+ *--------------------------------------------------------------------------
+ *
+ * mongoc_server_description_has_set_version --
+ *
+ *      Did this server's ismaster response have a "setVersion" field?
+ *
+ * Returns:
+ *      True if the server description's setVersion is set.
+ *
+ *--------------------------------------------------------------------------
+ */
+
+bool
+mongoc_server_description_has_set_version (mongoc_server_description_t *description)
+{
+   return description->set_version != MONGOC_NO_SET_VERSION;
+}
+
 /*
  *--------------------------------------------------------------------------
  *
@@ -267,6 +287,25 @@ mongoc_server_description_set_state (mongoc_server_description_t *description,
 }
 
 
+/*
+ *--------------------------------------------------------------------------
+ *
+ * mongoc_server_description_set_set_version --
+ *
+ *       Set the replica set version of this server.
+ *
+ * Side effects:
+ *       None.
+ *
+ *--------------------------------------------------------------------------
+ */
+void
+mongoc_server_description_set_set_version (mongoc_server_description_t *description,
+                                           int64_t set_version)
+{
+   description->set_version = set_version;
+}
+
 /*
  *--------------------------------------------------------------------------
  *
@@ -388,6 +427,9 @@ mongoc_server_description_handle_ismaster (
       } else if (strcmp ("setName", bson_iter_key (&iter)) == 0) {
          if (! BSON_ITER_HOLDS_UTF8 (&iter)) goto failure;
          sd->set_name = bson_iter_utf8 (&iter, NULL);
+      } else if (strcmp ("setVersion", bson_iter_key (&iter)) == 0) {
+         mongoc_server_description_set_set_version (sd,
+                                                    bson_iter_as_int64 (&iter));
       } else if (strcmp ("electionId", bson_iter_key (&iter)) == 0) {
          if (! BSON_ITER_HOLDS_OID (&iter)) goto failure;
          mongoc_server_description_set_election_id (sd, bson_iter_oid (&iter));
@@ -485,6 +527,7 @@ mongoc_server_description_new_copy (const mongoc_server_description_t *descripti
 
    /* wait for handle_ismaster to fill these in properly */
    copy->has_is_master = false;
+   copy->set_version = MONGOC_NO_SET_VERSION;
    bson_init_static (&copy->hosts, kMongocEmptyBson, sizeof (kMongocEmptyBson));
    bson_init_static (&copy->passives, kMongocEmptyBson, sizeof (kMongocEmptyBson));
    bson_init_static (&copy->arbiters, kMongocEmptyBson, sizeof (kMongocEmptyBson));

src/mongoc/mongoc-topology-description-private.h

@@ -38,6 +38,7 @@ typedef struct _mongoc_topology_description_t
    mongoc_topology_description_type_t type;
    mongoc_set_t                      *servers;
    char                              *set_name;
+   int64_t                            max_set_version;
    bson_oid_t                         max_election_id;
    bool                               compatible;
    char                              *compatibility_error;

src/mongoc/mongoc-topology-description.c

@@ -60,6 +60,7 @@ mongoc_topology_description_init (mongoc_topology_description_t     *description
    description->type = type;
    description->servers = mongoc_set_new(8, _mongoc_topology_server_dtor, NULL);
    description->set_name = NULL;
+   description->max_set_version = MONGOC_NO_SET_VERSION;
    description->compatible = true;
    description->compatibility_error = NULL;
    description->stale = true;
@@ -152,8 +153,8 @@ _mongoc_topology_description_has_primary (mongoc_topology_description_t *descrip
  *       than this server has.
  *
  * Returns:
- *       True if the topology description's max election id is greater
- *       than the server description's election_id.
+ *       True if the topology description's max replica set version plus
+ *       election id is later than the server description's.
  *
  * Side effects:
  *       None
@@ -164,7 +165,28 @@ static bool
 _mongoc_topology_description_later_election (mongoc_topology_description_t *td,
                                              mongoc_server_description_t   *sd)
 {
-   return bson_oid_compare (&td->max_election_id, &sd->election_id) > 0;
+   /* initially max_set_version is -1 and max_election_id is zeroed */
+   return td->max_set_version > sd->set_version ||
+      (td->max_set_version == sd->set_version &&
+         bson_oid_compare (&td->max_election_id, &sd->election_id) > 0);
+}
+
+/*
+ *--------------------------------------------------------------------------
+ *
+ * _mongoc_topology_description_set_max_set_version --
+ *
+ *       Remember that we've seen a new replica set version. Unconditionally
+ *       sets td->set_version to sd->set_version.
+ *
+ *--------------------------------------------------------------------------
+ */
+static void
+_mongoc_topology_description_set_max_set_version (
+   mongoc_topology_description_t *td,
+   mongoc_server_description_t   *sd)
+{
+   td->max_set_version = sd->set_version;
 }
 
 /*
@@ -590,6 +612,27 @@ _mongoc_topology_description_has_server_cb (void *item,
    return true;
 }
 
+/*
+ *--------------------------------------------------------------------------
+ *
+ * _mongoc_topology_description_has_set_version --
+ *
+ *       Whether @topology's max replica set version has been set.
+ *
+ * Returns:
+ *       True if the max setVersion was ever set.
+ *
+ * Side effects:
+ *       None.
+ *
+ *--------------------------------------------------------------------------
+ */
+static bool
+_mongoc_topology_description_has_set_version (mongoc_topology_description_t *td)
+{
+   return td->max_set_version != MONGOC_NO_SET_VERSION;
+}
+
 /*
  *--------------------------------------------------------------------------
  *
@@ -855,6 +898,7 @@ _mongoc_topology_description_invalidate_primaries_cb (void *item,
    if (server->id != data->primary->id &&
        server->type == MONGOC_SERVER_RS_PRIMARY) {
       mongoc_server_description_set_state (server, MONGOC_SERVER_UNKNOWN);
+      mongoc_server_description_set_set_version (server, MONGOC_NO_SET_VERSION);
       mongoc_server_description_set_election_id (server, NULL);
    }
    return true;
@@ -988,7 +1032,8 @@ _mongoc_topology_description_update_rs_from_primary (mongoc_topology_description
       }
    }
 
-   if (mongoc_server_description_has_election_id (server)) {
+   if (mongoc_server_description_has_set_version (server) &&
+       mongoc_server_description_has_election_id (server)) {
       /* Server Discovery And Monitoring Spec: "The client remembers the
        * greatest electionId reported by a primary, and distrusts primaries
        * with lesser electionIds. This prevents the client from oscillating
@@ -1005,6 +1050,12 @@ _mongoc_topology_description_update_rs_from_primary (mongoc_topology_description
       _mongoc_topology_description_set_max_election_id (topology, server);
    }
 
+   if (mongoc_server_description_has_set_version (server) &&
+         (! _mongoc_topology_description_has_set_version (topology) ||
+            server->set_version > topology->max_set_version)) {
+      _mongoc_topology_description_set_max_set_version (topology, server);
+   }
+
    /* 'Server' is the primary! Invalidate other primaries if found */
    data.primary = server;
    data.topology = topology;

tests/json/server_discovery_and_monitoring/rs/equal_electionids.json

@@ -7,13 +7,15 @@
                     "a:27017": {
                         "electionId": null, 
                         "setName": null, 
+                        "setVersion": null, 
                         "type": "Unknown"
                     }, 
                     "b:27017": {
                         "electionId": {
                             "$oid": "000000000000000000000001"
                         }, 
                         "setName": "rs", 
+                        "setVersion": 1, 
                         "type": "RSPrimary"
                     }
                 }, 
@@ -33,7 +35,8 @@
                         ], 
                         "ismaster": true, 
                         "ok": 1, 
-                        "setName": "rs"
+                        "setName": "rs", 
+                        "setVersion": 1
                     }
                 ], 
                 [
@@ -48,7 +51,8 @@
                         ], 
                         "ismaster": true, 
                         "ok": 1, 
-                        "setName": "rs"
+                        "setName": "rs", 
+                        "setVersion": 1
                     }
                 ]
             ]

tests/json/server_discovery_and_monitoring/rs/new_primary_new_electionid.json

@@ -1,5 +1,5 @@
 {
-    "description": "New primary with greater electionId", 
+    "description": "New primary with greater setVersion and electionId", 
     "phases": [
         {
             "outcome": {
@@ -9,6 +9,7 @@
                             "$oid": "000000000000000000000001"
                         }, 
                         "setName": "rs", 
+                        "setVersion": 1, 
                         "type": "RSPrimary"
                     }, 
                     "b:27017": {
@@ -33,7 +34,8 @@
                         ], 
                         "ismaster": true, 
                         "ok": 1, 
-                        "setName": "rs"
+                        "setName": "rs", 
+                        "setVersion": 1
                     }
                 ]
             ]
@@ -51,6 +53,7 @@
                             "$oid": "000000000000000000000002"
                         }, 
                         "setName": "rs", 
+                        "setVersion": 1, 
                         "type": "RSPrimary"
                     }
                 }, 
@@ -70,7 +73,8 @@
                         ], 
                         "ismaster": true, 
                         "ok": 1, 
-                        "setName": "rs"
+                        "setName": "rs", 
+                        "setVersion": 1
                     }
                 ]
             ]
@@ -88,6 +92,7 @@
                             "$oid": "000000000000000000000002"
                         }, 
                         "setName": "rs", 
+                        "setVersion": 1, 
                         "type": "RSPrimary"
                     }
                 }, 
@@ -107,7 +112,8 @@
                         ], 
                         "ismaster": true, 
                         "ok": 1, 
-                        "setName": "rs"
+                        "setName": "rs", 
+                        "setVersion": 1
                     }
                 ]
             ]