Re-architecture initial run

Author: imnasnainaec

database/init/00-replica-set.js

@@ -1,31 +1,93 @@
-// Initialize the replica set on first startup.
-// See: https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/
+// Ensure a single-node replica set is initialized and advertising the
+// expected host for this environment.
 //
 // MONGO_INITDB_REPLICA_HOST can be set to the resolvable hostname:port
-// used to advertise this member (e.g. the Kubernetes Service name "database:27017").
-// It defaults to "localhost:27017" for local development.
-try {
-  rs.status();
-} catch (e) {
-  print(`Replica set not yet initialized (${e}), initializing now...`);
-  const host = process.env.MONGO_INITDB_REPLICA_HOST || "localhost:27017";
-  rs.initiate({ _id: "rs0", members: [{ _id: 0, host: host }] });
-  // Wait for replica set to reach PRIMARY state before other init scripts run.
-  const maxWaitMs = 30000;
-  const intervalMs = 500;
-  let waited = 0;
-  let isPrimary = false;
-  while (!isPrimary && waited < maxWaitMs) {
-    sleep(intervalMs);
-    waited += intervalMs;
-    const status = rs.status();
-    isPrimary =
-      status.members !== undefined &&
-      status.members.some((m) => m.stateStr === "PRIMARY");
+// used to advertise this member (e.g. "database:27017" in Kubernetes).
+const host = process.env.MONGO_INITDB_REPLICA_HOST || "localhost:27017";
+const maxWaitMs = 60000;
+const intervalMs = 1000;
+const start = Date.now();
+
+function ensurePrimaryHost(forceReconfig) {
+  let conf;
+  try {
+    conf = rs.conf();
+  } catch (error) {
+    if (!forceReconfig) {
+      throw error;
+    }
+
+    conf = db.getSiblingDB("local").system.replset.findOne();
+    if (!conf) {
+      throw error;
+    }
   }
-  if (!isPrimary) {
-    throw new Error(
-      `Replica set did not reach PRIMARY state after ${maxWaitMs}ms`
-    );
+
+  if (!conf.members || conf.members.length === 0) {
+    throw new Error("Replica set config has no members");
   }
+
+  if (conf.members[0].host === host) {
+    return true;
+  }
+
+  if (conf.members[0].host !== host) {
+    print(`Updating replica set member host to ${host}`);
+    conf.members[0].host = host;
+    conf.version = (conf.version || 1) + 1;
+    if (forceReconfig) {
+      rs.reconfig(conf, { force: true });
+    } else {
+      rs.reconfig(conf);
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+while (Date.now() - start < maxWaitMs) {
+  try {
+    const hello = db.hello();
+    if (hello.isWritablePrimary) {
+      try {
+        const hostIsAligned = ensurePrimaryHost(false);
+        if (hostIsAligned) {
+          print("Replica set is PRIMARY with correct host");
+          quit(0);
+        }
+
+        print("Replica set host updated, waiting for PRIMARY");
+      } catch (error) {
+        print(`Host alignment deferred (${error})`);
+      }
+    }
+  } catch (error) {
+    print(`Replica set not writable yet (${error})`);
+  }
+
+  try {
+    const hostIsAligned = ensurePrimaryHost(true);
+    if (!hostIsAligned) {
+      print("Forced host alignment requested, waiting for PRIMARY");
+    }
+  } catch (error) {
+    print(`Forced host alignment deferred (${error})`);
+  }
+
+  try {
+    rs.initiate({ _id: "rs0", members: [{ _id: 0, host: host }] });
+    print(`Initialized replica set with host ${host}`);
+  } catch (error) {
+    if (!String(error).includes("already initialized")) {
+      print(`Replica set init deferred (${error})`);
+    }
+  }
+
+  sleep(intervalMs);
 }
+
+throw new Error(
+  `Replica set did not reach PRIMARY state with host ${host} after ${maxWaitMs}ms`
+);

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -57,17 +57,23 @@ spec:
                   - -c
                   - |
                     set -e
-                    max_attempts=60
-                    attempt=1
-                    while [ "$attempt" -le "$max_attempts" ]; do
-                      if mongosh --quiet --host 127.0.0.1 --eval "const host = '${MONGO_INITDB_REPLICA_HOST}'; try { rs.status(); const conf = rs.conf(); if (conf.members[0].host !== host) { print('Updating replica set member host to ' + host); conf.members[0].host = host; conf.version++; rs.reconfig(conf); } else { print('Replica set already initialized with correct host'); } } catch (e) { print('Initializing replica set (' + e + ')'); rs.initiate({_id: 'rs0', members: [{_id: 0, host: host}]}); }" ; then
-                        exit 0
+                    echo "[postStart] Waiting for mongod to accept connections"
+                    attempts=0
+                    until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do
+                      attempts=$((attempts + 1))
+                      if [ "${attempts}" -ge 120 ]; then
+                        echo "[postStart] Timed out waiting for mongod" >&2
+                        exit 1
                       fi
                       sleep 1
-                      attempt=$((attempt + 1))
                     done
-                    echo "Failed to ensure replica set initialization after ${max_attempts} attempts"
-                    exit 1
+                    echo "[postStart] Ensuring replica set host: ${MONGO_INITDB_REPLICA_HOST}"
+                    if ! mongosh --quiet --host 127.0.0.1 /docker-entrypoint-initdb.d/00-replica-set.js; then
+                      rc=$?
+                      echo "[postStart] 00-replica-set.js failed with exit code ${rc}" >&2
+                      exit ${rc}
+                    fi
+                    echo "[postStart] Replica set script completed successfully"
           env:
             - name: POD_IP
               valueFrom: