Add an e2e test to reproduce the issue that --force-new-cluster can't remove all members in a corner case

Signed-off-by: Benjamin Wang <[email protected]>

Author: ahrtr

tests/e2e/force_new_cluster_test.go

@@ -12,14 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cluster_proxy
+
 package e2e
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"go.etcd.io/bbolt"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
+	"go.etcd.io/etcd/server/v3/storage/datadir"
+	"go.etcd.io/etcd/server/v3/storage/schema"
 	"go.etcd.io/etcd/tests/v3/framework/config"
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 )
@@ -70,3 +78,63 @@ func TestForceNewCluster(t *testing.T) {
 		})
 	}
 }
+
+func TestForceNewCluster_MemberCount(t *testing.T) {
+	e2e.BeforeTest(t)
+
+	ctx := context.Background()
+
+	epc, promotedMembers := mustCreateNewClusterByPromotingMembers(t, e2e.CurrentVersion, 3, e2e.WithKeepDataDir(true))
+	require.Len(t, promotedMembers, 2)
+
+	// Wait for the backend TXN to sync/commit the data to disk, to ensure
+	// the consistent-index is persisted. Another way is to issue a snapshot
+	// command to forcibly commit the backend TXN.
+	time.Sleep(time.Second)
+
+	t.Log("Killing all the members")
+	require.NoError(t, epc.Kill())
+	require.NoError(t, epc.Wait(ctx))
+
+	m := epc.Procs[0]
+	t.Logf("Forcibly create a one-member cluster with member: %s", m.Config().Name)
+	m.Config().Args = append(m.Config().Args, "--force-new-cluster")
+	require.NoError(t, m.Start(ctx))
+
+	t.Log("Online checking the member count")
+	mresp, merr := m.Etcdctl().MemberList(ctx, false)
+	require.NoError(t, merr)
+	require.Len(t, mresp.Members, 1)
+
+	t.Log("Closing the member")
+	require.NoError(t, m.Close())
+	require.NoError(t, m.Wait(ctx))
+
+	t.Log("Offline checking the member count")
+	members := mustReadMembersFromBoltDB(t, m.Config().DataDirPath)
+	require.Len(t, members, 1)
+}
+
+func mustReadMembersFromBoltDB(t *testing.T, dataDir string) []*membership.Member {
+	dbPath := datadir.ToBackendFileName(dataDir)
+	db, err := bbolt.Open(dbPath, 0o400, &bbolt.Options{ReadOnly: true})
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, db.Close())
+	}()
+
+	var members []*membership.Member
+	_ = db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket(schema.Members.Name())
+		_ = b.ForEach(func(k, v []byte) error {
+			m := membership.Member{}
+			err := json.Unmarshal(v, &m)
+			require.NoError(t, err)
+			members = append(members, &m)
+			return nil
+		})
+		return nil
+	})
+
+	return members
+}

tests/framework/e2e/cluster.go

@@ -1002,6 +1002,38 @@ func (epc *EtcdProcessCluster) rollingStart(f func(ep EtcdProcess) error) error
 	return nil
 }
 
+func (epc *EtcdProcessCluster) Kill() (err error) {
+	for _, p := range epc.Procs {
+		if p == nil {
+			continue
+		}
+		if curErr := p.Kill(); curErr != nil {
+			if err != nil {
+				err = fmt.Errorf("%w; %w", err, curErr)
+			} else {
+				err = curErr
+			}
+		}
+	}
+	return err
+}
+
+func (epc *EtcdProcessCluster) Wait(ctx context.Context) error {
+	closedC := make(chan error, len(epc.Procs))
+	for i := range epc.Procs {
+		go func(n int) {
+			epc.Procs[n].Wait(ctx)
+			closedC <- epc.Procs[n].Wait(ctx)
+		}(i)
+	}
+	for range epc.Procs {
+		if err := <-closedC; err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (epc *EtcdProcessCluster) Stop() (err error) {
 	for _, p := range epc.Procs {
 		if p == nil {