Add more softassert.UnexpectedInternalErr (#8456)

## What changed?

Replace a few `serviceerror.NewInternal` with
`softassert.UnexpectedInternalErr`.

## Why?

Catch these issues in CICD and other testing.

## How did you test it?
- [ ] built
- [ ] run locally and tested manually
- [x] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

Author: stephanos

chasm/tree.go

@@ -658,7 +658,7 @@ func (n *Node) serialize() error {
 	case *persistencespb.ChasmNodeMetadata_PointerAttributes:
 		return n.serializePointerNode()
 	default:
-		return serviceerror.NewInternal("unknown node type")
+		return softassert.UnexpectedInternalErr(n.logger, "unknown node type", nil)
 	}
 }
 

service/history/api/startworkflow/api.go

@@ -21,6 +21,7 @@ import (
 	"go.temporal.io/server/common/persistence"
 	"go.temporal.io/server/common/persistence/visibility/manager"
 	"go.temporal.io/server/common/primitives"
+	"go.temporal.io/server/common/softassert"
 	"go.temporal.io/server/common/tasktoken"
 	"go.temporal.io/server/service/history/api"
 	"go.temporal.io/server/service/history/consts"
@@ -252,7 +253,11 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error)
 
 	workflowTaskInfo := mutableState.GetStartedWorkflowTask()
 	if s.requestEagerStart() && workflowTaskInfo == nil {
-		return nil, serviceerror.NewInternal("unexpected error: mutable state did not have a started workflow task")
+		return nil, softassert.UnexpectedInternalErr(
+			s.shardContext.GetLogger(),
+			"unexpected error: mutable state did not have a started workflow task",
+			nil,
+		)
 	}
 	workflowSnapshot, eventBatches, err := mutableState.CloseTransactionAsSnapshot(
 		historyi.TransactionPolicyActive,
@@ -261,7 +266,11 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error)
 		return nil, err
 	}
 	if len(eventBatches) != 1 {
-		return nil, serviceerror.NewInternal("unable to create 1st event batch")
+		return nil, softassert.UnexpectedInternalErr(
+			s.shardContext.GetLogger(),
+			"unable to create 1st event batch",
+			nil,
+		)
 	}
 
 	return &creationParams{

service/history/workflow/mutable_state_impl.go

@@ -2129,14 +2129,25 @@ func (ms *MutableStateImpl) IsTransientWorkflowTask() bool {
 
 func (ms *MutableStateImpl) ClearTransientWorkflowTask() error {
 	if !ms.HasStartedWorkflowTask() {
-		return serviceerror.NewInternal("cannot clear transient workflow task when task is missing")
+		return softassert.UnexpectedInternalErr(
+			ms.logger,
+			"cannot clear transient workflow task when task is missing",
+			nil,
+		)
 	}
 	if !ms.IsTransientWorkflowTask() {
-		return serviceerror.NewInternal("cannot clear transient workflow task when task is not transient")
+		return softassert.UnexpectedInternalErr(
+			ms.logger,
+			"cannot clear transient workflow task when task is not transient",
+			nil,
+		)
 	}
-	// this is transient workflow task
 	if ms.HasBufferedEvents() {
-		return serviceerror.NewInternal("cannot clear transient workflow task when there are buffered events")
+		return softassert.UnexpectedInternalErr(
+			ms.logger,
+			"cannot clear transient workflow task when there are buffered events",
+			nil,
+		)
 	}
 	// no buffered event
 	emptyWorkflowTaskInfo := &historyi.WorkflowTaskInfo{
@@ -6333,7 +6344,11 @@ func (ms *MutableStateImpl) CloseTransactionAsSnapshot(
 
 	if len(result.bufferEvents) > 0 {
 		// TODO do we need the functionality to generate snapshot with buffered events?
-		return nil, nil, serviceerror.NewInternal("cannot generate workflow snapshot with buffered events")
+		return nil, nil, softassert.UnexpectedInternalErr(
+			ms.logger,
+			"cannot generate workflow snapshot with buffered events",
+			nil,
+		)
 	}
 
 	workflowSnapshot := &persistence.WorkflowSnapshot{
@@ -7017,7 +7032,11 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks(
 
 	if transactionPolicy == historyi.TransactionPolicyPassive &&
 		len(ms.InsertTasks[tasks.CategoryReplication]) > 0 {
-		return serviceerror.NewInternal("should not generate replication task when close transaction as passive")
+		return softassert.UnexpectedInternalErr(
+			ms.logger,
+			"should not generate replication task when close transaction as passive",
+			nil,
+		)
 	}
 
 	return nil