fail assigned-queued shards directly in processExternalChanges

Author: ywangd

server/src/main/java/org/elasticsearch/snapshots/SnapshotsServiceUtils.java

@@ -413,7 +413,7 @@ public static ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus>
             .entrySet()) {
             SnapshotsInProgress.ShardSnapshotStatus shardStatus = shardSnapshotEntry.getValue();
             ShardId shardId = snapshotEntry.shardId(shardSnapshotEntry.getKey());
-            if (shardStatus.equals(SnapshotsInProgress.ShardSnapshotStatus.UNASSIGNED_QUEUED)) {
+            if (shardStatus.isUnassignedQueued() || shardStatus.isAssignedQueued()) {
                 // this shard snapshot is waiting for a previous snapshot to finish execution for this shard
                 final SnapshotsInProgress.ShardSnapshotStatus knownFailure = knownFailures.get(shardSnapshotEntry.getKey());
                 if (knownFailure == null) {
@@ -422,14 +422,30 @@ public static ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus>
                         // shard became unassigned while queued after a delete or clone operation so we can fail as missing here
                         assert snapshotEntry.partial();
                         snapshotChanged = true;
-                        logger.debug("failing snapshot of shard [{}] because index got deleted", shardId);
-                        shards.put(shardId, SnapshotsInProgress.ShardSnapshotStatus.MISSING);
-                        knownFailures.put(shardSnapshotEntry.getKey(), SnapshotsInProgress.ShardSnapshotStatus.MISSING);
+                        logger.debug("failing snapshot of shard [{}] with status [{}] because index got deleted", shardId, shardStatus);
+                        final SnapshotsInProgress.ShardSnapshotStatus newShardStatus = shardStatus.isAssignedQueued()
+                            ? new SnapshotsInProgress.ShardSnapshotStatus(
+                                shardStatus.nodeId(),
+                                SnapshotsInProgress.ShardState.FAILED,
+                                shardStatus.generation(),
+                                "shard is deleted"
+                            )
+                            : SnapshotsInProgress.ShardSnapshotStatus.MISSING;
+                        shards.put(shardId, newShardStatus);
+                        knownFailures.put(shardSnapshotEntry.getKey(), newShardStatus);
                     } else {
                         // if no failure is known for the shard we keep waiting
                         shards.put(shardId, shardStatus);
                     }
                 } else {
+                    assert shardStatus.isAssignedQueued() == false
+                        : shardId
+                            + " with status "
+                            + shardStatus
+                            + " has unexpected known failure "
+                            + knownFailure
+                            + " in snapshot entry "
+                            + snapshotEntry;
                     // If a failure is known for an execution we waited on for this shard then we fail with the same exception here
                     // as well
                     snapshotChanged = true;