Fix a racing condition in shard snapshot status update (elastic#130302) (elastic#130368)

The enqueued task can run before the code reaches the status update
line. When it happens, the status update can set the status backwards.
This PR fixes it by moving the status update before enqueuing the task.

Resolves: elastic#129752
(cherry picked from commit 6b0d534)

# Conflicts:
#	muted-tests.yml

Author: ywangd

muted-tests.yml

@@ -512,9 +512,6 @@ tests:
 - class: org.elasticsearch.test.apmintegration.TracesApmIT
   method: testApmIntegration
   issue: https://github.com/elastic/elasticsearch/issues/129651
-- class: org.elasticsearch.snapshots.SnapshotShutdownIT
-  method: testSnapshotShutdownProgressTracker
-  issue: https://github.com/elastic/elasticsearch/issues/129752
 - class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
   method: testUpdatingFileBasedRoleAffectsAllProjects
   issue: https://github.com/elastic/elasticsearch/issues/129775

server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java

@@ -411,6 +411,7 @@ private void startNewShardSnapshots(String localNodeId, SnapshotsInProgress.Entr
                 entry.version(),
                 entry.startTime()
             );
+            snapshotStatus.updateStatusDescription("shard snapshot enqueuing to start");
             startShardSnapshotTaskRunner.enqueueTask(new ActionListener<>() {
                 @Override
                 public void onResponse(Releasable releasable) {
@@ -429,7 +430,6 @@ public void onFailure(Exception e) {
                     assert false : wrapperException; // impossible
                 }
             });
-            snapshotStatus.updateStatusDescription("shard snapshot enqueued to start");
         }
 
         // apply some backpressure by reserving one SNAPSHOT thread for the startup work