fix sort

Author: anton.voskresensky

commands/snapshots.go

@@ -8,6 +8,8 @@ import (
 	"osctl/pkg/logging"
 	"osctl/pkg/opensearch"
 	"osctl/pkg/utils"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -55,6 +57,7 @@ func runSnapshot(cmd *cobra.Command, args []string) error {
 	var indicesToSnapshot []string
 	repoGroups := map[string]utils.SnapshotGroup{}
 	var unknownIndices []string
+	indexSizes := make(map[string]int64)
 
 	systemConfigs := make([]config.IndexConfig, 0)
 	regularConfigs := make([]config.IndexConfig, 0)
@@ -81,6 +84,9 @@ func runSnapshot(cmd *cobra.Command, args []string) error {
 
 		for _, idx := range allSystemIndices {
 			indexName := idx.Index
+			if size, err := strconv.ParseInt(idx.Size, 10, 64); err == nil {
+				indexSizes[indexName] = size
+			}
 			indexConfig := utils.FindMatchingIndexConfig(indexName, systemConfigs)
 			if indexConfig != nil && indexConfig.Snapshot && !indexConfig.ManualSnapshot {
 				utils.AddIndexToSnapshotGroups(indexName, *indexConfig, today, repoGroups, &indicesToSnapshot)
@@ -102,6 +108,9 @@ func runSnapshot(cmd *cobra.Command, args []string) error {
 
 		for _, idx := range allRegularIndices {
 			indexName := idx.Index
+			if size, err := strconv.ParseInt(idx.Size, 10, 64); err == nil {
+				indexSizes[indexName] = size
+			}
 			indexConfig := utils.FindMatchingIndexConfig(indexName, regularConfigs)
 			if indexConfig != nil && indexConfig.Snapshot && !indexConfig.ManualSnapshot {
 				utils.AddIndexToSnapshotGroups(indexName, *indexConfig, today, repoGroups, &indicesToSnapshot)
@@ -136,6 +145,17 @@ func runSnapshot(cmd *cobra.Command, args []string) error {
 		})
 	}
 
+	sort.Slice(snapshotGroups, func(i, j int) bool {
+		var sizeI, sizeJ int64
+		for _, idx := range snapshotGroups[i].Indices {
+			sizeI += indexSizes[idx]
+		}
+		for _, idx := range snapshotGroups[j].Indices {
+			sizeJ += indexSizes[idx]
+		}
+		return sizeI > sizeJ
+	})
+
 	if cfg.GetDryRun() {
 		existingMain, err := utils.GetSnapshotsIgnore404(client, defaultRepo, "*"+today+"*")
 		if err != nil {
@@ -340,6 +360,16 @@ func runSnapshot(cmd *cobra.Command, args []string) error {
 				perRepo[repo] = append(perRepo[repo], g)
 			}
 			for repo, groups := range perRepo {
+				sort.Slice(groups, func(i, j int) bool {
+					var sizeI, sizeJ int64
+					for _, idx := range groups[i].Indices {
+						sizeI += indexSizes[idx]
+					}
+					for _, idx := range groups[j].Indices {
+						sizeJ += indexSizes[idx]
+					}
+					return sizeI > sizeJ
+				})
 				existing, err := utils.GetSnapshotsIgnore404(client, repo, "*"+today+"*")
 				if err != nil {
 					logger.Error(fmt.Sprintf("Failed to get snapshots from repo repo=%s error=%v", repo, err))

commands/snapshotsbackfill.go

@@ -9,6 +9,7 @@ import (
 	"osctl/pkg/opensearch"
 	"osctl/pkg/utils"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -195,13 +196,17 @@ func runSnapshotsBackfill(cmd *cobra.Command, args []string) error {
 			indicesSet[idx] = true
 		}
 
+		indexSizes := make(map[string]int64)
 		pattern := "*" + dateKey + "*"
 		indicesWithSize, err := client.GetIndicesWithFields(pattern, "index,ss", "ss:asc")
 		if err != nil {
 			logger.Warn(fmt.Sprintf("Failed to get indices with size for date date=%s error=%v, using unsorted list", dateKey, err))
 		} else {
 			var sortedIndices []string
 			for _, idx := range indicesWithSize {
+				if size, err := strconv.ParseInt(idx.Size, 10, 64); err == nil {
+					indexSizes[idx.Index] = size
+				}
 				if indicesSet[idx.Index] {
 					sortedIndices = append(sortedIndices, idx.Index)
 					delete(indicesSet, idx.Index)
@@ -312,6 +317,17 @@ func runSnapshotsBackfill(cmd *cobra.Command, args []string) error {
 				Kind:         "unknown",
 			})
 		}
+
+		sort.Slice(snapshotGroups, func(i, j int) bool {
+			var sizeI, sizeJ int64
+			for _, idx := range snapshotGroups[i].Indices {
+				sizeI += indexSizes[idx]
+			}
+			for _, idx := range snapshotGroups[j].Indices {
+				sizeJ += indexSizes[idx]
+			}
+			return sizeI < sizeJ
+		})
 		if len(snapshotGroups) == 0 && len(repoGroups) == 0 {
 			logger.Info(fmt.Sprintf("No snapshots to create for date date=%s", dateKey))
 			continue
@@ -382,6 +398,16 @@ func runSnapshotsBackfill(cmd *cobra.Command, args []string) error {
 			filteredPerRepo := map[string][]utils.SnapshotGroup{}
 			inProgressPerRepo := make([]string, 0)
 			for repo, groups := range perRepo {
+				sort.Slice(groups, func(i, j int) bool {
+					var sizeI, sizeJ int64
+					for _, idx := range groups[i].Indices {
+						sizeI += indexSizes[idx]
+					}
+					for _, idx := range groups[j].Indices {
+						sizeJ += indexSizes[idx]
+					}
+					return sizeI < sizeJ
+				})
 				existing, err := utils.GetSnapshotsIgnore404(client, repo, "*"+snapshotDate+"*")
 				if err != nil {
 					existing = nil
@@ -598,6 +624,16 @@ func runSnapshotsBackfill(cmd *cobra.Command, args []string) error {
 					perRepo[repo] = append(perRepo[repo], g)
 				}
 				for repo, groups := range perRepo {
+					sort.Slice(groups, func(i, j int) bool {
+						var sizeI, sizeJ int64
+						for _, idx := range groups[i].Indices {
+							sizeI += indexSizes[idx]
+						}
+						for _, idx := range groups[j].Indices {
+							sizeJ += indexSizes[idx]
+						}
+						return sizeI < sizeJ
+					})
 					existing, err := utils.GetSnapshotsIgnore404(client, repo, "*"+snapshotDate+"*")
 					if err != nil {
 						logger.Error(fmt.Sprintf("Failed to get snapshots from repo repo=%s error=%v", repo, err))

pkg/utils/indices.go

@@ -45,10 +45,7 @@ func ShouldSkipIndex(indexName string) bool {
 }
 
 func ShouldSkipIndexRetention(indexName string) bool {
-	if strings.HasPrefix(indexName, ".") {
-		return true
-	}
-	return false
+	return strings.HasPrefix(indexName, ".")
 }
 
 func FindMatchingIndexConfig(indexName string, indicesConfig []config.IndexConfig) *config.IndexConfig {