Add installation deletion scheduling (#1110)

This change adds optional installation deletion scheduling logic
to the provisioner. Installations can now be created with a
configurable TTL which can be useful for quick testing and other
use-cases.

Installations with scheduled deletion will only be deleted if they
are not deletion-locked and the current time is past their scheduled
deletion time. Deletion-locked installations will be deleted once
their lock is removed if the scheduled deletion time has passed.

This is completely separate from the existing deleting pending
logic which allows for specifying how long an installation remains
in a deletion-pending state after it was scheduled for deletion.

Author: gabrieljackson

cmd/cloud/installation.go

@@ -37,6 +37,7 @@ func newCmdInstallation() *cobra.Command {
 	cmd.AddCommand(newCmdInstallationVolumeDelete())
 	cmd.AddCommand(newCmdInstallationUpdateDeletion())
 	cmd.AddCommand(newCmdInstallationCancelDeletion())
+	cmd.AddCommand(newCmdInstallationScheduleDeletion())
 	cmd.AddCommand(newCmdInstallationHibernate())
 	cmd.AddCommand(newCmdInstallationWakeup())
 	cmd.AddCommand(newCmdInstallationGet())
@@ -78,11 +79,16 @@ func executeInstallationCreateCmd(ctx context.Context, flags installationCreateF
 
 	envVarMap, err := parseEnvVarInput(flags.mattermostEnv, false)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to parse env var input")
 	}
 	priorityEnvVarMap, err := parseEnvVarInput(flags.priorityEnv, false)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to parse priority env var input")
+	}
+
+	var scheduledDeletionTime int64
+	if flags.scheduledDeletionTime > 0 {
+		scheduledDeletionTime = model.GetMillisAtTime(time.Now().Add(flags.scheduledDeletionTime))
 	}
 
 	request := &model.CreateInstallationRequest{
@@ -100,6 +106,7 @@ func executeInstallationCreateCmd(ctx context.Context, flags installationCreateF
 		PriorityEnv:               priorityEnvVarMap,
 		Annotations:               flags.annotations,
 		GroupSelectionAnnotations: flags.groupSelectionAnnotations,
+		ScheduledDeletionTime:     scheduledDeletionTime,
 	}
 
 	// For CLI to be backward compatible, if only one DNS is passed we use
@@ -359,6 +366,46 @@ func newCmdInstallationCancelDeletion() *cobra.Command {
 	return cmd
 }
 
+func newCmdInstallationScheduleDeletion() *cobra.Command {
+	var flags installationScheduledDeletionFlags
+
+	cmd := &cobra.Command{
+		Use:   "schedule-deletion",
+		Short: "Schedule an installation for future deletion.",
+		RunE: func(command *cobra.Command, args []string) error {
+			command.SilenceUsage = true
+			client := createClient(command.Context(), flags.clusterFlags)
+
+			request := &model.PatchInstallationScheduledDeletionRequest{}
+			if flags.scheduledDeletionTimeChanged {
+				var scheduledTimeMillis int64
+				if flags.scheduledDeletionTime > 0 {
+					scheduledTimeMillis = model.GetMillisAtTime(time.Now().Add(flags.scheduledDeletionTime))
+				}
+				request.ScheduledDeletionTime = &scheduledTimeMillis
+			}
+
+			if flags.dryRun {
+				return runDryRun(request)
+			}
+
+			installation, err := client.UpdateInstallationScheduledDeletion(flags.installationID, request)
+			if err != nil {
+				return errors.Wrap(err, "failed to update installation scheduled deletion")
+			}
+
+			return printJSON(installation)
+		},
+		PreRun: func(cmd *cobra.Command, args []string) {
+			flags.clusterFlags.addFlags(cmd)
+			flags.installationScheduledDeletionRequestOptionsChanged.addFlags(cmd)
+		},
+	}
+	flags.addFlags(cmd)
+
+	return cmd
+}
+
 func newCmdInstallationHibernate() *cobra.Command {
 	var flags installationHibernateFlags
 
@@ -802,7 +849,12 @@ func executeInstallationDeploymentReportCmd(ctx context.Context, flags installat
 	case model.InstallationStateDeleted:
 		output += fmt.Sprintf(" │ └ Deleted: %s\n", installation.DeletionDateString())
 	case model.InstallationStateDeletionPending:
-		output += fmt.Sprintf(" │ └ Scheduled Deletion: %s\n", installation.DeletionPendingExpiryCompleteTimeString())
+		output += fmt.Sprintf(" │ └ Deletion Pending Expiry: %s\n", installation.DeletionPendingExpiryCompleteTimeString())
+	default:
+		scheduledDeletion := installation.ScheculedDeletionCompleteTimeString()
+		if scheduledDeletion != "n/a" {
+			output += fmt.Sprintf(" │ └ Scheduled Deletion: %s\n", scheduledDeletion)
+		}
 	}
 	output += fmt.Sprintf(" ├ DNS: %s (primary)\n", installation.DNS) //nolint
 	if len(installation.DNSRecords) > 1 {

cmd/cloud/installation_flag.go

@@ -27,6 +27,7 @@ type installationCreateRequestOptions struct {
 	priorityEnv               []string
 	annotations               []string
 	groupSelectionAnnotations []string
+	scheduledDeletionTime     time.Duration
 }
 
 func (flags *installationCreateRequestOptions) addFlags(command *cobra.Command) {
@@ -44,6 +45,7 @@ func (flags *installationCreateRequestOptions) addFlags(command *cobra.Command)
 	command.Flags().StringArrayVar(&flags.priorityEnv, "priority-env", []string{}, "Env vars to add to the Mattermost App that take priority over group config. Accepts format: KEY_NAME=VALUE. Use the flag multiple times to set multiple env vars.")
 	command.Flags().StringArrayVar(&flags.annotations, "annotation", []string{}, "Additional annotations for the installation. Accepts multiple values, for example: '... --annotation abc --annotation def'")
 	command.Flags().StringArrayVar(&flags.groupSelectionAnnotations, "group-selection-annotation", []string{}, "Annotations for automatic group selection. Accepts multiple values, for example: '... --group-selection-annotation abc --group-selection-annotation def'")
+	command.Flags().DurationVar(&flags.scheduledDeletionTime, "scheduled-deletion-time", 0, "The time from now when the installation should be deleted. Use 0 for no scheduled deletion.")
 
 	_ = command.MarkFlagRequired("owner")
 }
@@ -524,3 +526,32 @@ type installationDeletionReportFlags struct {
 func (flags *installationDeletionReportFlags) addFlags(command *cobra.Command) {
 	command.Flags().IntVar(&flags.days, "days", 7, "The number of days include in the deletion report.")
 }
+
+type installationScheduledDeletionRequestOptions struct {
+	scheduledDeletionTime time.Duration
+}
+
+func (flags *installationScheduledDeletionRequestOptions) addFlags(command *cobra.Command) {
+	command.Flags().DurationVar(&flags.scheduledDeletionTime, "scheduled-deletion-time", 0, "The time from now when the installation should be deleted. Use 0 to cancel scheduled deletion.")
+}
+
+type installationScheduledDeletionRequestOptionsChanged struct {
+	scheduledDeletionTimeChanged bool
+}
+
+func (flags *installationScheduledDeletionRequestOptionsChanged) addFlags(command *cobra.Command) {
+	flags.scheduledDeletionTimeChanged = command.Flags().Changed("scheduled-deletion-time")
+}
+
+type installationScheduledDeletionFlags struct {
+	clusterFlags
+	installationScheduledDeletionRequestOptions
+	installationScheduledDeletionRequestOptionsChanged
+	installationID string
+}
+
+func (flags *installationScheduledDeletionFlags) addFlags(command *cobra.Command) {
+	flags.installationScheduledDeletionRequestOptions.addFlags(command)
+	command.Flags().StringVar(&flags.installationID, "installation", "", "The id of the installation to schedule deletion for.")
+	_ = command.MarkFlagRequired("installation")
+}

internal/api/installation.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"slices"
 	"time"
 
 	"github.com/mattermost/mattermost-cloud/internal/common"
@@ -52,6 +53,7 @@ func initInstallation(apiRouter *mux.Router, context *Context) {
 	installationRouter.Handle("", addContext(handleDeleteInstallation)).Methods("DELETE")
 	installationRouter.Handle("/deletion", addContext(handleUpdateInstallationDeletion)).Methods("PUT")
 	installationRouter.Handle("/deletion/cancel", addContext(handleCancelInstallationDeletion)).Methods("POST")
+	installationRouter.Handle("/deletion/schedule", addContext(handleUpdateInstallationScheduledDeletion)).Methods("PUT")
 	installationRouter.Handle("/annotations", addContext(handleAddInstallationAnnotations)).Methods("POST")
 	installationRouter.Handle("/annotation/{annotation-name}", addContext(handleDeleteInstallationAnnotation)).Methods("DELETE")
 
@@ -238,6 +240,7 @@ func handleCreateInstallation(c *Context, w http.ResponseWriter, r *http.Request
 		APISecurityLock:            createInstallationRequest.APISecurityLock,
 		MattermostEnv:              createInstallationRequest.MattermostEnv,
 		PriorityEnv:                createInstallationRequest.PriorityEnv,
+		ScheduledDeletionTime:      createInstallationRequest.ScheduledDeletionTime,
 		SingleTenantDatabaseConfig: createInstallationRequest.SingleTenantDatabaseConfig.ToDBConfig(createInstallationRequest.Database),
 		ExternalDatabaseConfig:     createInstallationRequest.ExternalDatabaseConfig.ToDBConfig(createInstallationRequest.Database),
 		CRVersion:                  model.DefaultCRVersion,
@@ -957,9 +960,17 @@ func handleCancelInstallationDeletion(c *Context, w http.ResponseWriter, r *http
 	}
 	defer unlockOnce()
 
-	err := updateInstallationState(c, installationDTO, newState)
+	// If the installation is scheduled for deletion, cancel the scheduled
+	// deletion so that it isn't immediately deleted again.
+	if installationDTO.ScheduledDeletionTime != 0 {
+		c.Logger.Debug("Removing scheduled deletion time as part of deletion cancellation")
+		installationDTO.ScheduledDeletionTime = 0
+	}
+	installationDTO.State = newState
+
+	err := c.Store.UpdateInstallation(installationDTO.Installation)
 	if err != nil {
-		c.Logger.WithError(err).Errorf("failed to update installation state to %q", newState)
+		c.Logger.WithError(err).Error("failed to update installation")
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
@@ -970,6 +981,53 @@ func handleCancelInstallationDeletion(c *Context, w http.ResponseWriter, r *http
 	w.WriteHeader(http.StatusAccepted)
 }
 
+// handleUpdateInstallationScheduledDeletion responds to PUT /api/installation/{installation}/deletion/schedule,
+// updating the scheduled deletion time of an installation.
+func handleUpdateInstallationScheduledDeletion(c *Context, w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	installationID := vars["installation"]
+	c.Logger = c.Logger.WithField("installation", installationID)
+
+	patchRequest, err := model.NewPatchInstallationScheduledDeletionRequestFromReader(r.Body)
+	if err != nil {
+		c.Logger.WithError(err).Error("failed to decode request")
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	installationDTO, status, unlockOnce := lockInstallation(c, installationID)
+	if status != 0 {
+		w.WriteHeader(status)
+		return
+	}
+	defer unlockOnce()
+
+	if installationDTO.APISecurityLock {
+		logSecurityLockConflict("installation", c.Logger)
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+
+	if slices.Contains(model.AllInstallationDeletionStates, installationDTO.State) {
+		c.Logger.Warnf("installation is in a deletion state, cannot update scheduled deletion time")
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if patchRequest.Apply(installationDTO.Installation) {
+		err := c.Store.UpdateInstallation(installationDTO.Installation)
+		if err != nil {
+			c.Logger.WithError(err).Error("failed to update installation")
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	outputJSON(c, w, installationDTO)
+}
+
 // handleAddInstallationAnnotations responds to POST /api/installation/{installation}/annotations,
 // adds the set of annotations to the Installation.
 func handleAddInstallationAnnotations(c *Context, w http.ResponseWriter, r *http.Request) {

internal/store/installation.go

@@ -30,7 +30,7 @@ func init() {
 			"Installation.CreateAt", "Installation.DeleteAt",
 			"Installation.DeletionPendingExpiry", "APISecurityLock", "LockAcquiredBy",
 			"LockAcquiredAt", "CRVersion", "Installation.DeletionLocked",
-			"AllowedIPRanges", "Volumes",
+			"AllowedIPRanges", "Volumes", "ScheduledDeletionTime",
 		).From(installationTable)
 }
 
@@ -346,6 +346,44 @@ func (sqlStore *SQLStore) GetUnlockedInstallationsPendingDeletion() ([]*model.In
 	return installations, nil
 }
 
+// GetUnlockedInstallationsWithScheduledDeletion returns unlocked installations
+// that are scheduled for deletion.
+func (sqlStore *SQLStore) GetUnlockedInstallationsWithScheduledDeletion() ([]*model.Installation, error) {
+	builder := installationSelect.
+		Where(sq.Eq{
+			"State": []string{model.InstallationStateStable, model.InstallationStateHibernating},
+		}).
+		Where("DeletionLocked = false").
+		Where("ScheduledDeletionTime > 0").
+		Where("LockAcquiredAt = 0").
+		OrderBy("CreateAt ASC")
+
+	var rawInstallationsOutput rawInstallations
+	err := sqlStore.selectBuilder(sqlStore.db, &rawInstallationsOutput, builder)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get installations pending deletion")
+	}
+
+	installations, err := rawInstallationsOutput.toInstallations()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, installation := range installations {
+		if !installation.IsInGroup() {
+			continue
+		}
+
+		group, err := sqlStore.GetGroup(*installation.GroupID)
+		if err != nil {
+			return nil, err
+		}
+		installation.MergeWithGroup(group, false)
+	}
+
+	return installations, nil
+}
+
 // GetSingleTenantDatabaseConfigForInstallation fetches single tenant database configuration
 // for specified installation.
 func (sqlStore *SQLStore) GetSingleTenantDatabaseConfigForInstallation(installationID string) (*model.SingleTenantDatabaseConfig, error) {
@@ -449,6 +487,7 @@ func (sqlStore *SQLStore) createInstallation(db execer, installation *model.Inst
 		"CreateAt":              installation.CreateAt,
 		"DeleteAt":              0,
 		"DeletionPendingExpiry": 0,
+		"ScheduledDeletionTime": installation.ScheduledDeletionTime,
 		"APISecurityLock":       installation.APISecurityLock,
 		"LockAcquiredBy":        nil,
 		"LockAcquiredAt":        0,
@@ -530,6 +569,7 @@ func (sqlStore *SQLStore) updateInstallation(db execer, installation *model.Inst
 			"State":                 installation.State,
 			"CRVersion":             installation.CRVersion,
 			"DeletionPendingExpiry": installation.DeletionPendingExpiry,
+			"ScheduledDeletionTime": installation.ScheduledDeletionTime,
 			"AllowedIPRanges":       installation.AllowedIPRanges,
 			"Volumes":               installation.Volumes,
 		}).

internal/store/migrations.go

@@ -2253,6 +2253,22 @@ var migrations = []migration{
 			return err
 		}
 
+		return nil
+	}},
+	{semver.MustParse("0.50.0"), semver.MustParse("0.51.0"), func(e execer) error {
+		_, err := e.Exec(`
+			ALTER TABLE Installation
+			ADD COLUMN ScheduledDeletionTime BIGINT NOT NULL DEFAULT '0';
+		`)
+		if err != nil {
+			return errors.Wrap(err, "failed to create ScheduledDeletionTime column")
+		}
+
+		_, err = e.Exec("ALTER TABLE Installation ALTER COLUMN ScheduledDeletionTime SET NOT NULL;")
+		if err != nil {
+			return errors.Wrap(err, "failed to remove not null constraint")
+		}
+
 		return nil
 	}},
 }