Merge pull request #147 from thin-edge/feat-delete-orphaned-cloud-services

feat: delete orphaned services from previously created containers

Author: reubenmiller

cli/run/cmd.go

@@ -48,6 +48,7 @@ func NewRunCommand(cliContext cli.Cli) *cobra.Command {
 				ServiceName:        cliContext.GetServiceName(),
 				EnableMetrics:      cliContext.MetricsEnabled(),
 				DeleteFromCloud:    cliContext.DeleteFromCloud(),
+				DeleteOrphans:      cliContext.DeleteOrphans(),
 				EnableEngineEvents: cliContext.EngineEventsEnabled(),
 
 				HTTPHost:       cliContext.GetHTTPHost(),
@@ -158,6 +159,7 @@ func NewRunCommand(cliContext cli.Cli) *cobra.Command {
 	// Feature flags
 	viper.SetDefault("events.enabled", true)
 	viper.SetDefault("delete_from_cloud.enabled", true)
+	viper.SetDefault("delete_from_cloud.orphans", true)
 
 	// thin-edge.io services
 	viper.SetDefault("client.http.host", "127.0.0.1")

packaging/config.toml

@@ -72,6 +72,8 @@ enabled = true
 [delete_from_cloud]
 # Enable/Disable the deletion of services from the cloud (using REST API) when a container is removed
 enabled = true
+# delete orphaned services (related to containers) from the cloud
+orphans = true
 
 [registry]
 # Path to the file containing container registry credentials

pkg/app/app.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/docker/docker/api/types/events"
 	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/reubenmiller/go-c8y/pkg/c8y"
 	"github.com/thin-edge/tedge-container-plugin/pkg/container"
 	"github.com/thin-edge/tedge-container-plugin/pkg/tedge"
 )
@@ -68,6 +69,7 @@ type Config struct {
 	EnableMetrics      bool
 	EnableEngineEvents bool
 	DeleteFromCloud    bool
+	DeleteOrphans      bool
 
 	HTTPHost string
 	HTTPPort uint16
@@ -167,6 +169,42 @@ func (a *App) DeleteLegacyService(deleteFromCloud bool) {
 	}
 }
 
+// Delete any unclaimed/orphaned cloud services which haven't been registered with
+func (a *App) DeleteOrphanedCloudServices(tedgeEntities map[string]tedge.Entity) error {
+	extID, _, err := a.client.CumulocityClient.Identity.GetExternalID(context.Background(), "c8y_Serial", a.Device.ExternalID())
+	if err != nil {
+		slog.Warn("Could not lookup device's managed object by its external id.", "err", err, "externalId", a.Device.ExternalID())
+	}
+
+	mos, _, err := a.client.CumulocityClient.Inventory.GetChildAdditions(context.Background(), extID.ManagedObject.ID, &c8y.ManagedObjectOptions{
+		Query:             fmt.Sprintf("type eq 'c8y_Service' and (serviceType eq '%s' or serviceType eq '%s')", container.ContainerType, container.ContainerGroupType),
+		PaginationOptions: *c8y.NewPaginationOptions(100),
+	})
+	if err != nil {
+		return err
+	}
+
+	slog.Info("Found cloud services.", "count", len(mos.References))
+
+	for _, ref := range mos.References {
+		target := a.client.Target.Service(ref.ManagedObject.Name)
+		slog.Info("Check if service is registered locally.", "topic-id", target.TopicID)
+
+		if _, found := tedgeEntities[target.TopicID]; !found {
+			slog.Info("Found orphaned cloud service.", "service", ref.ManagedObject.Name, "type", ref.ManagedObject.Type, "moID", ref.ManagedObject.ID)
+
+			if _, respErr := a.client.CumulocityClient.Inventory.Delete(context.Background(), ref.ManagedObject.ID); respErr != nil {
+				slog.Warn("Could not delete orphaned cloud service.", "err", respErr)
+			} else {
+				slog.Info("Successfully deleted orphaned cloud service.", "moID", ref.ManagedObject.ID)
+			}
+		} else {
+			slog.Info("Service is registered locally.", "topic-id", target.TopicID)
+		}
+	}
+	return nil
+}
+
 func (a *App) Subscribe() error {
 	topic := tedge.GetTopic(*a.Device.Service("+"), "cmd", "health", "check")
 	slog.Info("Listening to commands on topic.", "topic", topic)
@@ -465,19 +503,23 @@ func (a *App) doUpdate(filterOptions container.FilterOptions) error {
 		delete(existingServices, target.TopicID)
 
 		// Register using HTTP API
-		resp, err := tedgeClient.TedgeAPI.CreateEntity(context.Background(), tedge.Entity{
+		entity := tedge.Entity{
 			TedgeType:     tedge.EntityTypeService,
 			TedgeTopicID:  target.TopicID,
 			Name:          item.Name,
 			Type:          item.ServiceType,
 			TedgeParentID: tedgeClient.Parent.TopicID,
-		})
+		}
+		resp, err := tedgeClient.TedgeAPI.CreateEntity(context.Background(), entity)
 
 		if err == nil {
 			slog.Info("Registered container.", "topic", target.Topic(), "url", resp.RawResponse.Request.URL.String(), "status_code", resp.RawResponse.Status)
 		} else {
 			slog.Error("Failed to register container.", "topic", target.Topic(), "err", err)
 		}
+
+		// Manually add to entities store for re-use later without having to fetch a new list of entities
+		entities[entity.TedgeTopicID] = entity
 	}
 
 	// Publish health messages
@@ -530,6 +572,9 @@ func (a *App) doUpdate(filterOptions container.FilterOptions) error {
 				slog.Warn("Failed to deregister entity.", "err", err)
 			}
 
+			// Update entities map
+			delete(entities, staleTopicID)
+
 			// mark targets for deletion from the cloud, but don't delete them yet to give time
 			// for thin-edge.io to process the status updates
 			markedForDeletion = append(markedForDeletion, *target)
@@ -556,5 +601,12 @@ func (a *App) doUpdate(filterOptions container.FilterOptions) error {
 		}
 	}
 
+	// Delete orphaned cloud services
+	if a.config.DeleteFromCloud && a.config.DeleteOrphans {
+		if err := a.DeleteOrphanedCloudServices(entities); err != nil {
+			slog.Warn("Could not delete orphaned cloud services.", "err", err)
+		}
+	}
+
 	return nil
 }

pkg/cli/cli.go

@@ -155,6 +155,10 @@ func (c *Cli) DeleteFromCloud() bool {
 	return viper.GetBool("delete_from_cloud.enabled")
 }
 
+func (c *Cli) DeleteOrphans() bool {
+	return viper.GetBool("delete_from_cloud.orphans")
+}
+
 func (c *Cli) GetHTTPHost() string {
 	return viper.GetString("client.http.host")
 }

tests/operations.robot

@@ -129,6 +129,32 @@ Manual container created and then killed
     Cumulocity.Should Have Services    name=manualapp3    service_type=container    min_count=0    max_count=0    timeout=10
 
 
+Remove Orphaned Cloud Services
+    [Documentation]    Orphaned cloud services can occur if entities are deregistered manually when the tedge-container-plugin
+    ...    service is not running.
+    ${operation}=    Cumulocity.Execute Shell Command    sudo tedge-container engine docker run -d --name manualapp4 busybox sh -c 'exec sleep infinity'
+    Operation Should Be SUCCESSFUL    ${operation}    timeout=60
+    Cumulocity.Should Have Services    name=manualapp4    service_type=container    status=up
+
+    Stop Service    tedge-container-plugin
+
+    # Uninstall
+    ${operation}=    Cumulocity.Execute Shell Command    sudo tedge-container engine docker rm manualapp4 --force
+    Operation Should Be SUCCESSFUL    ${operation}
+
+    # Clear container service locally
+    ${operation}=    Cumulocity.Execute Shell Command    sudo tedge mqtt pub -r 'te/device/main/service/manualapp4' ''; sleep 1; sudo tedge http delete '/te/v1/entities/device/main/service/manualapp4'
+    Operation Should Be SUCCESSFUL    ${operation}
+
+    # Confirm that the cloud's service status has not changed
+    # Note: This could change once thin-edge.io supports deleting the cloud entities
+    Sleep    1s
+    Cumulocity.Should Have Services    name=manualapp4    service_type=container    status=up
+
+    # Start the service, and check that the service has been removed (without the explicit service type defined)
+    Start Service    tedge-container-plugin
+    Cumulocity.Should Have Services    name=manualapp4    min_count=0    max_count=0    timeout=10
+
 *** Keywords ***
 
 Suite Setup