Merge pull request #107 from bgokden/develop

Deletion of unused datasets

Author: bgokden

data/data.go

@@ -113,6 +113,7 @@ func (dt *Data) Close() error {
 		}
 	}
 	// return dt.DB.Close()
+	dt.DeletePath()
 	return nil
 }
 
@@ -123,6 +124,23 @@ func (dt *Data) DeletePath() error {
 	return nil
 }
 
+// Check if dataset is active
+func (dt *Data) CheckIfActive() bool {
+	if dt.N > 0 {
+		return true
+	}
+	if dt.Sources != nil && len(dt.Sources.Items()) > 0 {
+		sourceList := dt.Sources.Items()
+		for _, sourceItem := range sourceList { // Assumption is that random map runs are random enough
+			source := sourceItem.Object.(DataSource)
+			if source.GetDataInfo().N > 0 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // Run runs statistical calculation regularly
 func (dt *Data) Run() error {
 	if atomic.LoadInt32(&dt.Runs) >= 1 {

data/dataset.go

@@ -189,11 +189,20 @@ func (dts *Dataset) LoadIndex() error {
 
 func (dts *Dataset) ReloadRuns() error {
 	itemList := dts.DataList.Items()
-	for _, itemInterface := range itemList {
+	removeList := make([]string, 0, len(itemList))
+	for key, itemInterface := range itemList {
 		if item, ok := itemInterface.Object.(*Data); ok {
-			go item.Run()
+			if item.CheckIfActive() {
+				go item.Run()
+			} else {
+				item.Close()
+				removeList = append(removeList, key)
+			}
 		}
 	}
+	for _, key := range removeList {
+		dts.DataList.Delete(key)
+	}
 	return nil
 }
 

kubernetes.yml

@@ -44,7 +44,7 @@ spec:
   strategy:
     type: RollingUpdate
     rollingUpdate:
-      maxUnavailable: 10%
+      maxUnavailable: 5%
       maxSurge: 1
   selector:
     matchLabels:
@@ -56,7 +56,7 @@ spec:
     spec:
       containers:
         - name: veri
-          image: berkgokden/veri:v0.0.103
+          image: berkgokden/veri:v0.0.104
           args: ["serve", "--services=veriservice:10000"]
           imagePullPolicy: IfNotPresent
           ports: