Cleanup old networks and lock files after detecting reboot (#285)

Author: ashvindeodhar

cni/plugin.go

@@ -34,6 +34,18 @@ func NewPlugin(name, version string) (*Plugin, error) {
 		return nil, err
 	}
 
+	// Initialize logging.
+	log.SetName(plugin.Name)
+	log.SetLevel(log.LevelInfo)
+	err = log.SetTarget(log.TargetLogfile)
+	if err != nil {
+		log.Printf("[cni] Failed to configure logging, err:%v.\n", err)
+		return &Plugin{
+			Plugin:  plugin,
+			version: version,
+		}, err
+	}
+
 	return &Plugin{
 		Plugin:  plugin,
 		version: version,
@@ -45,14 +57,6 @@ func (plugin *Plugin) Initialize(config *common.PluginConfig) error {
 	// Initialize the base plugin.
 	plugin.Plugin.Initialize(config)
 
-	// Initialize logging.
-	log.SetName(plugin.Name)
-	log.SetLevel(log.LevelInfo)
-	err := log.SetTarget(log.TargetLogfile)
-	if err != nil {
-		log.Printf("[cni] Failed to configure logging, err:%v.\n", err)
-		return err
-	}
 	return nil
 }
 
@@ -164,6 +168,21 @@ func (plugin *Plugin) InitializeKeyValueStore(config *common.PluginConfig) error
 			log.Printf("[cni] Failed to create store: %v.", err)
 			return err
 		}
+
+		// Force unlock the json store if the lock file is left on the node after reboot
+		if lockFileModTime, err := plugin.Store.GetLockFileModificationTime(); err == nil {
+			rebootTime, err := platform.GetLastRebootTime()
+			log.Printf("[cni] reboot time %v storeLockFile mod time %v", rebootTime, lockFileModTime)
+			if err == nil && rebootTime.After(lockFileModTime) {
+				log.Printf("[cni] Detected Reboot")
+
+				if err := plugin.Store.Unlock(true); err != nil {
+					log.Printf("[cni] Failed to force unlock store due to error %v", err)
+				} else {
+					log.Printf("[cni] Force unlocked the store successfully")
+				}
+			}
+		}
 	}
 
 	// Acquire store lock.
@@ -180,7 +199,7 @@ func (plugin *Plugin) InitializeKeyValueStore(config *common.PluginConfig) error
 // Uninitialize key-value store
 func (plugin *Plugin) UninitializeKeyValueStore() error {
 	if plugin.Store != nil {
-		err := plugin.Store.Unlock()
+		err := plugin.Store.Unlock(false)
 		if err != nil {
 			log.Printf("[cni] Failed to unlock store: %v.", err)
 			return err

ipam/manager.go

@@ -105,7 +105,6 @@ func (am *addressManager) restore() error {
 	// Check if the VM is rebooted.
 	modTime, err := am.store.GetModificationTime()
 	if err == nil {
-
 		rebootTime, err := platform.GetLastRebootTime()
 		log.Printf("[ipam] reboot time %v store mod time %v", rebootTime, modTime)
 
@@ -147,6 +146,7 @@ func (am *addressManager) restore() error {
 		for _, as := range am.AddrSpaces {
 			for _, ap := range as.Pools {
 				ap.as = as
+				ap.RefCount = 0
 
 				for _, ar := range ap.Addresses {
 					ar.InUse = false

network/manager.go

@@ -129,6 +129,14 @@ func (nm *networkManager) restore() error {
 					return err
 				}
 
+				// Delete the networks left behind after reboot
+				for _, extIf := range nm.ExternalInterfaces {
+					for _, nw := range extIf.Networks {
+						log.Printf("[net] Deleting the network %s on reboot\n", nw.Id)
+						nm.deleteNetwork(nw.Id)
+					}
+				}
+
 				// Clear networkManager contents
 				nm.TimeStamp = time.Time{}
 				for extIfName := range nm.ExternalInterfaces {

platform/os_windows.go

@@ -65,7 +65,15 @@ func GetLastRebootTime() (time.Time, error) {
 	bootPM := strings.Contains(strings.Split(systemBootTime, " ")[2], "PM")
 
 	month := strings.Split(bootDate, "/")[0]
+	if len(month) < 2 {
+		month = "0" + month
+	}
+
 	day := strings.Split(bootDate, "/")[1]
+	if len(day) < 2 {
+		day = "0" + day
+	}
+
 	year := strings.Split(bootDate, "/")[2]
 	year = strings.Trim(year, ",")
 	hour := strings.Split(bootTime, ":")[0]

store/json.go

@@ -170,11 +170,11 @@ func (kvs *jsonFileStore) Lock(block bool) error {
 }
 
 // Unlock unlocks the store.
-func (kvs *jsonFileStore) Unlock() error {
+func (kvs *jsonFileStore) Unlock(forceUnlock bool) error {
 	kvs.Mutex.Lock()
 	defer kvs.Mutex.Unlock()
 
-	if !kvs.locked {
+	if !forceUnlock && !kvs.locked {
 		return ErrStoreNotLocked
 	}
 
@@ -202,3 +202,27 @@ func (kvs *jsonFileStore) GetModificationTime() (time.Time, error) {
 
 	return info.ModTime().UTC(), nil
 }
+
+// GetLockFileModificationTime returns the modification time of the lock file of the persistent store.
+func (kvs *jsonFileStore) GetLockFileModificationTime() (time.Time, error) {
+	kvs.Mutex.Lock()
+	defer kvs.Mutex.Unlock()
+
+	lockFileName := kvs.fileName + lockExtension
+
+	// Check if the file exists.
+	file, err := os.Open(lockFileName)
+	if err != nil {
+		return time.Time{}.UTC(), err
+	}
+
+	defer file.Close()
+
+	info, err := os.Stat(lockFileName)
+	if err != nil {
+		log.Printf("os.stat() for file %v failed: %v", lockFileName, err)
+		return time.Time{}.UTC(), err
+	}
+
+	return info.ModTime().UTC(), nil
+}

store/json_test.go

@@ -185,7 +185,7 @@ func TestLockingStoreGivesExclusiveAccess(t *testing.T) {
 	}
 
 	// Unlock the first store.
-	err = kvs.Unlock()
+	err = kvs.Unlock(false)
 	if err != nil {
 		t.Errorf("Failed to unlock first store: %v", err)
 	}
@@ -198,7 +198,7 @@ func TestLockingStoreGivesExclusiveAccess(t *testing.T) {
 	}
 
 	// Unlock the second store.
-	err = kvs2.Unlock()
+	err = kvs2.Unlock(false)
 	if err != nil {
 		t.Errorf("Failed to unlock second store: %v", err)
 	}

store/store.go

@@ -14,8 +14,9 @@ type KeyValueStore interface {
 	Write(key string, value interface{}) error
 	Flush() error
 	Lock(block bool) error
-	Unlock() error
+	Unlock(forceUnlock bool) error
 	GetModificationTime() (time.Time, error)
+	GetLockFileModificationTime() (time.Time, error)
 }
 
 var (