Updated network and address managers to ignore stale persisted state from previous boots

Author: ofiliz

common/utils.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os/exec"
+	"time"
 
 	"github.com/Azure/azure-container-networking/log"
 )
@@ -35,6 +36,26 @@ func LogNetworkInterfaces() {
 	}
 }
 
+// GetLastRebootTime returns the last time the system rebooted.
+func GetLastRebootTime() (time.Time, error) {
+	// Query last reboot time.
+	out, err := exec.Command("uptime", "-s").Output()
+	if err != nil {
+		log.Printf("Failed to query uptime, err:%v", err)
+		return time.Time{}, err
+	}
+
+	// Parse the output.
+	layout := "2006-01-02 15:04:05"
+	rebootTime, err := time.Parse(layout, string(out[:len(out)-1]))
+	if err != nil {
+		log.Printf("Failed to parse uptime, err:%v", err)
+		return time.Time{}, err
+	}
+
+	return rebootTime, nil
+}
+
 // ExecuteShellCommand executes a shell command.
 func ExecuteShellCommand(command string) error {
 	log.Debugf("[shell] %s", command)

ipam/manager.go

@@ -5,6 +5,7 @@ package ipam
 
 import (
 	"sync"
+	"time"
 
 	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/log"
@@ -19,6 +20,8 @@ const (
 
 // AddressManager manages the set of address spaces and pools allocated to containers.
 type addressManager struct {
+	Version    string
+	TimeStamp  time.Time
 	AddrSpaces map[string]*addressSpace `json:"AddressSpaces"`
 	store      store.KeyValueStore
 	source     addressConfigSource
@@ -64,6 +67,7 @@ func NewAddressManager() (AddressManager, error) {
 
 // Initialize configures address manager.
 func (am *addressManager) Initialize(config *common.PluginConfig, options map[string]interface{}) error {
+	am.Version = config.Version
 	am.store = config.Store
 	am.netApi, _ = config.NetApi.(network.NetworkManager)
 
@@ -91,8 +95,20 @@ func (am *addressManager) restore() error {
 		return nil
 	}
 
+	// After a reboot, all address resources are implicitly released.
+	// Ignore the persisted state if it is older than the last reboot time.
+	modTime, err := am.store.GetModificationTime()
+	if err == nil {
+		rebootTime, err := common.GetLastRebootTime()
+		if err == nil && rebootTime.After(modTime) {
+			log.Printf("[ipam] Ignoring stale state from store.")
+			log.Printf("[ipam] Store timestamp %v is older than boot timestamp %v.", modTime, rebootTime)
+			return nil
+		}
+	}
+
 	// Read any persisted state.
-	err := am.store.Read(storeKey, am)
+	err = am.store.Read(storeKey, am)
 	if err != nil {
 		if err == store.ErrKeyNotFound {
 			return nil
@@ -121,6 +137,9 @@ func (am *addressManager) save() error {
 		return nil
 	}
 
+	// Update time stamp.
+	am.TimeStamp = time.Now()
+
 	err := am.store.Write(storeKey, am)
 	if err == nil {
 		log.Printf("[ipam] Save succeeded.\n")

network/manager.go

@@ -5,6 +5,7 @@ package network
 
 import (
 	"sync"
+	"time"
 
 	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/log"
@@ -18,6 +19,8 @@ const (
 
 // NetworkManager manages the set of container networking resources.
 type networkManager struct {
+	Version            string
+	TimeStamp          time.Time
 	ExternalInterfaces map[string]*externalInterface
 	store              store.KeyValueStore
 	sync.Mutex
@@ -52,6 +55,7 @@ func NewNetworkManager() (NetworkManager, error) {
 
 // Initialize configures network manager.
 func (nm *networkManager) Initialize(config *common.PluginConfig) error {
+	nm.Version = config.Version
 	nm.store = config.Store
 
 	// Restore persisted state.
@@ -70,8 +74,20 @@ func (nm *networkManager) restore() error {
 		return nil
 	}
 
+	// After a reboot, all address resources are implicitly released.
+	// Ignore the persisted state if it is older than the last reboot time.
+	modTime, err := nm.store.GetModificationTime()
+	if err == nil {
+		rebootTime, err := common.GetLastRebootTime()
+		if err == nil && rebootTime.After(modTime) {
+			log.Printf("[net] Ignoring stale state from store.")
+			log.Printf("[net] Store timestamp %v is older than boot timestamp %v.", modTime, rebootTime)
+			return nil
+		}
+	}
+
 	// Read any persisted state.
-	err := nm.store.Read(storeKey, nm)
+	err = nm.store.Read(storeKey, nm)
 	if err != nil {
 		if err == store.ErrKeyNotFound {
 			// Considered successful.
@@ -101,6 +117,9 @@ func (nm *networkManager) save() error {
 		return nil
 	}
 
+	// Update time stamp.
+	nm.TimeStamp = time.Now()
+
 	err := nm.store.Write(storeKey, nm)
 	if err == nil {
 		log.Printf("[net] Save succeeded.\n")

store/json.go

@@ -102,7 +102,7 @@ func (kvs *jsonFileStore) Write(key string, value interface{}) error {
 	return kvs.flush()
 }
 
-// Flush commits in-memory state to backing store.
+// Flush commits in-memory state to persistent store.
 func (kvs *jsonFileStore) Flush() error {
 	kvs.Mutex.Lock()
 	defer kvs.Mutex.Unlock()
@@ -193,3 +193,13 @@ func (kvs *jsonFileStore) Unlock() error {
 
 	return nil
 }
+
+// GetModificationTime returns the modification time of the persistent store.
+func (kvs *jsonFileStore) GetModificationTime() (time.Time, error) {
+	info, err := os.Stat(kvs.fileName)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	return info.ModTime(), nil
+}

store/store.go

@@ -5,6 +5,7 @@ package store
 
 import (
 	"fmt"
+	"time"
 )
 
 // KeyValueStore represents a persistent store of (key,value) pairs.
@@ -14,6 +15,7 @@ type KeyValueStore interface {
 	Flush() error
 	Lock(block bool) error
 	Unlock() error
+	GetModificationTime() (time.Time, error)
 }
 
 var (