fix: Remove unreadable CNS state file upon CNS service start (#782)

This change removes unreadable state file ( azure-cns.json ) while starting the CNS service

Author: ashvindeodhar

cns/restserver/api_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"github.com/Azure/azure-container-networking/store"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -26,6 +27,7 @@ import (
 const (
 	defaultCnsURL   = "http://localhost:10090"
 	contentTypeJSON = "application/json"
+	cnsJsonFileName = "azure-cns.json"
 )
 
 type IPAddress struct {
@@ -125,7 +127,10 @@ func TestMain(m *testing.M) {
 	logger.InitLogger("testlogs", 0, 0, "./")
 
 	// Create the service.
-	startService()
+	if err = startService(); err != nil {
+		fmt.Printf("Failed to start CNS Service. Error: %v", err)
+		os.Exit(1)
+	}
 
 	// Setup mock nmagent server
 	u, err := url.Parse("tcp://" + nmagentEndpoint)
@@ -908,34 +913,50 @@ func setEnv(t *testing.T) *httptest.ResponseRecorder {
 	return w
 }
 
-func startService() {
+func startService() error {
 	var err error
 	// Create the service.
 	config := common.ServiceConfig{}
+	// Create the key value store.
+	if config.Store, err = store.NewJsonFileStore(cnsJsonFileName); err != nil {
+		logger.Errorf("Failed to create store file: %s, due to error %v\n", cnsJsonFileName, err)
+		return err
+	}
+
 	service, err = NewHTTPRestService(&config, fakes.NewFakeImdsClient(), fakes.NewFakeNMAgentClient())
 	if err != nil {
-		fmt.Printf("Failed to create CNS object %v\n", err)
-		os.Exit(1)
+		return err
 	}
 	svc = service.(*HTTPRestService)
 	svc.Name = "cns-test-server"
 	if err != nil {
 		logger.Errorf("Failed to create CNS object, err:%v.\n", err)
-		return
+		return err
 	}
 
 	svc.IPAMPoolMonitor = fakes.NewIPAMPoolMonitorFake()
 
 	if service != nil {
+		// Create empty azure-cns.json. CNS should start successfully by deleting this file
+		file, _ := os.Create(cnsJsonFileName)
+		file.Close()
+
 		err = service.Start(&config)
 		if err != nil {
 			logger.Errorf("Failed to start CNS, err:%v.\n", err)
-			return
+			return err
+		}
+
+		if _, err := os.Stat(cnsJsonFileName); err == nil || !os.IsNotExist(err) {
+			logger.Errorf("Failed to remove empty CNS state file: %s, err:%v", cnsJsonFileName, err)
+			return err
 		}
 	}
 
 	// Get the internal http mux as test hook.
 	mux = service.(*HTTPRestService).Listener.GetMux()
+
+	return nil
 }
 
 // IGNORE TEST AS IT IS FAILING. TODO:- Fix it https://msazure.visualstudio.com/One/_workitems/edit/7720083

cns/restserver/internalapi_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"reflect"
 	"strconv"
 	"testing"
@@ -545,5 +546,8 @@ func restartService() {
 	fmt.Println("Restart Service")
 
 	service.Stop()
-	startService()
+	if err := startService(); err != nil {
+		fmt.Printf("Failed to restart CNS Service. Error: %v", err)
+		os.Exit(1)
+	}
 }

cns/restserver/restserver.go

@@ -132,19 +132,13 @@ func NewHTTPRestService(config *common.ServiceConfig, imdsClientInterface imdscl
 
 // Start starts the CNS listener.
 func (service *HTTPRestService) Start(config *common.ServiceConfig) error {
-
 	err := service.Initialize(config)
 	if err != nil {
 		logger.Errorf("[Azure CNS]  Failed to initialize base service, err:%v.", err)
 		return err
 	}
 
-	err = service.restoreState()
-	if err != nil {
-		logger.Errorf("[Azure CNS]  Failed to restore service state, err:%v.", err)
-		return err
-	}
-
+	service.restoreState()
 	err = service.restoreNetworkState()
 	if err != nil {
 		logger.Errorf("[Azure CNS]  Failed to restore network state, err:%v.", err)

cns/restserver/util.go

@@ -74,13 +74,13 @@ func (service *HTTPRestService) saveState() error {
 }
 
 // restoreState restores CNS state from persistent store.
-func (service *HTTPRestService) restoreState() error {
+func (service *HTTPRestService) restoreState() {
 	logger.Printf("[Azure CNS] restoreState")
 
 	// Skip if a store is not provided.
 	if service.store == nil {
 		logger.Printf("[Azure CNS]  store not initialized.")
-		return nil
+		return
 	}
 
 	// Read any persisted state.
@@ -89,15 +89,16 @@ func (service *HTTPRestService) restoreState() error {
 		if err == store.ErrKeyNotFound {
 			// Nothing to restore.
 			logger.Printf("[Azure CNS]  No state to restore.\n")
-			return nil
+		} else {
+			logger.Errorf("[Azure CNS]  Failed to restore state, err:%v. Removing azure-cns.json", err)
+			service.store.Remove()
 		}
 
-		logger.Errorf("[Azure CNS]  Failed to restore state, err:%v\n", err)
-		return err
+		return
 	}
 
 	logger.Printf("[Azure CNS]  Restored state, %+v\n", service.state)
-	return nil
+	return
 }
 
 func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetworkContainerRequest) (int, string) {

store/json.go

@@ -148,7 +148,6 @@ func (kvs *jsonFileStore) flush() error {
 		return fmt.Errorf("temp file close failed with: %v", err)
 	}
 
-	log.Printf("renaming temp file %v to state file", tmpFileName)
 	// atomic replace
 	if err = platform.ReplaceFile(tmpFileName, kvs.fileName); err != nil {
 		return fmt.Errorf("rename temp file to state file failed:%v", err)
@@ -276,3 +275,11 @@ func (kvs *jsonFileStore) GetLockFileModificationTime() (time.Time, error) {
 func (kvs *jsonFileStore) GetLockFileName() string {
 	return kvs.fileName + lockExtension
 }
+
+func (kvs *jsonFileStore) Remove() {
+	kvs.Mutex.Lock()
+	if err := os.Remove(kvs.fileName); err != nil {
+		log.Errorf("could not remove file %s. Error: %v", kvs.fileName, err)
+	}
+	kvs.Mutex.Unlock()
+}

store/store.go

@@ -18,6 +18,7 @@ type KeyValueStore interface {
 	GetModificationTime() (time.Time, error)
 	GetLockFileModificationTime() (time.Time, error)
 	GetLockFileName() string
+	Remove()
 }
 
 var (

testutils/store_mock.go

@@ -47,3 +47,7 @@ func (store *KeyValueStoreMock) GetLockFileName() string {
 	return ""
 }
 
+func (store *KeyValueStoreMock) Remove() {
+	return
+}
+